def _write_item_annotations(self, item): key_id_data = self._key_id_data item_id = cast(item.attributes.get('frame'), int) if item_id is None or self._context._reindex: item_id = len(key_id_data['videos']) + 1 item_key = str(uuid.uuid4()) key_id_data['videos'][item_key] = item_id item_user_info = { k: item.attributes.get(k, default_v) for k, default_v in self._default_user_info.items() } item_ann_data = { 'description': item.attributes.get('description', ''), 'key': item_key, 'tags': [], 'objects': [], 'figures': [], } self._export_item_attributes(item, item_ann_data, item_user_info) self._export_item_annotations(item, item_ann_data, item_user_info) ann_path = osp.join(self._ann_dir, item.id + '.pcd.json') os.makedirs(osp.dirname(ann_path), exist_ok=True) dump_json_file(ann_path, item_ann_data, indent=True)
def save_meta_file(path, categories): dataset_meta = {} labels = [label.name for label in categories[AnnotationType.label]] dataset_meta['labels'] = labels if categories.get(AnnotationType.mask): label_map = {} segmentation_colors = [] for i, color in categories[AnnotationType.mask].colormap.items(): if color: segmentation_colors.append( [int(color[0]), int(color[1]), int(color[2])]) label_map[str(i)] = labels[i] dataset_meta['label_map'] = label_map dataset_meta['segmentation_colors'] = segmentation_colors bg_label = find(categories[AnnotationType.mask].colormap.items(), lambda x: x[1] == (0, 0, 0)) if bg_label is not None: dataset_meta['background_label'] = str(bg_label[0]) meta_file = path if osp.isdir(path): meta_file = get_meta_file(path) dump_json_file(meta_file, dataset_meta, indent=True)
def validate_command(args): has_sep = '--' in args._positionals if has_sep: pos = args._positionals.index('--') if 1 < pos: raise argparse.ArgumentError( None, message="Expected no more than 1 target argument") else: pos = 1 args.target = (args._positionals[:pos] or ['project'])[0] args.extra_args = args._positionals[pos + has_sep:] show_plugin_help = '-h' in args.extra_args or '--help' in args.extra_args project = None try: project = scope_add(load_project(args.project_dir)) except ProjectNotFoundError: if not show_plugin_help and args.project_dir: raise if project is not None: env = project.env else: env = Environment() try: validator_type = env.validators[args.task] except KeyError: raise CliException("Validator type '%s' is not found" % args.task) extra_args = validator_type.parse_cmdline(args.extra_args) dataset, target_project = parse_full_revpath(args.target, project) if target_project: scope_add(target_project) dst_file_name = f'validation-report' if args.subset_name is not None: dataset = dataset.get_subset(args.subset_name) dst_file_name += f'-{args.subset_name}' validator = validator_type(**extra_args) report = validator.validate(dataset) def _make_serializable(d): for key, val in list(d.items()): # tuple key to str if isinstance(key, tuple): d[str(key)] = val d.pop(key) if isinstance(val, dict): _make_serializable(val) _make_serializable(report) dst_file = generate_next_file_name(dst_file_name, ext='.json') log.info("Writing project validation results to '%s'" % dst_file) dump_json_file(dst_file, report, indent=True, allow_numpy=True)
def detect_format_command(args): project = None try: project = scope_add(load_project(args.project_dir)) except ProjectNotFoundError: if args.project_dir: raise if project is not None: env = project.env else: env = Environment() report = {'rejected_formats': {}} def rejection_callback( format_name: str, reason: RejectionReason, human_message: str, ): report['rejected_formats'][format_name] = { 'reason': reason.name, 'message': human_message, } detected_formats = detect_dataset_format( ((format_name, importer.detect) for format_name, importer in env.importers.items.items()), args.url, rejection_callback=rejection_callback, ) report['detected_formats'] = detected_formats if len(detected_formats) == 1: print(f"Detected format: {detected_formats[0]}") elif len(detected_formats) == 0: print("Unable to detect the format") else: print("Ambiguous dataset; detected the following formats:") print() for format_name in sorted(detected_formats): print(f"- {format_name}") if args.show_rejections: print() if report['rejected_formats']: print("The following formats were rejected:") print() for format_name, rejection in sorted( report['rejected_formats'].items()): print(f"{format_name}:") for line in rejection['message'].split('\n'): print(f" {line}") else: print("No formats were rejected.") if args.json_report: dump_json_file(args.json_report, report, indent=True)
def write(self, path): next_id = self._min_ann_id for ann in self.annotations: if not ann['id']: ann['id'] = next_id next_id += 1 dump_json_file(path, self._data)
def _write_key_id(self): objects = self._objects key_id_data = self._key_id_data key_id_data['objects'] = {v: k for k, v in objects.items()} dump_json_file(osp.join(self._save_dir, PointCloudPath.KEY_ID_FILE), key_id_data, indent=True)
def _write_meta(self): for tag in self._tag_meta.values(): if tag['value_type'] is None: tag['value_type'] = 'any_string' tag['classes'] = list(tag['classes']) self._meta_data['tags'] = list(self._tag_meta.values()) dump_json_file(osp.join(self._save_dir, PointCloudPath.META_FILE), self._meta_data, indent=True)
def write_meta_file(path, label_map): # Uses custom format with extra fields dataset_meta = {} labels = [] labels_dict = {} segmentation_colors = [] parts = {} actions = {} for i, (label_name, label_desc) in enumerate(label_map.items()): labels.append(label_name) if label_desc[0]: labels_dict[str(i)] = label_name segmentation_colors.append([ int(label_desc[0][0]), int(label_desc[0][1]), int(label_desc[0][2]) ]) parts[str(i)] = label_desc[1] actions[str(i)] = label_desc[2] dataset_meta['labels'] = labels if any(segmentation_colors): dataset_meta['label_map'] = labels_dict dataset_meta['segmentation_colors'] = segmentation_colors bg_label = find(label_map.items(), lambda x: x[1] == (0, 0, 0)) if bg_label is not None: dataset_meta['background_label'] = str(bg_label[0]) if any(parts): dataset_meta['parts'] = parts if any(actions): dataset_meta['actions'] = actions dump_json_file(get_meta_file(path), dataset_meta)
def save_merge_report(merger, path): item_errors = OrderedDict() source_errors = OrderedDict() all_errors = [] for e in merger.errors: if isinstance(e, DatasetQualityError): item_errors[str(e.item_id)] = item_errors.get(str(e.item_id), 0) + 1 elif isinstance(e, DatasetMergeError): for s in e.sources: source_errors[str(s)] = source_errors.get(s, 0) + 1 item_errors[str(e.item_id)] = item_errors.get(str(e.item_id), 0) + 1 all_errors.append(str(e)) errors = OrderedDict([ ('Item errors', item_errors), ('Source errors', source_errors), ('All errors', all_errors), ]) dump_json_file(path, errors, indent=True)
def stats_command(args): project = None try: project = scope_add(load_project(args.project_dir)) except ProjectNotFoundError: if args.project_dir: raise dataset, target_project = parse_full_revpath(args.target, project) if target_project: scope_add(target_project) if args.subset: dataset = dataset.get_subset(args.subset) stats = {} if args.image_stats: stats.update(compute_image_statistics(dataset)) if args.ann_stats: stats.update(compute_ann_statistics(dataset)) dst_file = generate_next_file_name('statistics', ext='.json') log.info("Writing project statistics to '%s'" % dst_file) dump_json_file(dst_file, stats, indent=True)
def _write_related_images(self, item): img_dir = self._related_images_dir for img in item.related_images: name = osp.splitext(osp.basename(img.path))[0] img_path = osp.join(img_dir, item.id + '_pcd', name + self._find_image_ext(img)) if img.has_data: img.save(img_path) img_data = { 'name': osp.basename(img_path), 'meta': { 'sensorsData': { 'extrinsicMatrix': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'intrinsicMatrix': [0, 0, 0, 0, 0, 0, 0, 0, 0] } } } dump_json_file(osp.join(img_dir, img_path + '.json'), img_data, indent=True)
def write(self, ann_file): dump_json_file(ann_file, self._data)
def diff_command(args): dst_dir = args.dst_dir if dst_dir: if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir): raise CliException("Directory '%s' already exists " "(pass --overwrite to overwrite)" % dst_dir) else: dst_dir = generate_next_file_name('diff') dst_dir = osp.abspath(dst_dir) if not osp.exists(dst_dir): on_error_do(rmtree, dst_dir, ignore_errors=True) os.makedirs(dst_dir) project = None try: project = scope_add(load_project(args.project_dir)) except ProjectNotFoundError: if args.project_dir: raise try: if not args.second_target: first_dataset = project.working_tree.make_dataset() second_dataset, target_project = \ parse_full_revpath(args.first_target, project) if target_project: scope_add(target_project) else: first_dataset, target_project = \ parse_full_revpath(args.first_target, project) if target_project: scope_add(target_project) second_dataset, target_project = \ parse_full_revpath(args.second_target, project) if target_project: scope_add(target_project) except Exception as e: raise CliException(str(e)) if args.method is ComparisonMethod.equality: if args.ignore_field: args.ignore_field = eq_default_if comparator = ExactComparator(match_images=args.match_images, ignored_fields=args.ignore_field, ignored_attrs=args.ignore_attr, ignored_item_attrs=args.ignore_item_attr) matches, mismatches, a_extra, b_extra, errors = \ comparator.compare_datasets(first_dataset, second_dataset) output = { "mismatches": mismatches, "a_extra_items": sorted(a_extra), "b_extra_items": sorted(b_extra), "errors": errors, } if args.all: output["matches"] = matches output_file = osp.join( dst_dir, generate_next_file_name('diff', ext='.json', basedir=dst_dir)) log.info("Saving diff to '%s'" % output_file) dump_json_file(output_file, output, indent=True) print("Found:") print("The first project has %s unmatched items" % len(a_extra)) print("The second project has %s unmatched items" % len(b_extra)) print("%s item conflicts" % len(errors)) print("%s matching annotations" % len(matches)) print("%s mismatching annotations" % len(mismatches)) elif args.method is ComparisonMethod.distance: comparator = DistanceComparator(iou_threshold=args.iou_thresh) with DiffVisualizer(save_dir=dst_dir, comparator=comparator, output_format=args.format) as visualizer: log.info("Saving diff to '%s'" % dst_dir) visualizer.save(first_dataset, second_dataset) return 0
def write(self, path): dump_json_file(path, self._data)