def _write_item_annotations(self, item):
key_id_data = self._key_id_data
item_id = cast(item.attributes.get('frame'), int)
if item_id is None or self._context._reindex:
item_id = len(key_id_data['videos']) + 1
item_key = str(uuid.uuid4())
key_id_data['videos'][item_key] = item_id
item_user_info = {
k: item.attributes.get(k, default_v)
for k, default_v in self._default_user_info.items()
}
item_ann_data = {
'description': item.attributes.get('description', ''),
'key': item_key,
'tags': [],
'objects': [],
'figures': [],
}
self._export_item_attributes(item, item_ann_data, item_user_info)
self._export_item_annotations(item, item_ann_data, item_user_info)
ann_path = osp.join(self._ann_dir, item.id + '.pcd.json')
os.makedirs(osp.dirname(ann_path), exist_ok=True)
dump_json_file(ann_path, item_ann_data, indent=True)
def save_meta_file(path, categories):
dataset_meta = {}
labels = [label.name for label in categories[AnnotationType.label]]
dataset_meta['labels'] = labels
if categories.get(AnnotationType.mask):
label_map = {}
segmentation_colors = []
for i, color in categories[AnnotationType.mask].colormap.items():
if color:
segmentation_colors.append(
[int(color[0]),
int(color[1]),
int(color[2])])
label_map[str(i)] = labels[i]
dataset_meta['label_map'] = label_map
dataset_meta['segmentation_colors'] = segmentation_colors
bg_label = find(categories[AnnotationType.mask].colormap.items(),
lambda x: x[1] == (0, 0, 0))
if bg_label is not None:
dataset_meta['background_label'] = str(bg_label[0])
meta_file = path
if osp.isdir(path):
meta_file = get_meta_file(path)
dump_json_file(meta_file, dataset_meta, indent=True)
def validate_command(args):
has_sep = '--' in args._positionals
if has_sep:
pos = args._positionals.index('--')
if 1 < pos:
raise argparse.ArgumentError(
None, message="Expected no more than 1 target argument")
else:
pos = 1
args.target = (args._positionals[:pos] or ['project'])[0]
args.extra_args = args._positionals[pos + has_sep:]
show_plugin_help = '-h' in args.extra_args or '--help' in args.extra_args
project = None
try:
project = scope_add(load_project(args.project_dir))
except ProjectNotFoundError:
if not show_plugin_help and args.project_dir:
raise
if project is not None:
env = project.env
else:
env = Environment()
try:
validator_type = env.validators[args.task]
except KeyError:
raise CliException("Validator type '%s' is not found" % args.task)
extra_args = validator_type.parse_cmdline(args.extra_args)
dataset, target_project = parse_full_revpath(args.target, project)
if target_project:
scope_add(target_project)
dst_file_name = f'validation-report'
if args.subset_name is not None:
dataset = dataset.get_subset(args.subset_name)
dst_file_name += f'-{args.subset_name}'
validator = validator_type(**extra_args)
report = validator.validate(dataset)
def _make_serializable(d):
for key, val in list(d.items()):
# tuple key to str
if isinstance(key, tuple):
d[str(key)] = val
d.pop(key)
if isinstance(val, dict):
_make_serializable(val)
_make_serializable(report)
dst_file = generate_next_file_name(dst_file_name, ext='.json')
log.info("Writing project validation results to '%s'" % dst_file)
dump_json_file(dst_file, report, indent=True, allow_numpy=True)
def detect_format_command(args):
project = None
try:
project = scope_add(load_project(args.project_dir))
except ProjectNotFoundError:
if args.project_dir:
raise
if project is not None:
env = project.env
else:
env = Environment()
report = {'rejected_formats': {}}
def rejection_callback(
format_name: str,
reason: RejectionReason,
human_message: str,
):
report['rejected_formats'][format_name] = {
'reason': reason.name,
'message': human_message,
}
detected_formats = detect_dataset_format(
((format_name, importer.detect)
for format_name, importer in env.importers.items.items()),
args.url,
rejection_callback=rejection_callback,
)
report['detected_formats'] = detected_formats
if len(detected_formats) == 1:
print(f"Detected format: {detected_formats[0]}")
elif len(detected_formats) == 0:
print("Unable to detect the format")
else:
print("Ambiguous dataset; detected the following formats:")
print()
for format_name in sorted(detected_formats):
print(f"- {format_name}")
if args.show_rejections:
print()
if report['rejected_formats']:
print("The following formats were rejected:")
print()
for format_name, rejection in sorted(
report['rejected_formats'].items()):
print(f"{format_name}:")
for line in rejection['message'].split('\n'):
print(f" {line}")
else:
print("No formats were rejected.")
if args.json_report:
dump_json_file(args.json_report, report, indent=True)
def write(self, path):
next_id = self._min_ann_id
for ann in self.annotations:
if not ann['id']:
ann['id'] = next_id
next_id += 1
dump_json_file(path, self._data)
def _write_key_id(self):
objects = self._objects
key_id_data = self._key_id_data
key_id_data['objects'] = {v: k for k, v in objects.items()}
dump_json_file(osp.join(self._save_dir, PointCloudPath.KEY_ID_FILE),
key_id_data,
indent=True)
def _write_meta(self):
for tag in self._tag_meta.values():
if tag['value_type'] is None:
tag['value_type'] = 'any_string'
tag['classes'] = list(tag['classes'])
self._meta_data['tags'] = list(self._tag_meta.values())
dump_json_file(osp.join(self._save_dir, PointCloudPath.META_FILE),
self._meta_data,
indent=True)
def write_meta_file(path, label_map):
# Uses custom format with extra fields
dataset_meta = {}
labels = []
labels_dict = {}
segmentation_colors = []
parts = {}
actions = {}
for i, (label_name, label_desc) in enumerate(label_map.items()):
labels.append(label_name)
if label_desc[0]:
labels_dict[str(i)] = label_name
segmentation_colors.append([
int(label_desc[0][0]),
int(label_desc[0][1]),
int(label_desc[0][2])
])
parts[str(i)] = label_desc[1]
actions[str(i)] = label_desc[2]
dataset_meta['labels'] = labels
if any(segmentation_colors):
dataset_meta['label_map'] = labels_dict
dataset_meta['segmentation_colors'] = segmentation_colors
bg_label = find(label_map.items(), lambda x: x[1] == (0, 0, 0))
if bg_label is not None:
dataset_meta['background_label'] = str(bg_label[0])
if any(parts):
dataset_meta['parts'] = parts
if any(actions):
dataset_meta['actions'] = actions
dump_json_file(get_meta_file(path), dataset_meta)
def save_merge_report(merger, path):
item_errors = OrderedDict()
source_errors = OrderedDict()
all_errors = []
for e in merger.errors:
if isinstance(e, DatasetQualityError):
item_errors[str(e.item_id)] = item_errors.get(str(e.item_id), 0) + 1
elif isinstance(e, DatasetMergeError):
for s in e.sources:
source_errors[str(s)] = source_errors.get(s, 0) + 1
item_errors[str(e.item_id)] = item_errors.get(str(e.item_id), 0) + 1
all_errors.append(str(e))
errors = OrderedDict([
('Item errors', item_errors),
('Source errors', source_errors),
('All errors', all_errors),
])
dump_json_file(path, errors, indent=True)
def stats_command(args):
project = None
try:
project = scope_add(load_project(args.project_dir))
except ProjectNotFoundError:
if args.project_dir:
raise
dataset, target_project = parse_full_revpath(args.target, project)
if target_project:
scope_add(target_project)
if args.subset:
dataset = dataset.get_subset(args.subset)
stats = {}
if args.image_stats:
stats.update(compute_image_statistics(dataset))
if args.ann_stats:
stats.update(compute_ann_statistics(dataset))
dst_file = generate_next_file_name('statistics', ext='.json')
log.info("Writing project statistics to '%s'" % dst_file)
dump_json_file(dst_file, stats, indent=True)
def _write_related_images(self, item):
img_dir = self._related_images_dir
for img in item.related_images:
name = osp.splitext(osp.basename(img.path))[0]
img_path = osp.join(img_dir, item.id + '_pcd',
name + self._find_image_ext(img))
if img.has_data:
img.save(img_path)
img_data = {
'name': osp.basename(img_path),
'meta': {
'sensorsData': {
'extrinsicMatrix':
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'intrinsicMatrix': [0, 0, 0, 0, 0, 0, 0, 0, 0]
}
}
}
dump_json_file(osp.join(img_dir, img_path + '.json'),
img_data,
indent=True)
def write(self, ann_file):
dump_json_file(ann_file, self._data)
def diff_command(args):
dst_dir = args.dst_dir
if dst_dir:
if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
raise CliException("Directory '%s' already exists "
"(pass --overwrite to overwrite)" % dst_dir)
else:
dst_dir = generate_next_file_name('diff')
dst_dir = osp.abspath(dst_dir)
if not osp.exists(dst_dir):
on_error_do(rmtree, dst_dir, ignore_errors=True)
os.makedirs(dst_dir)
project = None
try:
project = scope_add(load_project(args.project_dir))
except ProjectNotFoundError:
if args.project_dir:
raise
try:
if not args.second_target:
first_dataset = project.working_tree.make_dataset()
second_dataset, target_project = \
parse_full_revpath(args.first_target, project)
if target_project:
scope_add(target_project)
else:
first_dataset, target_project = \
parse_full_revpath(args.first_target, project)
if target_project:
scope_add(target_project)
second_dataset, target_project = \
parse_full_revpath(args.second_target, project)
if target_project:
scope_add(target_project)
except Exception as e:
raise CliException(str(e))
if args.method is ComparisonMethod.equality:
if args.ignore_field:
args.ignore_field = eq_default_if
comparator = ExactComparator(match_images=args.match_images,
ignored_fields=args.ignore_field,
ignored_attrs=args.ignore_attr,
ignored_item_attrs=args.ignore_item_attr)
matches, mismatches, a_extra, b_extra, errors = \
comparator.compare_datasets(first_dataset, second_dataset)
output = {
"mismatches": mismatches,
"a_extra_items": sorted(a_extra),
"b_extra_items": sorted(b_extra),
"errors": errors,
}
if args.all:
output["matches"] = matches
output_file = osp.join(
dst_dir,
generate_next_file_name('diff', ext='.json', basedir=dst_dir))
log.info("Saving diff to '%s'" % output_file)
dump_json_file(output_file, output, indent=True)
print("Found:")
print("The first project has %s unmatched items" % len(a_extra))
print("The second project has %s unmatched items" % len(b_extra))
print("%s item conflicts" % len(errors))
print("%s matching annotations" % len(matches))
print("%s mismatching annotations" % len(mismatches))
elif args.method is ComparisonMethod.distance:
comparator = DistanceComparator(iou_threshold=args.iou_thresh)
with DiffVisualizer(save_dir=dst_dir,
comparator=comparator,
output_format=args.format) as visualizer:
log.info("Saving diff to '%s'" % dst_dir)
visualizer.save(first_dataset, second_dataset)
return 0
def write(self, path):
dump_json_file(path, self._data)
