def test_can_find_bbox_with_wrong_label(self):
detections = 3
class_count = 2
item1 = DatasetItem(id=1,
annotations=[
Bbox(i * 10, 10, 10, 10, label=i)
for i in range(detections)
])
item2 = DatasetItem(id=2,
annotations=[
Bbox(i * 10,
10,
10,
10,
label=(i + 1) % class_count)
for i in range(detections)
])
iou_thresh = 0.5
comp = DistanceComparator(iou_threshold=iou_thresh)
result = comp.match_boxes(item1, item2)
matches, mispred, a_greater, b_greater = result
self.assertEqual(len(item1.annotations), len(mispred))
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(0, len(matches))
for a_bbox, b_bbox in mispred:
self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label)
def test_can_find_missing_boxes(self):
detections = 3
class_count = 2
item1 = DatasetItem(id=1,
annotations=[
Bbox(i * 10, 10, 10, 10, label=i)
for i in range(detections) if i % 2 == 0
])
item2 = DatasetItem(id=2,
annotations=[
Bbox(i * 10,
10,
10,
10,
label=(i + 1) % class_count)
for i in range(detections) if i % 2 == 1
])
iou_thresh = 0.5
comp = DistanceComparator(iou_threshold=iou_thresh)
result = comp.match_boxes(item1, item2)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(len(item1.annotations), len(a_greater))
self.assertEqual(len(item2.annotations), len(b_greater))
self.assertEqual(0, len(matches))
def diff_command(args):
first_project = load_project(args.project_dir)
second_project = load_project(args.other_project_dir)
comparator = DistanceComparator(iou_threshold=args.iou_thresh)
dst_dir = args.dst_dir
if dst_dir:
if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
raise CliException("Directory '%s' already exists "
"(pass --overwrite to overwrite)" % dst_dir)
else:
dst_dir = generate_next_file_name('%s-%s-diff' % (
first_project.config.project_name,
second_project.config.project_name)
)
dst_dir = osp.abspath(dst_dir)
log.info("Saving diff to '%s'" % dst_dir)
if not osp.exists(dst_dir):
on_error.do(shutil.rmtree, dst_dir, ignore_errors=True)
with DatasetDiffVisualizer(save_dir=dst_dir, comparator=comparator,
output_format=args.visualizer) as visualizer:
visualizer.save(
first_project.make_dataset(),
second_project.make_dataset())
return 0
def test_no_bbox_diff_with_same_item(self):
detections = 3
anns = [Bbox(i * 10, 10, 10, 10, label=i) for i in range(detections)]
item = DatasetItem(id=0, annotations=anns)
iou_thresh = 0.5
comp = DistanceComparator(iou_threshold=iou_thresh)
result = comp.match_boxes(item, item)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len(item.annotations), len(matches))
for a_bbox, b_bbox in matches:
self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
self.assertEqual(a_bbox.label, b_bbox.label)
def test_no_label_diff_with_same_item(self):
detections = 3
anns = [Label(i) for i in range(detections)]
item = DatasetItem(id=1, annotations=anns)
result = DistanceComparator().match_labels(item, item)
matches, a_greater, b_greater = result
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len(item.annotations), len(matches))
def test_can_find_wrong_label(self):
item1 = DatasetItem(id=1, annotations=[
Label(0),
Label(1),
Label(2),
])
item2 = DatasetItem(id=2, annotations=[
Label(2),
Label(3),
Label(4),
])
result = DistanceComparator().match_labels(item1, item2)
matches, a_greater, b_greater = result
self.assertEqual(2, len(a_greater))
self.assertEqual(2, len(b_greater))
self.assertEqual(1, len(matches))
def test_can_match_points(self):
item1 = DatasetItem(id=1,
annotations=[
Points([1, 2, 2, 0, 1, 1], label=0),
Points([3, 5, 5, 7, 5, 3], label=0),
])
item2 = DatasetItem(id=2,
annotations=[
Points([1.5, 2, 2, 0.5, 1, 1.5], label=0),
Points([5, 7, 7, 7, 7, 5], label=0),
])
result = DistanceComparator().match_points(item1, item2)
matches, mismatches, a_greater, b_greater = result
self.assertEqual(1, len(a_greater))
self.assertEqual(1, len(b_greater))
self.assertEqual(1, len(matches))
self.assertEqual(0, len(mismatches))
def diff_command(args):
dst_dir = args.dst_dir
if dst_dir:
if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
raise CliException("Directory '%s' already exists "
"(pass --overwrite to overwrite)" % dst_dir)
else:
dst_dir = generate_next_file_name('diff')
dst_dir = osp.abspath(dst_dir)
if not osp.exists(dst_dir):
on_error_do(rmtree, dst_dir, ignore_errors=True)
os.makedirs(dst_dir)
project = None
try:
project = scope_add(load_project(args.project_dir))
except ProjectNotFoundError:
if args.project_dir:
raise
try:
if not args.second_target:
first_dataset = project.working_tree.make_dataset()
second_dataset, target_project = \
parse_full_revpath(args.first_target, project)
if target_project:
scope_add(target_project)
else:
first_dataset, target_project = \
parse_full_revpath(args.first_target, project)
if target_project:
scope_add(target_project)
second_dataset, target_project = \
parse_full_revpath(args.second_target, project)
if target_project:
scope_add(target_project)
except Exception as e:
raise CliException(str(e))
if args.method is ComparisonMethod.equality:
if args.ignore_field:
args.ignore_field = eq_default_if
comparator = ExactComparator(match_images=args.match_images,
ignored_fields=args.ignore_field,
ignored_attrs=args.ignore_attr,
ignored_item_attrs=args.ignore_item_attr)
matches, mismatches, a_extra, b_extra, errors = \
comparator.compare_datasets(first_dataset, second_dataset)
output = {
"mismatches": mismatches,
"a_extra_items": sorted(a_extra),
"b_extra_items": sorted(b_extra),
"errors": errors,
}
if args.all:
output["matches"] = matches
output_file = osp.join(
dst_dir,
generate_next_file_name('diff', ext='.json', basedir=dst_dir))
log.info("Saving diff to '%s'" % output_file)
dump_json_file(output_file, output, indent=True)
print("Found:")
print("The first project has %s unmatched items" % len(a_extra))
print("The second project has %s unmatched items" % len(b_extra))
print("%s item conflicts" % len(errors))
print("%s matching annotations" % len(matches))
print("%s mismatching annotations" % len(mismatches))
elif args.method is ComparisonMethod.distance:
comparator = DistanceComparator(iou_threshold=args.iou_thresh)
with DiffVisualizer(save_dir=dst_dir,
comparator=comparator,
output_format=args.format) as visualizer:
log.info("Saving diff to '%s'" % dst_dir)
visualizer.save(first_dataset, second_dataset)
return 0
def test_can_compare_projects(self): # just a smoke test
label_categories1 = LabelCategories.from_iterable(['x', 'a', 'b', 'y'])
mask_categories1 = MaskCategories.make_default(len(label_categories1))
point_categories1 = PointsCategories()
for index, _ in enumerate(label_categories1.items):
point_categories1.add(index, ['cat1', 'cat2'], joints=[[0, 1]])
dataset1 = Dataset.from_iterable(
[
DatasetItem(
id=100,
subset='train',
image=np.ones((10, 6, 3)),
annotations=[
Caption('hello', id=1),
Caption('world', id=2, group=5),
Label(2, id=3, attributes={
'x': 1,
'y': '2',
}),
Bbox(1,
2,
3,
4,
label=0,
id=4,
z_order=1,
attributes={
'score': 1.0,
}),
Bbox(5, 6, 7, 8, id=5, group=5),
Points([1, 2, 2, 0, 1, 1], label=0, id=5, z_order=4),
Mask(label=3, id=5, z_order=2, image=np.ones((2, 3))),
]),
DatasetItem(id=21,
subset='train',
annotations=[
Caption('test'),
Label(2),
Bbox(1, 2, 3, 4, label=2, id=42, group=42)
]),
DatasetItem(
id=2,
subset='val',
annotations=[
PolyLine([1, 2, 3, 4, 5, 6, 7, 8], id=11, z_order=1),
Polygon([1, 2, 3, 4, 5, 6, 7, 8], id=12, z_order=4),
]),
DatasetItem(
id=42, subset='test', attributes={
'a1': 5,
'a2': '42'
}),
DatasetItem(id=42),
DatasetItem(id=43, image=Image(path='1/b/c.qq', size=(2, 4))),
],
categories={
AnnotationType.label: label_categories1,
AnnotationType.mask: mask_categories1,
AnnotationType.points: point_categories1,
})
label_categories2 = LabelCategories.from_iterable(['a', 'b', 'x', 'y'])
mask_categories2 = MaskCategories.make_default(len(label_categories2))
point_categories2 = PointsCategories()
for index, _ in enumerate(label_categories2.items):
point_categories2.add(index, ['cat1', 'cat2'], joints=[[0, 1]])
dataset2 = Dataset.from_iterable(
[
DatasetItem(
id=100,
subset='train',
image=np.ones((10, 6, 3)),
annotations=[
Caption('hello', id=1),
Caption('world', id=2, group=5),
Label(2, id=3, attributes={
'x': 1,
'y': '2',
}),
Bbox(1,
2,
3,
4,
label=1,
id=4,
z_order=1,
attributes={
'score': 1.0,
}),
Bbox(5, 6, 7, 8, id=5, group=5),
Points([1, 2, 2, 0, 1, 1], label=0, id=5, z_order=4),
Mask(label=3, id=5, z_order=2, image=np.ones((2, 3))),
]),
DatasetItem(id=21,
subset='train',
annotations=[
Caption('test'),
Label(2),
Bbox(1, 2, 3, 4, label=3, id=42, group=42)
]),
DatasetItem(
id=2,
subset='val',
annotations=[
PolyLine([1, 2, 3, 4, 5, 6, 7, 8], id=11, z_order=1),
Polygon([1, 2, 3, 4, 5, 6, 7, 8], id=12, z_order=4),
]),
DatasetItem(
id=42, subset='test', attributes={
'a1': 5,
'a2': '42'
}),
DatasetItem(id=42),
DatasetItem(id=43, image=Image(path='1/b/c.qq', size=(2, 4))),
],
categories={
AnnotationType.label: label_categories2,
AnnotationType.mask: mask_categories2,
AnnotationType.points: point_categories2,
})
with TestDir() as test_dir:
with DatasetDiffVisualizer(
save_dir=test_dir,
comparator=DistanceComparator(iou_threshold=0.8),
) as visualizer:
visualizer.save(dataset1, dataset2)
self.assertNotEqual(0, os.listdir(osp.join(test_dir)))