def test_no_bbox_diff_with_same_item(self):
detections = 3
anns = [
Bbox(i * 10, 10, 10, 10, label=i,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
]
item = DatasetItem(id=0, annotations=anns)
iou_thresh = 0.5
conf_thresh = 0.5
comp = Comparator(iou_threshold=iou_thresh, conf_threshold=conf_thresh)
result = comp.compare_item_bboxes(item, item)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len([it for it in item.annotations \
if conf_thresh < it.attributes['score']]),
len(matches))
for a_bbox, b_bbox in matches:
self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
self.assertEqual(a_bbox.label, b_bbox.label)
self.assertLess(conf_thresh, a_bbox.attributes['score'])
self.assertLess(conf_thresh, b_bbox.attributes['score'])
def test_can_find_missing_boxes(self):
detections = 3
class_count = 2
item1 = DatasetItem(id=1, annotations=[
Bbox(i * 10, 10, 10, 10, label=i,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections) if i % 2 == 0
])
item2 = DatasetItem(id=2, annotations=[
Bbox(i * 10, 10, 10, 10, label=(i + 1) % class_count,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections) if i % 2 == 1
])
iou_thresh = 0.5
conf_thresh = 0.5
comp = Comparator(iou_threshold=iou_thresh, conf_threshold=conf_thresh)
result = comp.compare_item_bboxes(item1, item2)
matches, mispred, a_greater, b_greater = result
self.assertEqual(0, len(mispred))
self.assertEqual(len([it for it in item1.annotations \
if conf_thresh < it.attributes['score']]),
len(a_greater))
self.assertEqual(len([it for it in item2.annotations \
if conf_thresh < it.attributes['score']]),
len(b_greater))
self.assertEqual(0, len(matches))
def test_can_find_bbox_with_wrong_label(self):
detections = 3
class_count = 2
item1 = DatasetItem(id=1, annotations=[
Bbox(i * 10, 10, 10, 10, label=i,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
])
item2 = DatasetItem(id=2, annotations=[
Bbox(i * 10, 10, 10, 10, label=(i + 1) % class_count,
attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
])
iou_thresh = 0.5
conf_thresh = 0.5
comp = Comparator(iou_threshold=iou_thresh, conf_threshold=conf_thresh)
result = comp.compare_item_bboxes(item1, item2)
matches, mispred, a_greater, b_greater = result
self.assertEqual(len([it for it in item1.annotations \
if conf_thresh < it.attributes['score']]),
len(mispred))
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(0, len(matches))
for a_bbox, b_bbox in mispred:
self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label)
self.assertLess(conf_thresh, a_bbox.attributes['score'])
self.assertLess(conf_thresh, b_bbox.attributes['score'])
def diff_command(args):
first_project = load_project(args.project_dir)
second_project = load_project(args.other_project_dir)
comparator = Comparator(
iou_threshold=args.iou_thresh,
conf_threshold=args.conf_thresh)
dst_dir = args.dst_dir
if dst_dir:
if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
raise CliException("Directory '%s' already exists "
"(pass --overwrite to overwrite)" % dst_dir)
else:
dst_dir = generate_next_file_name('%s-%s-diff' % (
first_project.config.project_name,
second_project.config.project_name)
)
dst_dir = osp.abspath(dst_dir)
log.info("Saving diff to '%s'" % dst_dir)
dst_dir_existed = osp.exists(dst_dir)
try:
visualizer = DiffVisualizer(save_dir=dst_dir, comparator=comparator,
output_format=args.format)
visualizer.save_dataset_diff(
first_project.make_dataset(),
second_project.make_dataset())
except BaseException:
if not dst_dir_existed and osp.isdir(dst_dir):
shutil.rmtree(dst_dir, ignore_errors=True)
raise
return 0
def test_no_label_diff_with_same_item(self):
detections = 3
anns = [
Label(i, attributes={'score': (1.0 + i) / detections}) \
for i in range(detections)
]
item = DatasetItem(id=1, annotations=anns)
conf_thresh = 0.5
comp = Comparator(conf_threshold=conf_thresh)
result = comp.compare_item_labels(item, item)
matches, a_greater, b_greater = result
self.assertEqual(0, len(a_greater))
self.assertEqual(0, len(b_greater))
self.assertEqual(len([it for it in item.annotations \
if conf_thresh < it.attributes['score']]),
len(matches))
def test_can_find_wrong_label(self):
item1 = DatasetItem(id=1, annotations=[
Label(0),
Label(1),
Label(2),
])
item2 = DatasetItem(id=2, annotations=[
Label(2),
Label(3),
Label(4),
])
conf_thresh = 0.5
comp = Comparator(conf_threshold=conf_thresh)
result = comp.compare_item_labels(item1, item2)
matches, a_greater, b_greater = result
self.assertEqual(2, len(a_greater))
self.assertEqual(2, len(b_greater))
self.assertEqual(1, len(matches))
def diff_command(args):
first_project = load_project(args.project_dir)
second_project = load_project(args.other_project_dir)
comparator = Comparator(iou_threshold=args.iou_thresh,
conf_threshold=args.conf_thresh)
save_dir = args.dst_dir
if save_dir is not None:
log.info("Saving diff to '%s'" % save_dir)
os.makedirs(osp.abspath(save_dir))
visualizer = DiffVisualizer(save_dir=save_dir,
comparator=comparator,
output_format=args.output_format)
visualizer.save_dataset_diff(first_project.make_dataset(),
second_project.make_dataset())
return 0