def test_correct_second_detection(self):
gts = [val for val in self.square_gt]
detections = [BBoxDetInst(self.default_label_list, [0, 0, 10, 10]),
BBoxDetInst(self.default_label_list, self.gt_box)]
evaluator = PDQ()
score = evaluator.score([(gts, detections)])
self.assertAlmostEqual(score, 0.5)
def test_run_score_multiple_times_on_single_pdq_instance(self):
two_detections = [BBoxDetInst(self.default_label_list, self.gt_box),
BBoxDetInst(self.default_label_list, self.gt_box)]
one_detection = [BBoxDetInst(self.default_label_list, self.gt_box)]
evaluator = PDQ()
score_two = evaluator.score([(self.square_gt, two_detections)])
score_one = evaluator.score([(self.square_gt, one_detection)])
self.assertAlmostEqual(score_two, 0.5)
self.assertAlmostEqual(score_one, 1.0)
def test_detection_no_gt(self):
detections = [BBoxDetInst(self.default_label_list, self.gt_box)]
gts = []
evaluator = PDQ()
score = evaluator.score([(gts, detections)])
self.assertEqual(score, 0)
def __iter__(self):
if self.n_imgs is not None:
dets_iter = islice(self.dict_dets, 0, self.n_imgs)
else:
dets_iter = iter(self.dict_dets)
for img_id, img_dets in enumerate(dets_iter):
yield [
# If the detection has a reported covariance above zero, create a Probabilistic Box
PBoxDetInst(
class_list=reorder_classes(det['label_probs'], self.class_assoc),
# Removed clamping boxes for detection output standardization for now.
# box=clamp_bbox(det['bbox'], det['img_size']),
box=det['bbox'],
covs=det['covars'] if self.cov_mat is None else self.cov_mat
)
if (self.cov_mat is not None or ('covars' in det and (np.sum(det['covars']) != 0)))
and self.override_cov != 0
# Otherwise create a standard bounding box
else
BBoxDetInst(
class_list=reorder_classes(det['label_probs'], self.class_assoc),
box=det['bbox']
)
for det in img_dets
# Regardless of type, ignore detections below given label threshold if provided
# Note that this includes label_prob of background classes
if self.label_threshold <= 0 or max(det['label_probs']) > self.label_threshold
]
def test_half_position_confidence(self):
detections = [
BBoxDetInst(self.square_label_list, self.square_gt_box, 0.5)
]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, detections)])
self.assertAlmostEqual(np.sqrt(0.5), score, 4)
def test_multiple_detections(self):
ten_detections = [
BBoxDetInst(self.square_label_list, self.square_gt_box)
for _ in range(10)
]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, ten_detections)])
self.assertAlmostEqual(score, 0.1)
def test_no_detections_for_image(self):
gts1 = [val for val in self.square_gt]
gts2 = [GroundTruthInstance(self.square_mask, 0, 1, 1)]
dets1 = [BBoxDetInst(self.default_label_list, self.gt_box)]
dets2 = []
evaluator = PDQ()
score = evaluator.score([(gts1, dets1), (gts2, dets2)])
self.assertAlmostEqual(score, 0.5)
def test_no_detections_for_image_with_too_small_gt(self):
gts1 = [val for val in self.square_gt]
small_mask = np.zeros(self.img_size, dtype=np.bool)
small_mask[500:504, 500:501] = True
gts2 = [GroundTruthInstance(small_mask, 0, 1, 1)]
dets1 = [BBoxDetInst(self.default_label_list, self.gt_box)]
dets2 = []
evaluator = PDQ()
score = evaluator.score([(gts1, dets1), (gts2, dets2)])
self.assertAlmostEqual(score, 1.0)
def test_detect_500_extra_pixels(self):
det_box = [val for val in self.gt_box]
det_box[2] += 1
detections = [BBoxDetInst(self.default_label_list, det_box)]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, detections)])
expected_spatial_quality = np.exp((_MAX_LOSS*500)/(500*500))
expected_gmean = np.sqrt(expected_spatial_quality)
self.assertAlmostEqual(expected_gmean, score, 4)
def test_multiple_missed_gts_too_small(self):
gts = [val for val in self.square_gt]
for i in range(9):
# Create small 2x2 boxes which are missed around an edge of the image (buffer of 2 pixels)
new_gt_mask = np.zeros(gts[0].segmentation_mask.shape, gts[0].segmentation_mask.dtype)
new_gt_mask[2:4, 2 + i * 4:4 + i * 4] = np.amax(gts[0].segmentation_mask)
gts.append(GroundTruthInstance(new_gt_mask, 0, 0, i+1))
detections = [BBoxDetInst(self.default_label_list, self.gt_box)]
evaluator = PDQ()
score = evaluator.score([(gts, detections)])
self.assertAlmostEqual(score, 1.0)
def test_cross_gt_detected_by_perfect_box_in_segment_mode(self):
detections = [BBoxDetInst(self.cross_label_list, self.cross_gt_box)]
evaluator = PDQ(segment_mode=True)
score = evaluator.score([(self.cross_gt, detections)])
expected_num_missed_pixels = np.sum(
np.logical_xor(self.square_mask, self.cross_mask))
expected_spatial_quality = np.exp(
(_MAX_LOSS * expected_num_missed_pixels) / np.sum(self.cross_mask))
expected_gmean = np.sqrt(expected_spatial_quality)
self.assertAlmostEqual(score, expected_gmean, 4)
def test_no_detections_for_image_with_small_and_big_gt_with_filtering(
self):
gts1 = [val for val in self.square_gt]
small_mask = np.zeros(self.img_size, dtype=np.bool)
small_mask[500:504, 500:501] = True
gts2 = [
GroundTruthInstance(self.square_mask, 0),
GroundTruthInstance(small_mask, 0)
]
dets1 = [BBoxDetInst(self.square_label_list, self.square_gt_box)]
dets2 = []
evaluator = PDQ(filter_gts=True)
score = evaluator.score([(gts1, dets1), (gts2, dets2)])
self.assertAlmostEqual(score, 0.5)
def test_no_overlap_box(self):
det_box = [val for val in self.gt_box]
box_width = (self.gt_box[2]+1) - self.gt_box[0]
det_box[0] += box_width
det_box[2] += box_width
detections = [BBoxDetInst(self.default_label_list, det_box)]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, detections)])
expected_spatial_quality = 0
expected_gmean = np.sqrt(expected_spatial_quality)
self.assertAlmostEqual(expected_gmean, score, 4)
def test_missed_gts_and_unmatched_detections(self):
gts = [val for val in self.square_gt]
for i in range(10):
# Create small 11x11 boxes which are missed around an edge of the image (buffer of 2 pixels)
new_gt_mask = np.zeros(gts[0].segmentation_mask.shape,
gts[0].segmentation_mask.dtype)
new_gt_mask[2:14, 2 + i * 14:14 + i * 14] = np.amax(
gts[0].segmentation_mask)
gts.append(GroundTruthInstance(new_gt_mask, 0))
detections = [
BBoxDetInst(self.square_label_list, self.square_gt_box)
for _ in range(10)
]
evaluator = PDQ()
score = evaluator.score([(gts, detections)])
self.assertAlmostEqual(score, 1 / 20.)
def test_half_position_and_label_confidences(self):
detections = [BBoxDetInst([0.5, 0.5], self.gt_box, 0.5)]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, detections)])
self.assertAlmostEqual(0.5, score, 4)
def test_perfect_bbox(self):
detections = [BBoxDetInst(self.default_label_list, self.gt_box)]
evaluator = PDQ()
score = evaluator.score([(self.square_gt, detections)])
self.assertAlmostEqual(score, 1, 4)
def test_cross_gt_detected_by_perfect_box_in_non_segment_mode(self):
detections = [BBoxDetInst(self.cross_label_list, self.cross_gt_box)]
evaluator = PDQ()
score = evaluator.score([(self.cross_gt, detections)])
self.assertAlmostEqual(score, 1, 4)
