class TestEvaluationSynthetic(unittest.TestCase): def setUp(self): self.d = Dataset(test_config, 'test_data2') self.d.load_from_json(test_data2) self.classes = self.d.classes det_gt = self.d.get_det_gt() # perfect detections scores = np.ones(det_gt.shape[0]) self.full_dets = det_gt.append_column('score', scores) # perfect detections, but only for class 'A' dets_just_A = self.d.get_det_gt_for_class('A') scores = np.ones(dets_just_A.shape[0]) self.partial_dets = dets_just_A.append_column('score', scores) def test_values(self): det_gt = self.d.get_det_gt() self.d.set_class_values('uniform') assert(np.all(self.d.values == 1. / 3 * np.ones(len(self.classes)))) ap = evaluation.compute_det_map(self.full_dets, det_gt, self.d.values) assert(ap == 1) ap = evaluation.compute_det_map( self.partial_dets, det_gt, self.d.values) assert_almost_equal(ap, 1 / 3.) self.d.set_class_values('inverse_prior') assert(np.all(self.d.values == np.array([0.25, 0.25, 0.5]))) ap = evaluation.compute_det_map(self.full_dets, det_gt, self.d.values) assert(ap == 1) ap = evaluation.compute_det_map( self.partial_dets, det_gt, self.d.values) assert_almost_equal(ap, 0.25) def test_compute_pr_multiclass(self): cols = ['x', 'y', 'w', 'h', 'cls_ind', 'img_ind', 'diff'] dets_cols = ['x', 'y', 'w', 'h', 'score', 'time', 'cls_ind', 'img_ind'] # two objects of different classes in the image, perfect detection arr = np.array( [[0, 0, 10, 10, 0, 0, 0], [10, 10, 10, 10, 1, 0, 0]]) gt = Table(arr, cols) dets_arr = np.array( [[0, 0, 10, 10, -1, -1, 0, 0], [10, 10, 10, 10, -1, -1, 1, 0]]) dets = Table(dets_arr, dets_cols) # make sure gt and gt_cols aren't modified gt_arr_copy = gt.arr.copy() gt_cols_copy = list(gt.cols) ap, rec, prec = evaluation.compute_det_pr(dets, gt) assert(np.all(gt.arr == gt_arr_copy)) assert(gt_cols_copy == gt.cols) correct_ap = 1 correct_rec = np.array([0.5, 1]) correct_prec = np.array([1, 1]) print((ap, rec, prec)) assert(correct_ap == ap) assert(np.all(correct_rec == rec)) assert(np.all(correct_prec == prec)) # some extra detections to generate false positives dets_arr = np.array( [[0, 0, 10, 10, -1, -1, 0, 0], [0, 0, 10, 10, 0, -1, 0, 0], [10, 10, 10, 10, 0, -1, 1, 0], [10, 10, 10, 10, -1, -1, 1, 0]]) dets = Table(dets_arr, dets_cols) ap, rec, prec = evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5, 1, 1, 1]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert(np.all(correct_rec == rec)) assert(np.all(correct_prec == prec)) # confirm that running on the same dets gives the same answer ap, rec, prec = evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5, 1, 1, 1]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert(np.all(correct_rec == rec)) assert(np.all(correct_prec == prec)) # now let's add two objects of a different class to gt to lower recall arr = np.array( [[0, 0, 10, 10, 0, 0, 0], [10, 10, 10, 10, 1, 0, 0], [20, 20, 10, 10, 2, 0, 0], [30, 30, 10, 10, 2, 0, 0]]) gt = Table(arr, cols) ap, rec, prec = evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.25, 0.5, 0.5, 0.5]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert(np.all(correct_rec == rec)) assert(np.all(correct_prec == prec)) # now call it with empty detections dets_arr = np.array([]) dets = Table(dets_arr, dets_cols) ap, rec, prec = evaluation.compute_det_pr(dets, gt) correct_ap = 0 correct_rec = np.array([0]) correct_prec = np.array([0]) print((ap, rec, prec)) assert(np.all(correct_ap == ap)) assert(np.all(correct_rec == rec)) assert(np.all(correct_prec == prec)) def test_plots(self): full_results_dirname = os.path.join(res_dir, 'full_dets_eval') partial_results_dirname = os.path.join(res_dir, 'partial_dets_eval') evaluation.evaluate_detections_whole( self.d, self.partial_dets, partial_results_dirname, force=True) assert(os.path.exists( os.path.join(partial_results_dirname, 'whole_dashboard.html'))) pngs = glob.glob(os.path.join(partial_results_dirname, '*.png')) assert(len(pngs) == 4) # 3 classes + 1 multiclass evaluation.evaluate_detections_whole( self.d, self.full_dets, full_results_dirname, force=True)
class TestDatasetJson(object): def setup(self): self.d = Dataset(test_config, 'test_data1') self.d.load_from_json(test_data1) self.classes = ["A", "B", "C"] def test_load(self): assert(self.d.num_images() == 4) assert(self.d.classes == self.classes) def test_get_det_gt(self): gt = self.d.get_det_gt(with_diff=True, with_trun=False) df = Table( np.array([[0., 0., 0., 0., 0., 0, 0, 0.], [1., 1., 1., 1., 1., 0, 0, 0.], [1., 1., 1., 0., 0., 0, 0, 1.], [0., 0., 0., 0., 1., 0, 0, 2.], [0., 0., 0., 0., 2., 0, 0, 3.], [1., 1., 1., 1., 2., 0, 0, 3.]]), ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']) print(gt) print(df) assert(gt == df) def test_get_cls_counts(self): arr = np.array( [[1, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 2]]) print(self.d.get_cls_counts()) assert(np.all(self.d.get_cls_counts() == arr)) def test_get_cls_ground_truth(self): table = Table( np.array([[True, True, False], [True, False, False], [False, True, False], [False, False, True]]), ["A", "B", "C"]) assert(self.d.get_cls_ground_truth() == table) def test_det_ground_truth_for_class(self): gt = self.d.get_det_gt_for_class("A", with_diff=True, with_trun=True) arr = np.array( [[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) # no diff or trun gt = self.d.get_det_gt_for_class("A", with_diff=False, with_trun=False) arr = np.array( [[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) def test_set_class_values(self): assert(np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_class_values('uniform') assert(np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_class_values('inverse_prior') print(self.d.values) assert(np.all(self.d.values == np.array([0.25, 0.25, 0.5])))