class TestDatasetJson: def setup(self): self.d = Dataset('test_data1', force=True) self.classes = ["A", "B", "C"] def test_load(self): assert (self.d.num_images() == 4) assert (self.d.classes == self.classes) def test_get_det_gt(self): gt = self.d.get_det_gt(with_diff=True, with_trun=False) df = Table( np.array([[0., 0., 0., 0., 0., 0, 0, 0.], [1., 1., 1., 1., 1., 0, 0, 0.], [1., 1., 1., 0., 0., 0, 0, 1.], [0., 0., 0., 0., 1., 0, 0, 2.], [0., 0., 0., 0., 2., 0, 0, 3.], [1., 1., 1., 1., 2., 0, 0, 3.]]), ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']) print(gt) print(df) assert (gt == df) def test_get_cls_counts_json(self): arr = np.array([[1, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 2]]) print(self.d.get_cls_counts()) assert (np.all(self.d.get_cls_counts() == arr)) def test_get_cls_ground_truth_json(self): table = Table( np.array([[True, True, False], [True, False, False], [False, True, False], [False, False, True]]), ["A", "B", "C"]) assert (self.d.get_cls_ground_truth() == table) def test_det_ground_truth_for_class_json(self): gt = self.d.get_det_gt_for_class("A", with_diff=True, with_trun=True) arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert (np.all(gt.arr == arr)) assert (gt.cols == cols) # no diff or trun gt = self.d.get_det_gt_for_class("A", with_diff=False, with_trun=False) arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert (np.all(gt.arr == arr)) assert (gt.cols == cols) def test_set_values(self): assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_values('uniform') assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_values('inverse_prior') print(self.d.values) assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
class TestEvaluationSynthetic: def __init__(self): self.d = Dataset('test_data2',force=True) self.classes = ["A","B","C"] self.det_gt = self.d.get_det_gt() def test(self): scores = np.ones(self.det_gt.shape[0]) dets = self.det_gt.append_column('score',scores) scores = np.ones(self.d.get_det_gt_for_class('A').shape[0]) dets_just_A = self.d.get_det_gt_for_class('A') dets_just_A = dets_just_A.append_column('score',scores) self.d.set_values('uniform') assert(np.all(self.d.values == 1./3 * np.ones(len(self.classes)))) dp = DatasetPolicy(self.d,self.d,detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets,self.det_gt) assert(ap==1) ap = ev.compute_det_map(dets_just_A,self.det_gt) print(ap) assert(ut.fequal(ap, 0.33333333333333)) self.d.set_values('inverse_prior') assert(np.all(self.d.values == np.array([0.25,0.25,0.5]))) dp = DatasetPolicy(self.d,self.d,detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets,self.det_gt) assert(ap==1) ap = ev.compute_det_map(dets_just_A,self.det_gt) print(ap) assert(ut.fequal(ap, 0.25))
class TestEvaluationSynthetic: def __init__(self): self.d = Dataset('test_data2', force=True) self.classes = ["A", "B", "C"] self.det_gt = self.d.get_det_gt() def test(self): scores = np.ones(self.det_gt.shape[0]) dets = self.det_gt.append_column('score', scores) scores = np.ones(self.d.get_det_gt_for_class('A').shape[0]) dets_just_A = self.d.get_det_gt_for_class('A') dets_just_A = dets_just_A.append_column('score', scores) self.d.set_values('uniform') assert (np.all(self.d.values == 1. / 3 * np.ones(len(self.classes)))) dp = DatasetPolicy(self.d, self.d, detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets, self.det_gt) assert (ap == 1) ap = ev.compute_det_map(dets_just_A, self.det_gt) print(ap) assert (ut.fequal(ap, 0.33333333333333)) self.d.set_values('inverse_prior') assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5]))) dp = DatasetPolicy(self.d, self.d, detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets, self.det_gt) assert (ap == 1) ap = ev.compute_det_map(dets_just_A, self.det_gt) print(ap) assert (ut.fequal(ap, 0.25))
def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False, with_trun=False) correct = np.matrix([[139., 200., 69., 102., 18., 0., 0., 0.], [123., 155., 93., 41., 17., 0., 0., 1.], [239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct)
def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False,with_trun=False) correct = np.matrix( [ [ 139., 200., 69., 102., 18., 0., 0., 0.], [ 123., 155., 93., 41., 17., 0., 0., 1.], [ 239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct)
class TestDatasetPolicy: def __init__(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train') self.weights_dataset_name = 'test_pascal_val' self.config = { 'suffix': 'default', 'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half 'policy_mode': 'random', 'bounds': None, 'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl } self.dp = DatasetPolicy( self.dataset,self.train_dataset,self.weights_dataset_name,**self.config) def test_run_on_dataset(self): # run on test dataset dets,clses,samples = self.dp.run_on_dataset(force=True) assert(len(samples) == clses.shape[0]) assert(len(samples) == self.dp.dataset.num_images()*len(self.dp.actions)) train_dets,train_clses,train_samples = self.dp.run_on_dataset(train=True,force=True) assert(len(train_samples) == train_clses.shape[0]) assert(len(train_samples) == self.dp.train_dataset.num_images()*len(self.dp.actions)) def test_unique_samples(self): "Test the correctness of making a list of samples unique." dets,clses,samples = self.dp.run_on_dataset(force=True) new_sample = copy.deepcopy(samples[11]) new_sample2 = copy.deepcopy(samples[11]) new_sample2.dt = -40 # an unreasonable value assert(new_sample in samples) assert(new_sample2 not in samples) def test_output_det_statistics(self): self.dp.output_det_statistics() def test_learn_weights(self): dataset = Dataset('full_pascal_val') train_dataset = Dataset('full_pascal_train') dataset.images = dataset.images[:20] train_dataset.images = train_dataset.images[:20] dp = DatasetPolicy(dataset,train_dataset,self.weights_dataset_name,**self.config) weights = dp.learn_weights() def test_regress(self): dets,clses,samples = self.dp.run_on_dataset(force=True) weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) samples += samples weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) samples += samples weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) def test_load_weights(self): modes = ['manual_1','manual_2','manual_3'] for mode in modes: print "%s weights:"%mode self.dp.weights_mode=mode self.dp.load_weights() print self.dp.get_reshaped_weights() assert(self.dp.weights.shape[0] == len(self.dp.actions)*BeliefState.num_features) self.dp.write_weights_image('temp_weights_%s.png'%mode) def test_perfect_detector(self): dets,clses,samples = self.dp.run_on_dataset(force=True) #embed() dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) gt = self.dataset.get_det_gt() gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) # TODO: does this make sense? dets.sort_by_column('x') gt.sort_by_column('x') print dets print gt assert(dets == gt) def test_load_dpm_detections(self): conf = dict(self.config) conf['detectors'] = ['dpm'] policy = DatasetPolicy(self.dataset,self.train_dataset,**conf) assert(policy.detectors == ['dpm']) dets = policy.load_ext_detections(self.dataset,'dpm_may25',force=True) dets = dets.with_column_omitted('time') # load the ground truth dets, processed in Matlab # (timely/data/test_support/concat_dets.m) filename = os.path.join(config.test_support_dir, 'val_dets.mat') dets_correct = Table( scipy.io.loadmat(filename)['dets'], ['x1','y1','x2','y2','dummy','dummy','dummy','dummy','score','cls_ind','img_ind'], 'dets_correct') dets_correct = dets_correct.subset( ['x1','y1','x2','y2','score','cls_ind','img_ind']) dets_correct.arr[:,:4] -= 1 dets_correct.arr[:,:4] = BoundingBox.convert_arr_from_corners( dets_correct.arr[:,:4]) dets_correct.cols = ['x','y','w','h','score','cls_ind','img_ind'] print('----mine:') print(dets) print('----correct:') print(dets_correct) assert(dets_correct == dets)
class TestDatasetJson: def setup(self): self.d = Dataset('test_data1',force=True) self.classes = ["A","B","C"] def test_load(self): assert(self.d.num_images() == 4) assert(self.d.classes == self.classes) def test_get_det_gt(self): gt = self.d.get_det_gt(with_diff=True,with_trun=False) df = Table( np.array([[ 0., 0., 0., 0., 0., 0, 0, 0.], [ 1., 1., 1., 1., 1., 0, 0, 0.], [ 1., 1., 1., 0., 0., 0, 0, 1.], [ 0., 0., 0., 0., 1., 0, 0, 2.], [ 0., 0., 0., 0., 2., 0, 0, 3.], [ 1., 1., 1., 1., 2., 0, 0, 3.]]), ['x','y','w','h','cls_ind','diff','trun','img_ind']) print(gt) print(df) assert(gt == df) def test_get_cls_counts_json(self): arr = np.array( [ [ 1, 1, 0], [ 1, 0, 0], [ 0, 1, 0], [ 0, 0, 2]]) print(self.d.get_cls_counts()) assert(np.all(self.d.get_cls_counts() == arr)) def test_get_cls_ground_truth_json(self): table = Table( np.array([ [ True, True, False], [ True, False, False], [ False, True, False], [ False, False, True] ]), ["A","B","C"]) assert(self.d.get_cls_ground_truth()==table) def test_det_ground_truth_for_class_json(self): gt = self.d.get_det_gt_for_class("A",with_diff=True,with_trun=True) arr = np.array( [[ 0., 0., 0., 0., 0., 0., 0, 0.], [ 1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x','y','w','h','cls_ind','diff','trun','img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) # no diff or trun gt = self.d.get_det_gt_for_class("A",with_diff=False,with_trun=False) arr = np.array( [[ 0., 0., 0., 0., 0., 0., 0, 0.], [ 1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x','y','w','h','cls_ind','diff','trun','img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) def test_set_values(self): assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes)))) self.d.set_values('uniform') assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes)))) self.d.set_values('inverse_prior') print(self.d.values) assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
class TestDatasetPolicy: def __init__(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train') self.weights_dataset_name = 'test_pascal_val' self.config = { 'suffix': 'default', 'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half 'policy_mode': 'random', 'bounds': None, 'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl } self.dp = DatasetPolicy(self.dataset, self.train_dataset, self.weights_dataset_name, **self.config) def test_run_on_dataset(self): # run on test dataset dets, clses, samples = self.dp.run_on_dataset(force=True) assert (len(samples) == clses.shape[0]) assert (len(samples) == self.dp.dataset.num_images() * len(self.dp.actions)) train_dets, train_clses, train_samples = self.dp.run_on_dataset( train=True, force=True) assert (len(train_samples) == train_clses.shape[0]) assert (len(train_samples) == self.dp.train_dataset.num_images() * len(self.dp.actions)) def test_unique_samples(self): "Test the correctness of making a list of samples unique." dets, clses, samples = self.dp.run_on_dataset(force=True) new_sample = copy.deepcopy(samples[11]) new_sample2 = copy.deepcopy(samples[11]) new_sample2.dt = -40 # an unreasonable value assert (new_sample in samples) assert (new_sample2 not in samples) def test_output_det_statistics(self): self.dp.output_det_statistics() def test_learn_weights(self): dataset = Dataset('full_pascal_val') train_dataset = Dataset('full_pascal_train') dataset.images = dataset.images[:20] train_dataset.images = train_dataset.images[:20] dp = DatasetPolicy(dataset, train_dataset, self.weights_dataset_name, **self.config) weights = dp.learn_weights() def test_regress(self): dets, clses, samples = self.dp.run_on_dataset(force=True) weights, error = self.dp.regress(samples, 'greedy') print "Weights after %d samples:\n %s" % (len(samples), weights) print "Error after %d samples: %s" % (len(samples), error) samples += samples weights, error = self.dp.regress(samples, 'greedy') print "Weights after %d samples:\n %s" % (len(samples), weights) print "Error after %d samples: %s" % (len(samples), error) samples += samples weights, error = self.dp.regress(samples, 'greedy') print "Weights after %d samples:\n %s" % (len(samples), weights) print "Error after %d samples: %s" % (len(samples), error) def test_load_weights(self): modes = ['manual_1', 'manual_2', 'manual_3'] for mode in modes: print "%s weights:" % mode self.dp.weights_mode = mode self.dp.load_weights() print self.dp.get_reshaped_weights() assert (self.dp.weights.shape[0] == len(self.dp.actions) * BeliefState.num_features) self.dp.write_weights_image('temp_weights_%s.png' % mode) def test_perfect_detector(self): dets, clses, samples = self.dp.run_on_dataset(force=True) #embed() dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) gt = self.dataset.get_det_gt() gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) # TODO: does this make sense? dets.sort_by_column('x') gt.sort_by_column('x') print dets print gt assert (dets == gt) def test_load_dpm_detections(self): conf = dict(self.config) conf['detectors'] = ['dpm'] policy = DatasetPolicy(self.dataset, self.train_dataset, **conf) assert (policy.detectors == ['dpm']) dets = policy.load_ext_detections(self.dataset, 'dpm_may25', force=True) dets = dets.with_column_omitted('time') # load the ground truth dets, processed in Matlab # (timely/data/test_support/concat_dets.m) filename = os.path.join(config.test_support_dir, 'val_dets.mat') dets_correct = Table( scipy.io.loadmat(filename)['dets'], [ 'x1', 'y1', 'x2', 'y2', 'dummy', 'dummy', 'dummy', 'dummy', 'score', 'cls_ind', 'img_ind' ], 'dets_correct') dets_correct = dets_correct.subset( ['x1', 'y1', 'x2', 'y2', 'score', 'cls_ind', 'img_ind']) dets_correct.arr[:, :4] -= 1 dets_correct.arr[:, :4] = BoundingBox.convert_arr_from_corners( dets_correct.arr[:, :4]) dets_correct.cols = ['x', 'y', 'w', 'h', 'score', 'cls_ind', 'img_ind'] print('----mine:') print(dets) print('----correct:') print(dets_correct) assert (dets_correct == dets)