class TestDatasetJson:
def setup(self):
self.d = Dataset('test_data1', force=True)
self.classes = ["A", "B", "C"]
def test_load(self):
assert (self.d.num_images() == 4)
assert (self.d.classes == self.classes)
def test_get_det_gt(self):
gt = self.d.get_det_gt(with_diff=True, with_trun=False)
df = Table(
np.array([[0., 0., 0., 0., 0., 0, 0, 0.],
[1., 1., 1., 1., 1., 0, 0, 0.],
[1., 1., 1., 0., 0., 0, 0, 1.],
[0., 0., 0., 0., 1., 0, 0, 2.],
[0., 0., 0., 0., 2., 0, 0, 3.],
[1., 1., 1., 1., 2., 0, 0, 3.]]),
['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'])
print(gt)
print(df)
assert (gt == df)
def test_get_cls_counts_json(self):
arr = np.array([[1, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 2]])
print(self.d.get_cls_counts())
assert (np.all(self.d.get_cls_counts() == arr))
def test_get_cls_ground_truth_json(self):
table = Table(
np.array([[True, True, False], [True, False, False],
[False, True, False], [False, False, True]]),
["A", "B", "C"])
assert (self.d.get_cls_ground_truth() == table)
def test_det_ground_truth_for_class_json(self):
gt = self.d.get_det_gt_for_class("A", with_diff=True, with_trun=True)
arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.],
[1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']
print(gt.arr)
assert (np.all(gt.arr == arr))
assert (gt.cols == cols)
# no diff or trun
gt = self.d.get_det_gt_for_class("A", with_diff=False, with_trun=False)
arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.],
[1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']
print(gt.arr)
assert (np.all(gt.arr == arr))
assert (gt.cols == cols)
def test_set_values(self):
assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes))))
self.d.set_values('uniform')
assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes))))
self.d.set_values('inverse_prior')
print(self.d.values)
assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
class TestEvaluationSynthetic:
def __init__(self):
self.d = Dataset('test_data2',force=True)
self.classes = ["A","B","C"]
self.det_gt = self.d.get_det_gt()
def test(self):
scores = np.ones(self.det_gt.shape[0])
dets = self.det_gt.append_column('score',scores)
scores = np.ones(self.d.get_det_gt_for_class('A').shape[0])
dets_just_A = self.d.get_det_gt_for_class('A')
dets_just_A = dets_just_A.append_column('score',scores)
self.d.set_values('uniform')
assert(np.all(self.d.values == 1./3 * np.ones(len(self.classes))))
dp = DatasetPolicy(self.d,self.d,detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets,self.det_gt)
assert(ap==1)
ap = ev.compute_det_map(dets_just_A,self.det_gt)
print(ap)
assert(ut.fequal(ap, 0.33333333333333))
self.d.set_values('inverse_prior')
assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
dp = DatasetPolicy(self.d,self.d,detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets,self.det_gt)
assert(ap==1)
ap = ev.compute_det_map(dets_just_A,self.det_gt)
print(ap)
assert(ut.fequal(ap, 0.25))
class TestEvaluationSynthetic:
def __init__(self):
self.d = Dataset('test_data2', force=True)
self.classes = ["A", "B", "C"]
self.det_gt = self.d.get_det_gt()
def test(self):
scores = np.ones(self.det_gt.shape[0])
dets = self.det_gt.append_column('score', scores)
scores = np.ones(self.d.get_det_gt_for_class('A').shape[0])
dets_just_A = self.d.get_det_gt_for_class('A')
dets_just_A = dets_just_A.append_column('score', scores)
self.d.set_values('uniform')
assert (np.all(self.d.values == 1. / 3 * np.ones(len(self.classes))))
dp = DatasetPolicy(self.d, self.d, detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets, self.det_gt)
assert (ap == 1)
ap = ev.compute_det_map(dets_just_A, self.det_gt)
print(ap)
assert (ut.fequal(ap, 0.33333333333333))
self.d.set_values('inverse_prior')
assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
dp = DatasetPolicy(self.d, self.d, detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets, self.det_gt)
assert (ap == 1)
ap = ev.compute_det_map(dets_just_A, self.det_gt)
print(ap)
assert (ut.fequal(ap, 0.25))
def test_ground_truth_test(self):
d = Dataset('test_pascal_val')
gt = d.get_det_gt(with_diff=False, with_trun=False)
correct = np.matrix([[139., 200., 69., 102., 18., 0., 0., 0.],
[123., 155., 93., 41., 17., 0., 0., 1.],
[239., 156., 69., 50., 8., 0., 0., 1.]])
print(gt)
assert np.all(gt.arr == correct)
def test_ground_truth_test(self):
d = Dataset('test_pascal_val')
gt = d.get_det_gt(with_diff=False,with_trun=False)
correct = np.matrix(
[ [ 139., 200., 69., 102., 18., 0., 0., 0.],
[ 123., 155., 93., 41., 17., 0., 0., 1.],
[ 239., 156., 69., 50., 8., 0., 0., 1.]])
print(gt)
assert np.all(gt.arr == correct)
class TestDatasetPolicy:
def __init__(self):
self.dataset = Dataset('test_pascal_val')
self.train_dataset = Dataset('test_pascal_train')
self.weights_dataset_name = 'test_pascal_val'
self.config = {
'suffix': 'default',
'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half
'policy_mode': 'random',
'bounds': None,
'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl
}
self.dp = DatasetPolicy(
self.dataset,self.train_dataset,self.weights_dataset_name,**self.config)
def test_run_on_dataset(self):
# run on test dataset
dets,clses,samples = self.dp.run_on_dataset(force=True)
assert(len(samples) == clses.shape[0])
assert(len(samples) == self.dp.dataset.num_images()*len(self.dp.actions))
train_dets,train_clses,train_samples = self.dp.run_on_dataset(train=True,force=True)
assert(len(train_samples) == train_clses.shape[0])
assert(len(train_samples) == self.dp.train_dataset.num_images()*len(self.dp.actions))
def test_unique_samples(self):
"Test the correctness of making a list of samples unique."
dets,clses,samples = self.dp.run_on_dataset(force=True)
new_sample = copy.deepcopy(samples[11])
new_sample2 = copy.deepcopy(samples[11])
new_sample2.dt = -40 # an unreasonable value
assert(new_sample in samples)
assert(new_sample2 not in samples)
def test_output_det_statistics(self):
self.dp.output_det_statistics()
def test_learn_weights(self):
dataset = Dataset('full_pascal_val')
train_dataset = Dataset('full_pascal_train')
dataset.images = dataset.images[:20]
train_dataset.images = train_dataset.images[:20]
dp = DatasetPolicy(dataset,train_dataset,self.weights_dataset_name,**self.config)
weights = dp.learn_weights()
def test_regress(self):
dets,clses,samples = self.dp.run_on_dataset(force=True)
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
samples += samples
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
samples += samples
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
def test_load_weights(self):
modes = ['manual_1','manual_2','manual_3']
for mode in modes:
print "%s weights:"%mode
self.dp.weights_mode=mode
self.dp.load_weights()
print self.dp.get_reshaped_weights()
assert(self.dp.weights.shape[0] == len(self.dp.actions)*BeliefState.num_features)
self.dp.write_weights_image('temp_weights_%s.png'%mode)
def test_perfect_detector(self):
dets,clses,samples = self.dp.run_on_dataset(force=True)
#embed()
dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
gt = self.dataset.get_det_gt()
gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
# TODO: does this make sense?
dets.sort_by_column('x')
gt.sort_by_column('x')
print dets
print gt
assert(dets == gt)
def test_load_dpm_detections(self):
conf = dict(self.config)
conf['detectors'] = ['dpm']
policy = DatasetPolicy(self.dataset,self.train_dataset,**conf)
assert(policy.detectors == ['dpm'])
dets = policy.load_ext_detections(self.dataset,'dpm_may25',force=True)
dets = dets.with_column_omitted('time')
# load the ground truth dets, processed in Matlab
# (timely/data/test_support/concat_dets.m)
filename = os.path.join(config.test_support_dir, 'val_dets.mat')
dets_correct = Table(
scipy.io.loadmat(filename)['dets'],
['x1','y1','x2','y2','dummy','dummy','dummy','dummy','score','cls_ind','img_ind'],
'dets_correct')
dets_correct = dets_correct.subset(
['x1','y1','x2','y2','score','cls_ind','img_ind'])
dets_correct.arr[:,:4] -= 1
dets_correct.arr[:,:4] = BoundingBox.convert_arr_from_corners(
dets_correct.arr[:,:4])
dets_correct.cols = ['x','y','w','h','score','cls_ind','img_ind']
print('----mine:')
print(dets)
print('----correct:')
print(dets_correct)
assert(dets_correct == dets)
class TestDatasetJson:
def setup(self):
self.d = Dataset('test_data1',force=True)
self.classes = ["A","B","C"]
def test_load(self):
assert(self.d.num_images() == 4)
assert(self.d.classes == self.classes)
def test_get_det_gt(self):
gt = self.d.get_det_gt(with_diff=True,with_trun=False)
df = Table(
np.array([[ 0., 0., 0., 0., 0., 0, 0, 0.],
[ 1., 1., 1., 1., 1., 0, 0, 0.],
[ 1., 1., 1., 0., 0., 0, 0, 1.],
[ 0., 0., 0., 0., 1., 0, 0, 2.],
[ 0., 0., 0., 0., 2., 0, 0, 3.],
[ 1., 1., 1., 1., 2., 0, 0, 3.]]),
['x','y','w','h','cls_ind','diff','trun','img_ind'])
print(gt)
print(df)
assert(gt == df)
def test_get_cls_counts_json(self):
arr = np.array(
[ [ 1, 1, 0],
[ 1, 0, 0],
[ 0, 1, 0],
[ 0, 0, 2]])
print(self.d.get_cls_counts())
assert(np.all(self.d.get_cls_counts() == arr))
def test_get_cls_ground_truth_json(self):
table = Table(
np.array([ [ True, True, False],
[ True, False, False],
[ False, True, False],
[ False, False, True] ]), ["A","B","C"])
assert(self.d.get_cls_ground_truth()==table)
def test_det_ground_truth_for_class_json(self):
gt = self.d.get_det_gt_for_class("A",with_diff=True,with_trun=True)
arr = np.array(
[[ 0., 0., 0., 0., 0., 0., 0, 0.],
[ 1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x','y','w','h','cls_ind','diff','trun','img_ind']
print(gt.arr)
assert(np.all(gt.arr == arr))
assert(gt.cols == cols)
# no diff or trun
gt = self.d.get_det_gt_for_class("A",with_diff=False,with_trun=False)
arr = np.array(
[[ 0., 0., 0., 0., 0., 0., 0, 0.],
[ 1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x','y','w','h','cls_ind','diff','trun','img_ind']
print(gt.arr)
assert(np.all(gt.arr == arr))
assert(gt.cols == cols)
def test_set_values(self):
assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes))))
self.d.set_values('uniform')
assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes))))
self.d.set_values('inverse_prior')
print(self.d.values)
assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
class TestDatasetPolicy:
def __init__(self):
self.dataset = Dataset('test_pascal_val')
self.train_dataset = Dataset('test_pascal_train')
self.weights_dataset_name = 'test_pascal_val'
self.config = {
'suffix': 'default',
'detectors':
['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half
'policy_mode': 'random',
'bounds': None,
'weights_mode':
'manual_1' # manual_1, manual_2, manual_3, greedy, rl
}
self.dp = DatasetPolicy(self.dataset, self.train_dataset,
self.weights_dataset_name, **self.config)
def test_run_on_dataset(self):
# run on test dataset
dets, clses, samples = self.dp.run_on_dataset(force=True)
assert (len(samples) == clses.shape[0])
assert (len(samples) == self.dp.dataset.num_images() *
len(self.dp.actions))
train_dets, train_clses, train_samples = self.dp.run_on_dataset(
train=True, force=True)
assert (len(train_samples) == train_clses.shape[0])
assert (len(train_samples) == self.dp.train_dataset.num_images() *
len(self.dp.actions))
def test_unique_samples(self):
"Test the correctness of making a list of samples unique."
dets, clses, samples = self.dp.run_on_dataset(force=True)
new_sample = copy.deepcopy(samples[11])
new_sample2 = copy.deepcopy(samples[11])
new_sample2.dt = -40 # an unreasonable value
assert (new_sample in samples)
assert (new_sample2 not in samples)
def test_output_det_statistics(self):
self.dp.output_det_statistics()
def test_learn_weights(self):
dataset = Dataset('full_pascal_val')
train_dataset = Dataset('full_pascal_train')
dataset.images = dataset.images[:20]
train_dataset.images = train_dataset.images[:20]
dp = DatasetPolicy(dataset, train_dataset, self.weights_dataset_name,
**self.config)
weights = dp.learn_weights()
def test_regress(self):
dets, clses, samples = self.dp.run_on_dataset(force=True)
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
samples += samples
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
samples += samples
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
def test_load_weights(self):
modes = ['manual_1', 'manual_2', 'manual_3']
for mode in modes:
print "%s weights:" % mode
self.dp.weights_mode = mode
self.dp.load_weights()
print self.dp.get_reshaped_weights()
assert (self.dp.weights.shape[0] == len(self.dp.actions) *
BeliefState.num_features)
self.dp.write_weights_image('temp_weights_%s.png' % mode)
def test_perfect_detector(self):
dets, clses, samples = self.dp.run_on_dataset(force=True)
#embed()
dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
gt = self.dataset.get_det_gt()
gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
# TODO: does this make sense?
dets.sort_by_column('x')
gt.sort_by_column('x')
print dets
print gt
assert (dets == gt)
def test_load_dpm_detections(self):
conf = dict(self.config)
conf['detectors'] = ['dpm']
policy = DatasetPolicy(self.dataset, self.train_dataset, **conf)
assert (policy.detectors == ['dpm'])
dets = policy.load_ext_detections(self.dataset,
'dpm_may25',
force=True)
dets = dets.with_column_omitted('time')
# load the ground truth dets, processed in Matlab
# (timely/data/test_support/concat_dets.m)
filename = os.path.join(config.test_support_dir, 'val_dets.mat')
dets_correct = Table(
scipy.io.loadmat(filename)['dets'], [
'x1', 'y1', 'x2', 'y2', 'dummy', 'dummy', 'dummy', 'dummy',
'score', 'cls_ind', 'img_ind'
], 'dets_correct')
dets_correct = dets_correct.subset(
['x1', 'y1', 'x2', 'y2', 'score', 'cls_ind', 'img_ind'])
dets_correct.arr[:, :4] -= 1
dets_correct.arr[:, :4] = BoundingBox.convert_arr_from_corners(
dets_correct.arr[:, :4])
dets_correct.cols = ['x', 'y', 'w', 'h', 'score', 'cls_ind', 'img_ind']
print('----mine:')
print(dets)
print('----correct:')
print(dets_correct)
assert (dets_correct == dets)
