class TestDatasetJson:
def setup(self):
self.d = Dataset('test_data1', force=True)
self.classes = ["A", "B", "C"]
def test_load(self):
assert (self.d.num_images() == 4)
assert (self.d.classes == self.classes)
def test_get_det_gt(self):
gt = self.d.get_det_gt(with_diff=True, with_trun=False)
df = Table(
np.array([[0., 0., 0., 0., 0., 0, 0, 0.],
[1., 1., 1., 1., 1., 0, 0, 0.],
[1., 1., 1., 0., 0., 0, 0, 1.],
[0., 0., 0., 0., 1., 0, 0, 2.],
[0., 0., 0., 0., 2., 0, 0, 3.],
[1., 1., 1., 1., 2., 0, 0, 3.]]),
['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'])
print(gt)
print(df)
assert (gt == df)
def test_get_cls_counts_json(self):
arr = np.array([[1, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 2]])
print(self.d.get_cls_counts())
assert (np.all(self.d.get_cls_counts() == arr))
def test_get_cls_ground_truth_json(self):
table = Table(
np.array([[True, True, False], [True, False, False],
[False, True, False], [False, False, True]]),
["A", "B", "C"])
assert (self.d.get_cls_ground_truth() == table)
def test_det_ground_truth_for_class_json(self):
gt = self.d.get_det_gt_for_class("A", with_diff=True, with_trun=True)
arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.],
[1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']
print(gt.arr)
assert (np.all(gt.arr == arr))
assert (gt.cols == cols)
# no diff or trun
gt = self.d.get_det_gt_for_class("A", with_diff=False, with_trun=False)
arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.],
[1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']
print(gt.arr)
assert (np.all(gt.arr == arr))
assert (gt.cols == cols)
def test_set_values(self):
assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes))))
self.d.set_values('uniform')
assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes))))
self.d.set_values('inverse_prior')
print(self.d.values)
assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
class TestEvaluationSynthetic:
def __init__(self):
self.d = Dataset('test_data2',force=True)
self.classes = ["A","B","C"]
self.det_gt = self.d.get_det_gt()
def test(self):
scores = np.ones(self.det_gt.shape[0])
dets = self.det_gt.append_column('score',scores)
scores = np.ones(self.d.get_det_gt_for_class('A').shape[0])
dets_just_A = self.d.get_det_gt_for_class('A')
dets_just_A = dets_just_A.append_column('score',scores)
self.d.set_values('uniform')
assert(np.all(self.d.values == 1./3 * np.ones(len(self.classes))))
dp = DatasetPolicy(self.d,self.d,detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets,self.det_gt)
assert(ap==1)
ap = ev.compute_det_map(dets_just_A,self.det_gt)
print(ap)
assert(ut.fequal(ap, 0.33333333333333))
self.d.set_values('inverse_prior')
assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
dp = DatasetPolicy(self.d,self.d,detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets,self.det_gt)
assert(ap==1)
ap = ev.compute_det_map(dets_just_A,self.det_gt)
print(ap)
assert(ut.fequal(ap, 0.25))
class TestEvaluationSynthetic:
def __init__(self):
self.d = Dataset('test_data2', force=True)
self.classes = ["A", "B", "C"]
self.det_gt = self.d.get_det_gt()
def test(self):
scores = np.ones(self.det_gt.shape[0])
dets = self.det_gt.append_column('score', scores)
scores = np.ones(self.d.get_det_gt_for_class('A').shape[0])
dets_just_A = self.d.get_det_gt_for_class('A')
dets_just_A = dets_just_A.append_column('score', scores)
self.d.set_values('uniform')
assert (np.all(self.d.values == 1. / 3 * np.ones(len(self.classes))))
dp = DatasetPolicy(self.d, self.d, detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets, self.det_gt)
assert (ap == 1)
ap = ev.compute_det_map(dets_just_A, self.det_gt)
print(ap)
assert (ut.fequal(ap, 0.33333333333333))
self.d.set_values('inverse_prior')
assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
dp = DatasetPolicy(self.d, self.d, detector='perfect')
ev = Evaluation(dp)
ap = ev.compute_det_map(dets, self.det_gt)
assert (ap == 1)
ap = ev.compute_det_map(dets_just_A, self.det_gt)
print(ap)
assert (ut.fequal(ap, 0.25))
class TestDatasetPascal:
def setup(self):
self.d = Dataset('test_pascal_train', force=True)
def test_ground_truth_pascal_train(self):
assert (self.d.num_classes() == 20)
assert ('dog' in self.d.classes)
def test_ground_truth_for_class_pascal(self):
correct = np.array([[48., 240., 148., 132., 11., 0., 1., 0.]])
ans = self.d.get_det_gt_for_class("dog")
print ans
assert np.all(ans.arr == correct)
def test_neg_samples(self):
# unlimited negative examples
indices = self.d.get_neg_samples_for_class("dog",
with_diff=True,
with_trun=True)
correct = np.array([1, 2])
assert (np.all(indices == correct))
# maximum 1 negative example
indices = self.d.get_neg_samples_for_class("dog",
1,
with_diff=True,
with_trun=True)
correct1 = np.array([1])
correct2 = np.array([2])
print(indices)
assert (np.all(indices == correct1) or np.all(indices == correct2))
def test_pos_samples(self):
indices = self.d.get_pos_samples_for_class("dog")
correct = np.array([0])
assert (np.all(indices == correct))
def test_ground_truth_test(self):
d = Dataset('test_pascal_val')
gt = d.get_det_gt(with_diff=False, with_trun=False)
correct = np.matrix([[139., 200., 69., 102., 18., 0., 0., 0.],
[123., 155., 93., 41., 17., 0., 0., 1.],
[239., 156., 69., 50., 8., 0., 0., 1.]])
print(gt)
assert np.all(gt.arr == correct)
def test_get_pos_windows(self):
d = Dataset('test_pascal_val')
# TODO
def test_kfold(self):
"""
'sizes' here are empirical values over the trainval set.
"""
d = Dataset('full_pascal_trainval')
numfolds = 4
d.create_folds(numfolds)
cls = 'dog'
sizes = [314, 308, 321, 320]
for i in range(len(d.folds)):
d.next_folds()
pos = d.get_pos_samples_for_fold_class(cls)
neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0])
assert (pos.shape[0] == sizes[i])
assert (neg.shape[0] == sizes[i])
class TestDatasetPascal:
def setup(self):
self.d = Dataset('test_pascal_train',force=True)
def test_ground_truth_pascal_train(self):
assert(self.d.num_classes() == 20)
assert('dog' in self.d.classes)
def test_ground_truth_for_class_pascal(self):
correct = np.array(
[[ 48., 240., 148., 132., 11., 0., 1., 0.]])
ans = self.d.get_det_gt_for_class("dog")
print ans
assert np.all(ans.arr == correct)
def test_neg_samples(self):
# unlimited negative examples
indices = self.d.get_neg_samples_for_class("dog",with_diff=True,with_trun=True)
correct = np.array([1,2])
assert(np.all(indices == correct))
# maximum 1 negative example
indices = self.d.get_neg_samples_for_class("dog",1,with_diff=True,with_trun=True)
correct1 = np.array([1])
correct2 = np.array([2])
print(indices)
assert(np.all(indices == correct1) or np.all(indices == correct2))
def test_pos_samples(self):
indices = self.d.get_pos_samples_for_class("dog")
correct = np.array([0])
assert(np.all(indices == correct))
def test_ground_truth_test(self):
d = Dataset('test_pascal_val')
gt = d.get_det_gt(with_diff=False,with_trun=False)
correct = np.matrix(
[ [ 139., 200., 69., 102., 18., 0., 0., 0.],
[ 123., 155., 93., 41., 17., 0., 0., 1.],
[ 239., 156., 69., 50., 8., 0., 0., 1.]])
print(gt)
assert np.all(gt.arr == correct)
def test_get_pos_windows(self):
d = Dataset('test_pascal_val')
# TODO
def test_kfold(self):
"""
'sizes' here are empirical values over the trainval set.
"""
d = Dataset('full_pascal_trainval')
numfolds = 4
d.create_folds(numfolds)
cls = 'dog'
sizes = [314, 308, 321, 320]
for i in range(len(d.folds)):
d.next_folds()
pos = d.get_pos_samples_for_fold_class(cls)
neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0])
assert(pos.shape[0] == sizes[i])
assert(neg.shape[0] == sizes[i])
class TestDatasetJson:
def setup(self):
self.d = Dataset('test_data1',force=True)
self.classes = ["A","B","C"]
def test_load(self):
assert(self.d.num_images() == 4)
assert(self.d.classes == self.classes)
def test_get_det_gt(self):
gt = self.d.get_det_gt(with_diff=True,with_trun=False)
df = Table(
np.array([[ 0., 0., 0., 0., 0., 0, 0, 0.],
[ 1., 1., 1., 1., 1., 0, 0, 0.],
[ 1., 1., 1., 0., 0., 0, 0, 1.],
[ 0., 0., 0., 0., 1., 0, 0, 2.],
[ 0., 0., 0., 0., 2., 0, 0, 3.],
[ 1., 1., 1., 1., 2., 0, 0, 3.]]),
['x','y','w','h','cls_ind','diff','trun','img_ind'])
print(gt)
print(df)
assert(gt == df)
def test_get_cls_counts_json(self):
arr = np.array(
[ [ 1, 1, 0],
[ 1, 0, 0],
[ 0, 1, 0],
[ 0, 0, 2]])
print(self.d.get_cls_counts())
assert(np.all(self.d.get_cls_counts() == arr))
def test_get_cls_ground_truth_json(self):
table = Table(
np.array([ [ True, True, False],
[ True, False, False],
[ False, True, False],
[ False, False, True] ]), ["A","B","C"])
assert(self.d.get_cls_ground_truth()==table)
def test_det_ground_truth_for_class_json(self):
gt = self.d.get_det_gt_for_class("A",with_diff=True,with_trun=True)
arr = np.array(
[[ 0., 0., 0., 0., 0., 0., 0, 0.],
[ 1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x','y','w','h','cls_ind','diff','trun','img_ind']
print(gt.arr)
assert(np.all(gt.arr == arr))
assert(gt.cols == cols)
# no diff or trun
gt = self.d.get_det_gt_for_class("A",with_diff=False,with_trun=False)
arr = np.array(
[[ 0., 0., 0., 0., 0., 0., 0, 0.],
[ 1., 1., 1., 0., 0., 0., 0., 1.]])
cols = ['x','y','w','h','cls_ind','diff','trun','img_ind']
print(gt.arr)
assert(np.all(gt.arr == arr))
assert(gt.cols == cols)
def test_set_values(self):
assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes))))
self.d.set_values('uniform')
assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes))))
self.d.set_values('inverse_prior')
print(self.d.values)
assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
