def __init__(self, dataset, L, numfolds=4): self.d = Dataset(dataset) self.e = Extractor() self.dense_codebook = self.e.get_codebook(self.d, 'dsift') self.sparse_codebook = self.e.get_codebook(self.d, 'sift') self.L = L self.numfolds = numfolds
def test_learn_weights(self): dataset = Dataset('full_pascal_val') train_dataset = Dataset('full_pascal_train') dataset.images = dataset.images[:20] train_dataset.images = train_dataset.images[:20] dp = DatasetPolicy(dataset,train_dataset,self.weights_dataset_name,**self.config) weights = dp.learn_weights()
def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False, with_trun=False) correct = np.matrix([[139., 200., 69., 102., 18., 0., 0., 0.], [123., 155., 93., 41., 17., 0., 0., 1.], [239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct)
def __init__(self): self.d = Dataset('full_pascal_trainval') self.d_val = Dataset('full_pascal_test') self.cls = 'dog' suffix = 'default' self.csc = CSCClassifier(suffix, self.cls, self.d, self.d_val) csc_test = np.load(config.get_ext_dets_filename(self.d, 'csc_default')) self.dets = csc_test[()]
def __init__(self): self.csc_trainval = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'csc_trainval'), 'r')) self.csc_test = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'csc_test'), 'r')) self.ext_csc_test = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'ext_csc_test'), 'r')) self.ext_csc_trainval = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'ext_csc_trainval'), 'r')) self.d_train = Dataset('full_pascal_trainval') self.trainval_gt = self.d_train.get_cls_ground_truth() self.d_test = Dataset('full_pascal_test') self.test_gt = self.d_test.get_cls_ground_truth()
def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False,with_trun=False) correct = np.matrix( [ [ 139., 200., 69., 102., 18., 0., 0., 0.], [ 123., 155., 93., 41., 17., 0., 0., 1.], [ 239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct)
def compute_error_vs_iterations(suffix, num_images, dataset): # assemble truth d = Dataset(dataset) truth = d.get_cls_ground_truth().arr truth = np.random.permutation(truth)[:num_images, :] num_classes = truth.shape[1] tt = ut.TicToc() lbp_times = [0] + [10**x for x in range(3)] lbp_times += [1000 + 1000 * x for x in range(10)] lbp_times += [10**x for x in [5]] #lbp_times = [3000] all_scores = np.zeros((num_classes, len(lbp_times), num_classes)) all_times = np.zeros((num_classes, len(lbp_times))) counter = 0 # do inference for itdex in range(len(lbp_times)): fm = FastinfModel(d, 'perfect', num_classes, lbp_time=lbp_times[itdex]) for rowdex in range(comm_rank, truth.shape[0], comm_size): # parallel obs = truth[rowdex, :].astype(int) taken = np.zeros(num_classes).astype(int) for num_obser in range(num_classes): counter += 1 taken[np.argmax(fm.p_c - taken)] = 1 tt.tic() fm.update_with_observations(taken, obs) utime = tt.toc(quiet=True) curr_score = compute_sq_error(obs, fm.p_c) all_scores[num_obser, itdex, :] = np.add(all_scores[num_obser, itdex, :], curr_score) all_times[num_obser, itdex] += utime print '%d is at %d / %d :'%(comm_rank, counter, len(lbp_times)* \ num_classes*num_images/float(comm_size)),curr_score all_scores /= num_images all_times /= num_images safebarrier(comm) all_scores = comm.reduce(all_scores) all_times = comm.reduce(all_times) if comm_rank == 0: #parallel outfile = open('all_scores_' + suffix, 'w') cPickle.dump(all_scores, outfile) outfile.close() outfile = open('all_times_' + suffix, 'w') cPickle.dump(all_times, outfile) outfile.close()
class ClassifierConfig(): def __init__(self, dataset, L, numfolds=4): self.d = Dataset(dataset) self.e = Extractor() self.dense_codebook = self.e.get_codebook(self.d, 'dsift') self.sparse_codebook = self.e.get_codebook(self.d, 'sift') self.L = L self.numfolds = numfolds def kfold(self): train_idx, val_idx = KFold(len(len(self.d.images), self.numfolds)) self.d.create_folds(self.numfolds)
def compute_error_vs_iterations(suffix, num_images, dataset): # assemble truth d = Dataset(dataset) truth = d.get_cls_ground_truth().arr truth = np.random.permutation(truth)[:num_images,:] num_classes = truth.shape[1] tt = ut.TicToc() lbp_times = [0] + [10**x for x in range(3)] lbp_times += [1000+1000*x for x in range(10)] lbp_times += [10**x for x in [5]] #lbp_times = [3000] all_scores = np.zeros((num_classes, len(lbp_times), num_classes)) all_times = np.zeros((num_classes, len(lbp_times))) counter = 0 # do inference for itdex in range(len(lbp_times)): fm = FastinfModel(d, 'perfect', num_classes, lbp_time=lbp_times[itdex]) for rowdex in range(comm_rank, truth.shape[0], comm_size): # parallel obs = truth[rowdex,:].astype(int) taken = np.zeros(num_classes).astype(int) for num_obser in range(num_classes): counter += 1 taken[np.argmax(fm.p_c-taken)] = 1 tt.tic() fm.update_with_observations(taken, obs) utime = tt.toc(quiet=True) curr_score = compute_sq_error(obs, fm.p_c) all_scores[num_obser, itdex, :] = np.add(all_scores[num_obser, itdex, :], curr_score) all_times[num_obser, itdex] += utime print '%d is at %d / %d :'%(comm_rank, counter, len(lbp_times)* \ num_classes*num_images/float(comm_size)),curr_score all_scores /= num_images all_times /= num_images safebarrier(comm) all_scores = comm.reduce(all_scores) all_times = comm.reduce(all_times) if comm_rank == 0: #parallel outfile = open('all_scores_'+suffix,'w') cPickle.dump(all_scores,outfile) outfile.close() outfile = open('all_times_'+suffix,'w') cPickle.dump(all_times,outfile) outfile.close()
def __init__(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train') self.weights_dataset_name = 'test_pascal_val' self.config = { 'suffix': 'default', 'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half 'policy_mode': 'random', 'bounds': None, 'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl } self.dp = DatasetPolicy(self.dataset, self.train_dataset, self.weights_dataset_name, **self.config)
def run(): dataset = Dataset('full_pascal_test') train_dataset = Dataset('full_pascal_trainval') cls = 'dog' rtype = '1big_2small' args = 0.5 detector = 'csc_default' from synthetic.dataset_policy import DatasetPolicy all_dets = DatasetPolicy.load_ext_detections(dataset, detector) cls_ind = dataset.get_ind(cls) dets = all_dets.filter_on_column('cls_ind',cls_ind,omit=True) ext_det = ExternalDetectorRegions(dataset, train_dataset, cls, dets, detector, rtype, args) img = dataset.images[13] # Just some random image...where did the get_image_by_name go? print img.size print ext_det.detect(img, 0) print ext_det.detect(img, 1) print ext_det.detect(img, 2)
class TestEvaluationSynthetic: def __init__(self): self.d = Dataset('test_data2', force=True) self.classes = ["A", "B", "C"] self.det_gt = self.d.get_det_gt() def test(self): scores = np.ones(self.det_gt.shape[0]) dets = self.det_gt.append_column('score', scores) scores = np.ones(self.d.get_det_gt_for_class('A').shape[0]) dets_just_A = self.d.get_det_gt_for_class('A') dets_just_A = dets_just_A.append_column('score', scores) self.d.set_values('uniform') assert (np.all(self.d.values == 1. / 3 * np.ones(len(self.classes)))) dp = DatasetPolicy(self.d, self.d, detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets, self.det_gt) assert (ap == 1) ap = ev.compute_det_map(dets_just_A, self.det_gt) print(ap) assert (ut.fequal(ap, 0.33333333333333)) self.d.set_values('inverse_prior') assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5]))) dp = DatasetPolicy(self.d, self.d, detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets, self.det_gt) assert (ap == 1) ap = ev.compute_det_map(dets_just_A, self.det_gt) print(ap) assert (ut.fequal(ap, 0.25))
class TestEvaluationSynthetic: def __init__(self): self.d = Dataset('test_data2',force=True) self.classes = ["A","B","C"] self.det_gt = self.d.get_det_gt() def test(self): scores = np.ones(self.det_gt.shape[0]) dets = self.det_gt.append_column('score',scores) scores = np.ones(self.d.get_det_gt_for_class('A').shape[0]) dets_just_A = self.d.get_det_gt_for_class('A') dets_just_A = dets_just_A.append_column('score',scores) self.d.set_values('uniform') assert(np.all(self.d.values == 1./3 * np.ones(len(self.classes)))) dp = DatasetPolicy(self.d,self.d,detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets,self.det_gt) assert(ap==1) ap = ev.compute_det_map(dets_just_A,self.det_gt) print(ap) assert(ut.fequal(ap, 0.33333333333333)) self.d.set_values('inverse_prior') assert(np.all(self.d.values == np.array([0.25,0.25,0.5]))) dp = DatasetPolicy(self.d,self.d,detector='perfect') ev = Evaluation(dp) ap = ev.compute_det_map(dets,self.det_gt) assert(ap==1) ap = ev.compute_det_map(dets_just_A,self.det_gt) print(ap) assert(ut.fequal(ap, 0.25))
def run(): dataset = Dataset('full_pascal_test') train_dataset = Dataset('full_pascal_trainval') cls = 'dog' rtype = '1big_2small' args = 0.5 detector = 'csc_default' from synthetic.dataset_policy import DatasetPolicy all_dets = DatasetPolicy.load_ext_detections(dataset, detector) cls_ind = dataset.get_ind(cls) dets = all_dets.filter_on_column('cls_ind', cls_ind, omit=True) ext_det = ExternalDetectorRegions(dataset, train_dataset, cls, dets, detector, rtype, args) img = dataset.images[ 13] # Just some random image...where did the get_image_by_name go? print img.size print ext_det.detect(img, 0) print ext_det.detect(img, 1) print ext_det.detect(img, 2)
def __init__(self): self.csc_trainval = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'csc_trainval'), 'r')) self.csc_test = cPickle.load( open(os.path.join(config.get_ext_test_support_dir(), 'csc_test'), 'r')) self.ext_csc_test = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'ext_csc_test'), 'r')) self.ext_csc_trainval = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'ext_csc_trainval'), 'r')) self.d_train = Dataset('full_pascal_trainval') self.trainval_gt = self.d_train.get_cls_ground_truth() self.d_test = Dataset('full_pascal_test') self.test_gt = self.d_test.get_cls_ground_truth()
def __init__(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train') self.weights_dataset_name = 'test_pascal_val' self.config = { 'suffix': 'default', 'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half 'policy_mode': 'random', 'bounds': None, 'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl } self.dp = DatasetPolicy( self.dataset,self.train_dataset,self.weights_dataset_name,**self.config)
def test_kfold(self): """ 'sizes' here are empirical values over the trainval set. """ d = Dataset('full_pascal_trainval') numfolds = 4 d.create_folds(numfolds) cls = 'dog' sizes = [314, 308, 321, 320] for i in range(len(d.folds)): d.next_folds() pos = d.get_pos_samples_for_fold_class(cls) neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0]) assert (pos.shape[0] == sizes[i]) assert (neg.shape[0] == sizes[i])
def test_learn_weights(self): dataset = Dataset('full_pascal_val') train_dataset = Dataset('full_pascal_train') dataset.images = dataset.images[:20] train_dataset.images = train_dataset.images[:20] dp = DatasetPolicy(dataset, train_dataset, self.weights_dataset_name, **self.config) weights = dp.learn_weights()
def test(): dataset = Dataset('full_pascal_trainval') fm = FastinfModel(dataset, 'perfect', 20) # NOTE: just took values from a run of the thing prior_correct = [ float(x) for x in "0.050543 0.053053 0.073697 0.038331 0.050954 0.041879 0.16149\ 0.068721 0.10296 0.026837 0.043779 0.087683 0.063447 0.052205\ 0.41049 0.051664 0.014211 0.068361 0.056969 0.05046".split() ] np.testing.assert_almost_equal(fm.p_c, prior_correct, 4) observations = np.zeros(20) taken = np.zeros(20) fm.update_with_observations(taken, observations) np.testing.assert_almost_equal(fm.p_c, prior_correct, 4) observations[5] = 1 taken[5] = 1 fm.update_with_observations(taken, observations) print fm.p_c correct = [ float(x) for x in "0.027355 0.11855 0.027593 0.026851 0.012569 0.98999 0.52232\ 0.017783 0.010806 0.015199 0.0044641 0.02389 0.033602 0.089089\ 0.50297 0.0083272 0.0088274 0.0098522 0.034259 0.0086298".split() ] np.testing.assert_almost_equal(fm.p_c, correct, 4) observations[15] = 0 taken[15] = 1 fm.update_with_observations(taken, observations) correct = [ float(x) for x in "2.73590000e-02 1.19030000e-01 2.75500000e-02 2.68760000e-02 \ 1.23920000e-02 9.90200000e-01 5.25320000e-01 1.76120000e-02 \ 1.05030000e-02 1.52130000e-02 4.26410000e-03 2.38250000e-02 \ 3.36870000e-02 8.96450000e-02 5.04300000e-01 8.71880000e-05 \ 8.82630000e-03 9.55290000e-03 3.43240000e-02 8.44510000e-03".split() ] np.testing.assert_almost_equal(fm.p_c, correct) # reinit_marginals fm.reset() np.testing.assert_equal(fm.p_c, prior_correct) print(fm.cache)
def test_kfold(self): """ 'sizes' here are empirical values over the trainval set. """ d = Dataset('full_pascal_trainval') numfolds = 4 d.create_folds(numfolds) cls = 'dog' sizes = [314, 308, 321, 320] for i in range(len(d.folds)): d.next_folds() pos = d.get_pos_samples_for_fold_class(cls) neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0]) assert(pos.shape[0] == sizes[i]) assert(neg.shape[0] == sizes[i])
Created on Nov 21, 2011 @author: Tobias Baumgartner ''' from common_imports import * from common_mpi import * import synthetic.config as config from synthetic.dataset import Dataset from synthetic.classifier import Classifier if __name__=='__main__': train_set = 'full_pascal_train' train_dataset = Dataset(train_set) images = train_dataset.images classes = config.pascal_classes suffix = 'default' filename = config.get_ext_dets_filename(train_dataset, 'csc_'+suffix) csc_train = np.load(filename) csc_train = csc_train[()] csc_train = csc_train.subset(['score', 'cls_ind', 'img_ind']) score = csc_train.subset(['score']).arr classif = Classifier() csc_train.arr = classif.normalize_dpm_scores(csc_train.arr) numpos = train_dataset.get_ground_truth().shape[0] threshs = np.arange(0,1.01,0.05)
def setup(self): self.d = Dataset('test_pascal_train',force=True)
def test_get_pos_windows(self): d = Dataset('test_pascal_val')
""" Runner script to output cooccurrence statistics for the synthetic and PASCAL datasets. """ from synthetic.common_imports import * from synthetic.dataset import Dataset datasets = [ 'synthetic', 'full_pascal_train', 'full_pascal_trainval', 'full_pascal_val', 'full_pascal_test' ] for dataset in datasets: d = Dataset(dataset) f = d.plot_coocurrence() f = d.plot_coocurrence(second_order=True) f = d.plot_distribution()
def setup(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train')
class TestDatasetJson: def setup(self): self.d = Dataset('test_data1',force=True) self.classes = ["A","B","C"] def test_load(self): assert(self.d.num_images() == 4) assert(self.d.classes == self.classes) def test_get_det_gt(self): gt = self.d.get_det_gt(with_diff=True,with_trun=False) df = Table( np.array([[ 0., 0., 0., 0., 0., 0, 0, 0.], [ 1., 1., 1., 1., 1., 0, 0, 0.], [ 1., 1., 1., 0., 0., 0, 0, 1.], [ 0., 0., 0., 0., 1., 0, 0, 2.], [ 0., 0., 0., 0., 2., 0, 0, 3.], [ 1., 1., 1., 1., 2., 0, 0, 3.]]), ['x','y','w','h','cls_ind','diff','trun','img_ind']) print(gt) print(df) assert(gt == df) def test_get_cls_counts_json(self): arr = np.array( [ [ 1, 1, 0], [ 1, 0, 0], [ 0, 1, 0], [ 0, 0, 2]]) print(self.d.get_cls_counts()) assert(np.all(self.d.get_cls_counts() == arr)) def test_get_cls_ground_truth_json(self): table = Table( np.array([ [ True, True, False], [ True, False, False], [ False, True, False], [ False, False, True] ]), ["A","B","C"]) assert(self.d.get_cls_ground_truth()==table) def test_det_ground_truth_for_class_json(self): gt = self.d.get_det_gt_for_class("A",with_diff=True,with_trun=True) arr = np.array( [[ 0., 0., 0., 0., 0., 0., 0, 0.], [ 1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x','y','w','h','cls_ind','diff','trun','img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) # no diff or trun gt = self.d.get_det_gt_for_class("A",with_diff=False,with_trun=False) arr = np.array( [[ 0., 0., 0., 0., 0., 0., 0, 0.], [ 1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x','y','w','h','cls_ind','diff','trun','img_ind'] print(gt.arr) assert(np.all(gt.arr == arr)) assert(gt.cols == cols) def test_set_values(self): assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes)))) self.d.set_values('uniform') assert(np.all(self.d.values == 1/3. * np.ones(len(self.classes)))) self.d.set_values('inverse_prior') print(self.d.values) assert(np.all(self.d.values == np.array([0.25,0.25,0.5])))
class TestDatasetPolicy: def __init__(self): self.dataset = Dataset('test_pascal_val') self.train_dataset = Dataset('test_pascal_train') self.weights_dataset_name = 'test_pascal_val' self.config = { 'suffix': 'default', 'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half 'policy_mode': 'random', 'bounds': None, 'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl } self.dp = DatasetPolicy( self.dataset,self.train_dataset,self.weights_dataset_name,**self.config) def test_run_on_dataset(self): # run on test dataset dets,clses,samples = self.dp.run_on_dataset(force=True) assert(len(samples) == clses.shape[0]) assert(len(samples) == self.dp.dataset.num_images()*len(self.dp.actions)) train_dets,train_clses,train_samples = self.dp.run_on_dataset(train=True,force=True) assert(len(train_samples) == train_clses.shape[0]) assert(len(train_samples) == self.dp.train_dataset.num_images()*len(self.dp.actions)) def test_unique_samples(self): "Test the correctness of making a list of samples unique." dets,clses,samples = self.dp.run_on_dataset(force=True) new_sample = copy.deepcopy(samples[11]) new_sample2 = copy.deepcopy(samples[11]) new_sample2.dt = -40 # an unreasonable value assert(new_sample in samples) assert(new_sample2 not in samples) def test_output_det_statistics(self): self.dp.output_det_statistics() def test_learn_weights(self): dataset = Dataset('full_pascal_val') train_dataset = Dataset('full_pascal_train') dataset.images = dataset.images[:20] train_dataset.images = train_dataset.images[:20] dp = DatasetPolicy(dataset,train_dataset,self.weights_dataset_name,**self.config) weights = dp.learn_weights() def test_regress(self): dets,clses,samples = self.dp.run_on_dataset(force=True) weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) samples += samples weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) samples += samples weights,error = self.dp.regress(samples,'greedy') print "Weights after %d samples:\n %s"%(len(samples),weights) print "Error after %d samples: %s"%(len(samples),error) def test_load_weights(self): modes = ['manual_1','manual_2','manual_3'] for mode in modes: print "%s weights:"%mode self.dp.weights_mode=mode self.dp.load_weights() print self.dp.get_reshaped_weights() assert(self.dp.weights.shape[0] == len(self.dp.actions)*BeliefState.num_features) self.dp.write_weights_image('temp_weights_%s.png'%mode) def test_perfect_detector(self): dets,clses,samples = self.dp.run_on_dataset(force=True) #embed() dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) gt = self.dataset.get_det_gt() gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind']) # TODO: does this make sense? dets.sort_by_column('x') gt.sort_by_column('x') print dets print gt assert(dets == gt) def test_load_dpm_detections(self): conf = dict(self.config) conf['detectors'] = ['dpm'] policy = DatasetPolicy(self.dataset,self.train_dataset,**conf) assert(policy.detectors == ['dpm']) dets = policy.load_ext_detections(self.dataset,'dpm_may25',force=True) dets = dets.with_column_omitted('time') # load the ground truth dets, processed in Matlab # (timely/data/test_support/concat_dets.m) filename = os.path.join(config.test_support_dir, 'val_dets.mat') dets_correct = Table( scipy.io.loadmat(filename)['dets'], ['x1','y1','x2','y2','dummy','dummy','dummy','dummy','score','cls_ind','img_ind'], 'dets_correct') dets_correct = dets_correct.subset( ['x1','y1','x2','y2','score','cls_ind','img_ind']) dets_correct.arr[:,:4] -= 1 dets_correct.arr[:,:4] = BoundingBox.convert_arr_from_corners( dets_correct.arr[:,:4]) dets_correct.cols = ['x','y','w','h','score','cls_ind','img_ind'] print('----mine:') print(dets) print('----correct:') print(dets_correct) assert(dets_correct == dets)
class TestEvaluationPerfect: def __init__(self): self.csc_trainval = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'csc_trainval'), 'r')) self.csc_test = cPickle.load( open(os.path.join(config.get_ext_test_support_dir(), 'csc_test'), 'r')) self.ext_csc_test = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'ext_csc_test'), 'r')) self.ext_csc_trainval = cPickle.load( open( os.path.join(config.get_ext_test_support_dir(), 'ext_csc_trainval'), 'r')) self.d_train = Dataset('full_pascal_trainval') self.trainval_gt = self.d_train.get_cls_ground_truth() self.d_test = Dataset('full_pascal_test') self.test_gt = self.d_test.get_cls_ground_truth() def setup(self): train_dataset = Dataset('test_pascal_train', force=True) dataset = Dataset('test_pascal_val', force=True) self.dp = DatasetPolicy(dataset, train_dataset, detector='perfect') self.evaluation = Evaluation(self.dp) def test_compute_pr_multiclass(self): cols = ['x', 'y', 'w', 'h', 'cls_ind', 'img_ind', 'diff'] dets_cols = ['x', 'y', 'w', 'h', 'score', 'time', 'cls_ind', 'img_ind'] # two objects of different classes in the image, perfect detection arr = np.array([[0, 0, 10, 10, 0, 0, 0], [10, 10, 10, 10, 1, 0, 0]]) gt = Table(arr, cols) dets_arr = np.array([[0, 0, 10, 10, -1, -1, 0, 0], [10, 10, 10, 10, -1, -1, 1, 0]]) dets = Table(dets_arr, dets_cols) # make sure gt and gt_cols aren't modified gt_arr_copy = gt.arr.copy() gt_cols_copy = list(gt.cols) ap, rec, prec = self.evaluation.compute_det_pr(dets, gt) assert (np.all(gt.arr == gt_arr_copy)) assert (gt_cols_copy == gt.cols) correct_ap = 1 correct_rec = np.array([0.5, 1]) correct_prec = np.array([1, 1]) print((ap, rec, prec)) assert (correct_ap == ap) assert (np.all(correct_rec == rec)) assert (np.all(correct_prec == prec)) # some extra detections to generate false positives dets_arr = np.array([[0, 0, 10, 10, -1, -1, 0, 0], [0, 0, 10, 10, 0, -1, 0, 0], [10, 10, 10, 10, 0, -1, 1, 0], [10, 10, 10, 10, -1, -1, 1, 0]]) dets = Table(dets_arr, dets_cols) ap, rec, prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5, 1, 1, 1]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert (np.all(correct_rec == rec)) assert (np.all(correct_prec == prec)) # confirm that running on the same dets gives the same answer ap, rec, prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5, 1, 1, 1]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert (np.all(correct_rec == rec)) assert (np.all(correct_prec == prec)) # now let's add two objects of a different class to gt to lower recall arr = np.array([[0, 0, 10, 10, 0, 0, 0], [10, 10, 10, 10, 1, 0, 0], [20, 20, 10, 10, 2, 0, 0], [30, 30, 10, 10, 2, 0, 0]]) gt = Table(arr, cols) ap, rec, prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.25, 0.5, 0.5, 0.5]) correct_prec = np.array([1, 1, 2. / 3, 0.5]) print((ap, rec, prec)) assert (np.all(correct_rec == rec)) assert (np.all(correct_prec == prec)) # now call it with empty detections dets_arr = np.array([]) dets = Table(dets_arr, dets_cols) ap, rec, prec = self.evaluation.compute_det_pr(dets, gt) correct_ap = 0 correct_rec = np.array([0]) correct_prec = np.array([0]) print((ap, rec, prec)) assert (np.all(correct_ap == ap)) assert (np.all(correct_rec == rec)) assert (np.all(correct_prec == prec)) def test_compute_cls_map(self): res = Evaluation.compute_cls_map(self.csc_trainval, self.trainval_gt) assert (round(res, 11) == 0.47206391958) def test_compute_cls_map_half(self): table_csc_half = Table() table_csc_half.cols = list(self.csc_trainval.cols) for _ in range(10): rand_inds = np.random.permutation(range(5011))[:2500] table_csc_half.arr = self.csc_trainval.arr[rand_inds, :] res = Evaluation.compute_cls_map(table_csc_half, self.trainval_gt) assert (round(res, 6) > .45) def test_compute_cls_map_gt(self): res = Evaluation.compute_cls_map(self.trainval_gt, self.trainval_gt) assert (round(res, 6) == 1) def test_compute_cls_map_gt_half(self): rand_inds = np.random.permutation(range(5011))[:2500] table_gt_half = Table() table_gt_half.arr = np.hstack( (self.trainval_gt.arr, np.array(np.arange(5011), ndmin=2).T)) table_gt_half.arr = table_gt_half.arr[rand_inds, :] table_gt_half.cols = list(self.trainval_gt.cols) + ['img_ind'] res = Evaluation.compute_cls_map(table_gt_half, self.trainval_gt) assert (round(res, 6) == 1) def test_compute_cls_map_random_clf(self): clf_table = Table() num_test = 10 ress = np.zeros((num_test, )) for idx in range(num_test): clf_table.arr = np.hstack( (np.random.rand(5011, 20), np.array(np.arange(5011), ndmin=2).T)) clf_table.cols = list(self.trainval_gt.cols) + ['img_ind'] res = Evaluation.compute_cls_map(clf_table, self.trainval_gt) ress[idx] = res assert (np.mean(ress) < 0.09) def test_other_scores(self): print 'csc_test', Evaluation.compute_cls_map(self.csc_test, self.test_gt) print 'csc_trainval', Evaluation.compute_cls_map( self.csc_trainval, self.trainval_gt) print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test, self.test_gt) print 'ext_trainval', Evaluation.compute_cls_map( self.ext_csc_trainval, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_chi2_20') ext_table_chi2_20 = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_test', Evaluation.compute_cls_map( ext_table_chi2_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_chi2_20') ext_table_chi2_20_tv = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_trainval', Evaluation.compute_cls_map( ext_table_chi2_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_rbf_20') ext_table_rbf_20 = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_test', Evaluation.compute_cls_map( ext_table_rbf_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_rbf_20') ext_table_rbf_20_tv = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_trainval', Evaluation.compute_cls_map( ext_table_rbf_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_linear_20') ext_linear_20_test = cPickle.load(open(filename, 'r')) print 'ext_linear_test', Evaluation.compute_cls_map( ext_linear_20_test, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_linear_20') ext_table_linear_20 = cPickle.load(open(filename, 'r')) print 'ext_linear_20_trainval', Evaluation.compute_cls_map( ext_table_linear_20, self.trainval_gt) filename = 'tab_linear_5' ext_tab_lin_5 = cPickle.load(open(filename, 'r')) print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map( ext_tab_lin_5, self.trainval_gt)
def __init__(self): self.d = Dataset('test_data2', force=True) self.classes = ["A", "B", "C"] self.det_gt = self.d.get_det_gt()
codebook = e.get_codebook(d, 'sift') ut.makedirs(join(config.data_dir, 'jumping_window','lookup')) train_jumping_windows(d, codebook, use_scale=use_scale,trun=True,diff=False, feature=feature) debug = True just_eval = True if just_eval: basedir = join(config.data_dir, 'jumping_window') foldname_det = join(basedir, 'detections') foldname_lookup = join(basedir, 'lookup') ut.makedirs(foldname_det) print 'start testing on node', comm_rank dtest = Dataset('full_pascal_test') #for cls_idx, cls in enumerate(all_classes): for cls_idx, cls in enumerate([all_classes[0]]): #cls=all_classes gt_t = dtest.get_ground_truth_for_class(cls, with_diff=False, with_trun=True) e = Extractor() codebook = e.get_codebook(dtest, 'sift') filename_lookup = join(foldname_lookup,cls) store_file = open(filename_lookup, 'r') bbinfo = cPickle.load(store_file) test_gt = gt_t.arr npos = test_gt.shape[0] test_imgs = test_gt[:,gt_t.cols.index('img_ind')]
print table cPickle.dump(table, open('table','w')) print 'saved' return table def conv(d_train, table_arr): table = Table() #table_arr = cPickle.load(open('table_linear_5','r')) table.arr = np.hstack((table_arr, np.array(np.arange(table_arr.shape[0]),ndmin=2).T)) table.cols = d_train.classes + ['img_ind'] print table #cPickle.dump(table, open('tab_linear_5','w')) return table if __name__=='__main__': d_train = Dataset('full_pascal_trainval') d_val = Dataset('full_pascal_val') train_gt = d_train.get_cls_ground_truth() val_gt = d_val.get_cls_ground_truth() if comm_rank == 0: filename = os.path.join(config.get_classifier_dataset_dirname(CSCClassifier('default','dog', d_train, d_val), d_train),'crossval.txt') kernels = ['linear'] Cs = [50] settings = list(itertools.product(kernels, Cs)) for setin in settings: kernel = setin[0]
# testY = load_from_mat('testY.mat', 'testY') # print testY # model = train_svm(x, y) # print 'result:' # result = svm_predict(x0, model) # print result # d = Dataset('full_pascal_trainval') # d.evaluate_get_pos_windows(0.5) #if False: randomize = not os.path.exists('/home/tobibaum') d = Dataset('full_pascal_train') dtest = Dataset('full_pascal_val') e = Extractor() classes = config.pascal_classes num_words = 3000 iters = 10 feature_type = 'dsift' codebook_samples = 15 num_pos = 'max' testsize = 'max' kernel = 'chi2' # num_pos = 3 # testsize = 4
def main(): parser = argparse.ArgumentParser( description='Execute different functions of our system') parser.add_argument('--first_n', type=int, help='only take the first N images in the datasets') parser.add_argument('--name', help='name for this run', default='default', choices=['default', 'nolateral', 'nohal', 'halfsize']) parser.add_argument('--force', action='store_true', default=False, help='force overwrite') args = parser.parse_args() print(args) #configuration class class config(object): pass cfg = config() cfg.testname = "../ctfdet/data/finalRL/%s2_test" #object model cfg.bottomup = False #use complete search cfg.resize = 1.0 #resize the input image cfg.hallucinate = True #use HOGs up to 4 pixels cfg.initr = 1 #initial radious of the CtF search cfg.ratio = 1 #radious at the next levels cfg.deform = True #use deformation cfg.usemrf = True #use lateral constraints if args.name == 'default': cfg # sticking with the default params elif args.name == 'nolateral': cfg.usemrf = False elif args.name == 'nohal': cfg.hallucinate = False elif args.name == 'halfsize': cfg.resize = 0.5 # f**k it, do both test_datasets = ['val', 'test', 'train'] for test_dataset in test_datasets: # Load the dataset dataset = Dataset('full_pascal_' + test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # create directory for storing cached detections dirname = './temp_data' if os.path.exists('/u/sergeyk'): dirname = '/u/vis/x1/sergeyk/object_detection' dirname = dirname + '/ctfdets/%s' % (args.name) ut.makedirs(dirname) num_images = len(dataset.images) for img_ind in range(comm_rank, num_images, comm_size): # check for existing det image = dataset.images[img_ind] filename = os.path.join(dirname, image.name + '.npy') if os.path.exists(filename) and not args.force: #table = np.load(filename)[()] continue #read the image imname = dataset.get_image_filename(img_ind) img = util2.myimread(imname, resize=cfg.resize) #compute the hog pyramid f = pyrHOG2.pyrHOG(img, interv=10, savedir="", notsave=True, notload=True, hallucinate=cfg.hallucinate, cformat=True) #for each class all_dets = [] for ccls in dataset.classes: t = time.time() cls_ind = dataset.get_ind(ccls) print "%s Img %d/%d Class: %s" % (test_dataset, img_ind + 1, num_images, ccls) #load the class model m = util2.load("%s%d.model" % (cfg.testname % ccls, 7)) res = [] t1 = time.time() #for each aspect for clm, m in enumerate(m): #scan the image with left and right models res.append( pyrHOG2RL.detectflip(f, m, None, hallucinate=cfg.hallucinate, initr=cfg.initr, ratio=cfg.ratio, deform=cfg.deform, bottomup=cfg.bottomup, usemrf=cfg.usemrf, small=False, cl=clm)) fuse = [] numhog = 0 #fuse the detections for mix in res: tr = mix[0] fuse += mix[1] numhog += mix[3] rfuse = tr.rank(fuse, maxnum=300) nfuse = tr.cluster(rfuse, ovr=0.3, inclusion=False) #print "Number of computed HOGs:",numhog time_elapsed = time.time() - t print "Elapsed time: %.3f s" % time_elapsed bboxes = [nf['bbox'] for nf in nfuse] scores = [nf['scr'] for nf in nfuse] assert (len(bboxes) == len(scores)) if len(bboxes) > 0: arr = np.zeros((len(bboxes), 7)) arr[:, :4] = BoundingBox.convert_arr_from_corners( np.array(bboxes)) arr[:, 4] = scores arr[:, 5] = time_elapsed arr[:, 6] = cls_ind all_dets.append(arr) cols = ['x', 'y', 'w', 'h', 'score', 'time', 'cls_ind'] if len(all_dets) > 0: all_dets = np.concatenate(all_dets, 0) else: all_dets = np.array([]) table = Table(all_dets, cols) np.save(filename, table)
class TestDatasetJson: def setup(self): self.d = Dataset('test_data1', force=True) self.classes = ["A", "B", "C"] def test_load(self): assert (self.d.num_images() == 4) assert (self.d.classes == self.classes) def test_get_det_gt(self): gt = self.d.get_det_gt(with_diff=True, with_trun=False) df = Table( np.array([[0., 0., 0., 0., 0., 0, 0, 0.], [1., 1., 1., 1., 1., 0, 0, 0.], [1., 1., 1., 0., 0., 0, 0, 1.], [0., 0., 0., 0., 1., 0, 0, 2.], [0., 0., 0., 0., 2., 0, 0, 3.], [1., 1., 1., 1., 2., 0, 0, 3.]]), ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind']) print(gt) print(df) assert (gt == df) def test_get_cls_counts_json(self): arr = np.array([[1, 1, 0], [1, 0, 0], [0, 1, 0], [0, 0, 2]]) print(self.d.get_cls_counts()) assert (np.all(self.d.get_cls_counts() == arr)) def test_get_cls_ground_truth_json(self): table = Table( np.array([[True, True, False], [True, False, False], [False, True, False], [False, False, True]]), ["A", "B", "C"]) assert (self.d.get_cls_ground_truth() == table) def test_det_ground_truth_for_class_json(self): gt = self.d.get_det_gt_for_class("A", with_diff=True, with_trun=True) arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert (np.all(gt.arr == arr)) assert (gt.cols == cols) # no diff or trun gt = self.d.get_det_gt_for_class("A", with_diff=False, with_trun=False) arr = np.array([[0., 0., 0., 0., 0., 0., 0, 0.], [1., 1., 1., 0., 0., 0., 0., 1.]]) cols = ['x', 'y', 'w', 'h', 'cls_ind', 'diff', 'trun', 'img_ind'] print(gt.arr) assert (np.all(gt.arr == arr)) assert (gt.cols == cols) def test_set_values(self): assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_values('uniform') assert (np.all(self.d.values == 1 / 3. * np.ones(len(self.classes)))) self.d.set_values('inverse_prior') print(self.d.values) assert (np.all(self.d.values == np.array([0.25, 0.25, 0.5])))
class TestDatasetPascal: def setup(self): self.d = Dataset('test_pascal_train', force=True) def test_ground_truth_pascal_train(self): assert (self.d.num_classes() == 20) assert ('dog' in self.d.classes) def test_ground_truth_for_class_pascal(self): correct = np.array([[48., 240., 148., 132., 11., 0., 1., 0.]]) ans = self.d.get_det_gt_for_class("dog") print ans assert np.all(ans.arr == correct) def test_neg_samples(self): # unlimited negative examples indices = self.d.get_neg_samples_for_class("dog", with_diff=True, with_trun=True) correct = np.array([1, 2]) assert (np.all(indices == correct)) # maximum 1 negative example indices = self.d.get_neg_samples_for_class("dog", 1, with_diff=True, with_trun=True) correct1 = np.array([1]) correct2 = np.array([2]) print(indices) assert (np.all(indices == correct1) or np.all(indices == correct2)) def test_pos_samples(self): indices = self.d.get_pos_samples_for_class("dog") correct = np.array([0]) assert (np.all(indices == correct)) def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False, with_trun=False) correct = np.matrix([[139., 200., 69., 102., 18., 0., 0., 0.], [123., 155., 93., 41., 17., 0., 0., 1.], [239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct) def test_get_pos_windows(self): d = Dataset('test_pascal_val') # TODO def test_kfold(self): """ 'sizes' here are empirical values over the trainval set. """ d = Dataset('full_pascal_trainval') numfolds = 4 d.create_folds(numfolds) cls = 'dog' sizes = [314, 308, 321, 320] for i in range(len(d.folds)): d.next_folds() pos = d.get_pos_samples_for_fold_class(cls) neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0]) assert (pos.shape[0] == sizes[i]) assert (neg.shape[0] == sizes[i])
return np.zeros((1,intervals+1)) dpm = feats.subset(['score', 'cls_ind', 'img_ind']) img_dpm = dpm.filter_on_column('img_ind', img, omit=True) if img_dpm.arr.size == 0: print 'empty vector' return np.zeros((1,intervals+1)) cls_dpm = img_dpm.filter_on_column('cls_ind', cls, omit=True) hist = self.compute_histogram(cls_dpm.arr, intervals, lower, upper) vector = np.zeros((1, intervals+1)) vector[0,0:-1] = hist vector[0,-1] = img_dpm.shape[0] return vector if __name__=='__main__': train_set = 'full_pascal_train' train_dataset = Dataset(train_set) dpm_dir = os.path.join(config.res_dir, 'dpm_dets') filename = os.path.join(dpm_dir, train_set + '_dets_all_may25_DP.npy') dpm_train = np.load(filename) dpm_train = dpm_train[()] dpm_train = dpm_train.subset(['score', 'cls_ind', 'img_ind']) dpm_classif = DPMClassifier() dpm_train.arr = dpm_classif.normalize_dpm_scores(dpm_train.arr) val_set = 'full_pascal_val' test_dataset = Dataset(val_set) dpm_test_dir = os.path.join(config.res_dir, 'dpm_dets') filename = os.path.join(dpm_dir, val_set + '_dets_all_may25_DP.npy') dpm_test = np.load(filename) dpm_test = dpm_test[()] dpm_test = dpm_test.subset(['score', 'cls_ind', 'img_ind'])
def main(): parser = argparse.ArgumentParser(description="Run experiments with the timely detection system.") parser.add_argument( "--test_dataset", choices=["val", "test", "trainval"], default="val", help="""Dataset to use for testing. Run on val until final runs. The training dataset is inferred (val->train; test->trainval; trainval->trainval).""", ) parser.add_argument("--first_n", type=int, help="only take the first N images in the test dataset") parser.add_argument("--first_n_train", type=int, help="only take the first N images in the train dataset") parser.add_argument( "--config", help="""Config file name that specifies the experiments to run. Give name such that the file is configs/#{name}.json or configs/#{name}/ In the latter case, all files within the directory will be loaded.""", ) parser.add_argument("--suffix", help="Overwrites the suffix in the config(s).") parser.add_argument("--bounds10", action="store_true", default=False, help="set bounds to [0,10]") parser.add_argument("--bounds515", action="store_true", default=False, help="set bounds to [5,15]") parser.add_argument("--force", action="store_true", default=False, help="force overwrite") parser.add_argument("--wholeset_prs", action="store_true", default=False, help="evaluate in the final p-r regime") parser.add_argument( "--no_apvst", action="store_true", default=False, help="do NOT evaluate in the ap vs. time regime" ) parser.add_argument( "--det_configs", action="store_true", default=False, help="output detector statistics to det_configs" ) parser.add_argument("--inverse_prior", action="store_true", default=False, help="use inverse prior class values") args = parser.parse_args() print(args) # If config file is not given, just run one experiment using default config if not args.config: configs = [DatasetPolicy.default_config] else: configs = load_configs(args.config) # Load the dataset dataset = Dataset("full_pascal_" + args.test_dataset) if args.first_n: dataset.images = dataset.images[: args.first_n] # Infer train_dataset if args.test_dataset == "test": train_dataset = Dataset("full_pascal_trainval") elif args.test_dataset == "val": train_dataset = Dataset("full_pascal_train") elif args.test_dataset == "trainval": train_dataset = Dataset("full_pascal_trainval") else: None # impossible by argparse settings # Only need to set training dataset values; evaluation gets it from there if args.inverse_prior: train_dataset.set_values("inverse_prior") # TODO: hack if args.first_n_train: train_dataset.images = train_dataset.images[: args.first_n_train] # In both the above cases, we use the val dataset for weights weights_dataset_name = "full_pascal_val" dets_tables = [] dets_tables_whole = [] clses_tables_whole = [] all_bounds = [] plot_infos = [] for config_f in configs: if args.suffix: config_f["suffix"] = args.suffix if args.bounds10: config_f["bounds"] = [0, 10] if args.bounds515: config_f["bounds"] = [5, 15] assert not (args.bounds10 and args.bounds515) if args.inverse_prior: config_f["suffix"] += "_inverse_prior" config_f["values"] = "inverse_prior" dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name, **config_f) ev = Evaluation(dp) all_bounds.append(dp.bounds) plot_infos.append(dict((k, config_f[k]) for k in ("label", "line", "color") if k in config_f)) # output the det configs first if args.det_configs: dp.output_det_statistics() # evaluate in the AP vs. Time regime, unless told not to if not args.no_apvst: dets_table = ev.evaluate_vs_t(None, None, force=args.force) # dets_table_whole,clses_table_whole = ev.evaluate_vs_t_whole(None,None,force=args.force) if comm_rank == 0: dets_tables.append(dets_table) # dets_tables_whole.append(dets_table_whole) # clses_tables_whole.append(clses_table_whole) # optionally, evaluate in the standard PR regime if args.wholeset_prs: ev.evaluate_detections_whole(None, force=args.force) # and plot the comparison if multiple config files were given if not args.no_apvst and len(configs) > 1 and comm_rank == 0: # filename of the final plot is the config file name dirname = config.get_evals_dir(dataset.get_name()) filename = args.config if args.inverse_prior: filename += "_inverse_prior" # det avg ff = opjoin(dirname, "%s_det_avg.png" % filename) ff_nl = opjoin(dirname, "%s_det_avg_nl.png" % filename) # make sure directory exists ut.makedirs(os.path.dirname(ff)) Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) if False: # det whole ff = opjoin(dirname, "%s_det_whole.png" % filename) ff_nl = opjoin(dirname, "%s_det_whole_nl.png" % filename) Evaluation.plot_ap_vs_t( dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos ) Evaluation.plot_ap_vs_t( dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos ) # cls whole ff = opjoin(dirname, "%s_cls_whole.png" % filename) ff_nl = opjoin(dirname, "%s_cls_whole_nl.png" % filename) Evaluation.plot_ap_vs_t( clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos ) Evaluation.plot_ap_vs_t( clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos )
def __init__(self): self.clf = Classifier() self.d = Dataset('full_pascal_trainval')
def main(): parser = argparse.ArgumentParser( description="Run experiments with the timely detection system.") parser.add_argument('--test_dataset', choices=['val','test','trainval'], default='val', help="""Dataset to use for testing. Run on val until final runs. The training dataset is inferred (val->train; test->trainval; trainval->trainval).""") parser.add_argument('--first_n', type=int, help='only take the first N images in the test dataset') parser.add_argument('--first_n_train', type=int, help='only take the first N images in the train dataset') parser.add_argument('--config', help="""Config file name that specifies the experiments to run. Give name such that the file is configs/#{name}.json or configs/#{name}/ In the latter case, all files within the directory will be loaded.""") parser.add_argument('--suffix', help="Overwrites the suffix in the config(s).") parser.add_argument('--bounds10', action='store_true', default=False, help='set bounds to [0,10]') parser.add_argument('--bounds515', action='store_true', default=False, help='set bounds to [5,15]') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--wholeset_prs', action='store_true', default=False, help='evaluate in the final p-r regime') parser.add_argument('--no_apvst', action='store_true', default=False, help='do NOT evaluate in the ap vs. time regime') parser.add_argument('--det_configs', action='store_true', default=False, help='output detector statistics to det_configs') parser.add_argument('--inverse_prior', action='store_true', default=False, help='use inverse prior class values') args = parser.parse_args() print(args) # If config file is not given, just run one experiment using default config if not args.config: configs = [DatasetPolicy.default_config] else: configs = load_configs(args.config) # Load the dataset dataset = Dataset('full_pascal_'+args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset=='test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset=='val': train_dataset = Dataset('full_pascal_train') elif args.test_dataset=='trainval': train_dataset = Dataset('full_pascal_trainval') else: None # impossible by argparse settings # Only need to set training dataset values; evaluation gets it from there if args.inverse_prior: train_dataset.set_values('inverse_prior') # TODO: hack if args.first_n_train: train_dataset.images = train_dataset.images[:args.first_n_train] # In both the above cases, we use the val dataset for weights weights_dataset_name = 'full_pascal_val' dets_tables = [] dets_tables_whole = [] clses_tables_whole = [] all_bounds = [] plot_infos = [] for config_f in configs: if args.suffix: config_f['suffix'] = args.suffix if args.bounds10: config_f['bounds'] = [0,10] if args.bounds515: config_f['bounds'] = [5,15] assert(not (args.bounds10 and args.bounds515)) if args.inverse_prior: config_f['suffix'] += '_inverse_prior' config_f['values'] = 'inverse_prior' dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name, **config_f) ev = Evaluation(dp) all_bounds.append(dp.bounds) plot_infos.append(dict((k,config_f[k]) for k in ('label','line','color') if k in config_f)) # output the det configs first if args.det_configs: dp.output_det_statistics() # evaluate in the AP vs. Time regime, unless told not to if not args.no_apvst: dets_table = ev.evaluate_vs_t(None,None,force=args.force) #dets_table_whole,clses_table_whole = ev.evaluate_vs_t_whole(None,None,force=args.force) if comm_rank==0: dets_tables.append(dets_table) #dets_tables_whole.append(dets_table_whole) #clses_tables_whole.append(clses_table_whole) # optionally, evaluate in the standard PR regime if args.wholeset_prs: ev.evaluate_detections_whole(None,force=args.force) # and plot the comparison if multiple config files were given if not args.no_apvst and len(configs)>1 and comm_rank==0: # filename of the final plot is the config file name dirname = config.get_evals_dir(dataset.get_name()) filename = args.config if args.inverse_prior: filename += '_inverse_prior' # det avg ff = opjoin(dirname, '%s_det_avg.png'%filename) ff_nl = opjoin(dirname, '%s_det_avg_nl.png'%filename) # make sure directory exists ut.makedirs(os.path.dirname(ff)) Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) if False: # det whole ff = opjoin(dirname, '%s_det_whole.png'%filename) ff_nl = opjoin(dirname, '%s_det_whole_nl.png'%filename) Evaluation.plot_ap_vs_t(dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) # cls whole ff = opjoin(dirname, '%s_cls_whole.png'%filename) ff_nl = opjoin(dirname, '%s_cls_whole_nl.png'%filename) Evaluation.plot_ap_vs_t(clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
from common_imports import * from common_mpi import * import synthetic.config as config from synthetic.dataset import Dataset from synthetic.extractor import Extractor if __name__ == '__main__': d = Dataset('full_pascal_trainval') feature_type = 'dsift' numpos = 15 num_words = 3000 iterations = 8 e = Extractor() all_classes = config.pascal_classes # for cls_idx in range(comm_rank, len(all_classes), comm_size): # PARALLEL # #for cls in all_classes: # cls = all_classes[cls_idx] # print cls #d, feature_type, num_words=3000,iterations=10, force_new=False, kmeansBatch=True e.get_codebook(d, feature_type, numpos, iterations, force_new=False, kmeansBatch=True)
dets_seq.append(cls_dets) cols = [ 'x', 'y', 'w', 'h', 'dummy', 'dummy', 'dummy', 'dummy', 'score', 'time', 'cls_ind' ] # NMS detections per class individually dets_mc = ut.collect(dets_seq, Detector.nms_detections, {'cols': cols}) dets_mc[:, :4] = BoundingBox.clipboxes_arr( dets_mc[:, :4], (0, 0, image.size[0] - 1, image.size[1] - 1)) time_elapsed = time.time() - t print("On image %s, took %.3f s" % (image.name, time_elapsed)) return dets_mc if __name__ == '__main__': train_d = Dataset('full_pascal_trainval') just_combine = False for ds in ['full_pascal_trainval']: # 'full_pascal_test' eval_d = Dataset(ds) dp = DatasetPolicy(eval_d, train_d, detectors=['csc_default']) test_table = np.zeros((len(eval_d.images), len(dp.actions))) if not just_combine: for img_idx in range(comm_rank, len(eval_d.images), comm_size): img = eval_d.images[img_idx] for act_idx, act in enumerate(dp.actions): print '%s on %d for act %d' % (img.name, comm_rank, act_idx) score = act.obj.get_observations(img)['score']
class TestDatasetPascal: def setup(self): self.d = Dataset('test_pascal_train',force=True) def test_ground_truth_pascal_train(self): assert(self.d.num_classes() == 20) assert('dog' in self.d.classes) def test_ground_truth_for_class_pascal(self): correct = np.array( [[ 48., 240., 148., 132., 11., 0., 1., 0.]]) ans = self.d.get_det_gt_for_class("dog") print ans assert np.all(ans.arr == correct) def test_neg_samples(self): # unlimited negative examples indices = self.d.get_neg_samples_for_class("dog",with_diff=True,with_trun=True) correct = np.array([1,2]) assert(np.all(indices == correct)) # maximum 1 negative example indices = self.d.get_neg_samples_for_class("dog",1,with_diff=True,with_trun=True) correct1 = np.array([1]) correct2 = np.array([2]) print(indices) assert(np.all(indices == correct1) or np.all(indices == correct2)) def test_pos_samples(self): indices = self.d.get_pos_samples_for_class("dog") correct = np.array([0]) assert(np.all(indices == correct)) def test_ground_truth_test(self): d = Dataset('test_pascal_val') gt = d.get_det_gt(with_diff=False,with_trun=False) correct = np.matrix( [ [ 139., 200., 69., 102., 18., 0., 0., 0.], [ 123., 155., 93., 41., 17., 0., 0., 1.], [ 239., 156., 69., 50., 8., 0., 0., 1.]]) print(gt) assert np.all(gt.arr == correct) def test_get_pos_windows(self): d = Dataset('test_pascal_val') # TODO def test_kfold(self): """ 'sizes' here are empirical values over the trainval set. """ d = Dataset('full_pascal_trainval') numfolds = 4 d.create_folds(numfolds) cls = 'dog' sizes = [314, 308, 321, 320] for i in range(len(d.folds)): d.next_folds() pos = d.get_pos_samples_for_fold_class(cls) neg = d.get_neg_samples_for_fold_class(cls, pos.shape[0]) assert(pos.shape[0] == sizes[i]) assert(neg.shape[0] == sizes[i])
def setup(self): self.d = Dataset('test_data1', force=True) self.classes = ["A", "B", "C"]
def __init__(self): self.d = Dataset('test_data2',force=True) self.classes = ["A","B","C"] self.det_gt = self.d.get_det_gt()
def setup(self): self.d = Dataset('test_pascal_train', force=True)
class TestEvaluationPerfect: def __init__(self): self.csc_trainval = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'csc_trainval'), 'r')) self.csc_test = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'csc_test'), 'r')) self.ext_csc_test = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'ext_csc_test'), 'r')) self.ext_csc_trainval = cPickle.load(open(os.path.join(config.get_ext_test_support_dir(), 'ext_csc_trainval'), 'r')) self.d_train = Dataset('full_pascal_trainval') self.trainval_gt = self.d_train.get_cls_ground_truth() self.d_test = Dataset('full_pascal_test') self.test_gt = self.d_test.get_cls_ground_truth() def setup(self): train_dataset = Dataset('test_pascal_train',force=True) dataset = Dataset('test_pascal_val',force=True) self.dp = DatasetPolicy(dataset,train_dataset,detector='perfect') self.evaluation = Evaluation(self.dp) def test_compute_pr_multiclass(self): cols = ['x','y','w','h','cls_ind','img_ind','diff'] dets_cols = ['x', 'y', 'w', 'h', 'score', 'time', 'cls_ind', 'img_ind'] # two objects of different classes in the image, perfect detection arr = np.array( [ [0,0,10,10,0,0,0], [10,10,10,10,1,0,0] ]) gt = Table(arr,cols) dets_arr = np.array( [ [0,0,10,10,-1,-1,0,0], [10,10,10,10,-1,-1,1,0] ]) dets = Table(dets_arr,dets_cols) # make sure gt and gt_cols aren't modified gt_arr_copy = gt.arr.copy() gt_cols_copy = list(gt.cols) ap,rec,prec = self.evaluation.compute_det_pr(dets, gt) assert(np.all(gt.arr == gt_arr_copy)) assert(gt_cols_copy == gt.cols) correct_ap = 1 correct_rec = np.array([0.5,1]) correct_prec = np.array([1,1]) print((ap, rec, prec)) assert(correct_ap == ap) assert(np.all(correct_rec==rec)) assert(np.all(correct_prec==prec)) # some extra detections to generate false positives dets_arr = np.array( [ [0,0,10,10,-1,-1,0,0], [0,0,10,10,0,-1,0,0], [10,10,10,10,0,-1,1,0], [10,10,10,10,-1,-1,1,0] ]) dets = Table(dets_arr,dets_cols) ap,rec,prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5,1,1,1]) correct_prec = np.array([1,1,2./3,0.5]) print((ap, rec, prec)) assert(np.all(correct_rec==rec)) assert(np.all(correct_prec==prec)) # confirm that running on the same dets gives the same answer ap,rec,prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.5,1,1,1]) correct_prec = np.array([1,1,2./3,0.5]) print((ap, rec, prec)) assert(np.all(correct_rec==rec)) assert(np.all(correct_prec==prec)) # now let's add two objects of a different class to gt to lower recall arr = np.array( [ [0,0,10,10,0,0,0], [10,10,10,10,1,0,0], [20,20,10,10,2,0,0], [30,30,10,10,2,0,0] ]) gt = Table(arr,cols) ap,rec,prec = self.evaluation.compute_det_pr(dets, gt) correct_rec = np.array([0.25,0.5,0.5,0.5]) correct_prec = np.array([1,1,2./3,0.5]) print((ap, rec, prec)) assert(np.all(correct_rec==rec)) assert(np.all(correct_prec==prec)) # now call it with empty detections dets_arr = np.array([]) dets = Table(dets_arr,dets_cols) ap,rec,prec = self.evaluation.compute_det_pr(dets, gt) correct_ap = 0 correct_rec = np.array([0]) correct_prec = np.array([0]) print((ap, rec, prec)) assert(np.all(correct_ap==ap)) assert(np.all(correct_rec==rec)) assert(np.all(correct_prec==prec)) def test_compute_cls_map(self): res = Evaluation.compute_cls_map(self.csc_trainval, self.trainval_gt) assert(round(res,11) == 0.47206391958) def test_compute_cls_map_half(self): table_csc_half = Table() table_csc_half.cols = list(self.csc_trainval.cols) for _ in range(10): rand_inds = np.random.permutation(range(5011))[:2500] table_csc_half.arr = self.csc_trainval.arr[rand_inds,:] res = Evaluation.compute_cls_map(table_csc_half, self.trainval_gt) assert(round(res,6) > .45) def test_compute_cls_map_gt(self): res = Evaluation.compute_cls_map(self.trainval_gt, self.trainval_gt) assert(round(res,6) == 1) def test_compute_cls_map_gt_half(self): rand_inds = np.random.permutation(range(5011))[:2500] table_gt_half = Table() table_gt_half.arr = np.hstack((self.trainval_gt.arr,np.array(np.arange(5011), ndmin=2).T)) table_gt_half.arr = table_gt_half.arr[rand_inds,:] table_gt_half.cols = list(self.trainval_gt.cols) + ['img_ind'] res = Evaluation.compute_cls_map(table_gt_half, self.trainval_gt) assert(round(res,6) == 1) def test_compute_cls_map_random_clf(self): clf_table = Table() num_test = 10 ress = np.zeros((num_test,)) for idx in range(num_test): clf_table.arr = np.hstack((np.random.rand(5011, 20),np.array(np.arange(5011), ndmin=2).T)) clf_table.cols = list(self.trainval_gt.cols) + ['img_ind'] res = Evaluation.compute_cls_map(clf_table, self.trainval_gt) ress[idx] = res assert(np.mean(ress) < 0.09) def test_other_scores(self): print 'csc_test', Evaluation.compute_cls_map(self.csc_test, self.test_gt) print 'csc_trainval', Evaluation.compute_cls_map(self.csc_trainval, self.trainval_gt) print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test, self.test_gt) print 'ext_trainval', Evaluation.compute_cls_map(self.ext_csc_trainval, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_chi2_20') ext_table_chi2_20 = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_test', Evaluation.compute_cls_map(ext_table_chi2_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_chi2_20') ext_table_chi2_20_tv = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_trainval', Evaluation.compute_cls_map(ext_table_chi2_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_rbf_20') ext_table_rbf_20 = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_test', Evaluation.compute_cls_map(ext_table_rbf_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_rbf_20') ext_table_rbf_20_tv = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_trainval', Evaluation.compute_cls_map(ext_table_rbf_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_linear_20') ext_linear_20_test = cPickle.load(open(filename, 'r')) print 'ext_linear_test', Evaluation.compute_cls_map(ext_linear_20_test, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_linear_20') ext_table_linear_20 = cPickle.load(open(filename, 'r')) print 'ext_linear_20_trainval', Evaluation.compute_cls_map(ext_table_linear_20, self.trainval_gt) filename = 'tab_linear_5' ext_tab_lin_5 = cPickle.load(open(filename, 'r')) print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map(ext_tab_lin_5, self.trainval_gt)
def main(): parser = argparse.ArgumentParser(description="Execute different functions of our system") parser.add_argument("--first_n", type=int, help="only take the first N images in the datasets") parser.add_argument( "--name", help="name for this run", default="default", choices=["default", "nolateral", "nohal", "halfsize"] ) parser.add_argument("--force", action="store_true", default=False, help="force overwrite") args = parser.parse_args() print (args) # configuration class class config(object): pass cfg = config() cfg.testname = "../ctfdet/data/finalRL/%s2_test" # object model cfg.bottomup = False # use complete search cfg.resize = 1.0 # resize the input image cfg.hallucinate = True # use HOGs up to 4 pixels cfg.initr = 1 # initial radious of the CtF search cfg.ratio = 1 # radious at the next levels cfg.deform = True # use deformation cfg.usemrf = True # use lateral constraints if args.name == "default": cfg # sticking with the default params elif args.name == "nolateral": cfg.usemrf = False elif args.name == "nohal": cfg.hallucinate = False elif args.name == "halfsize": cfg.resize = 0.5 # f**k it, do both test_datasets = ["val", "test", "train"] for test_dataset in test_datasets: # Load the dataset dataset = Dataset("full_pascal_" + test_dataset) if args.first_n: dataset.images = dataset.images[: args.first_n] # create directory for storing cached detections dirname = "./temp_data" if os.path.exists("/u/sergeyk"): dirname = "/u/vis/x1/sergeyk/object_detection" dirname = dirname + "/ctfdets/%s" % (args.name) ut.makedirs(dirname) num_images = len(dataset.images) for img_ind in range(comm_rank, num_images, comm_size): # check for existing det image = dataset.images[img_ind] filename = os.path.join(dirname, image.name + ".npy") if os.path.exists(filename) and not args.force: # table = np.load(filename)[()] continue # read the image imname = dataset.get_image_filename(img_ind) img = util2.myimread(imname, resize=cfg.resize) # compute the hog pyramid f = pyrHOG2.pyrHOG( img, interv=10, savedir="", notsave=True, notload=True, hallucinate=cfg.hallucinate, cformat=True ) # for each class all_dets = [] for ccls in dataset.classes: t = time.time() cls_ind = dataset.get_ind(ccls) print "%s Img %d/%d Class: %s" % (test_dataset, img_ind + 1, num_images, ccls) # load the class model m = util2.load("%s%d.model" % (cfg.testname % ccls, 7)) res = [] t1 = time.time() # for each aspect for clm, m in enumerate(m): # scan the image with left and right models res.append( pyrHOG2RL.detectflip( f, m, None, hallucinate=cfg.hallucinate, initr=cfg.initr, ratio=cfg.ratio, deform=cfg.deform, bottomup=cfg.bottomup, usemrf=cfg.usemrf, small=False, cl=clm, ) ) fuse = [] numhog = 0 # fuse the detections for mix in res: tr = mix[0] fuse += mix[1] numhog += mix[3] rfuse = tr.rank(fuse, maxnum=300) nfuse = tr.cluster(rfuse, ovr=0.3, inclusion=False) # print "Number of computed HOGs:",numhog time_elapsed = time.time() - t print "Elapsed time: %.3f s" % time_elapsed bboxes = [nf["bbox"] for nf in nfuse] scores = [nf["scr"] for nf in nfuse] assert len(bboxes) == len(scores) if len(bboxes) > 0: arr = np.zeros((len(bboxes), 7)) arr[:, :4] = BoundingBox.convert_arr_from_corners(np.array(bboxes)) arr[:, 4] = scores arr[:, 5] = time_elapsed arr[:, 6] = cls_ind all_dets.append(arr) cols = ["x", "y", "w", "h", "score", "time", "cls_ind"] if len(all_dets) > 0: all_dets = np.concatenate(all_dets, 0) else: all_dets = np.array([]) table = Table(all_dets, cols) np.save(filename, table)
def setup(self): self.d = Dataset('test_data1',force=True) self.classes = ["A","B","C"]
def setup(self): train_dataset = Dataset('test_pascal_train', force=True) dataset = Dataset('test_pascal_val', force=True) self.dp = DatasetPolicy(dataset, train_dataset, detector='perfect') self.evaluation = Evaluation(self.dp)
def main(): parser = argparse.ArgumentParser(description='Execute different functions of our system') parser.add_argument('mode', choices=[ 'window_stats', 'evaluate_metaparams', 'evaluate_jw', 'evaluate_get_pos_windows', 'train_svm', 'extract_sift','extract_assignments','extract_codebook', 'evaluate_jw_grid', 'final_metaparams', 'assemble_dpm_dets','ctfdet','assemble_ctf_dets' ]) parser.add_argument('--test_dataset', choices=['val','test','train'], default='test', help='dataset to use for testing. the training dataset \ is automatically inferred (val->train and test->trainval).') parser.add_argument('--first_n', type=int, help='only take the first N images in the datasets') parser.add_argument('--bounds', type=str, help='the start_time and deadline_time for the ImagePolicy and corresponding evaluation. ex: (1,5)') parser.add_argument('--name', help='name for this run') parser.add_argument('--priors', default='random', help= \ "list of choice for the policy for selecting the next action. choose from random, oracle,fixed_order, no_smooth, backoff. ex: --priors=random,oracle,no_smooth") parser.add_argument('--compare_evals', action='store_true', default=False, help='plot all the priors modes given on same plot'), parser.add_argument('--detector', choices=['perfect','perfect_with_noise', 'dpm','ctf'], default='perfect', help='detector type') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--gist', action='store_true', default=False, help='use GIST as one of the actions') parser.add_argument('--clear_tmp', action='store_true', default=False, help='clear the cached windows folder before running'), parser.add_argument('--feature_type', choices=['sift','dsift'], default='dsift', help='use this feature type'), parser.add_argument('--kernel', choices=['chi2','rbf'], default='chi2', help='kernel to train svm on'), args = parser.parse_args() if args.priors: args.priors = args.priors.split(',') if args.bounds: args.bounds = [float(x) for x in re.findall(r'\d+', args.bounds)] assert(len(args.bounds)==2) print(args) # Load the dataset dataset = Dataset('full_pascal_'+args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset=='test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset=='val': train_dataset = Dataset('full_pascal_train') else: print("Impossible, setting train_dataset to dataset") train_dataset = dataset # Create window generator sw = SlidingWindows(dataset,train_dataset) if args.clear_tmp: dirname = config.get_sliding_windows_cached_dir(train_dataset.get_name()) shutil.rmtree(dirname) dirname = config.get_sliding_windows_cached_dir(dataset.get_name()) shutil.rmtree(dirname) if args.mode=='assemble_dpm_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,suffix='dpm_may25') if args.mode=='assemble_ctf_dets': policy = DatasetPolicy(dataset,train_dataset,sw) dets = policy.load_ext_detections(dataset,'ctf','ctf_default') dets = policy.load_ext_detections(dataset,'ctf','ctf_nohal') dets = policy.load_ext_detections(dataset,'ctf', 'ctf_halfsize') if args.mode=='evaluate_get_pos_windows': evaluate_get_pos_windows(train_dataset) return if args.mode=='window_stats': "Compute and plot the statistics of ground truth window parameters." results = SlidingWindows.get_dataset_window_stats(train_dataset,plot=True) if args.mode=='ctfdet': """Run Pedersoli's detector on the dataset and assemble into one Table.""" run_pedersoli(dataset) if args.mode=='evaluate_jw': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ # TODO hack: both sw and jw should subclass something like WindowGenerator jw = JumpingWindowsDetector(use_scale=True) sw.jw = jw #classes = dataset.classes classes = ['car'] # classes = ['bicycle' ,'car','horse', 'sofa',\ # 'bird', 'chair', 'motorbike', 'train',\ # 'boat', 'cow', 'person', 'tvmonitor',\ # 'bottle','diningtable', 'pottedplant',\ # 'bus','dog' ,'sheep'] for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in dataset.classes: cls = classes[cls_idx] dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='evaluate_jw_grid': """ Evaluate the jumping window approach by producing plots of recall vs. #windows. """ sw = SlidingWindows(dataset,train_dataset) jw = JumpingWindowsDetectorGrid() sw.jw = jw for cls in dataset.classes: dirname = config.get_jumping_windows_dir(dataset.get_name()) filename = os.path.join(dirname,'%s'%cls) if os.path.isfile(config.data_dir + 'JumpingWindows/'+cls): sw.evaluate_recall(cls, filename, metaparams=None, mode='jw', plot=True) if args.mode=='train_svm': randomize = not os.path.exists('/home/tobibaum') d = Dataset('full_pascal_train') dtest = Dataset('full_pascal_val') e = Extractor() classes = config.pascal_classes num_words = 3000 iters = 5 feature_type = 'dsift' codebook_samples = 15 num_pos = 'max' testsize = 'max' if args.first_n: num_pos = args.first_n testsize = 1.5*num_pos kernel = args.kernel if comm_rank == 0: ut.makedirs(config.data_dir + 'features/' + feature_type + '/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/codebooks/times/') ut.makedirs(config.data_dir + 'features/' + feature_type + '/svms/train_times/') for cls_idx in range(comm_rank, len(classes), comm_size): #for cls in classes: cls = classes[cls_idx] codebook = e.get_codebook(d, feature_type) pos_arr = d.get_pos_windows(cls) neg_arr = d.get_neg_windows(pos_arr.shape[0], cls, max_overlap=0) if not num_pos == 'max': if not randomize: pos_arr = pos_arr[:num_pos] neg_arr = pos_arr[:num_pos] else: rand = np.random.random_integers(0, pos_arr.shape[0] - 1, size=num_pos) pos_arr = pos_arr[rand] rand = np.random.random_integers(0, neg_arr.shape[0] - 1, size=num_pos) neg_arr = neg_arr[rand] pos_table = Table(pos_arr, ['x','y','w','h','img_ind']) neg_table = Table(neg_arr, pos_table.cols) train_with_hard_negatives(d, dtest, num_words,codebook_samples,codebook,\ cls, pos_table, neg_table,feature_type, \ iterations=iters, kernel=kernel, L=2, \ testsize=testsize,randomize=randomize) if args.mode=='evaluate_metaparams': """ Grid search over metaparams values for get_windows_new, with the AUC of recall vs. # windows evaluation. """ sw.grid_search_over_metaparams() return if args.mode=='final_metaparams': dirname = config.get_sliding_windows_metaparams_dir(train_dataset.get_name()) # currently these are the best auc/complexity params best_params_for_classes = [ (62,15,12,'importance',0), #aeroplane (83,15,12,'importance',0), #bicycle (62,15,12,'importance',0), #bird (62,15,12,'importance',0), #boat (125,12,12,'importance',0), #bottle (83,12,9,'importance',0), #bus (125,15,9,'importance',0), #car (125,12,12,'linear',0), #cat (125,15,9,'importance',0), #chair (125,9,6,'importance',0), #cow (125,15,6,'linear',0), #diningtable (62,15,12,'importance',0), #dog (83,15,6,'importance',0), #horse (83,12,6,'importance',0), #motorbike (83,15,12,'importance',0), #person (83,15,6,'importance',0), #pottedplant (83,15,12,'importance',0), #sheep (83,9,6,'importance',0), #sofa (62,12,6,'importance',0), #train (62,12,12,'importance',0), #tvmonitor (125,9,12,'importance',0) #all ] # ACTUALLY THEY ARE ALL THE SAME! cheap_params = (62, 9, 6, 'importance', 0) for i in range(comm_rank,dataset.num_classes(),comm_size): cls = dataset.classes[i] best_params = best_params_for_classes[i] #samples,num_scales,num_ratios,mode,priority,cls = cheap_params metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 0 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) metaparams = { 'samples_per_500px': samples, 'num_scales': num_scales, 'num_ratios': num_ratios, 'mode': mode, 'priority': 1 } filename = '%s_%d_%d_%d_%s_%d'%( cls, metaparams['samples_per_500px'], metaparams['num_scales'], metaparams['num_ratios'], metaparams['mode'], metaparams['priority']) filename = os.path.join(dirname,filename) tables = sw.evaluate_recall(cls,filename,metaparams,'sw',plot=True,force=False) return if args.mode=='extract_sift': e=Extractor() e.extract_all(['sift'], ['full_pascal_trainval','full_pascal_test'], 0, 0) if args.mode=='extract_assignments': e=Extractor() feature_type = 'sift' for image_set in ['full_pascal_trainval','full_pascal_test']: d = Dataset(image_set) codebook = e.get_codebook(d, feature_type) print 'codebook loaded' for img_ind in range(comm_rank,len(d.images),comm_size): img = d.images[img_ind] #for img in d.images: e.get_assignments(np.array([0,0,img.size[0],img.size[1]]), feature_type, \ codebook, img) if args.mode=='extract_codebook': d = Dataset('full_pascal_trainval') e = Extractor() codebook = e.get_codebook(d, args.feature_type)