def test_other_scores(self): print 'csc_test', Evaluation.compute_cls_map(self.csc_test, self.test_gt) print 'csc_trainval', Evaluation.compute_cls_map(self.csc_trainval, self.trainval_gt) print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test, self.test_gt) print 'ext_trainval', Evaluation.compute_cls_map(self.ext_csc_trainval, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_chi2_20') ext_table_chi2_20 = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_test', Evaluation.compute_cls_map(ext_table_chi2_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_chi2_20') ext_table_chi2_20_tv = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_trainval', Evaluation.compute_cls_map(ext_table_chi2_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_rbf_20') ext_table_rbf_20 = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_test', Evaluation.compute_cls_map(ext_table_rbf_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_rbf_20') ext_table_rbf_20_tv = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_trainval', Evaluation.compute_cls_map(ext_table_rbf_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_linear_20') ext_linear_20_test = cPickle.load(open(filename, 'r')) print 'ext_linear_test', Evaluation.compute_cls_map(ext_linear_20_test, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_linear_20') ext_table_linear_20 = cPickle.load(open(filename, 'r')) print 'ext_linear_20_trainval', Evaluation.compute_cls_map(ext_table_linear_20, self.trainval_gt) filename = 'tab_linear_5' ext_tab_lin_5 = cPickle.load(open(filename, 'r')) print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map(ext_tab_lin_5, self.trainval_gt)
def test_other_scores(self): print 'csc_test', Evaluation.compute_cls_map(self.csc_test, self.test_gt) print 'csc_trainval', Evaluation.compute_cls_map( self.csc_trainval, self.trainval_gt) print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test, self.test_gt) print 'ext_trainval', Evaluation.compute_cls_map( self.ext_csc_trainval, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_chi2_20') ext_table_chi2_20 = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_test', Evaluation.compute_cls_map( ext_table_chi2_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_chi2_20') ext_table_chi2_20_tv = cPickle.load(open(filename, 'r')) print 'ext_chi2_20_trainval', Evaluation.compute_cls_map( ext_table_chi2_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_rbf_20') ext_table_rbf_20 = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_test', Evaluation.compute_cls_map( ext_table_rbf_20, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_rbf_20') ext_table_rbf_20_tv = cPickle.load(open(filename, 'r')) print 'ext_rbf_20_trainval', Evaluation.compute_cls_map( ext_table_rbf_20_tv, self.trainval_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_linear_20') ext_linear_20_test = cPickle.load(open(filename, 'r')) print 'ext_linear_test', Evaluation.compute_cls_map( ext_linear_20_test, self.test_gt) filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_linear_20') ext_table_linear_20 = cPickle.load(open(filename, 'r')) print 'ext_linear_20_trainval', Evaluation.compute_cls_map( ext_table_linear_20, self.trainval_gt) filename = 'tab_linear_5' ext_tab_lin_5 = cPickle.load(open(filename, 'r')) print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map( ext_tab_lin_5, self.trainval_gt)
train_gt = d_train.get_cls_ground_truth() val_gt = d_val.get_cls_ground_truth() if mpi.comm_rank == 0: filename = os.path.join(config.get_classifier_dataset_dirname(CSCClassifier('default','dog', d_train, d_val), d_train),'crossval.txt') kernels = ['linear'] Cs = [50] settings = list(itertools.product(kernels, Cs)) for setin in settings: kernel = setin[0] C = setin[1] #train_csc_svms(d_train, d_val, kernel, C) safebarrier(comm) table_arr = test_csc_svm(d_train, d_val) safebarrier(comm) if mpi.comm_rank == 0: table = conv(d_val, table_arr) cPickle.dump(table, open(os.path.join(config.get_ext_dets_foldname(d_val),'table'),'w')) res = Evaluation.compute_cls_map(table, val_gt) res_file = open(filename,'a') res_file.write('%s, C=%d - %f\n'%(kernel, C, res)) res_file.close() print res
'crossval.txt') kernels = ['linear'] Cs = [50] settings = list(itertools.product(kernels, Cs)) for setin in settings: kernel = setin[0] C = setin[1] #train_csc_svms(d_train, d_val, kernel, C) safebarrier(comm) table_arr = test_csc_svm(d_train, d_val) safebarrier(comm) if mpi.comm_rank == 0: table = conv(d_val, table_arr) cPickle.dump( table, open( os.path.join(config.get_ext_dets_foldname(d_val), 'table'), 'w')) res = Evaluation.compute_cls_map(table, val_gt) res_file = open(filename, 'a') res_file.write('%s, C=%d - %f\n' % (kernel, C, res)) res_file.close() print res
eval_d = Dataset(ds) dp = DatasetPolicy(eval_d, train_d, detectors=['csc_default']) test_table = np.zeros((len(eval_d.images), len(dp.actions))) if not just_combine: for img_idx in range(comm_rank, len(eval_d.images), comm_size): img = eval_d.images[img_idx] for act_idx, act in enumerate(dp.actions): print '%s on %d for act %d' % (img.name, comm_rank, act_idx) score = act.obj.get_observations(img)['score'] print '%s, score: %f' % (img.name, score) test_table[img_idx, act_idx] = score dirname = ut.makedirs( os.path.join(config.get_ext_dets_foldname(eval_d), 'dp', 'agent_wise')) filename = os.path.join(dirname, 'table_%d' % comm_rank) np.savetxt(filename, test_table) safebarrier(comm) if comm_rank == 0: for i in range(comm_size - 1): filename = os.path.join(dirname, 'table_%d' % (i + 1)) test_table += np.loadtxt(filename) dirname = ut.makedirs( os.path.join(config.get_ext_dets_foldname(eval_d), 'dp')) filename = os.path.join(dirname, 'table_linear_20') tab_test_table = Table() tab_test_table.cols = list(eval_d.classes) + ['img_ind']
def run_fastinf_different_settings(d, ms, rs, suffixs, num_bins = 5): settings = list(itertools.product(suffixs, ms, rs)) table_gt = d.get_cls_ground_truth().arr.astype(int) print 'run with a total of %d settings'%len(settings) for setindx in range(comm_rank, len(settings), comm_size): second_table = None setin = settings[setindx] suffix = setin[0] m = str(setin[1]) r1 = str(setin[2]) print 'node %d runs %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1) filename = config.get_fastinf_mrf_file(d, suffix) data_filename = config.get_fastinf_data_file(d, suffix) if suffix == 'perfect': table = np.hstack((table_gt, table_gt)) bounds = np.tile(np.linspace(0, 1, num_bins+1),(table_gt.shape[1],1)) print bounds elif suffix == 'GIST': table = gist_classify_dataset(d) bounds, discr_table = discretize_table(table, num_bins) table = np.hstack((table_gt, discr_table)) elif suffix == 'CSC': filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table') print filename_csc if not os.path.exists(filename_csc): raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc) orig_table = cPickle.load(open(filename_csc,'r')) if isinstance(orig_table, Table): orig_table = orig_table.arr[:,:-1] bounds, discr_table = discretize_table(orig_table, num_bins) table = np.hstack((table_gt, discr_table)) elif suffix == 'CSC_regions': rm = RegionModel("1big_2small", 0.5) detector = 'csc_default' from synthetic.dataset_policy import DatasetPolicy orig_table = DatasetPolicy.load_ext_detections(d, detector) gt = d.get_det_gt().copy() # we need to spice up the gt by a score of 1 for each class (results in less code) gt.cols.append('score') gt.arr = np.hstack((gt.arr, np.ones((gt.shape[0], 1)))) table_gt_region = create_regioned_table(rm, gt, d.images, len(d.classes)) # At this point we need to split them for the different regions orig_table_region = create_regioned_table(rm, orig_table, d.images, len(d.classes)) bounds, discr_table_region = discretize_table(orig_table_region, num_bins) table = np.hstack((table_gt_region, discr_table_region)) elif suffix == 'GIST_CSC': filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table') if not os.path.exists(filename_csc): raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc) orig_table = cPickle.load(open(filename_csc,'r')) if isinstance(orig_table, Table): orig_table = orig_table.arr[:,:-1] bounds, discr_table = discretize_table(orig_table, num_bins) table = np.hstack((table_gt, discr_table)) store_bound(d, 'CSC', bounds) second_table = gist_classify_dataset(d) sec_bounds, second_table = discretize_table(second_table, num_bins) store_bound(d, 'GIST', sec_bounds) full_bound = np.hstack((sec_bounds, bounds)) store_bound(d, 'GIST_CSC', full_bound) if not suffix == 'GIST_CSC': store_bound(d, suffix, bounds) print 'set up table on %d, write out mrf for %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1) write_out_mrf(table, num_bins, filename, data_filename, second_table=second_table) add_sets = ['-m',m] if not r1 == '': add_sets += ['-r1', r1] if not second_table == None: sec_bound_file = '%s_secbounds'%filename for s in add_sets: sec_bound_file += '_'+s np.savetxt(sec_bound_file, sec_bounds) print '%d start running lbp for %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1) filename_out = config.get_fastinf_res_file(d, suffix, m, r1) execute_lbp(filename, data_filename, filename_out, add_settings=add_sets)
for ds in ['full_pascal_trainval']: # 'full_pascal_test' eval_d = Dataset(ds) dp = DatasetPolicy(eval_d, train_d, detectors=['csc_default']) test_table = np.zeros((len(eval_d.images), len(dp.actions))) if not just_combine: for img_idx in range(comm_rank, len(eval_d.images), comm_size): img = eval_d.images[img_idx] for act_idx, act in enumerate(dp.actions): print '%s on %d for act %d'%(img.name, comm_rank, act_idx) score = act.obj.get_observations(img)['score'] print '%s, score: %f'%(img.name, score) test_table[img_idx, act_idx] = score dirname = ut.makedirs(os.path.join(config.get_ext_dets_foldname(eval_d), 'dp','agent_wise')) filename = os.path.join(dirname,'table_%d'%comm_rank) np.savetxt(filename, test_table) safebarrier(comm) if comm_rank == 0: for i in range(comm_size-1): filename = os.path.join(dirname,'table_%d'%(i+1)) test_table += np.loadtxt(filename) dirname = ut.makedirs(os.path.join(config.get_ext_dets_foldname(eval_d), 'dp')) filename = os.path.join(dirname,'table_linear_20') tab_test_table = Table() tab_test_table.cols = list(eval_d.classes) + ['img_ind'] tab_test_table.arr = np.hstack((test_table, np.array(np.arange(test_table.shape[0]),ndmin=2).T))
def run_fastinf_different_settings(d, ms, rs, suffixs, num_bins = 5): settings = list(itertools.product(suffixs, ms, rs)) table_gt = d.get_cls_ground_truth().arr.astype(int) print 'run with a total of %d settings'%len(settings) for setindx in range(mpi.comm_rank, len(settings), mpi.comm_size): second_table = None setin = settings[setindx] suffix = setin[0] m = str(setin[1]) r1 = str(setin[2]) print 'node %d runs %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1) filename = config.get_fastinf_mrf_file(d, suffix) data_filename = config.get_fastinf_data_file(d, suffix) if suffix == 'perfect': table = np.hstack((table_gt, table_gt)) bounds = np.tile(np.linspace(0, 1, num_bins+1),(table_gt.shape[1],1)) print bounds elif suffix == 'GIST': table = gist_classify_dataset(d) bounds, discr_table = discretize_table(table, num_bins) table = np.hstack((table_gt, discr_table)) elif suffix == 'CSC': filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table') print filename_csc if not os.path.exists(filename_csc): raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc) orig_table = cPickle.load(open(filename_csc,'r')) if isinstance(orig_table, Table): orig_table = orig_table.arr[:,:-1] bounds, discr_table = discretize_table(orig_table, num_bins) table = np.hstack((table_gt, discr_table)) elif suffix == 'CSC_regions': rm = RegionModel("1big_2small", 0.5) detector = 'csc_default' from synthetic.dataset_policy import DatasetPolicy orig_table = DatasetPolicy.load_ext_detections(d, detector) gt = d.get_det_gt().copy() # we need to spice up the gt by a score of 1 for each class (results in less code) gt.cols.append('score') gt.arr = np.hstack((gt.arr, np.ones((gt.shape[0], 1)))) table_gt_region = create_regioned_table(rm, gt, d.images, len(d.classes)) # At this point we need to split them for the different regions orig_table_region = create_regioned_table(rm, orig_table, d.images, len(d.classes)) bounds, discr_table_region = discretize_table(orig_table_region, num_bins) table = np.hstack((table_gt_region, discr_table_region)) elif suffix == 'GIST_CSC': filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table') if not os.path.exists(filename_csc): raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc) orig_table = cPickle.load(open(filename_csc,'r')) if isinstance(orig_table, Table): orig_table = orig_table.arr[:,:-1] bounds, discr_table = discretize_table(orig_table, num_bins) table = np.hstack((table_gt, discr_table)) store_bound(d, 'CSC', bounds) second_table = gist_classify_dataset(d) sec_bounds, second_table = discretize_table(second_table, num_bins) store_bound(d, 'GIST', sec_bounds) full_bound = np.hstack((sec_bounds, bounds)) store_bound(d, 'GIST_CSC', full_bound) if not suffix == 'GIST_CSC': store_bound(d, suffix, bounds) print 'set up table on %d, write out mrf for %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1) write_out_mrf(table, num_bins, filename, data_filename, second_table=second_table) add_sets = ['-m',m] if not r1 == '': add_sets += ['-r1', r1] if not second_table == None: sec_bound_file = '%s_secbounds'%filename for s in add_sets: sec_bound_file += '_'+s np.savetxt(sec_bound_file, sec_bounds) print '%d start running lbp for %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1) filename_out = config.get_fastinf_res_file(d, suffix, m, r1) execute_lbp(filename, data_filename, filename_out, add_settings=add_sets)