def test_other_scores(self):
print 'csc_test', Evaluation.compute_cls_map(self.csc_test, self.test_gt)
print 'csc_trainval', Evaluation.compute_cls_map(self.csc_trainval, self.trainval_gt)
print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test, self.test_gt)
print 'ext_trainval', Evaluation.compute_cls_map(self.ext_csc_trainval, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_chi2_20')
ext_table_chi2_20 = cPickle.load(open(filename, 'r'))
print 'ext_chi2_20_test', Evaluation.compute_cls_map(ext_table_chi2_20, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_chi2_20')
ext_table_chi2_20_tv = cPickle.load(open(filename, 'r'))
print 'ext_chi2_20_trainval', Evaluation.compute_cls_map(ext_table_chi2_20_tv, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_rbf_20')
ext_table_rbf_20 = cPickle.load(open(filename, 'r'))
print 'ext_rbf_20_test', Evaluation.compute_cls_map(ext_table_rbf_20, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_rbf_20')
ext_table_rbf_20_tv = cPickle.load(open(filename, 'r'))
print 'ext_rbf_20_trainval', Evaluation.compute_cls_map(ext_table_rbf_20_tv, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test), 'dp', 'table_linear_20')
ext_linear_20_test = cPickle.load(open(filename, 'r'))
print 'ext_linear_test', Evaluation.compute_cls_map(ext_linear_20_test, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train), 'dp', 'table_linear_20')
ext_table_linear_20 = cPickle.load(open(filename, 'r'))
print 'ext_linear_20_trainval', Evaluation.compute_cls_map(ext_table_linear_20, self.trainval_gt)
filename = 'tab_linear_5'
ext_tab_lin_5 = cPickle.load(open(filename, 'r'))
print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map(ext_tab_lin_5, self.trainval_gt)
def test_other_scores(self):
print 'csc_test', Evaluation.compute_cls_map(self.csc_test,
self.test_gt)
print 'csc_trainval', Evaluation.compute_cls_map(
self.csc_trainval, self.trainval_gt)
print 'ext_test', Evaluation.compute_cls_map(self.ext_csc_test,
self.test_gt)
print 'ext_trainval', Evaluation.compute_cls_map(
self.ext_csc_trainval, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test),
'dp', 'table_chi2_20')
ext_table_chi2_20 = cPickle.load(open(filename, 'r'))
print 'ext_chi2_20_test', Evaluation.compute_cls_map(
ext_table_chi2_20, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train),
'dp', 'table_chi2_20')
ext_table_chi2_20_tv = cPickle.load(open(filename, 'r'))
print 'ext_chi2_20_trainval', Evaluation.compute_cls_map(
ext_table_chi2_20_tv, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test),
'dp', 'table_rbf_20')
ext_table_rbf_20 = cPickle.load(open(filename, 'r'))
print 'ext_rbf_20_test', Evaluation.compute_cls_map(
ext_table_rbf_20, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train),
'dp', 'table_rbf_20')
ext_table_rbf_20_tv = cPickle.load(open(filename, 'r'))
print 'ext_rbf_20_trainval', Evaluation.compute_cls_map(
ext_table_rbf_20_tv, self.trainval_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_test),
'dp', 'table_linear_20')
ext_linear_20_test = cPickle.load(open(filename, 'r'))
print 'ext_linear_test', Evaluation.compute_cls_map(
ext_linear_20_test, self.test_gt)
filename = os.path.join(config.get_ext_dets_foldname(self.d_train),
'dp', 'table_linear_20')
ext_table_linear_20 = cPickle.load(open(filename, 'r'))
print 'ext_linear_20_trainval', Evaluation.compute_cls_map(
ext_table_linear_20, self.trainval_gt)
filename = 'tab_linear_5'
ext_tab_lin_5 = cPickle.load(open(filename, 'r'))
print 'ext_tab_lin_5_trainval', Evaluation.compute_cls_map(
ext_tab_lin_5, self.trainval_gt)
train_gt = d_train.get_cls_ground_truth()
val_gt = d_val.get_cls_ground_truth()
if mpi.comm_rank == 0:
filename = os.path.join(config.get_classifier_dataset_dirname(CSCClassifier('default','dog', d_train, d_val), d_train),'crossval.txt')
kernels = ['linear']
Cs = [50]
settings = list(itertools.product(kernels, Cs))
for setin in settings:
kernel = setin[0]
C = setin[1]
#train_csc_svms(d_train, d_val, kernel, C)
safebarrier(comm)
table_arr = test_csc_svm(d_train, d_val)
safebarrier(comm)
if mpi.comm_rank == 0:
table = conv(d_val, table_arr)
cPickle.dump(table, open(os.path.join(config.get_ext_dets_foldname(d_val),'table'),'w'))
res = Evaluation.compute_cls_map(table, val_gt)
res_file = open(filename,'a')
res_file.write('%s, C=%d - %f\n'%(kernel, C, res))
res_file.close()
print res
'crossval.txt')
kernels = ['linear']
Cs = [50]
settings = list(itertools.product(kernels, Cs))
for setin in settings:
kernel = setin[0]
C = setin[1]
#train_csc_svms(d_train, d_val, kernel, C)
safebarrier(comm)
table_arr = test_csc_svm(d_train, d_val)
safebarrier(comm)
if mpi.comm_rank == 0:
table = conv(d_val, table_arr)
cPickle.dump(
table,
open(
os.path.join(config.get_ext_dets_foldname(d_val), 'table'),
'w'))
res = Evaluation.compute_cls_map(table, val_gt)
res_file = open(filename, 'a')
res_file.write('%s, C=%d - %f\n' % (kernel, C, res))
res_file.close()
print res
eval_d = Dataset(ds)
dp = DatasetPolicy(eval_d, train_d, detectors=['csc_default'])
test_table = np.zeros((len(eval_d.images), len(dp.actions)))
if not just_combine:
for img_idx in range(comm_rank, len(eval_d.images), comm_size):
img = eval_d.images[img_idx]
for act_idx, act in enumerate(dp.actions):
print '%s on %d for act %d' % (img.name, comm_rank,
act_idx)
score = act.obj.get_observations(img)['score']
print '%s, score: %f' % (img.name, score)
test_table[img_idx, act_idx] = score
dirname = ut.makedirs(
os.path.join(config.get_ext_dets_foldname(eval_d), 'dp',
'agent_wise'))
filename = os.path.join(dirname, 'table_%d' % comm_rank)
np.savetxt(filename, test_table)
safebarrier(comm)
if comm_rank == 0:
for i in range(comm_size - 1):
filename = os.path.join(dirname, 'table_%d' % (i + 1))
test_table += np.loadtxt(filename)
dirname = ut.makedirs(
os.path.join(config.get_ext_dets_foldname(eval_d), 'dp'))
filename = os.path.join(dirname, 'table_linear_20')
tab_test_table = Table()
tab_test_table.cols = list(eval_d.classes) + ['img_ind']
def run_fastinf_different_settings(d, ms, rs, suffixs, num_bins = 5):
settings = list(itertools.product(suffixs, ms, rs))
table_gt = d.get_cls_ground_truth().arr.astype(int)
print 'run with a total of %d settings'%len(settings)
for setindx in range(comm_rank, len(settings), comm_size):
second_table = None
setin = settings[setindx]
suffix = setin[0]
m = str(setin[1])
r1 = str(setin[2])
print 'node %d runs %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1)
filename = config.get_fastinf_mrf_file(d, suffix)
data_filename = config.get_fastinf_data_file(d, suffix)
if suffix == 'perfect':
table = np.hstack((table_gt, table_gt))
bounds = np.tile(np.linspace(0, 1, num_bins+1),(table_gt.shape[1],1))
print bounds
elif suffix == 'GIST':
table = gist_classify_dataset(d)
bounds, discr_table = discretize_table(table, num_bins)
table = np.hstack((table_gt, discr_table))
elif suffix == 'CSC':
filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table')
print filename_csc
if not os.path.exists(filename_csc):
raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc)
orig_table = cPickle.load(open(filename_csc,'r'))
if isinstance(orig_table, Table):
orig_table = orig_table.arr[:,:-1]
bounds, discr_table = discretize_table(orig_table, num_bins)
table = np.hstack((table_gt, discr_table))
elif suffix == 'CSC_regions':
rm = RegionModel("1big_2small", 0.5)
detector = 'csc_default'
from synthetic.dataset_policy import DatasetPolicy
orig_table = DatasetPolicy.load_ext_detections(d, detector)
gt = d.get_det_gt().copy()
# we need to spice up the gt by a score of 1 for each class (results in less code)
gt.cols.append('score')
gt.arr = np.hstack((gt.arr, np.ones((gt.shape[0], 1))))
table_gt_region = create_regioned_table(rm, gt, d.images, len(d.classes))
# At this point we need to split them for the different regions
orig_table_region = create_regioned_table(rm, orig_table, d.images, len(d.classes))
bounds, discr_table_region = discretize_table(orig_table_region, num_bins)
table = np.hstack((table_gt_region, discr_table_region))
elif suffix == 'GIST_CSC':
filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table')
if not os.path.exists(filename_csc):
raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc)
orig_table = cPickle.load(open(filename_csc,'r'))
if isinstance(orig_table, Table):
orig_table = orig_table.arr[:,:-1]
bounds, discr_table = discretize_table(orig_table, num_bins)
table = np.hstack((table_gt, discr_table))
store_bound(d, 'CSC', bounds)
second_table = gist_classify_dataset(d)
sec_bounds, second_table = discretize_table(second_table, num_bins)
store_bound(d, 'GIST', sec_bounds)
full_bound = np.hstack((sec_bounds, bounds))
store_bound(d, 'GIST_CSC', full_bound)
if not suffix == 'GIST_CSC':
store_bound(d, suffix, bounds)
print 'set up table on %d, write out mrf for %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1)
write_out_mrf(table, num_bins, filename, data_filename, second_table=second_table)
add_sets = ['-m',m]
if not r1 == '':
add_sets += ['-r1', r1]
if not second_table == None:
sec_bound_file = '%s_secbounds'%filename
for s in add_sets:
sec_bound_file += '_'+s
np.savetxt(sec_bound_file, sec_bounds)
print '%d start running lbp for %s, m=%s, r1=%s'%(comm_rank, suffix, m, r1)
filename_out = config.get_fastinf_res_file(d, suffix, m, r1)
execute_lbp(filename, data_filename, filename_out, add_settings=add_sets)
for ds in ['full_pascal_trainval']: # 'full_pascal_test'
eval_d = Dataset(ds)
dp = DatasetPolicy(eval_d, train_d, detectors=['csc_default'])
test_table = np.zeros((len(eval_d.images), len(dp.actions)))
if not just_combine:
for img_idx in range(comm_rank, len(eval_d.images), comm_size):
img = eval_d.images[img_idx]
for act_idx, act in enumerate(dp.actions):
print '%s on %d for act %d'%(img.name, comm_rank, act_idx)
score = act.obj.get_observations(img)['score']
print '%s, score: %f'%(img.name, score)
test_table[img_idx, act_idx] = score
dirname = ut.makedirs(os.path.join(config.get_ext_dets_foldname(eval_d), 'dp','agent_wise'))
filename = os.path.join(dirname,'table_%d'%comm_rank)
np.savetxt(filename, test_table)
safebarrier(comm)
if comm_rank == 0:
for i in range(comm_size-1):
filename = os.path.join(dirname,'table_%d'%(i+1))
test_table += np.loadtxt(filename)
dirname = ut.makedirs(os.path.join(config.get_ext_dets_foldname(eval_d), 'dp'))
filename = os.path.join(dirname,'table_linear_20')
tab_test_table = Table()
tab_test_table.cols = list(eval_d.classes) + ['img_ind']
tab_test_table.arr = np.hstack((test_table, np.array(np.arange(test_table.shape[0]),ndmin=2).T))
def run_fastinf_different_settings(d, ms, rs, suffixs, num_bins = 5):
settings = list(itertools.product(suffixs, ms, rs))
table_gt = d.get_cls_ground_truth().arr.astype(int)
print 'run with a total of %d settings'%len(settings)
for setindx in range(mpi.comm_rank, len(settings), mpi.comm_size):
second_table = None
setin = settings[setindx]
suffix = setin[0]
m = str(setin[1])
r1 = str(setin[2])
print 'node %d runs %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1)
filename = config.get_fastinf_mrf_file(d, suffix)
data_filename = config.get_fastinf_data_file(d, suffix)
if suffix == 'perfect':
table = np.hstack((table_gt, table_gt))
bounds = np.tile(np.linspace(0, 1, num_bins+1),(table_gt.shape[1],1))
print bounds
elif suffix == 'GIST':
table = gist_classify_dataset(d)
bounds, discr_table = discretize_table(table, num_bins)
table = np.hstack((table_gt, discr_table))
elif suffix == 'CSC':
filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table')
print filename_csc
if not os.path.exists(filename_csc):
raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc)
orig_table = cPickle.load(open(filename_csc,'r'))
if isinstance(orig_table, Table):
orig_table = orig_table.arr[:,:-1]
bounds, discr_table = discretize_table(orig_table, num_bins)
table = np.hstack((table_gt, discr_table))
elif suffix == 'CSC_regions':
rm = RegionModel("1big_2small", 0.5)
detector = 'csc_default'
from synthetic.dataset_policy import DatasetPolicy
orig_table = DatasetPolicy.load_ext_detections(d, detector)
gt = d.get_det_gt().copy()
# we need to spice up the gt by a score of 1 for each class (results in less code)
gt.cols.append('score')
gt.arr = np.hstack((gt.arr, np.ones((gt.shape[0], 1))))
table_gt_region = create_regioned_table(rm, gt, d.images, len(d.classes))
# At this point we need to split them for the different regions
orig_table_region = create_regioned_table(rm, orig_table, d.images, len(d.classes))
bounds, discr_table_region = discretize_table(orig_table_region, num_bins)
table = np.hstack((table_gt_region, discr_table_region))
elif suffix == 'GIST_CSC':
filename_csc = os.path.join(config.get_ext_dets_foldname(d),'table')
if not os.path.exists(filename_csc):
raise RuntimeWarning('The csc classification could not be loaded from %s'%filename_csc)
orig_table = cPickle.load(open(filename_csc,'r'))
if isinstance(orig_table, Table):
orig_table = orig_table.arr[:,:-1]
bounds, discr_table = discretize_table(orig_table, num_bins)
table = np.hstack((table_gt, discr_table))
store_bound(d, 'CSC', bounds)
second_table = gist_classify_dataset(d)
sec_bounds, second_table = discretize_table(second_table, num_bins)
store_bound(d, 'GIST', sec_bounds)
full_bound = np.hstack((sec_bounds, bounds))
store_bound(d, 'GIST_CSC', full_bound)
if not suffix == 'GIST_CSC':
store_bound(d, suffix, bounds)
print 'set up table on %d, write out mrf for %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1)
write_out_mrf(table, num_bins, filename, data_filename, second_table=second_table)
add_sets = ['-m',m]
if not r1 == '':
add_sets += ['-r1', r1]
if not second_table == None:
sec_bound_file = '%s_secbounds'%filename
for s in add_sets:
sec_bound_file += '_'+s
np.savetxt(sec_bound_file, sec_bounds)
print '%d start running lbp for %s, m=%s, r1=%s'%(mpi.comm_rank, suffix, m, r1)
filename_out = config.get_fastinf_res_file(d, suffix, m, r1)
execute_lbp(filename, data_filename, filename_out, add_settings=add_sets)
