def main():
parser = argparse.ArgumentParser(description="Run experiments with the timely detection system.")
parser.add_argument(
"--test_dataset",
choices=["val", "test", "trainval"],
default="val",
help="""Dataset to use for testing. Run on val until final runs.
The training dataset is inferred (val->train; test->trainval; trainval->trainval).""",
)
parser.add_argument("--first_n", type=int, help="only take the first N images in the test dataset")
parser.add_argument("--first_n_train", type=int, help="only take the first N images in the train dataset")
parser.add_argument(
"--config",
help="""Config file name that specifies the experiments to run.
Give name such that the file is configs/#{name}.json or configs/#{name}/
In the latter case, all files within the directory will be loaded.""",
)
parser.add_argument("--suffix", help="Overwrites the suffix in the config(s).")
parser.add_argument("--bounds10", action="store_true", default=False, help="set bounds to [0,10]")
parser.add_argument("--bounds515", action="store_true", default=False, help="set bounds to [5,15]")
parser.add_argument("--force", action="store_true", default=False, help="force overwrite")
parser.add_argument("--wholeset_prs", action="store_true", default=False, help="evaluate in the final p-r regime")
parser.add_argument(
"--no_apvst", action="store_true", default=False, help="do NOT evaluate in the ap vs. time regime"
)
parser.add_argument(
"--det_configs", action="store_true", default=False, help="output detector statistics to det_configs"
)
parser.add_argument("--inverse_prior", action="store_true", default=False, help="use inverse prior class values")
args = parser.parse_args()
print(args)
# If config file is not given, just run one experiment using default config
if not args.config:
configs = [DatasetPolicy.default_config]
else:
configs = load_configs(args.config)
# Load the dataset
dataset = Dataset("full_pascal_" + args.test_dataset)
if args.first_n:
dataset.images = dataset.images[: args.first_n]
# Infer train_dataset
if args.test_dataset == "test":
train_dataset = Dataset("full_pascal_trainval")
elif args.test_dataset == "val":
train_dataset = Dataset("full_pascal_train")
elif args.test_dataset == "trainval":
train_dataset = Dataset("full_pascal_trainval")
else:
None # impossible by argparse settings
# Only need to set training dataset values; evaluation gets it from there
if args.inverse_prior:
train_dataset.set_values("inverse_prior")
# TODO: hack
if args.first_n_train:
train_dataset.images = train_dataset.images[: args.first_n_train]
# In both the above cases, we use the val dataset for weights
weights_dataset_name = "full_pascal_val"
dets_tables = []
dets_tables_whole = []
clses_tables_whole = []
all_bounds = []
plot_infos = []
for config_f in configs:
if args.suffix:
config_f["suffix"] = args.suffix
if args.bounds10:
config_f["bounds"] = [0, 10]
if args.bounds515:
config_f["bounds"] = [5, 15]
assert not (args.bounds10 and args.bounds515)
if args.inverse_prior:
config_f["suffix"] += "_inverse_prior"
config_f["values"] = "inverse_prior"
dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name, **config_f)
ev = Evaluation(dp)
all_bounds.append(dp.bounds)
plot_infos.append(dict((k, config_f[k]) for k in ("label", "line", "color") if k in config_f))
# output the det configs first
if args.det_configs:
dp.output_det_statistics()
# evaluate in the AP vs. Time regime, unless told not to
if not args.no_apvst:
dets_table = ev.evaluate_vs_t(None, None, force=args.force)
# dets_table_whole,clses_table_whole = ev.evaluate_vs_t_whole(None,None,force=args.force)
if comm_rank == 0:
dets_tables.append(dets_table)
# dets_tables_whole.append(dets_table_whole)
# clses_tables_whole.append(clses_table_whole)
# optionally, evaluate in the standard PR regime
if args.wholeset_prs:
ev.evaluate_detections_whole(None, force=args.force)
# and plot the comparison if multiple config files were given
if not args.no_apvst and len(configs) > 1 and comm_rank == 0:
# filename of the final plot is the config file name
dirname = config.get_evals_dir(dataset.get_name())
filename = args.config
if args.inverse_prior:
filename += "_inverse_prior"
# det avg
ff = opjoin(dirname, "%s_det_avg.png" % filename)
ff_nl = opjoin(dirname, "%s_det_avg_nl.png" % filename)
# make sure directory exists
ut.makedirs(os.path.dirname(ff))
Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
if False:
# det whole
ff = opjoin(dirname, "%s_det_whole.png" % filename)
ff_nl = opjoin(dirname, "%s_det_whole_nl.png" % filename)
Evaluation.plot_ap_vs_t(
dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos
)
Evaluation.plot_ap_vs_t(
dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos
)
# cls whole
ff = opjoin(dirname, "%s_cls_whole.png" % filename)
ff_nl = opjoin(dirname, "%s_cls_whole_nl.png" % filename)
Evaluation.plot_ap_vs_t(
clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos
)
Evaluation.plot_ap_vs_t(
clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos
)
def main():
parser = argparse.ArgumentParser(
description="Run experiments with the timely detection system.")
parser.add_argument('--test_dataset',
choices=['val','test','trainval'],
default='val',
help="""Dataset to use for testing. Run on val until final runs.
The training dataset is inferred (val->train; test->trainval; trainval->trainval).""")
parser.add_argument('--first_n', type=int,
help='only take the first N images in the test dataset')
parser.add_argument('--first_n_train', type=int,
help='only take the first N images in the train dataset')
parser.add_argument('--config',
help="""Config file name that specifies the experiments to run.
Give name such that the file is configs/#{name}.json or configs/#{name}/
In the latter case, all files within the directory will be loaded.""")
parser.add_argument('--suffix',
help="Overwrites the suffix in the config(s).")
parser.add_argument('--bounds10', action='store_true',
default=False, help='set bounds to [0,10]')
parser.add_argument('--bounds515', action='store_true',
default=False, help='set bounds to [5,15]')
parser.add_argument('--force', action='store_true',
default=False, help='force overwrite')
parser.add_argument('--wholeset_prs', action='store_true',
default=False, help='evaluate in the final p-r regime')
parser.add_argument('--no_apvst', action='store_true',
default=False, help='do NOT evaluate in the ap vs. time regime')
parser.add_argument('--det_configs', action='store_true',
default=False, help='output detector statistics to det_configs')
parser.add_argument('--inverse_prior', action='store_true',
default=False, help='use inverse prior class values')
args = parser.parse_args()
print(args)
# If config file is not given, just run one experiment using default config
if not args.config:
configs = [DatasetPolicy.default_config]
else:
configs = load_configs(args.config)
# Load the dataset
dataset = Dataset('full_pascal_'+args.test_dataset)
if args.first_n:
dataset.images = dataset.images[:args.first_n]
# Infer train_dataset
if args.test_dataset=='test':
train_dataset = Dataset('full_pascal_trainval')
elif args.test_dataset=='val':
train_dataset = Dataset('full_pascal_train')
elif args.test_dataset=='trainval':
train_dataset = Dataset('full_pascal_trainval')
else:
None # impossible by argparse settings
# Only need to set training dataset values; evaluation gets it from there
if args.inverse_prior:
train_dataset.set_values('inverse_prior')
# TODO: hack
if args.first_n_train:
train_dataset.images = train_dataset.images[:args.first_n_train]
# In both the above cases, we use the val dataset for weights
weights_dataset_name = 'full_pascal_val'
dets_tables = []
dets_tables_whole = []
clses_tables_whole = []
all_bounds = []
plot_infos = []
for config_f in configs:
if args.suffix:
config_f['suffix'] = args.suffix
if args.bounds10:
config_f['bounds'] = [0,10]
if args.bounds515:
config_f['bounds'] = [5,15]
assert(not (args.bounds10 and args.bounds515))
if args.inverse_prior:
config_f['suffix'] += '_inverse_prior'
config_f['values'] = 'inverse_prior'
dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name, **config_f)
ev = Evaluation(dp)
all_bounds.append(dp.bounds)
plot_infos.append(dict((k,config_f[k]) for k in ('label','line','color') if k in config_f))
# output the det configs first
if args.det_configs:
dp.output_det_statistics()
# evaluate in the AP vs. Time regime, unless told not to
if not args.no_apvst:
dets_table = ev.evaluate_vs_t(None,None,force=args.force)
#dets_table_whole,clses_table_whole = ev.evaluate_vs_t_whole(None,None,force=args.force)
if comm_rank==0:
dets_tables.append(dets_table)
#dets_tables_whole.append(dets_table_whole)
#clses_tables_whole.append(clses_table_whole)
# optionally, evaluate in the standard PR regime
if args.wholeset_prs:
ev.evaluate_detections_whole(None,force=args.force)
# and plot the comparison if multiple config files were given
if not args.no_apvst and len(configs)>1 and comm_rank==0:
# filename of the final plot is the config file name
dirname = config.get_evals_dir(dataset.get_name())
filename = args.config
if args.inverse_prior:
filename += '_inverse_prior'
# det avg
ff = opjoin(dirname, '%s_det_avg.png'%filename)
ff_nl = opjoin(dirname, '%s_det_avg_nl.png'%filename)
# make sure directory exists
ut.makedirs(os.path.dirname(ff))
Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
if False:
# det whole
ff = opjoin(dirname, '%s_det_whole.png'%filename)
ff_nl = opjoin(dirname, '%s_det_whole_nl.png'%filename)
Evaluation.plot_ap_vs_t(dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
# cls whole
ff = opjoin(dirname, '%s_cls_whole.png'%filename)
ff_nl = opjoin(dirname, '%s_cls_whole_nl.png'%filename)
Evaluation.plot_ap_vs_t(clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
class TestDatasetPolicy:
def __init__(self):
self.dataset = Dataset('test_pascal_val')
self.train_dataset = Dataset('test_pascal_train')
self.weights_dataset_name = 'test_pascal_val'
self.config = {
'suffix': 'default',
'detectors': ['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half
'policy_mode': 'random',
'bounds': None,
'weights_mode': 'manual_1' # manual_1, manual_2, manual_3, greedy, rl
}
self.dp = DatasetPolicy(
self.dataset,self.train_dataset,self.weights_dataset_name,**self.config)
def test_run_on_dataset(self):
# run on test dataset
dets,clses,samples = self.dp.run_on_dataset(force=True)
assert(len(samples) == clses.shape[0])
assert(len(samples) == self.dp.dataset.num_images()*len(self.dp.actions))
train_dets,train_clses,train_samples = self.dp.run_on_dataset(train=True,force=True)
assert(len(train_samples) == train_clses.shape[0])
assert(len(train_samples) == self.dp.train_dataset.num_images()*len(self.dp.actions))
def test_unique_samples(self):
"Test the correctness of making a list of samples unique."
dets,clses,samples = self.dp.run_on_dataset(force=True)
new_sample = copy.deepcopy(samples[11])
new_sample2 = copy.deepcopy(samples[11])
new_sample2.dt = -40 # an unreasonable value
assert(new_sample in samples)
assert(new_sample2 not in samples)
def test_output_det_statistics(self):
self.dp.output_det_statistics()
def test_learn_weights(self):
dataset = Dataset('full_pascal_val')
train_dataset = Dataset('full_pascal_train')
dataset.images = dataset.images[:20]
train_dataset.images = train_dataset.images[:20]
dp = DatasetPolicy(dataset,train_dataset,self.weights_dataset_name,**self.config)
weights = dp.learn_weights()
def test_regress(self):
dets,clses,samples = self.dp.run_on_dataset(force=True)
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
samples += samples
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
samples += samples
weights,error = self.dp.regress(samples,'greedy')
print "Weights after %d samples:\n %s"%(len(samples),weights)
print "Error after %d samples: %s"%(len(samples),error)
def test_load_weights(self):
modes = ['manual_1','manual_2','manual_3']
for mode in modes:
print "%s weights:"%mode
self.dp.weights_mode=mode
self.dp.load_weights()
print self.dp.get_reshaped_weights()
assert(self.dp.weights.shape[0] == len(self.dp.actions)*BeliefState.num_features)
self.dp.write_weights_image('temp_weights_%s.png'%mode)
def test_perfect_detector(self):
dets,clses,samples = self.dp.run_on_dataset(force=True)
#embed()
dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
gt = self.dataset.get_det_gt()
gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
# TODO: does this make sense?
dets.sort_by_column('x')
gt.sort_by_column('x')
print dets
print gt
assert(dets == gt)
def test_load_dpm_detections(self):
conf = dict(self.config)
conf['detectors'] = ['dpm']
policy = DatasetPolicy(self.dataset,self.train_dataset,**conf)
assert(policy.detectors == ['dpm'])
dets = policy.load_ext_detections(self.dataset,'dpm_may25',force=True)
dets = dets.with_column_omitted('time')
# load the ground truth dets, processed in Matlab
# (timely/data/test_support/concat_dets.m)
filename = os.path.join(config.test_support_dir, 'val_dets.mat')
dets_correct = Table(
scipy.io.loadmat(filename)['dets'],
['x1','y1','x2','y2','dummy','dummy','dummy','dummy','score','cls_ind','img_ind'],
'dets_correct')
dets_correct = dets_correct.subset(
['x1','y1','x2','y2','score','cls_ind','img_ind'])
dets_correct.arr[:,:4] -= 1
dets_correct.arr[:,:4] = BoundingBox.convert_arr_from_corners(
dets_correct.arr[:,:4])
dets_correct.cols = ['x','y','w','h','score','cls_ind','img_ind']
print('----mine:')
print(dets)
print('----correct:')
print(dets_correct)
assert(dets_correct == dets)
class TestDatasetPolicy:
def __init__(self):
self.dataset = Dataset('test_pascal_val')
self.train_dataset = Dataset('test_pascal_train')
self.weights_dataset_name = 'test_pascal_val'
self.config = {
'suffix': 'default',
'detectors':
['perfect'], # perfect,perfect_with_noise,dpm,csc_default,csc_half
'policy_mode': 'random',
'bounds': None,
'weights_mode':
'manual_1' # manual_1, manual_2, manual_3, greedy, rl
}
self.dp = DatasetPolicy(self.dataset, self.train_dataset,
self.weights_dataset_name, **self.config)
def test_run_on_dataset(self):
# run on test dataset
dets, clses, samples = self.dp.run_on_dataset(force=True)
assert (len(samples) == clses.shape[0])
assert (len(samples) == self.dp.dataset.num_images() *
len(self.dp.actions))
train_dets, train_clses, train_samples = self.dp.run_on_dataset(
train=True, force=True)
assert (len(train_samples) == train_clses.shape[0])
assert (len(train_samples) == self.dp.train_dataset.num_images() *
len(self.dp.actions))
def test_unique_samples(self):
"Test the correctness of making a list of samples unique."
dets, clses, samples = self.dp.run_on_dataset(force=True)
new_sample = copy.deepcopy(samples[11])
new_sample2 = copy.deepcopy(samples[11])
new_sample2.dt = -40 # an unreasonable value
assert (new_sample in samples)
assert (new_sample2 not in samples)
def test_output_det_statistics(self):
self.dp.output_det_statistics()
def test_learn_weights(self):
dataset = Dataset('full_pascal_val')
train_dataset = Dataset('full_pascal_train')
dataset.images = dataset.images[:20]
train_dataset.images = train_dataset.images[:20]
dp = DatasetPolicy(dataset, train_dataset, self.weights_dataset_name,
**self.config)
weights = dp.learn_weights()
def test_regress(self):
dets, clses, samples = self.dp.run_on_dataset(force=True)
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
samples += samples
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
samples += samples
weights, error = self.dp.regress(samples, 'greedy')
print "Weights after %d samples:\n %s" % (len(samples), weights)
print "Error after %d samples: %s" % (len(samples), error)
def test_load_weights(self):
modes = ['manual_1', 'manual_2', 'manual_3']
for mode in modes:
print "%s weights:" % mode
self.dp.weights_mode = mode
self.dp.load_weights()
print self.dp.get_reshaped_weights()
assert (self.dp.weights.shape[0] == len(self.dp.actions) *
BeliefState.num_features)
self.dp.write_weights_image('temp_weights_%s.png' % mode)
def test_perfect_detector(self):
dets, clses, samples = self.dp.run_on_dataset(force=True)
#embed()
dets = dets.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
gt = self.dataset.get_det_gt()
gt = gt.subset(['x', 'y', 'w', 'h', 'cls_ind', 'img_ind'])
# TODO: does this make sense?
dets.sort_by_column('x')
gt.sort_by_column('x')
print dets
print gt
assert (dets == gt)
def test_load_dpm_detections(self):
conf = dict(self.config)
conf['detectors'] = ['dpm']
policy = DatasetPolicy(self.dataset, self.train_dataset, **conf)
assert (policy.detectors == ['dpm'])
dets = policy.load_ext_detections(self.dataset,
'dpm_may25',
force=True)
dets = dets.with_column_omitted('time')
# load the ground truth dets, processed in Matlab
# (timely/data/test_support/concat_dets.m)
filename = os.path.join(config.test_support_dir, 'val_dets.mat')
dets_correct = Table(
scipy.io.loadmat(filename)['dets'], [
'x1', 'y1', 'x2', 'y2', 'dummy', 'dummy', 'dummy', 'dummy',
'score', 'cls_ind', 'img_ind'
], 'dets_correct')
dets_correct = dets_correct.subset(
['x1', 'y1', 'x2', 'y2', 'score', 'cls_ind', 'img_ind'])
dets_correct.arr[:, :4] -= 1
dets_correct.arr[:, :4] = BoundingBox.convert_arr_from_corners(
dets_correct.arr[:, :4])
dets_correct.cols = ['x', 'y', 'w', 'h', 'score', 'cls_ind', 'img_ind']
print('----mine:')
print(dets)
print('----correct:')
print(dets_correct)
assert (dets_correct == dets)
