def __init__(self,config,dataset_policy=None,dataset=None,name='default'):
"""
Must have either dataset_policy or dataset and name.
If dataset_policy is given, dataset and name are ignored.
"""
assert(dataset_policy or (dataset and name))
if dataset_policy:
self.dp = dataset_policy
self.dataset = dataset_policy.dataset
self.name = dataset_policy.get_config_name()
else:
self.dataset = dataset
self.name = name
self.time_intervals = Evaluation.TIME_INTERVALS
self.min_overlap = Evaluation.MIN_OVERLAP
# Determine filenames and create directories
self.results_path = config.get_evals_dp_dir(self.dp)
self.det_apvst_data_fname = os.path.join(self.results_path, 'det_apvst_table.npy')
self.det_apvst_data_whole_fname = os.path.join(self.results_path, 'det_apvst_table_whole.npy')
self.cls_apvst_data_whole_fname = os.path.join(self.results_path, 'cls_apvst_table_whole.npy')
self.det_apvst_png_fname = os.path.join(self.results_path, 'det_apvst.png')
self.det_apvst_png_whole_fname = os.path.join(self.results_path, 'det_apvst_whole.png')
self.cls_apvst_png_whole_fname = os.path.join(self.results_path, 'cls_apvst_whole.png')
self.dashboard_filename = os.path.join(self.results_path, 'dashboard_%s.html')
self.wholeset_aps_filename = os.path.join(self.results_path, 'aps_whole.txt')
wholeset_dirname = ut.makedirs(os.path.join(self.results_path, 'wholeset_detailed'))
self.pr_whole_png_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.png')
self.pr_whole_txt_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.txt')
def __init__(self,
config,
dataset_policy=None,
dataset=None,
name='default'):
"""
Must have either dataset_policy or dataset and name.
If dataset_policy is given, dataset and name are ignored.
"""
assert (dataset_policy or (dataset and name))
if dataset_policy:
self.dp = dataset_policy
self.dataset = dataset_policy.dataset
self.name = dataset_policy.get_config_name()
else:
self.dataset = dataset
self.name = name
self.time_intervals = Evaluation.TIME_INTERVALS
self.min_overlap = Evaluation.MIN_OVERLAP
# Determine filenames and create directories
self.results_path = config.get_evals_dp_dir(self.dp)
self.det_apvst_data_fname = os.path.join(self.results_path,
'det_apvst_table.npy')
self.det_apvst_data_whole_fname = os.path.join(
self.results_path, 'det_apvst_table_whole.npy')
self.cls_apvst_data_whole_fname = os.path.join(
self.results_path, 'cls_apvst_table_whole.npy')
self.det_apvst_png_fname = os.path.join(self.results_path,
'det_apvst.png')
self.det_apvst_png_whole_fname = os.path.join(self.results_path,
'det_apvst_whole.png')
self.cls_apvst_png_whole_fname = os.path.join(self.results_path,
'cls_apvst_whole.png')
self.dashboard_filename = os.path.join(self.results_path,
'dashboard_%s.html')
self.wholeset_aps_filename = os.path.join(self.results_path,
'aps_whole.txt')
wholeset_dirname = ut.makedirs(
os.path.join(self.results_path, 'wholeset_detailed'))
self.pr_whole_png_filename = os.path.join(wholeset_dirname,
'pr_whole_%s.png')
self.pr_whole_txt_filename = os.path.join(wholeset_dirname,
'pr_whole_%s.txt')
def evaluate_detections_whole(dataset, dets, dirname, force=False):
"""
Output evaluations of detections over the whole dataset in multi-class and
per-class regimes.
Will distribute per-class evaluation computation over MPI if available.
Args:
dataset (skvisutils.Dataset): the dataset of the detections
dets (skpyutils.Table): the detections
dirname (string): path to directory where the evaluations will be output
force (bool): if False, will not re-compute PR curves if files are there
Returns:
none, but outputs plot files and a single HTML dashboard.
"""
aps = evaluate_per_class_detections(dataset, dets, dirname, force)
if mpi.comm_rank == 0:
# TODO: re-run per-class to make sure they plot?
ap_mc = evaluate_multiclass_detections(dataset, dets, dirname, force)
# Write out the information to a single overview file
dirname = skutil.makedirs(dirname)
filename = os.path.join(dirname, "aps_whole.txt")
with open(filename, "w") as f:
f.write("Multiclass: %.3f\n" % ap_mc)
f.write(",".join(dataset.classes) + "\n")
f.write(",".join(["%.3f" % ap for ap in aps]) + "\n")
# Assemble everything in one HTML file, the Dashboard
names = list(dataset.classes) + ["mean ap", "multiclass"]
aps = aps.tolist() + [np.mean(aps), ap_mc]
pngs = glob.glob(os.path.join(dirname, "*.png"))
aps_to_print = ["%.3f" % ap for ap in aps]
template_filename = os.path.join(SUPPORT_DIR, "dashboard_template.html")
template = Template(filename=template_filename)
filename = os.path.join(dirname, "whole_dashboard.html")
with open(filename, "w") as f:
f.write(template.render(support_dir=SUPPORT_DIR, names=names, aps=aps_to_print, pngs=pngs))
mpi.safebarrier()
def compute_pr(dets, gt, name, dirname, force=False, plot=False):
dirname = skutil.makedirs(dirname)
filename = os.path.join(dirname, "pr_whole_{}.txt".format(name))
ap = None
if force or not os.path.exists(filename):
ap, rec, prec = compute_det_pr(dets, gt)
if plot:
try:
plot_filename = os.path.join(dirname, "pr_whole_{0}.png".format(name))
plot_pr(ap, rec, prec, name, plot_filename, force)
except:
print("compute_and_plot_pr: could not plot!")
with open(filename, "w") as f:
f.write("%f\n" % ap)
for i in range(np.shape(rec)[0]):
f.write("%f %f\n" % (rec[i], prec[i]))
else:
with open(filename) as f:
ap = float(f.readline())
return ap
def main():
parser = argparse.ArgumentParser(
description="Run experiments with the timely detection system.")
parser.add_argument(
'--test_dataset',
choices=['val', 'test', 'trainval'],
default='val',
help="""Dataset to use for testing. Run on val until final runs.
The training dataset is inferred (val->train; test->trainval; trainval->trainval)."""
)
parser.add_argument(
'--first_n',
type=int,
help='only take the first N images in the test dataset')
parser.add_argument(
'--first_n_train',
type=int,
help='only take the first N images in the train dataset')
parser.add_argument(
'--config',
help="""Config file name that specifies the experiments to run.
Give name such that the file is configs/#{name}.json or configs/#{name}/
In the latter case, all files within the directory will be loaded.""")
parser.add_argument('--suffix',
help="Overwrites the suffix in the config(s).")
parser.add_argument('--bounds10',
action='store_true',
default=False,
help='set bounds to [0,10]')
parser.add_argument('--bounds515',
action='store_true',
default=False,
help='set bounds to [5,15]')
parser.add_argument('--force',
action='store_true',
default=False,
help='force overwrite')
parser.add_argument('--wholeset_prs',
action='store_true',
default=False,
help='evaluate in the final p-r regime')
parser.add_argument('--no_apvst',
action='store_true',
default=False,
help='do NOT evaluate in the ap vs. time regime')
parser.add_argument('--det_configs',
action='store_true',
default=False,
help='output detector statistics to det_configs')
parser.add_argument('--inverse_prior',
action='store_true',
default=False,
help='use inverse prior class values')
args = parser.parse_args()
print(args)
# If config file is not given, just run one experiment using default config
if not args.config:
configs = [DatasetPolicy.default_config]
else:
configs = load_configs(args.config)
# Load the dataset
dataset = Dataset('full_pascal_' + args.test_dataset)
if args.first_n:
dataset.images = dataset.images[:args.first_n]
# Infer train_dataset
if args.test_dataset == 'test':
train_dataset = Dataset('full_pascal_trainval')
elif args.test_dataset == 'val':
train_dataset = Dataset('full_pascal_train')
elif args.test_dataset == 'trainval':
train_dataset = Dataset('full_pascal_trainval')
else:
None # impossible by argparse settings
# Only need to set training dataset values; evaluation gets it from there
if args.inverse_prior:
train_dataset.set_values('inverse_prior')
# TODO: hack
if args.first_n_train:
train_dataset.images = train_dataset.images[:args.first_n_train]
# In both the above cases, we use the val dataset for weights
weights_dataset_name = 'full_pascal_val'
dets_tables = []
dets_tables_whole = []
clses_tables_whole = []
all_bounds = []
plot_infos = []
for config_f in configs:
if args.suffix:
config_f['suffix'] = args.suffix
if args.bounds10:
config_f['bounds'] = [0, 10]
if args.bounds515:
config_f['bounds'] = [5, 15]
assert (not (args.bounds10 and args.bounds515))
if args.inverse_prior:
config_f['suffix'] += '_inverse_prior'
config_f['values'] = 'inverse_prior'
dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name,
**config_f)
ev = Evaluation(config, dp)
all_bounds.append(dp.bounds)
plot_infos.append(
dict((k, config_f[k]) for k in ('label', 'line', 'color')
if k in config_f))
# output the det configs first
if args.det_configs:
dp.output_det_statistics()
# evaluate in the AP vs. Time regime, unless told not to
if not args.no_apvst:
dets_table = ev.evaluate_vs_t(None, None, force=args.force)
# dets_table_whole,clses_table_whole =
# ev.evaluate_vs_t_whole(None,None,force=args.force)
if mpi.mpi.comm_rank == 0:
dets_tables.append(dets_table)
# dets_tables_whole.append(dets_table_whole)
# clses_tables_whole.append(clses_table_whole)
# optionally, evaluate in the standard PR regime
if args.wholeset_prs:
ev.evaluate_detections_whole(None, force=args.force)
# and plot the comparison if multiple config files were given
if not args.no_apvst and len(configs) > 1 and mpi.mpi.comm_rank == 0:
# filename of the final plot is the config file name
dirname = config.get_evals_dir(dataset.get_name())
filename = args.config
if args.inverse_prior:
filename += '_inverse_prior'
# det avg
ff = os.path.join(dirname, '%s_det_avg.png' % filename)
ff_nl = os.path.join(dirname, '%s_det_avg_nl.png' % filename)
# make sure directory exists
skutil.makedirs(os.path.dirname(ff))
Evaluation.plot_ap_vs_t(dets_tables,
ff,
all_bounds,
with_legend=True,
force=True,
plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables,
ff_nl,
all_bounds,
with_legend=False,
force=True,
plot_infos=plot_infos)
if False:
# det whole
ff = os.path.join(dirname, '%s_det_whole.png' % filename)
ff_nl = os.path.join(dirname, '%s_det_whole_nl.png' % filename)
Evaluation.plot_ap_vs_t(dets_tables_whole,
ff,
all_bounds,
with_legend=True,
force=True,
plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables_whole,
ff_nl,
all_bounds,
with_legend=False,
force=True,
plot_infos=plot_infos)
# cls whole
ff = os.path.join(dirname, '%s_cls_whole.png' % filename)
ff_nl = os.path.join(dirname, '%s_cls_whole_nl.png' % filename)
Evaluation.plot_ap_vs_t(clses_tables_whole,
ff,
all_bounds,
with_legend=True,
force=True,
plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(clses_tables_whole,
ff_nl,
all_bounds,
with_legend=False,
force=True,
plot_infos=plot_infos)
def main():
parser = argparse.ArgumentParser(
description="Run experiments with the timely detection system.")
parser.add_argument('--test_dataset',
choices=['val', 'test', 'trainval'],
default='val',
help="""Dataset to use for testing. Run on val until final runs.
The training dataset is inferred (val->train; test->trainval; trainval->trainval).""")
parser.add_argument('--first_n', type=int,
help='only take the first N images in the test dataset')
parser.add_argument('--first_n_train', type=int,
help='only take the first N images in the train dataset')
parser.add_argument('--config',
help="""Config file name that specifies the experiments to run.
Give name such that the file is configs/#{name}.json or configs/#{name}/
In the latter case, all files within the directory will be loaded.""")
parser.add_argument('--suffix',
help="Overwrites the suffix in the config(s).")
parser.add_argument('--bounds10', action='store_true',
default=False, help='set bounds to [0,10]')
parser.add_argument('--bounds515', action='store_true',
default=False, help='set bounds to [5,15]')
parser.add_argument('--force', action='store_true',
default=False, help='force overwrite')
parser.add_argument('--wholeset_prs', action='store_true',
default=False, help='evaluate in the final p-r regime')
parser.add_argument('--no_apvst', action='store_true',
default=False, help='do NOT evaluate in the ap vs. time regime')
parser.add_argument('--det_configs', action='store_true',
default=False, help='output detector statistics to det_configs')
parser.add_argument('--inverse_prior', action='store_true',
default=False, help='use inverse prior class values')
args = parser.parse_args()
print(args)
# If config file is not given, just run one experiment using default config
if not args.config:
configs = [DatasetPolicy.default_config]
else:
configs = load_configs(args.config)
# Load the dataset
dataset = Dataset('full_pascal_' + args.test_dataset)
if args.first_n:
dataset.images = dataset.images[:args.first_n]
# Infer train_dataset
if args.test_dataset == 'test':
train_dataset = Dataset('full_pascal_trainval')
elif args.test_dataset == 'val':
train_dataset = Dataset('full_pascal_train')
elif args.test_dataset == 'trainval':
train_dataset = Dataset('full_pascal_trainval')
else:
None # impossible by argparse settings
# Only need to set training dataset values; evaluation gets it from there
if args.inverse_prior:
train_dataset.set_values('inverse_prior')
# TODO: hack
if args.first_n_train:
train_dataset.images = train_dataset.images[:args.first_n_train]
# In both the above cases, we use the val dataset for weights
weights_dataset_name = 'full_pascal_val'
dets_tables = []
dets_tables_whole = []
clses_tables_whole = []
all_bounds = []
plot_infos = []
for config_f in configs:
if args.suffix:
config_f['suffix'] = args.suffix
if args.bounds10:
config_f['bounds'] = [0, 10]
if args.bounds515:
config_f['bounds'] = [5, 15]
assert(not (args.bounds10 and args.bounds515))
if args.inverse_prior:
config_f['suffix'] += '_inverse_prior'
config_f['values'] = 'inverse_prior'
dp = DatasetPolicy(
dataset, train_dataset, weights_dataset_name, **config_f)
ev = Evaluation(config, dp)
all_bounds.append(dp.bounds)
plot_infos.append(dict((k, config_f[k]) for k in (
'label', 'line', 'color') if k in config_f))
# output the det configs first
if args.det_configs:
dp.output_det_statistics()
# evaluate in the AP vs. Time regime, unless told not to
if not args.no_apvst:
dets_table = ev.evaluate_vs_t(None, None, force=args.force)
# dets_table_whole,clses_table_whole =
# ev.evaluate_vs_t_whole(None,None,force=args.force)
if mpi.mpi.comm_rank == 0:
dets_tables.append(dets_table)
# dets_tables_whole.append(dets_table_whole)
# clses_tables_whole.append(clses_table_whole)
# optionally, evaluate in the standard PR regime
if args.wholeset_prs:
ev.evaluate_detections_whole(None, force=args.force)
# and plot the comparison if multiple config files were given
if not args.no_apvst and len(configs) > 1 and mpi.mpi.comm_rank == 0:
# filename of the final plot is the config file name
dirname = config.get_evals_dir(dataset.get_name())
filename = args.config
if args.inverse_prior:
filename += '_inverse_prior'
# det avg
ff = os.path.join(dirname, '%s_det_avg.png' % filename)
ff_nl = os.path.join(dirname, '%s_det_avg_nl.png' % filename)
# make sure directory exists
skutil.makedirs(os.path.dirname(ff))
Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True,
force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds,
with_legend=False, force=True, plot_infos=plot_infos)
if False:
# det whole
ff = os.path.join(dirname, '%s_det_whole.png' % filename)
ff_nl = os.path.join(dirname, '%s_det_whole_nl.png' % filename)
Evaluation.plot_ap_vs_t(dets_tables_whole, ff, all_bounds,
with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(dets_tables_whole, ff_nl, all_bounds,
with_legend=False, force=True, plot_infos=plot_infos)
# cls whole
ff = os.path.join(dirname, '%s_cls_whole.png' % filename)
ff_nl = os.path.join(dirname, '%s_cls_whole_nl.png' % filename)
Evaluation.plot_ap_vs_t(clses_tables_whole, ff, all_bounds,
with_legend=True, force=True, plot_infos=plot_infos)
Evaluation.plot_ap_vs_t(clses_tables_whole, ff_nl, all_bounds,
with_legend=False, force=True, plot_infos=plot_infos)
