def __init__(self,config,dataset_policy=None,dataset=None,name='default'): """ Must have either dataset_policy or dataset and name. If dataset_policy is given, dataset and name are ignored. """ assert(dataset_policy or (dataset and name)) if dataset_policy: self.dp = dataset_policy self.dataset = dataset_policy.dataset self.name = dataset_policy.get_config_name() else: self.dataset = dataset self.name = name self.time_intervals = Evaluation.TIME_INTERVALS self.min_overlap = Evaluation.MIN_OVERLAP # Determine filenames and create directories self.results_path = config.get_evals_dp_dir(self.dp) self.det_apvst_data_fname = os.path.join(self.results_path, 'det_apvst_table.npy') self.det_apvst_data_whole_fname = os.path.join(self.results_path, 'det_apvst_table_whole.npy') self.cls_apvst_data_whole_fname = os.path.join(self.results_path, 'cls_apvst_table_whole.npy') self.det_apvst_png_fname = os.path.join(self.results_path, 'det_apvst.png') self.det_apvst_png_whole_fname = os.path.join(self.results_path, 'det_apvst_whole.png') self.cls_apvst_png_whole_fname = os.path.join(self.results_path, 'cls_apvst_whole.png') self.dashboard_filename = os.path.join(self.results_path, 'dashboard_%s.html') self.wholeset_aps_filename = os.path.join(self.results_path, 'aps_whole.txt') wholeset_dirname = ut.makedirs(os.path.join(self.results_path, 'wholeset_detailed')) self.pr_whole_png_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.png') self.pr_whole_txt_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.txt')
def __init__(self, config, dataset_policy=None, dataset=None, name='default'): """ Must have either dataset_policy or dataset and name. If dataset_policy is given, dataset and name are ignored. """ assert (dataset_policy or (dataset and name)) if dataset_policy: self.dp = dataset_policy self.dataset = dataset_policy.dataset self.name = dataset_policy.get_config_name() else: self.dataset = dataset self.name = name self.time_intervals = Evaluation.TIME_INTERVALS self.min_overlap = Evaluation.MIN_OVERLAP # Determine filenames and create directories self.results_path = config.get_evals_dp_dir(self.dp) self.det_apvst_data_fname = os.path.join(self.results_path, 'det_apvst_table.npy') self.det_apvst_data_whole_fname = os.path.join( self.results_path, 'det_apvst_table_whole.npy') self.cls_apvst_data_whole_fname = os.path.join( self.results_path, 'cls_apvst_table_whole.npy') self.det_apvst_png_fname = os.path.join(self.results_path, 'det_apvst.png') self.det_apvst_png_whole_fname = os.path.join(self.results_path, 'det_apvst_whole.png') self.cls_apvst_png_whole_fname = os.path.join(self.results_path, 'cls_apvst_whole.png') self.dashboard_filename = os.path.join(self.results_path, 'dashboard_%s.html') self.wholeset_aps_filename = os.path.join(self.results_path, 'aps_whole.txt') wholeset_dirname = ut.makedirs( os.path.join(self.results_path, 'wholeset_detailed')) self.pr_whole_png_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.png') self.pr_whole_txt_filename = os.path.join(wholeset_dirname, 'pr_whole_%s.txt')
def evaluate_detections_whole(dataset, dets, dirname, force=False): """ Output evaluations of detections over the whole dataset in multi-class and per-class regimes. Will distribute per-class evaluation computation over MPI if available. Args: dataset (skvisutils.Dataset): the dataset of the detections dets (skpyutils.Table): the detections dirname (string): path to directory where the evaluations will be output force (bool): if False, will not re-compute PR curves if files are there Returns: none, but outputs plot files and a single HTML dashboard. """ aps = evaluate_per_class_detections(dataset, dets, dirname, force) if mpi.comm_rank == 0: # TODO: re-run per-class to make sure they plot? ap_mc = evaluate_multiclass_detections(dataset, dets, dirname, force) # Write out the information to a single overview file dirname = skutil.makedirs(dirname) filename = os.path.join(dirname, "aps_whole.txt") with open(filename, "w") as f: f.write("Multiclass: %.3f\n" % ap_mc) f.write(",".join(dataset.classes) + "\n") f.write(",".join(["%.3f" % ap for ap in aps]) + "\n") # Assemble everything in one HTML file, the Dashboard names = list(dataset.classes) + ["mean ap", "multiclass"] aps = aps.tolist() + [np.mean(aps), ap_mc] pngs = glob.glob(os.path.join(dirname, "*.png")) aps_to_print = ["%.3f" % ap for ap in aps] template_filename = os.path.join(SUPPORT_DIR, "dashboard_template.html") template = Template(filename=template_filename) filename = os.path.join(dirname, "whole_dashboard.html") with open(filename, "w") as f: f.write(template.render(support_dir=SUPPORT_DIR, names=names, aps=aps_to_print, pngs=pngs)) mpi.safebarrier()
def compute_pr(dets, gt, name, dirname, force=False, plot=False): dirname = skutil.makedirs(dirname) filename = os.path.join(dirname, "pr_whole_{}.txt".format(name)) ap = None if force or not os.path.exists(filename): ap, rec, prec = compute_det_pr(dets, gt) if plot: try: plot_filename = os.path.join(dirname, "pr_whole_{0}.png".format(name)) plot_pr(ap, rec, prec, name, plot_filename, force) except: print("compute_and_plot_pr: could not plot!") with open(filename, "w") as f: f.write("%f\n" % ap) for i in range(np.shape(rec)[0]): f.write("%f %f\n" % (rec[i], prec[i])) else: with open(filename) as f: ap = float(f.readline()) return ap
def main(): parser = argparse.ArgumentParser( description="Run experiments with the timely detection system.") parser.add_argument( '--test_dataset', choices=['val', 'test', 'trainval'], default='val', help="""Dataset to use for testing. Run on val until final runs. The training dataset is inferred (val->train; test->trainval; trainval->trainval).""" ) parser.add_argument( '--first_n', type=int, help='only take the first N images in the test dataset') parser.add_argument( '--first_n_train', type=int, help='only take the first N images in the train dataset') parser.add_argument( '--config', help="""Config file name that specifies the experiments to run. Give name such that the file is configs/#{name}.json or configs/#{name}/ In the latter case, all files within the directory will be loaded.""") parser.add_argument('--suffix', help="Overwrites the suffix in the config(s).") parser.add_argument('--bounds10', action='store_true', default=False, help='set bounds to [0,10]') parser.add_argument('--bounds515', action='store_true', default=False, help='set bounds to [5,15]') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--wholeset_prs', action='store_true', default=False, help='evaluate in the final p-r regime') parser.add_argument('--no_apvst', action='store_true', default=False, help='do NOT evaluate in the ap vs. time regime') parser.add_argument('--det_configs', action='store_true', default=False, help='output detector statistics to det_configs') parser.add_argument('--inverse_prior', action='store_true', default=False, help='use inverse prior class values') args = parser.parse_args() print(args) # If config file is not given, just run one experiment using default config if not args.config: configs = [DatasetPolicy.default_config] else: configs = load_configs(args.config) # Load the dataset dataset = Dataset('full_pascal_' + args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset == 'test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset == 'val': train_dataset = Dataset('full_pascal_train') elif args.test_dataset == 'trainval': train_dataset = Dataset('full_pascal_trainval') else: None # impossible by argparse settings # Only need to set training dataset values; evaluation gets it from there if args.inverse_prior: train_dataset.set_values('inverse_prior') # TODO: hack if args.first_n_train: train_dataset.images = train_dataset.images[:args.first_n_train] # In both the above cases, we use the val dataset for weights weights_dataset_name = 'full_pascal_val' dets_tables = [] dets_tables_whole = [] clses_tables_whole = [] all_bounds = [] plot_infos = [] for config_f in configs: if args.suffix: config_f['suffix'] = args.suffix if args.bounds10: config_f['bounds'] = [0, 10] if args.bounds515: config_f['bounds'] = [5, 15] assert (not (args.bounds10 and args.bounds515)) if args.inverse_prior: config_f['suffix'] += '_inverse_prior' config_f['values'] = 'inverse_prior' dp = DatasetPolicy(dataset, train_dataset, weights_dataset_name, **config_f) ev = Evaluation(config, dp) all_bounds.append(dp.bounds) plot_infos.append( dict((k, config_f[k]) for k in ('label', 'line', 'color') if k in config_f)) # output the det configs first if args.det_configs: dp.output_det_statistics() # evaluate in the AP vs. Time regime, unless told not to if not args.no_apvst: dets_table = ev.evaluate_vs_t(None, None, force=args.force) # dets_table_whole,clses_table_whole = # ev.evaluate_vs_t_whole(None,None,force=args.force) if mpi.mpi.comm_rank == 0: dets_tables.append(dets_table) # dets_tables_whole.append(dets_table_whole) # clses_tables_whole.append(clses_table_whole) # optionally, evaluate in the standard PR regime if args.wholeset_prs: ev.evaluate_detections_whole(None, force=args.force) # and plot the comparison if multiple config files were given if not args.no_apvst and len(configs) > 1 and mpi.mpi.comm_rank == 0: # filename of the final plot is the config file name dirname = config.get_evals_dir(dataset.get_name()) filename = args.config if args.inverse_prior: filename += '_inverse_prior' # det avg ff = os.path.join(dirname, '%s_det_avg.png' % filename) ff_nl = os.path.join(dirname, '%s_det_avg_nl.png' % filename) # make sure directory exists skutil.makedirs(os.path.dirname(ff)) Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) if False: # det whole ff = os.path.join(dirname, '%s_det_whole.png' % filename) ff_nl = os.path.join(dirname, '%s_det_whole_nl.png' % filename) Evaluation.plot_ap_vs_t(dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) # cls whole ff = os.path.join(dirname, '%s_cls_whole.png' % filename) ff_nl = os.path.join(dirname, '%s_cls_whole_nl.png' % filename) Evaluation.plot_ap_vs_t(clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)
def main(): parser = argparse.ArgumentParser( description="Run experiments with the timely detection system.") parser.add_argument('--test_dataset', choices=['val', 'test', 'trainval'], default='val', help="""Dataset to use for testing. Run on val until final runs. The training dataset is inferred (val->train; test->trainval; trainval->trainval).""") parser.add_argument('--first_n', type=int, help='only take the first N images in the test dataset') parser.add_argument('--first_n_train', type=int, help='only take the first N images in the train dataset') parser.add_argument('--config', help="""Config file name that specifies the experiments to run. Give name such that the file is configs/#{name}.json or configs/#{name}/ In the latter case, all files within the directory will be loaded.""") parser.add_argument('--suffix', help="Overwrites the suffix in the config(s).") parser.add_argument('--bounds10', action='store_true', default=False, help='set bounds to [0,10]') parser.add_argument('--bounds515', action='store_true', default=False, help='set bounds to [5,15]') parser.add_argument('--force', action='store_true', default=False, help='force overwrite') parser.add_argument('--wholeset_prs', action='store_true', default=False, help='evaluate in the final p-r regime') parser.add_argument('--no_apvst', action='store_true', default=False, help='do NOT evaluate in the ap vs. time regime') parser.add_argument('--det_configs', action='store_true', default=False, help='output detector statistics to det_configs') parser.add_argument('--inverse_prior', action='store_true', default=False, help='use inverse prior class values') args = parser.parse_args() print(args) # If config file is not given, just run one experiment using default config if not args.config: configs = [DatasetPolicy.default_config] else: configs = load_configs(args.config) # Load the dataset dataset = Dataset('full_pascal_' + args.test_dataset) if args.first_n: dataset.images = dataset.images[:args.first_n] # Infer train_dataset if args.test_dataset == 'test': train_dataset = Dataset('full_pascal_trainval') elif args.test_dataset == 'val': train_dataset = Dataset('full_pascal_train') elif args.test_dataset == 'trainval': train_dataset = Dataset('full_pascal_trainval') else: None # impossible by argparse settings # Only need to set training dataset values; evaluation gets it from there if args.inverse_prior: train_dataset.set_values('inverse_prior') # TODO: hack if args.first_n_train: train_dataset.images = train_dataset.images[:args.first_n_train] # In both the above cases, we use the val dataset for weights weights_dataset_name = 'full_pascal_val' dets_tables = [] dets_tables_whole = [] clses_tables_whole = [] all_bounds = [] plot_infos = [] for config_f in configs: if args.suffix: config_f['suffix'] = args.suffix if args.bounds10: config_f['bounds'] = [0, 10] if args.bounds515: config_f['bounds'] = [5, 15] assert(not (args.bounds10 and args.bounds515)) if args.inverse_prior: config_f['suffix'] += '_inverse_prior' config_f['values'] = 'inverse_prior' dp = DatasetPolicy( dataset, train_dataset, weights_dataset_name, **config_f) ev = Evaluation(config, dp) all_bounds.append(dp.bounds) plot_infos.append(dict((k, config_f[k]) for k in ( 'label', 'line', 'color') if k in config_f)) # output the det configs first if args.det_configs: dp.output_det_statistics() # evaluate in the AP vs. Time regime, unless told not to if not args.no_apvst: dets_table = ev.evaluate_vs_t(None, None, force=args.force) # dets_table_whole,clses_table_whole = # ev.evaluate_vs_t_whole(None,None,force=args.force) if mpi.mpi.comm_rank == 0: dets_tables.append(dets_table) # dets_tables_whole.append(dets_table_whole) # clses_tables_whole.append(clses_table_whole) # optionally, evaluate in the standard PR regime if args.wholeset_prs: ev.evaluate_detections_whole(None, force=args.force) # and plot the comparison if multiple config files were given if not args.no_apvst and len(configs) > 1 and mpi.mpi.comm_rank == 0: # filename of the final plot is the config file name dirname = config.get_evals_dir(dataset.get_name()) filename = args.config if args.inverse_prior: filename += '_inverse_prior' # det avg ff = os.path.join(dirname, '%s_det_avg.png' % filename) ff_nl = os.path.join(dirname, '%s_det_avg_nl.png' % filename) # make sure directory exists skutil.makedirs(os.path.dirname(ff)) Evaluation.plot_ap_vs_t(dets_tables, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) if False: # det whole ff = os.path.join(dirname, '%s_det_whole.png' % filename) ff_nl = os.path.join(dirname, '%s_det_whole_nl.png' % filename) Evaluation.plot_ap_vs_t(dets_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(dets_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos) # cls whole ff = os.path.join(dirname, '%s_cls_whole.png' % filename) ff_nl = os.path.join(dirname, '%s_cls_whole_nl.png' % filename) Evaluation.plot_ap_vs_t(clses_tables_whole, ff, all_bounds, with_legend=True, force=True, plot_infos=plot_infos) Evaluation.plot_ap_vs_t(clses_tables_whole, ff_nl, all_bounds, with_legend=False, force=True, plot_infos=plot_infos)