def evaluate_per_class_detections(dataset, dets, dirname, force=False): """ Output evaluations of detections over the dataset in per-class regime. Will distribute per-class evaluation computation over MPI if available. Args: dataset (skvisutils.Dataset): the dataset of the detections dets (skpyutils.Table): the detections dirname (string): path to directory where the evaluations will be output force (bool): if False, will not re-compute PR curves if files are there Returns: aps (list of floats), and outputs plots and supporting files to dirname """ num_classes = len(dataset.classes) dist_aps = np.zeros(num_classes) for cls_ind in range(mpi.comm_rank, num_classes, mpi.comm_size): object_class = dataset.classes[cls_ind] cls_dets = dets.filter_on_column("cls_ind", cls_ind) cls_gt = dataset.get_det_gt_for_class(object_class, with_diff=True) ap = compute_and_plot_pr(cls_dets, cls_gt, object_class, dirname, force) dist_aps[cls_ind] = ap aps = np.zeros(num_classes) mpi.safebarrier() mpi.comm.Reduce(dist_aps, aps) return aps
def evaluate_detections_whole(dataset, dets, dirname, force=False): """ Output evaluations of detections over the whole dataset in multi-class and per-class regimes. Will distribute per-class evaluation computation over MPI if available. Args: dataset (skvisutils.Dataset): the dataset of the detections dets (skpyutils.Table): the detections dirname (string): path to directory where the evaluations will be output force (bool): if False, will not re-compute PR curves if files are there Returns: none, but outputs plot files and a single HTML dashboard. """ aps = evaluate_per_class_detections(dataset, dets, dirname, force) if mpi.comm_rank == 0: # TODO: re-run per-class to make sure they plot? ap_mc = evaluate_multiclass_detections(dataset, dets, dirname, force) # Write out the information to a single overview file dirname = skutil.makedirs(dirname) filename = os.path.join(dirname, "aps_whole.txt") with open(filename, "w") as f: f.write("Multiclass: %.3f\n" % ap_mc) f.write(",".join(dataset.classes) + "\n") f.write(",".join(["%.3f" % ap for ap in aps]) + "\n") # Assemble everything in one HTML file, the Dashboard names = list(dataset.classes) + ["mean ap", "multiclass"] aps = aps.tolist() + [np.mean(aps), ap_mc] pngs = glob.glob(os.path.join(dirname, "*.png")) aps_to_print = ["%.3f" % ap for ap in aps] template_filename = os.path.join(SUPPORT_DIR, "dashboard_template.html") template = Template(filename=template_filename) filename = os.path.join(dirname, "whole_dashboard.html") with open(filename, "w") as f: f.write(template.render(support_dir=SUPPORT_DIR, names=names, aps=aps_to_print, pngs=pngs)) mpi.safebarrier()