def evaluate_per_class_detections(dataset, dets, dirname, force=False):
"""
Output evaluations of detections over the dataset in per-class regime.
Will distribute per-class evaluation computation over MPI if available.
Args:
dataset (skvisutils.Dataset): the dataset of the detections
dets (skpyutils.Table): the detections
dirname (string): path to directory where the evaluations will be output
force (bool): if False, will not re-compute PR curves if files are there
Returns:
aps (list of floats), and outputs plots and supporting files to dirname
"""
num_classes = len(dataset.classes)
dist_aps = np.zeros(num_classes)
for cls_ind in range(mpi.comm_rank, num_classes, mpi.comm_size):
object_class = dataset.classes[cls_ind]
cls_dets = dets.filter_on_column("cls_ind", cls_ind)
cls_gt = dataset.get_det_gt_for_class(object_class, with_diff=True)
ap = compute_and_plot_pr(cls_dets, cls_gt, object_class, dirname, force)
dist_aps[cls_ind] = ap
aps = np.zeros(num_classes)
mpi.safebarrier()
mpi.comm.Reduce(dist_aps, aps)
return aps
def evaluate_detections_whole(dataset, dets, dirname, force=False):
"""
Output evaluations of detections over the whole dataset in multi-class and
per-class regimes.
Will distribute per-class evaluation computation over MPI if available.
Args:
dataset (skvisutils.Dataset): the dataset of the detections
dets (skpyutils.Table): the detections
dirname (string): path to directory where the evaluations will be output
force (bool): if False, will not re-compute PR curves if files are there
Returns:
none, but outputs plot files and a single HTML dashboard.
"""
aps = evaluate_per_class_detections(dataset, dets, dirname, force)
if mpi.comm_rank == 0:
# TODO: re-run per-class to make sure they plot?
ap_mc = evaluate_multiclass_detections(dataset, dets, dirname, force)
# Write out the information to a single overview file
dirname = skutil.makedirs(dirname)
filename = os.path.join(dirname, "aps_whole.txt")
with open(filename, "w") as f:
f.write("Multiclass: %.3f\n" % ap_mc)
f.write(",".join(dataset.classes) + "\n")
f.write(",".join(["%.3f" % ap for ap in aps]) + "\n")
# Assemble everything in one HTML file, the Dashboard
names = list(dataset.classes) + ["mean ap", "multiclass"]
aps = aps.tolist() + [np.mean(aps), ap_mc]
pngs = glob.glob(os.path.join(dirname, "*.png"))
aps_to_print = ["%.3f" % ap for ap in aps]
template_filename = os.path.join(SUPPORT_DIR, "dashboard_template.html")
template = Template(filename=template_filename)
filename = os.path.join(dirname, "whole_dashboard.html")
with open(filename, "w") as f:
f.write(template.render(support_dir=SUPPORT_DIR, names=names, aps=aps_to_print, pngs=pngs))
mpi.safebarrier()
