def evaluate_workflow(coco_eval: COCOeval, cat_ids: List[int],
cat_names: List[str], out_dir: str) -> Dict[str, float]:
"""Execute evaluation."""
n_tit = 12 # number of evaluation titles
n_cls = len(cat_ids) # 10/8 classes for BDD100K detection/tracking
n_thr = 10 # [.5:.05:.95] T=10 IoU thresholds for evaluation
n_rec = 101 # [0:.01:1] R=101 recall thresholds for evaluation
n_area = 4 # A=4 object area ranges for evaluation
n_mdet = 3 # [1 10 100] M=3 thresholds on max detections per image
eval_param = {
"params": {
"imgIds": [],
"catIds": [],
"iouThrs":
np.linspace(
0.5,
0.95,
int(np.round((0.95 - 0.5) / 0.05) + 1),
endpoint=True,
).tolist(),
"recThrs":
np.linspace(
0.0,
1.00,
int(np.round((1.00 - 0.0) / 0.01) + 1),
endpoint=True,
).tolist(),
"maxDets": [1, 10, 100],
"areaRng": [
[0**2, 1e5**2],
[0**2, 32**2],
[32**2, 96**2],
[96**2, 1e5**2],
],
"useSegm":
0,
"useCats":
1,
},
"date":
datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3],
"counts": [n_thr, n_rec, n_cls, n_area, n_mdet],
"precision": -np.ones(
(n_thr, n_rec, n_cls, n_area, n_mdet), order="F"),
"recall": -np.ones((n_thr, n_cls, n_area, n_mdet), order="F"),
}
stats_all = -np.ones((n_cls, n_tit))
for i, (cat_id, cat_name) in enumerate(zip(cat_ids, cat_names)):
print("\nEvaluate category: %s" % cat_name)
coco_eval.params.catIds = [cat_id]
# coco_eval.params.useSegm = ann_type == "segm"
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
stats_all[i, :] = coco_eval.stats
eval_param["precision"][:, :,
i, :, :] = coco_eval.eval["precision"].reshape(
(n_thr, n_rec, n_area, n_mdet))
eval_param["recall"][:, i, :, :] = coco_eval.eval["recall"].reshape(
(n_thr, n_area, n_mdet))
# Print evaluation results
stats = np.zeros((n_tit, 1))
print("\nOverall performance")
coco_eval.eval = eval_param
coco_eval.summarize()
for i in range(n_tit):
column = stats_all[:, i]
if len(column > -1) == 0:
stats[i] = -1
else:
stats[i] = np.mean(column[column > -1], axis=0)
score_titles = [
"AP",
"AP_50",
"AP_75",
"AP_small",
"AP_medium",
"AP_large",
"AR_max_1",
"AR_max_10",
"AR_max_100",
"AR_small",
"AR_medium",
"AR_large",
]
scores: Dict[str, float] = {}
for title, stat in zip(score_titles, stats):
scores[title] = stat.item()
if out_dir != "none":
write_eval(out_dir, scores, eval_param)
return scores
