def evaluation(submit_file, eval_ann, threshold=0.1, confidence=0.3):
assert osp.isfile(submit_file)
assert osp.isfile(eval_ann)
with open(eval_ann, 'r', encoding='utf-8') as f:
gt_annotations = json.loads(f.read(), object_pairs_hook=OrderedDict)
with open(submit_file, 'r', encoding='utf-8') as f:
preds = json.loads(f.read(), object_pairs_hook=OrderedDict)
tp, fp, npos = 0, 0, 0
""" for validation the annotation is same as the trainset. """
for name, pred_ann in preds.items():
gt_annotation = gt_annotations[name] # is a list
# npos should filter the ignored box.
# npos += len(gt_annotation)
for gt_polygon_tran in gt_annotation:
if gt_polygon_tran["illegibility"]:
continue
npos += 1
cover = set()
# compare each pred_polygon with each gt_polygon
for pred_polygon_pro in pred_ann:
pred_polygon = np.array(
pred_polygon_pro["points"]) # shape: [n, 2]
pred_prob = np.float32(pred_polygon_pro["confidence"])
if pred_prob < confidence: # confidence threshold.
continue
pred_polygon = plg.Polygon(pred_polygon)
flag = False
is_ignore = False
for gt_id, gt_polygon_tran in enumerate(gt_annotation):
gt_illegibility = gt_polygon_tran["illegibility"]
gt_polygon = np.array(gt_polygon_tran["points"]).reshape(
-1, 2).astype(np.int64)
gt_polygon = plg.Polyogn(gt_polygon)
union = get_union(pred_polygon, gt_polygon)
intersection = get_intersection(pred_polygon, gt_polygon)
if intersection * 1.0 / union >= threshold and flag is False:
if gt_id not in cover:
flag = True
if gt_illegibility:
is_ignore = True
cover.add(gt_id)
if flag:
tp += 0.0 if is_ignore else 1.0
else:
fp += 1.0
precision = tp / (fp + tp)
recall = tp / npos # recall do not calculate the ignored ones.
hmean = 0 if (
precision +
recall) == 0 else 2 * precision * recall / (precision + recall)
print("p: {:f} r: {:f} h: {:f}".format(precision, recall, hmean))
