def evaluation(submit_file, eval_ann, threshold=0.1, confidence=0.3): assert osp.isfile(submit_file) assert osp.isfile(eval_ann) with open(eval_ann, 'r', encoding='utf-8') as f: gt_annotations = json.loads(f.read(), object_pairs_hook=OrderedDict) with open(submit_file, 'r', encoding='utf-8') as f: preds = json.loads(f.read(), object_pairs_hook=OrderedDict) tp, fp, npos = 0, 0, 0 """ for validation the annotation is same as the trainset. """ for name, pred_ann in preds.items(): gt_annotation = gt_annotations[name] # is a list # npos should filter the ignored box. # npos += len(gt_annotation) for gt_polygon_tran in gt_annotation: if gt_polygon_tran["illegibility"]: continue npos += 1 cover = set() # compare each pred_polygon with each gt_polygon for pred_polygon_pro in pred_ann: pred_polygon = np.array( pred_polygon_pro["points"]) # shape: [n, 2] pred_prob = np.float32(pred_polygon_pro["confidence"]) if pred_prob < confidence: # confidence threshold. continue pred_polygon = plg.Polygon(pred_polygon) flag = False is_ignore = False for gt_id, gt_polygon_tran in enumerate(gt_annotation): gt_illegibility = gt_polygon_tran["illegibility"] gt_polygon = np.array(gt_polygon_tran["points"]).reshape( -1, 2).astype(np.int64) gt_polygon = plg.Polyogn(gt_polygon) union = get_union(pred_polygon, gt_polygon) intersection = get_intersection(pred_polygon, gt_polygon) if intersection * 1.0 / union >= threshold and flag is False: if gt_id not in cover: flag = True if gt_illegibility: is_ignore = True cover.add(gt_id) if flag: tp += 0.0 if is_ignore else 1.0 else: fp += 1.0 precision = tp / (fp + tp) recall = tp / npos # recall do not calculate the ignored ones. hmean = 0 if ( precision + recall) == 0 else 2 * precision * recall / (precision + recall) print("p: {:f} r: {:f} h: {:f}".format(precision, recall, hmean))