def calc_ner_f1(true_dir, pred_dir): total_tp = 0 total_fp = 0 total_fn = 0 files = os.listdir(true_dir) for doc in files: if doc.endswith(".ann"): true_doc = os.path.join(true_dir, doc) pred_doc = os.path.join(pred_dir, doc) true_data = read_file(true_doc) true_ners = sorted(true_data.ners, key=lambda x: x[1]) pred_ners = [] if os.path.exists(pred_doc): try: pred_data = read_file(pred_doc) pred_ners = sorted(pred_data.ners, key=lambda x: x[1]) except: pass tp, fp, fn = cacl_ner_tp_fp_fn(true_ners, pred_ners) total_tp += tp total_fp += fp total_fn += fn precision, recall = compute_precision_and_recall(total_tp, total_fp, total_fn) f_measure = 2 * precision * recall / (precision + recall) return f_measure
def calc_rels_f1(true_dir, pred_dir): total_tp = 0 total_fp = 0 total_fn = 0 files = os.listdir(true_dir) for doc in files: if doc.endswith(".ann"): # print(doc) true_doc = os.path.join(true_dir, doc) pred_doc = os.path.join(pred_dir, doc) true_data = read_file(true_doc) if not os.path.exists(pred_doc): if len(true_data.relations) == 0: continue else: total_fn += len(true_data.relations) continue try: pred_data = read_file(pred_doc) except: if len(true_data.relations) == 0: continue else: total_fn += len(true_data.relations) continue if len(pred_data.relations) == 0: total_fn += len(true_data.relations) continue if len(true_data.relations) == 0: total_fp += len(pred_data.relations) continue tp, fp, fn = cacl_rel_tp_fp_fn(true_data, pred_data) total_tp += tp total_fp += fp total_fn += fn # print(total_tp, total_fp, total_fn) precision, recall = compute_precision_and_recall(total_tp, total_fp, total_fn) f_measure = 2 * precision * recall / (precision + recall) return f_measure
false_negative += len(true_ners) - i false_positive += len(pred_ners) - j return true_positive, false_positive, false_negative total_tp = 0 total_fp = 0 total_fn = 0 for doc in os.listdir(true_dir): if doc.endswith(".ann"): true_doc = os.path.join(true_dir, doc) pred_doc = os.path.join(predict_dir, doc) true_data = read_file(true_doc) pred_data = read_file(pred_doc) true_ners = sorted(true_data.ners, key=lambda x: x[1]) pred_ners = sorted(pred_data.ners, key=lambda x: x[1]) tp, fp, fn = cacl_ner_tp_fp_fn(true_ners, pred_ners) print("{} : TP - {} / FP - {} / FN - {}".format(doc, tp, fp, fn)) total_tp += tp total_fp += fp total_fn += fn precision, recall = compute_precision_and_recall(total_tp, total_fp, total_fn) f_measure = 2 * precision * recall / (precision + recall)