def calc_ner_f1(true_dir, pred_dir):
total_tp = 0
total_fp = 0
total_fn = 0
files = os.listdir(true_dir)
for doc in files:
if doc.endswith(".ann"):
true_doc = os.path.join(true_dir, doc)
pred_doc = os.path.join(pred_dir, doc)
true_data = read_file(true_doc)
true_ners = sorted(true_data.ners, key=lambda x: x[1])
pred_ners = []
if os.path.exists(pred_doc):
try:
pred_data = read_file(pred_doc)
pred_ners = sorted(pred_data.ners, key=lambda x: x[1])
except:
pass
tp, fp, fn = cacl_ner_tp_fp_fn(true_ners, pred_ners)
total_tp += tp
total_fp += fp
total_fn += fn
precision, recall = compute_precision_and_recall(total_tp, total_fp,
total_fn)
f_measure = 2 * precision * recall / (precision + recall)
return f_measure
def calc_rels_f1(true_dir, pred_dir):
total_tp = 0
total_fp = 0
total_fn = 0
files = os.listdir(true_dir)
for doc in files:
if doc.endswith(".ann"):
# print(doc)
true_doc = os.path.join(true_dir, doc)
pred_doc = os.path.join(pred_dir, doc)
true_data = read_file(true_doc)
if not os.path.exists(pred_doc):
if len(true_data.relations) == 0:
continue
else:
total_fn += len(true_data.relations)
continue
try:
pred_data = read_file(pred_doc)
except:
if len(true_data.relations) == 0:
continue
else:
total_fn += len(true_data.relations)
continue
if len(pred_data.relations) == 0:
total_fn += len(true_data.relations)
continue
if len(true_data.relations) == 0:
total_fp += len(pred_data.relations)
continue
tp, fp, fn = cacl_rel_tp_fp_fn(true_data, pred_data)
total_tp += tp
total_fp += fp
total_fn += fn
# print(total_tp, total_fp, total_fn)
precision, recall = compute_precision_and_recall(total_tp, total_fp,
total_fn)
f_measure = 2 * precision * recall / (precision + recall)
return f_measure
false_negative += len(true_ners) - i
false_positive += len(pred_ners) - j
return true_positive, false_positive, false_negative
total_tp = 0
total_fp = 0
total_fn = 0
for doc in os.listdir(true_dir):
if doc.endswith(".ann"):
true_doc = os.path.join(true_dir, doc)
pred_doc = os.path.join(predict_dir, doc)
true_data = read_file(true_doc)
pred_data = read_file(pred_doc)
true_ners = sorted(true_data.ners, key=lambda x: x[1])
pred_ners = sorted(pred_data.ners, key=lambda x: x[1])
tp, fp, fn = cacl_ner_tp_fp_fn(true_ners, pred_ners)
print("{} : TP - {} / FP - {} / FN - {}".format(doc, tp, fp, fn))
total_tp += tp
total_fp += fp
total_fn += fn
precision, recall = compute_precision_and_recall(total_tp, total_fp, total_fn)
f_measure = 2 * precision * recall / (precision + recall)