def evaluate(insts: List[Instance],
predictions: List[List[str]],
print_details: bool = False) -> None:
"""
Evaluation
"""
p_dict = defaultdict(int)
total_entity_dict = defaultdict(int)
total_predict_dict = defaultdict(int)
for i, inst in enumerate(insts):
ground_truth = inst.ground_truth
prediction = predictions[i]
inst.prediction = prediction
output_spans = set()
start = -1
for i in range(len(ground_truth)):
if ground_truth[i].startswith("B-"):
start = i
if ground_truth[i].startswith("E-"):
end = i
output_spans.add(Span(start, end, ground_truth[i][2:]))
total_entity_dict[ground_truth[i][2:]] += 1
if ground_truth[i].startswith("S-"):
output_spans.add(Span(i, i, ground_truth[i][2:]))
total_entity_dict[ground_truth[i][2:]] += 1
predict_spans = set()
start = -1
for i in range(len(prediction)):
if prediction[i].startswith("B-"):
start = i
if prediction[i].startswith("E-"):
end = i
predict_spans.add(Span(start, end, prediction[i][2:]))
total_predict_dict[prediction[i][2:]] += 1
if prediction[i].startswith("S-"):
predict_spans.add(Span(i, i, prediction[i][2:]))
total_predict_dict[prediction[i][2:]] += 1
correct_spans = predict_spans.intersection(output_spans)
for span in correct_spans:
p_dict[span.type] += 1
if print_details:
for key in total_entity_dict:
precision, recall, fscore = get_metric(
p_num=p_dict[key],
total_predicted_num=total_predict_dict[key],
total_num=total_entity_dict[key])
print("[%s] Prec.: %.2f, Rec.: %.2f, F1: %.2f" %
(key, precision, recall, fscore))
total_p = sum(list(p_dict.values()))
total_predict = sum(list(total_predict_dict.values()))
total_entity = sum(list(total_entity_dict.values()))
precision, recall, fscore = get_metric(p_num=total_p,
total_predicted_num=total_predict,
total_num=total_entity)
print("[Total] Prec.: %.2f, Rec.: %.2f, F1: %.2f" %
(precision, recall, fscore),
flush=True)
def get_spans(output):
output_spans = set()
start = -1
for i in range(len(output)):
if output[i].startswith("B-"):
start = i
if output[i].startswith("E-"):
end = i
output_spans.add(Span(start, end, output[i][2:]))
if output[i].startswith("S-"):
output_spans.add(Span(i, i, output[i][2:]))
return output_spans