def _pred_ans_by_thresh(pred_dir, pred_file_prefix, candi_thresh):
predictions_path = os.path.join(pred_dir,
f"{pred_file_prefix}_predictions.json")
cls_path = os.path.join(pred_dir,
f"{pred_file_prefix}_cls_preditions.json")
pred_text = read_model_prediction(predictions_path)
pred_prob = read_cls_prediction(cls_path)
for q_id, q_prob in pred_prob.items():
if q_prob < candi_thresh:
pred_text[q_id] = "no answer"
return pred_text
def _evaluate(raw_data_path, pred_data_path, tag=None):
ref_ans = read_mrc_dataset(raw_data_path, tag=tag)
assert len(ref_ans) > 0, 'Find no sample with tag - {}'.format(tag)
pred_ans = read_model_prediction(pred_data_path)
F1, EM, ans_score, TOTAL, SKIP = evaluate(ref_ans, pred_ans, verbose=False)
print_metrics(F1, EM, ans_score, TOTAL, SKIP, tag)