def evaluate_results(predictions,
qids,
questions,
answers,
args,
evidences,
scores,
titles,
se_positions=None):
# Filter if there's candidate
if args.candidate_path is not None:
candidates = set()
with open(args.candidate_path) as f:
for line in f:
line = line.strip().lower()
candidates.add(line)
logger.info(
f'{len(candidates)} candidates are loaded from {args.candidate_path}'
)
topk_preds = [
list(
filter(
lambda x: (x in candidates) or (x.lower() in candidates),
a)) for a in predictions
]
topk_preds = [
a[:args.top_k] if len(a) > 0 else [''] for a in topk_preds
]
predictions = topk_preds[:]
top1_preds = [a[0] for a in topk_preds]
else:
predictions = [
a[:args.top_k] if len(a) > 0 else [''] for a in predictions
]
top1_preds = [a[0] for a in predictions]
no_ans = sum([a == '' for a in top1_preds])
logger.info(f'no_ans/all: {no_ans}, {len(top1_preds)}')
logger.info(f'Evaluating {len(top1_preds)} answers')
# Get em/f1
f1s, ems = [], []
for prediction, groundtruth in zip(top1_preds, answers):
if len(groundtruth) == 0:
f1s.append(0)
ems.append(0)
continue
f1s.append(max([f1_score(prediction, gt)[0] for gt in groundtruth]))
ems.append(
max([exact_match_score(prediction, gt) for gt in groundtruth]))
final_f1, final_em = np.mean(f1s), np.mean(ems)
if not args.regex:
logger.info('EM: %.2f, F1: %.2f' % (final_em * 100, final_f1 * 100))
def long_answer_em(prediction, groundtruths, se_pos):
em = []
for gt in groundtruths:
start = prediction.find(gt)
if start != -1 and se_pos[0] >= start and se_pos[1] <= start + len(
gt):
em.append(1)
else:
em.append(0)
return max(em)
# Top 1/k em (or regex em)
exact_match_topk = 0
exact_match_top1 = 0
f1_score_topk = 0
f1_score_top1 = 0
redundant_topk = 0
pred_out = {}
if "question_type" in args and "4" in args.question_type:
exact_match_long_topk = 0
exact_match_long_top1 = 0
for i in range(len(predictions)):
# For debugging
if i < 10:
logger.info(f'{i+1}) {questions[i]}')
logger.info(
f'=> groundtruths: {answers[i]}, top 5 prediction: {predictions[i][:5]}'
)
match_fn = drqa_regex_match_score if args.regex else drqa_exact_match_score
em_topk = max([
drqa_metric_max_over_ground_truths(match_fn, prediction,
answers[i])
for prediction in predictions[i][:args.top_k]
])
em_top1 = drqa_metric_max_over_ground_truths(match_fn, top1_preds[i],
answers[i])
exact_match_topk += em_topk
exact_match_top1 += em_top1
if "question_type" in args and "4" in args.question_type:
em_long_topk = [
long_answer_em(prediction, answers[i], se)
for prediction, se in zip(predictions[i][:args.top_k],
se_positions[i][:args.top_k])
]
exact_match_long_topk += max(em_long_topk)
exact_match_long_top1 += em_long_topk[0]
if i < 10:
logger.info(
f"Top 1 score {exact_match_long_top1}, top 10 score {exact_match_long_topk}"
)
# Compute top-k redundancy (could be ill-defined for regex)
rd_topk = sum([
drqa_metric_max_over_ground_truths(match_fn, prediction,
[predictions[i][0]])
for prediction in predictions[i][:args.top_k]
])
redundant_topk += rd_topk
f1_topk = 0
f1_top1 = 0
if not args.regex:
match_fn = lambda x, y: f1_score(x, y)[0]
f1_topk = max([
drqa_metric_max_over_ground_truths(match_fn, prediction,
answers[i])
for prediction in predictions[i][:args.top_k]
])
f1_top1 = drqa_metric_max_over_ground_truths(
match_fn, top1_preds[i], answers[i])
f1_score_topk += f1_topk
f1_score_top1 += f1_top1
# Score statistics
assert len(predictions[i]) <= args.top_k
pred_out[qids[i]] = {
'question': questions[i],
'answer': answers[i],
'prediction': predictions[i],
'score': scores[i],
'title': titles[i],
'evidence': evidences[i] if evidences is not None else '',
'em_top1': bool(em_top1),
f'em_top{args.top_k}': bool(em_topk),
'f1_top1': f1_top1,
f'f1_top{args.top_k}': f1_topk,
'se_pos': se_positions[i] if se_positions is not None else
(-1, -1),
'rd_topk': rd_topk,
}
if "question_type" in args and "4" in args.question_type:
exact_match_top1 = exact_match_long_top1
exact_match_topk = exact_match_long_topk
total = len(predictions)
exact_match_top1 = 100.0 * exact_match_top1 / total
f1_score_top1 = 100.0 * f1_score_top1 / total
logger.info({
'exact_match_top1': exact_match_top1,
'f1_score_top1': f1_score_top1
})
exact_match_topk = 100.0 * exact_match_topk / total
f1_score_topk = 100.0 * f1_score_topk / total
logger.info({
f'exact_match_top{args.top_k}': exact_match_topk,
f'f1_score_top{args.top_k}': f1_score_topk
})
redundant_topk = redundant_topk / total
logger.info({f'redundancy of top{args.top_k}': redundant_topk})
# Dump predictions
if len(args.load_dir) == 0:
pred_dir = os.path.join(os.environ['SAVE_DIR'], 'pred')
else:
pred_dir = os.path.join(args.load_dir, 'pred')
if not os.path.exists(pred_dir):
os.makedirs(pred_dir)
if args.save_pred:
pred_path = os.path.join(
pred_dir,
os.path.splitext(os.path.basename(args.test_path))[0] +
f'_{total}_top{args.top_k}.pred')
logger.info(f'Saving prediction file to {pred_path}')
with open(pred_path, 'w') as f:
json.dump(pred_out, f)
logger.info(
f"Results for {args.test_path}: exact match top 1: {exact_match_top1} | f1 score top 1: {f1_score_top1} | exact match top {args.top_k}: {exact_match_topk} | f1 score top {args.top_k}: {f1_score_topk}"
)
if args.wandb:
wandb.log({
"Test Path": args.test_path,
"Model": args.load_dir,
"Top 1 EM": exact_match_top1,
"Top 1 F1": f1_score_top1,
f"Top {args.top_k} EM": exact_match_topk,
f"Top {args.top_k} F1": f1_score_topk
})
wandb_table = wandb.Table(columns=[
"Model", "Question Type", "Top 1 EM", "Top 1 F1",
f"Top {args.top_k} EM", f"Top {args.top_k} F1"
])
wandb_table.add_data(args.load_dir, args.question_type,
exact_match_top1, f1_score_top1, exact_match_topk,
f1_score_topk)
wandb.log({args.test_path: wandb_table})
# Evaluate passage retrieval
if args.eval_psg:
evaluate_results_psg(pred_path, args)
return exact_match_top1, f1_score_top1, exact_match_topk, f1_score_topk
def evaluate_results(predictions,
qids,
questions,
answers,
args,
evidences,
scores,
titles,
q_tokens=None):
wandb.init(project="DensePhrases (open)",
mode="online" if args.wandb else "disabled")
wandb.config.update(args)
# Filter if there's candidate
if args.candidate_path is not None:
candidates = set()
with open(args.candidate_path) as f:
for line in f:
line = line.strip().lower()
candidates.add(line)
logger.info(
f'{len(candidates)} candidates are loaded from {args.candidate_path}'
)
topk_preds = [
list(
filter(
lambda x: (x in candidates) or (x.lower() in candidates),
a)) for a in predictions
]
topk_preds = [
a[:args.top_k] if len(a) > 0 else [''] for a in topk_preds
]
predictions = topk_preds[:]
top1_preds = [a[0] for a in topk_preds]
else:
predictions = [
a[:args.top_k] if len(a) > 0 else [''] for a in predictions
]
top1_preds = [a[0] for a in predictions]
no_ans = sum([a == '' for a in top1_preds])
logger.info(f'no_ans/all: {no_ans}, {len(top1_preds)}')
logger.info(f'Evaluating {len(top1_preds)} answers.')
# Get em/f1
f1s, ems = [], []
for prediction, groundtruth in zip(top1_preds, answers):
if len(groundtruth) == 0:
f1s.append(0)
ems.append(0)
continue
f1s.append(max([f1_score(prediction, gt)[0] for gt in groundtruth]))
ems.append(
max([exact_match_score(prediction, gt) for gt in groundtruth]))
final_f1, final_em = np.mean(f1s), np.mean(ems)
logger.info('EM: %.2f, F1: %.2f' % (final_em * 100, final_f1 * 100))
# Top 1/k em (or regex em)
exact_match_topk = 0
exact_match_top1 = 0
f1_score_topk = 0
f1_score_top1 = 0
pred_out = {}
for i in range(len(predictions)):
# For debugging
if i < 3:
logger.info(f'{i+1}) {questions[i]}')
logger.info(
f'=> groundtruths: {answers[i]}, top 5 prediction: {predictions[i][:5]}'
)
match_fn = drqa_regex_match_score if args.regex else drqa_exact_match_score
em_topk = max([
drqa_metric_max_over_ground_truths(match_fn, prediction,
answers[i])
for prediction in predictions[i][:args.top_k]
])
em_top1 = drqa_metric_max_over_ground_truths(match_fn, top1_preds[i],
answers[i])
exact_match_topk += em_topk
exact_match_top1 += em_top1
f1_topk = 0
f1_top1 = 0
if not args.regex:
match_fn = lambda x, y: f1_score(x, y)[0]
f1_topk = max([
drqa_metric_max_over_ground_truths(match_fn, prediction,
answers[i])
for prediction in predictions[i][:args.top_k]
])
f1_top1 = drqa_metric_max_over_ground_truths(
match_fn, top1_preds[i], answers[i])
f1_score_topk += f1_topk
f1_score_top1 += f1_top1
pred_out[qids[i]] = {
'question': questions[i],
'answer': answers[i],
'prediction': predictions[i],
'score': scores[i],
'title': titles[i],
'evidence': evidences[i] if evidences is not None else '',
'em_top1': bool(em_top1),
f'em_top{args.top_k}': bool(em_topk),
'f1_top1': f1_top1,
f'f1_top{args.top_k}': f1_topk,
'q_tokens': q_tokens[i] if q_tokens is not None else ['']
}
total = len(predictions)
exact_match_top1 = 100.0 * exact_match_top1 / total
f1_score_top1 = 100.0 * f1_score_top1 / total
logger.info({
'exact_match_top1': exact_match_top1,
'f1_score_top1': f1_score_top1
})
exact_match_topk = 100.0 * exact_match_topk / total
f1_score_topk = 100.0 * f1_score_topk / total
logger.info({
f'exact_match_top{args.top_k}': exact_match_topk,
f'f1_score_top{args.top_k}': f1_score_topk
})
wandb.log({
"Top1 EM": exact_match_top1,
"Top1 F1": f1_score_top1,
"Topk EM": exact_match_topk,
"Topk F1": f1_score_topk
})
# Dump predictions
if len(args.query_encoder_path) == 0:
pred_dir = os.path.join(os.environ['DPH_SAVE_DIR'], 'pred')
else:
pred_dir = os.path.join(args.query_encoder_path, 'pred')
if not os.path.exists(pred_dir):
os.makedirs(pred_dir)
pred_path = os.path.join(
pred_dir,
os.path.splitext(os.path.basename(args.test_path))[0] +
f'_{total}.pred')
logger.info(f'Saving prediction file to {pred_path}')
with open(pred_path, 'w') as f:
json.dump(pred_out, f)
return exact_match_top1
def main(args):
with open(args.input_path, encoding='utf-8') as f:
data = json.load(f)
output_data = []
nlp = English()
sentencizer = nlp.add_pipe('sentencizer')
for i, sample_id in enumerate(tqdm(data, desc="Processing Inputs")):
sample = data[sample_id]
question = sample['question']
answers = sample['answer']
predictions = sample['prediction'][0:args.top_k]
titles = sample['title'][0:args.top_k]
evidences = sample['evidence'][0:args.top_k]
scores = sample['score'][0:args.top_k]
se_pos = sample['se_pos'][0:args.top_k]
f1s = [
max([f1_score(prediction, gt)[0] for gt in answers])
for prediction in predictions
]
ems = [
max([exact_match_score(prediction, gt) for gt in answers])
for prediction in predictions
]
# not a useful example if there are no exact match
if max(ems) < 1:
continue
is_impossible = []
start_pos = []
end_pos = []
match_fn = drqa_regex_match_score if args.regex else drqa_exact_match_score
# only check if prediction is matched in a golden answer in the answer list
for pred_idx, pred in enumerate(predictions):
if pred != "" and drqa_metric_max_over_ground_truths(
match_fn, pred, answers):
is_impossible.append(False)
answer_start = find_substring_and_return_first(
pred, evidences[pred_idx])
start_pos.append(answer_start)
end_pos.append(answer_start + len(pred) - 1)
else:
is_impossible.append(True)
start_pos.append(-1)
end_pos.append(-1)
docs = [nlp(evi) for evi in evidences]
spans = [
doc.char_span(
se_pos[j][0],
se_pos[j][1] if se_pos[j][1] >= se_pos[j][0] else se_pos[j][0],
alignment_mode='expand') for j, doc in enumerate(docs)
]
sents = [
doc[span[0].sent.start:span[-1].sent.end].text
if len(span) > 0 else "" for doc, span in zip(docs, spans)
]
# could add score in here if we somehow make the reader also take the score into account
output_data.append({
'question': question,
'answer': answers,
'prediction': predictions,
'title': [title[0] for title in titles],
'evidence': evidences,
'is_impossible': is_impossible,
'start_pos': start_pos,
'end_pos': end_pos,
'score': scores,
'f1s': f1s,
'exact_matches': ems,
'se_pos': se_pos,
'sentence': sents,
})
with open(args.output_path, 'w', encoding='utf-8') as f:
json.dump({'data': output_data}, f)
def recall(x, y):
return f1_score(x, y)[2]
def precision(x, y):
return f1_score(x, y)[1]
def f1(x, y):
return f1_score(x, y)[0]