if g_length not in f1_length:
f1_length[g_length] = AverageMeter()
# if g_length == 0:
# print(data)
if predict_s < truth_s and predict_e > truth_e:
matching_more_set.append({'prediction': data['answer'], 'answer': evaluator.gold_data[(data['id'], data['turn_id'])]})
span_start_dis_cnt[truth_s - predict_s] += 1
span_end_dis_cnt[truth_e - predict_e] += 1
dis_diff_cnt[g_length][0][truth_s - predict_s] += 1
dis_diff_cnt[g_length][1][truth_e - predict_e] += 1
f1_length[g_length].update(evaluator.compute_turn_score(data['id'], data['turn_id'], data['answer'])['f1'], 1)
print('wrong type predict: {} / {} = {}'.format(wrong_type, len(predictions), wrong_type * 1.0 / len(predictions)))
print('F1 distribution on gold answer length(most common = 5):')
f1_cnt = Counter()
for ans_len in f1_length:
f1_cnt[ans_len] = f1_length[ans_len].avg
print(json.dumps(f1_cnt.most_common(5), indent=2))
print('F1 distribution on gold answer length(all):')
f1_dict = {ans_len: f1_length[ans_len].avg for ans_len in f1_length}
sorted_f1_length = sorted(f1_dict.items(), key=lambda d: d[0])
for ans_len in sorted_f1_length:
print('gold answer length: {}, F1: {}, questions num: {}'.format(ans_len[0], ans_len[1], f1_length[ans_len[0]].count))
questions = data['questions']
answers = data['answers']
for question, answer in zip(questions, answers):
dev_map[(data['id'], question['turn_id'])] = (question['input_text'],
answer['input_text'])
flow_higher = []
no_flow_higher = []
equality = []
max_f1 = AverageMeter()
for h, nh in zip(with_flow, no_flow):
assert h['id'] == nh['id'] and h['turn_id'] == nh['turn_id']
with_flow_f1 = evaluator.compute_turn_score(h['id'], h['turn_id'],
h['answer'])['f1']
no_flow_f1 = evaluator.compute_turn_score(nh['id'], nh['turn_id'],
nh['answer'])['f1']
story_id = h['id']
turn_id = h['turn_id']
flow_answer = h['answer']
no_flow_answer = nh['answer']
question, gold = dev_map[(story_id, turn_id)]
max_f1.update(max(with_flow_f1, no_flow_f1), 1)
ent = {
'id': story_id,
'turn_id': turn_id,
'question': question,