if g_length not in f1_length: f1_length[g_length] = AverageMeter() # if g_length == 0: # print(data) if predict_s < truth_s and predict_e > truth_e: matching_more_set.append({'prediction': data['answer'], 'answer': evaluator.gold_data[(data['id'], data['turn_id'])]}) span_start_dis_cnt[truth_s - predict_s] += 1 span_end_dis_cnt[truth_e - predict_e] += 1 dis_diff_cnt[g_length][0][truth_s - predict_s] += 1 dis_diff_cnt[g_length][1][truth_e - predict_e] += 1 f1_length[g_length].update(evaluator.compute_turn_score(data['id'], data['turn_id'], data['answer'])['f1'], 1) print('wrong type predict: {} / {} = {}'.format(wrong_type, len(predictions), wrong_type * 1.0 / len(predictions))) print('F1 distribution on gold answer length(most common = 5):') f1_cnt = Counter() for ans_len in f1_length: f1_cnt[ans_len] = f1_length[ans_len].avg print(json.dumps(f1_cnt.most_common(5), indent=2)) print('F1 distribution on gold answer length(all):') f1_dict = {ans_len: f1_length[ans_len].avg for ans_len in f1_length} sorted_f1_length = sorted(f1_dict.items(), key=lambda d: d[0]) for ans_len in sorted_f1_length: print('gold answer length: {}, F1: {}, questions num: {}'.format(ans_len[0], ans_len[1], f1_length[ans_len[0]].count))
questions = data['questions'] answers = data['answers'] for question, answer in zip(questions, answers): dev_map[(data['id'], question['turn_id'])] = (question['input_text'], answer['input_text']) flow_higher = [] no_flow_higher = [] equality = [] max_f1 = AverageMeter() for h, nh in zip(with_flow, no_flow): assert h['id'] == nh['id'] and h['turn_id'] == nh['turn_id'] with_flow_f1 = evaluator.compute_turn_score(h['id'], h['turn_id'], h['answer'])['f1'] no_flow_f1 = evaluator.compute_turn_score(nh['id'], nh['turn_id'], nh['answer'])['f1'] story_id = h['id'] turn_id = h['turn_id'] flow_answer = h['answer'] no_flow_answer = nh['answer'] question, gold = dev_map[(story_id, turn_id)] max_f1.update(max(with_flow_f1, no_flow_f1), 1) ent = { 'id': story_id, 'turn_id': turn_id, 'question': question,