def write_sentence_predictions(all_examples, all_features: List[QAFullInputFeatures], all_results: List[WeightResult], output_prediction_file=None, null_score_diff_threshold=0.0): """Write final predictions to the json file and log-odds of null if needed.""" logger.info("Writing predictions to: %s" % output_prediction_file) sentence_pred = utils.AverageMeter() example_index_to_features = collections.defaultdict(list) for feature in all_features: example_index_to_features[feature.example_index].append(feature) unique_id_to_result = {} for result in all_results: unique_id_to_result[result.unique_id] = result all_predictions = collections.OrderedDict() for (example_index, example) in enumerate(all_examples): features = example_index_to_features[example_index] max_weight_logit = 0 max_weight_feature_index = -1 max_weight_index = -1 for (feature_index, feature) in enumerate(features): result = unique_id_to_result[feature.unique_id] feature_max_weight = result.max_weight if feature_max_weight > max_weight_logit: max_weight_logit = feature_max_weight max_weight_feature_index = feature_index max_weight_index = result.max_weight_index target_feature = features[max_weight_feature_index] sentence_id = target_feature.meta_data['span_sen_to_orig_sen_map'][ max_weight_index] target_result = unique_id_to_result[target_feature.unique_id] all_predictions[example.qas_id] = { 'sentence_id': sentence_id, 'max_weight': max_weight_logit, 'max_weight_index': max_weight_index, 'sentence_logits': target_result.sentence_logits, 'feature_gold_sentence': target_feature.sentence_id } if sentence_id == example.sentence_id: sentence_pred.update(1, 1) else: sentence_pred.update(0, 1) logger.info('Sentence Prediction Result: {}'.format(sentence_pred.avg)) if output_prediction_file is not None: with open(output_prediction_file, 'w') as f: json.dump(all_predictions, f, indent=2) return {'predictions': all_predictions, 'acc': sentence_pred.avg}
def read(self, input_file, dialog_turns: int = 2, remove_evidence=False, remove_question=False, remove_passage=False, remove_dict=None): """ :param input_file: input file to load data. The format is in CoQA style :param dialog_turns: Decide how many turns' questions and answers will be appended before current question. """ logger.info('Reading data set from {}...'.format(input_file)) logger.info('Read parameters:') logger.info('Dialog turns: {}'.format(dialog_turns)) logger.info('Remove evidence during test: {}'.format(remove_evidence)) logger.info('Remove dict: {}'.format(remove_dict)) with open(input_file, "r", encoding='utf-8') as reader: input_data = json.load(reader)['data'] def is_whitespace(ch): if ch == " " or ch == "\t" or ch == "\r" or ch == "\n" or ord( ch) == 0x202F: return True return False if remove_dict is None: remove_dict = {} else: remove_dict = json.load(open(remove_dict, 'r')) logger.info(len(remove_dict)) rule_labels_acc = utils.AverageMeter() examples = [] for paragraph in input_data: paragraph_text = paragraph["story"] story_id = paragraph['id'] doc_tokens = [] prev_is_whitespace = True char_to_word_offset = [] for c in paragraph_text: if is_whitespace(c): prev_is_whitespace = True else: if prev_is_whitespace: doc_tokens.append(c) else: doc_tokens[-1] += c prev_is_whitespace = False char_to_word_offset.append(len(doc_tokens) - 1) # Split context into sentences sentence_start_list, sentence_end_list = utils.split_sentence( paragraph_text, self.sentence_tokenizer) sentence_span_list = [] for c_start, c_end in zip(sentence_start_list, sentence_end_list): t_start = char_to_word_offset[c_start] t_end = char_to_word_offset[c_end] sentence_span_list.append((t_start, t_end)) questions = paragraph['questions'] answers = paragraph['answers'] for i, (question, answer) in enumerate(zip(questions, answers)): question_text = question['input_text'] # We are only concerned about questions with Yes/No as answers answer_type = utils.normalize_answer(answer['input_text']) if answer_type not in ['yes', 'no']: continue if story_id in remove_dict and str(i + 1) in remove_dict[story_id]: continue if answer_type == 'yes': answer_choice = 0 else: answer_choice = 1 for j in range(dialog_turns): pre_idx = i - (j + 1) if pre_idx >= 0: question_text = questions[pre_idx][ 'input_text'] + '<Q>' + answers[pre_idx][ 'input_text'] + '<A>' + question_text qas_id = story_id + '--' + str(i + 1) # Add rationale start and end as extra supervised label. rationale_start_position = char_to_word_offset[ answer['span_start']] rationale_end_position = char_to_word_offset[answer['span_end'] - 1] sentence_id = utils.find_evidence_sentence( sentence_span_list, rationale_start_position, rationale_end_position) # Remove evidence sentence for experiments while evaluation only. if remove_evidence and sentence_id != -1 and 'train' not in input_file: evi_token_s, evi_token_e = sentence_span_list[sentence_id] new_doc_tokens = doc_tokens[:evi_token_s] + doc_tokens[ (evi_token_e + 1):] rationale_start_position = rationale_end_position = -1 reduce_offset = evi_token_e - evi_token_s + 1 new_sentence_span_list = sentence_span_list[:sentence_id] + [ (s - reduce_offset, e - reduce_offset) for s, e in sentence_span_list[(sentence_id + 1):] ] sentence_id = -1 else: new_doc_tokens = doc_tokens new_sentence_span_list = sentence_span_list if 'sentence_id' in question: pseudo_sentence_id = question['sentence_id'] if pseudo_sentence_id == sentence_id: rule_labels_acc.update(1) else: rule_labels_acc.update(0) sentence_id = pseudo_sentence_id # example = SQuADFullExample( # qas_id=qas_id, # question_text=question_text, # doc_tokens=doc_tokens, # sentence_span_list=sentence_span_list, # orig_answer_text="", # start_position=None, # end_position=None, # sentence_id=sentence_id, # is_impossible=answer_choice, # ral_start_position=rationale_start_position, # ral_end_position=rationale_end_position) example = SQuADFullExample( qas_id=qas_id, question_text=question_text, doc_tokens=new_doc_tokens, sentence_span_list=new_sentence_span_list, orig_answer_text="", start_position=None, end_position=None, sentence_id=sentence_id, is_impossible=answer_choice, ral_start_position=rationale_start_position, ral_end_position=rationale_end_position) examples.append(example) if rule_labels_acc.count > 0: logger.info('Read labels generated by rules.') logger.info(f'Accuracy of labels: {rule_labels_acc.avg}') return examples
def write_sentence_predictions(all_examples, all_features: List[QAFullInputFeatures], all_results: List[WeightResult], output_prediction_file=None, null_score_diff_threshold=0.0): logger.info("Writing evidence id predictions to: %s" % output_prediction_file) sentence_pred = utils.AverageMeter() yesno_pred = utils.AverageMeter() example_index_to_features = collections.defaultdict(list) for feature in all_features: example_index_to_features[feature.example_index].append(feature) unique_id_to_result = {} for result in all_results: unique_id_to_result[result.unique_id] = result all_predictions = collections.OrderedDict() for (example_index, example) in enumerate(all_examples): features = example_index_to_features[example_index] max_diff = -1000000 max_diff_feature_index = 0 for (feature_index, feature) in enumerate(features): result = unique_id_to_result[feature.unique_id] choice_logits = result.choice_logits non_null_logit = choice_logits[1] + choice_logits[2] null_logit = choice_logits[0] diff = non_null_logit - null_logit if diff > max_diff: max_diff = diff max_diff_feature_index = feature_index target_feature = features[max_diff_feature_index] target_result = unique_id_to_result[target_feature.unique_id] sentence_id_pred = target_feature.meta_data[ 'span_sen_to_orig_sen_map'][target_result.max_weight_index] target_choice_prob = utils.compute_softmax([ target_result.choice_logits[1], target_result.choice_logits[2] ]) if target_choice_prob[0] > target_choice_prob[1]: target_choice = 0 else: target_choice = 1 if target_choice == example.is_impossible: yesno_pred.update(1, 1) else: yesno_pred.update(0, 1) if sentence_id_pred == example.sentence_id: sentence_pred.update(1, 1) else: sentence_pred.update(0, 1) # Find top 5 sentences sentence_logits = target_result.sentence_logits sentence_num = len(target_feature.sentence_span_list) sentence_logits_item = [ (i, x) for i, x in enumerate(sentence_logits[:sentence_num]) ] # Find sentence gold_span_sen_id = -1 for p_sen_idx, orig_sen_id in enumerate( target_feature.meta_data["span_sen_to_orig_sen_map"]): if orig_sen_id == example.sentence_id: gold_span_sen_id = p_sen_idx if gold_span_sen_id != -1: gold_span_sen_logit = target_result.sentence_logits[ gold_span_sen_id] else: gold_span_sen_logit = 0 sorted_sentence_logits_item = sorted(sentence_logits_item, key=lambda x: x[1], reverse=True) top_5_sentences = [ target_feature.meta_data["span_sen_to_orig_sen_map"][x[0]] for x in sorted_sentence_logits_item[:5] ] assert sentence_id_pred in top_5_sentences all_predictions[example.qas_id] = { 'sentence_id': sentence_id_pred, 'gold_sentence_logit': gold_span_sen_logit, 'max_weight': target_result.max_weight, 'max_weight_index': target_result.max_weight_index, 'yesno_pred': target_choice, 'yesno_gold': example.is_impossible, 'sentence_logits': target_result.sentence_logits, 'feature_gold_sentence': target_feature.sentence_id, 'gold_sentence_id': example.sentence_id, 'top_5_sentence_ids': top_5_sentences, 'yesno_pred_prob': target_choice_prob, 'doc_span_index': target_feature.doc_span_index, 'span_sen_to_orig_sen_map': target_feature.meta_data['span_sen_to_orig_sen_map'] } logger.info(f'Yesno Prediction Result: {yesno_pred.avg}') logger.info('Sentence Prediction Result: {}'.format(sentence_pred.avg)) if output_prediction_file is not None: with open(output_prediction_file, 'w') as f: json.dump(all_predictions, f, indent=2) return {'predictions': all_predictions, 'acc': sentence_pred.avg}
def write_sentence_predictions(all_examples, all_features: List[QAFullInputFeatures], all_results: List[FullResult], output_prediction_file=None, weight_threshold: float = 0.0, label_threshold: float = 0.0): logger.info("Writing evidence id predictions to: %s" % output_prediction_file) sentence_pred_cnt = utils.AverageMeter() example_index_to_features = collections.defaultdict(list) for feature in all_features: example_index_to_features[feature.example_index].append(feature) unique_id_to_result = {} for result in all_results: unique_id_to_result[result.unique_id] = result all_predictions = collections.OrderedDict() for (example_index, example) in enumerate(all_examples): features = example_index_to_features[example_index] max_diff = -1000000 max_diff_feature_index = 0 for (feature_index, feature) in enumerate(features): result = unique_id_to_result[feature.unique_id] choice_logits = result.choice_logits non_null_logit = choice_logits[1] + choice_logits[2] null_logit = choice_logits[0] diff = non_null_logit - null_logit if diff > max_diff: max_diff = diff max_diff_feature_index = feature_index target_feature = features[max_diff_feature_index] target_result = unique_id_to_result[target_feature.unique_id] sentence_id_pred = target_feature.meta_data['span_sen_to_orig_sen_map'][target_result.max_weight_index] if target_result.max_weight <= weight_threshold: sentence_id_pred = -1 target_choice_prob = utils.compute_softmax([target_result.choice_logits[1], target_result.choice_logits[2]]) if target_choice_prob[0] > target_choice_prob[1]: target_choice = 0 else: target_choice = 1 if target_choice == example.is_impossible: if target_choice_prob[target_choice] <= label_threshold: sentence_id_pred = -1 else: sentence_id_pred = -1 if sentence_id_pred != -1: sentence_pred_cnt.update(val=1, n=1) else: sentence_pred_cnt.update(val=0, n=1) continue all_predictions[example.qas_id] = { 'sentence_id': sentence_id_pred, 'max_weight': target_result.max_weight, 'doc_span_index': target_feature.doc_span_index, 'doc_span_sentence_id': target_result.max_weight_index } assert sentence_pred_cnt.sum == len(all_predictions) logger.info(f'Labeled evidence ids {sentence_pred_cnt.sum}/{sentence_pred_cnt.count} = {sentence_pred_cnt.avg} in total.') if output_prediction_file is not None: with open(output_prediction_file, 'w') as f: json.dump(all_predictions, f, indent=2) return all_predictions