def predict_sentence_ids(all_examples, all_features: List[SQuADFullInputFeatures], all_results: List[WeightResultChoice],
output_prediction_file=None,
weight_threshold: float = 0.0, only_correct: bool = False, label_threshold: float = 0.0):
"""
:param all_results:
:param all_examples:
:param all_features:
:param output_prediction_file:
:param weight_threshold: The threshold for attention weights, only id predictions with a higher weight than this will
be added.
:param only_correct: If true, only id predictions with final choices predicted which are correct will be added.
Otherwise, all the id predictions will be added no matter if the yes/no prediction is correct.
:param label_threshold: Only make sense while only_correct=True, which means that only the id predictions with true
yes/no prediction and the probability for yes/no is higher than this will be added.
:return:
"""
logger.info("Predicting sentence id to: %s" % output_prediction_file)
logger.info("Weight threshold: {}".format(weight_threshold))
logger.info("Use ids with true yes/no prediction only: {}".format(only_correct))
logger.info("Yes/No prediction probability threshold : {}, only make sense while only_correct=True.".format(label_threshold))
example_index_to_features = collections.defaultdict(list)
for feature in all_features:
example_index_to_features[feature.example_index].append(feature)
unique_id_to_result = {}
for result in all_results:
unique_id_to_result[result.unique_id] = result
all_predictions = collections.OrderedDict()
total = 0
no_label = 0
correct = 0
for (example_index, example) in enumerate(all_examples):
features = example_index_to_features[example_index]
max_diff = -1000000
max_diff_feature_index = 0
max_yes_logit = 0
max_no_logit = 0
for (feature_index, feature) in enumerate(features):
result = unique_id_to_result[feature.unique_id]
choice_logits = result.choice_logits
non_null_logit = choice_logits[1] + choice_logits[2]
null_logit = choice_logits[0]
diff = non_null_logit - null_logit
if diff > max_diff:
max_diff = diff
max_diff_feature_index = feature_index
max_yes_logit = choice_logits[1]
max_no_logit = choice_logits[2]
# if max_diff > null_score_diff_threshold:
# final_text = 'unknown'
# Here we only consider questions with Yes/No as answers
target_feature = features[max_diff_feature_index]
sentence_id = unique_id_to_result[target_feature.unique_id].max_weight_index
max_weight = unique_id_to_result[target_feature.unique_id].max_weight
# Yes/No prediction restriction
yesno_scores = utils.compute_softmax([max_yes_logit, max_no_logit])
if max_yes_logit > max_no_logit:
final_choice = 0
prediction_prob = yesno_scores[0]
else:
final_choice = 1
prediction_prob = yesno_scores[1]
if only_correct:
if (final_choice + 1) != target_feature.is_impossible:
sentence_id = -1
if prediction_prob <= label_threshold:
sentence_id = -1
if max_weight <= weight_threshold:
sentence_id = -1
feature_doc_span_index = target_feature.doc_span_index
all_predictions[example.qas_id] = {'sentence_id': sentence_id,
'doc_span_index': feature_doc_span_index,
'weight': max_weight,
'choice_prediction': 'yes' if final_choice == 0 else 'no',
'prediction_prob': prediction_prob}
total += 1
if sentence_id == -1:
no_label += 1
else:
if sentence_id == target_feature.sentence_id:
correct += 1
logger.info("Labeling {} instances of {} in total".format(total - no_label, len(all_examples)))
logger.info(
"Labeling accuracy: Correct / Total: {} / {}, {}".format(correct, total - no_label, correct * 1.0 / (total - no_label + 1e-15)))
if output_prediction_file is not None:
with open(output_prediction_file, 'w') as f:
json.dump(all_predictions, f, indent=2)
return all_predictions
def write_sentence_predictions(all_examples,
all_features: List[QAFullInputFeatures],
all_results: List[WeightResult],
output_prediction_file=None,
null_score_diff_threshold=0.0):
logger.info("Writing evidence id predictions to: %s" %
output_prediction_file)
sentence_pred = utils.AverageMeter()
yesno_pred = utils.AverageMeter()
example_index_to_features = collections.defaultdict(list)
for feature in all_features:
example_index_to_features[feature.example_index].append(feature)
unique_id_to_result = {}
for result in all_results:
unique_id_to_result[result.unique_id] = result
all_predictions = collections.OrderedDict()
for (example_index, example) in enumerate(all_examples):
features = example_index_to_features[example_index]
max_diff = -1000000
max_diff_feature_index = 0
for (feature_index, feature) in enumerate(features):
result = unique_id_to_result[feature.unique_id]
choice_logits = result.choice_logits
non_null_logit = choice_logits[1] + choice_logits[2]
null_logit = choice_logits[0]
diff = non_null_logit - null_logit
if diff > max_diff:
max_diff = diff
max_diff_feature_index = feature_index
target_feature = features[max_diff_feature_index]
target_result = unique_id_to_result[target_feature.unique_id]
sentence_id_pred = target_feature.meta_data[
'span_sen_to_orig_sen_map'][target_result.max_weight_index]
target_choice_prob = utils.compute_softmax([
target_result.choice_logits[1], target_result.choice_logits[2]
])
if target_choice_prob[0] > target_choice_prob[1]:
target_choice = 0
else:
target_choice = 1
if target_choice == example.is_impossible:
yesno_pred.update(1, 1)
else:
yesno_pred.update(0, 1)
if sentence_id_pred == example.sentence_id:
sentence_pred.update(1, 1)
else:
sentence_pred.update(0, 1)
# Find top 5 sentences
sentence_logits = target_result.sentence_logits
sentence_num = len(target_feature.sentence_span_list)
sentence_logits_item = [
(i, x) for i, x in enumerate(sentence_logits[:sentence_num])
]
# Find sentence
gold_span_sen_id = -1
for p_sen_idx, orig_sen_id in enumerate(
target_feature.meta_data["span_sen_to_orig_sen_map"]):
if orig_sen_id == example.sentence_id:
gold_span_sen_id = p_sen_idx
if gold_span_sen_id != -1:
gold_span_sen_logit = target_result.sentence_logits[
gold_span_sen_id]
else:
gold_span_sen_logit = 0
sorted_sentence_logits_item = sorted(sentence_logits_item,
key=lambda x: x[1],
reverse=True)
top_5_sentences = [
target_feature.meta_data["span_sen_to_orig_sen_map"][x[0]]
for x in sorted_sentence_logits_item[:5]
]
assert sentence_id_pred in top_5_sentences
all_predictions[example.qas_id] = {
'sentence_id':
sentence_id_pred,
'gold_sentence_logit':
gold_span_sen_logit,
'max_weight':
target_result.max_weight,
'max_weight_index':
target_result.max_weight_index,
'yesno_pred':
target_choice,
'yesno_gold':
example.is_impossible,
'sentence_logits':
target_result.sentence_logits,
'feature_gold_sentence':
target_feature.sentence_id,
'gold_sentence_id':
example.sentence_id,
'top_5_sentence_ids':
top_5_sentences,
'yesno_pred_prob':
target_choice_prob,
'doc_span_index':
target_feature.doc_span_index,
'span_sen_to_orig_sen_map':
target_feature.meta_data['span_sen_to_orig_sen_map']
}
logger.info(f'Yesno Prediction Result: {yesno_pred.avg}')
logger.info('Sentence Prediction Result: {}'.format(sentence_pred.avg))
if output_prediction_file is not None:
with open(output_prediction_file, 'w') as f:
json.dump(all_predictions, f, indent=2)
return {'predictions': all_predictions, 'acc': sentence_pred.avg}
def predict_sentence_ids(all_examples,
all_features: List[SQuADFullInputFeatures],
all_results: List[RawOutput],
output_prediction_file=None,
weight_threshold: float = 0.0,
only_correct: bool = False,
label_threshold: float = 0.0):
"""
:param all_results:
:param all_examples:
:param all_features:
:param output_prediction_file:
:param weight_threshold: The threshold for attention weights, only id predictions with a higher weight than this will
be added.
:param only_correct: If true, only id predictions with final choices predicted which are correct will be added.
Otherwise, all the id predictions will be added no matter if the yes/no prediction is correct.
:param label_threshold: Only make sense while only_correct=True, which means that only the id predictions with true
yes/no prediction and the probability for yes/no is higher than this will be added.
:return:
"""
logger.info("Predicting sentence id to: %s" % output_prediction_file)
logger.info("Weight threshold: {}".format(weight_threshold))
logger.info("Use ids with true yes/no prediction only: {}".format(
only_correct))
logger.info(
"Yes/No prediction probability threshold : {}, only make sense while only_correct=True."
.format(label_threshold))
example_index_to_features = collections.defaultdict(list)
for feature in all_features:
example_index_to_features[feature.example_index].append(feature)
unique_id_to_result = {}
for result in all_results:
unique_id_to_result[result.unique_id] = result
all_predictions = collections.OrderedDict()
no_label = 0
total = 0
for (example_index, example) in enumerate(all_examples):
features = example_index_to_features[example_index]
max_diff = -1000000
max_diff_feature_index = 0
max_yes_logit = 0
max_no_logit = 0
for (feature_index, feature) in enumerate(features):
result = unique_id_to_result[feature.unique_id]
choice_logits = result.model_output['choice_logits']
non_null_logit = choice_logits[1] + choice_logits[2]
null_logit = choice_logits[0]
diff = non_null_logit - null_logit
if diff > max_diff:
max_diff = diff
max_diff_feature_index = feature_index
max_yes_logit = choice_logits[1]
max_no_logit = choice_logits[2]
# Yes/No prediction restriction
yesno_scores = utils.compute_softmax([max_yes_logit, max_no_logit])
if max_yes_logit > max_no_logit:
final_choice = 0
prediction_prob = yesno_scores[0]
else:
final_choice = 1
prediction_prob = yesno_scores[1]
target_feature = features[max_diff_feature_index]
evidence = unique_id_to_result[
target_feature.unique_id].model_output['evidence']
evidence_value = evidence['value']
if (only_correct and final_choice + 1 == target_feature.is_impossible and prediction_prob > label_threshold) \
or not only_correct:
if evidence_value > weight_threshold:
sentence_id = evidence['sentences']
else:
sentence_id = []
else:
sentence_id = []
if not sentence_id:
no_label += 1
feature_doc_span_index = target_feature.doc_span_index
all_predictions[example.qas_id] = {
'sentence_id': sentence_id,
'doc_span_index': feature_doc_span_index,
'weight': evidence_value,
'choice_prediction': 'yes' if final_choice == 0 else 'no',
'prediction_prob': prediction_prob
}
total += 1
logger.info("Labeling {} instances of {} in total".format(
total - no_label, len(all_examples)))
if output_prediction_file is not None:
with open(output_prediction_file, 'w') as f:
json.dump(all_predictions, f, indent=2)
return all_predictions
def write_sentence_predictions(all_examples, all_features: List[QAFullInputFeatures], all_results: List[FullResult],
output_prediction_file=None, weight_threshold: float = 0.0, label_threshold: float = 0.0):
logger.info("Writing evidence id predictions to: %s" % output_prediction_file)
sentence_pred_cnt = utils.AverageMeter()
example_index_to_features = collections.defaultdict(list)
for feature in all_features:
example_index_to_features[feature.example_index].append(feature)
unique_id_to_result = {}
for result in all_results:
unique_id_to_result[result.unique_id] = result
all_predictions = collections.OrderedDict()
for (example_index, example) in enumerate(all_examples):
features = example_index_to_features[example_index]
max_diff = -1000000
max_diff_feature_index = 0
for (feature_index, feature) in enumerate(features):
result = unique_id_to_result[feature.unique_id]
choice_logits = result.choice_logits
non_null_logit = choice_logits[1] + choice_logits[2]
null_logit = choice_logits[0]
diff = non_null_logit - null_logit
if diff > max_diff:
max_diff = diff
max_diff_feature_index = feature_index
target_feature = features[max_diff_feature_index]
target_result = unique_id_to_result[target_feature.unique_id]
sentence_id_pred = target_feature.meta_data['span_sen_to_orig_sen_map'][target_result.max_weight_index]
if target_result.max_weight <= weight_threshold:
sentence_id_pred = -1
target_choice_prob = utils.compute_softmax([target_result.choice_logits[1], target_result.choice_logits[2]])
if target_choice_prob[0] > target_choice_prob[1]:
target_choice = 0
else:
target_choice = 1
if target_choice == example.is_impossible:
if target_choice_prob[target_choice] <= label_threshold:
sentence_id_pred = -1
else:
sentence_id_pred = -1
if sentence_id_pred != -1:
sentence_pred_cnt.update(val=1, n=1)
else:
sentence_pred_cnt.update(val=0, n=1)
continue
all_predictions[example.qas_id] = {
'sentence_id': sentence_id_pred,
'max_weight': target_result.max_weight,
'doc_span_index': target_feature.doc_span_index,
'doc_span_sentence_id': target_result.max_weight_index
}
assert sentence_pred_cnt.sum == len(all_predictions)
logger.info(f'Labeled evidence ids {sentence_pred_cnt.sum}/{sentence_pred_cnt.count} = {sentence_pred_cnt.avg} in total.')
if output_prediction_file is not None:
with open(output_prediction_file, 'w') as f:
json.dump(all_predictions, f, indent=2)
return all_predictions