def label_to_pred(labels):
"""Convert a list of gold human annotations to a perfect prediction."""
gold_has_short_answer = util.gold_has_short_answer(labels)
gold_has_long_answer = util.gold_has_long_answer(labels)
# We did not put `long_answer` and `yes_no_answer`, and they should be
# considered as null when loading from input.
pred = {
'example_id': labels[0].example_id,
'short_answers': [],
'short_answers_score': random.random(),
'long_answer_score': random.random()
}
keep_answer = random.random() <= FLAGS.desired_recall
for label in labels:
if gold_has_short_answer and keep_answer:
pred['short_answers_score'] *= 2
if not util.is_null_span_list(label.short_answer_span_list):
pred['short_answers'] = ([{
'start_token': span.start_token_idx,
'end_token': span.end_token_idx,
'start_byte': span.start_byte,
'end_byte': span.end_byte
} for span in label.short_answer_span_list])
pred['yes_no_answer'] = 'none'
elif label.yes_no_answer != 'none':
pred['short_answers'] = []
pred['yes_no_answer'] = label.yes_no_answer
if (gold_has_long_answer
and not label.long_answer_span.is_null_span()
and keep_answer):
pred['long_answer'] = {
'start_token': label.long_answer_span.start_token_idx,
'end_token': label.long_answer_span.end_token_idx,
'start_byte': label.long_answer_span.start_byte,
'end_byte': label.long_answer_span.end_byte
}
pred['long_answer_score'] *= 2
if FLAGS.generate_false_positives:
if not gold_has_short_answer:
pred['short_answers'] = [{
'start_token': 0,
'end_token': 1,
'start_byte': -1,
'end_byte': -1
}]
if not gold_has_long_answer:
pred['long_answer_start_token'] = 0
pred['long_answer_end_token'] = 1
return pred
def score_short_answer(gold_label_list, pred_label, score_thres):
"""Scores a short answer as correct or not.
1) First decide if there is a gold short answer with SHORT_NO_NULL_THRESHOLD.
2) The prediction will get a match if:
a. There is a gold short answer.
b. The prediction span *set* match exactly with *one* of the non-null gold
short answer span *set*.
Args:
gold_label_list: A list of NQLabel.
pred_label: A single NQLabel.
score_thres: score threshold
Returns:
gold_has_answer, pred_has_answer, is_correct, score
"""
# There is a gold short answer if gold_label_list not empty and non null
# answers is over the threshold (sum over annotators).
gold_has_answer = util.gold_has_short_answer(gold_label_list)
is_correct = False
score = pred_label.short_score
# There is a pred long answer if pred_label is not empty and short answer
# set is not empty.
pred_has_answer = pred_label and (
(not util.is_null_span_list(pred_label.short_answer_span_list))
or pred_label.yes_no_answer != 'none') and score >= score_thres
# Both sides have short answers, which contains yes/no questions.
if gold_has_answer and pred_has_answer:
if pred_label.yes_no_answer != 'none': # System thinks its y/n questions.
for gold_label in gold_label_list:
if pred_label.yes_no_answer == gold_label.yes_no_answer:
is_correct = True
break
else:
for gold_label in gold_label_list:
if util.span_set_equal(gold_label.short_answer_span_list,
pred_label.short_answer_span_list):
is_correct = True
break
return gold_has_answer, pred_has_answer, is_correct, score
def score_short_answer(gold_label_list, pred_label):
"""Scores a short answer as correct or not.
1) First decide if there is a gold short answer with SHORT_NO_NULL_THRESHOLD.
2) The prediction will get a match if:
a. There is a gold short answer.
b. The prediction span *set* match exactly with *one* of the non-null gold
short answer span *set*.
Args:
gold_label_list: A list of NQLabel.
pred_label: A single NQLabel.
Returns:
gold_has_answer, pred_has_answer, f1, score
"""
# There is a gold short answer if gold_label_list not empty and non null
# answers is over the threshold (sum over annotators).
gold_has_answer = util.gold_has_short_answer(gold_label_list)
# There is a pred long answer if pred_label is not empty and short answer
# set is not empty.
pred_has_answer = pred_label and (
(not util.is_null_span_list(pred_label.short_answer_span_list))
or pred_label.yes_no_answer != 'none')
f1 = 0
p = 0
r = 0
score = pred_label.short_score
# Both sides have short answers, which contains yes/no questions.
if gold_has_answer and pred_has_answer:
if pred_label.yes_no_answer != 'none': # System thinks its y/n questions.
for gold_label in gold_label_list:
if pred_label.yes_no_answer == gold_label.yes_no_answer:
f1 = 1
p = 1
r = 1
break
else:
for gold_label in gold_label_list:
# if util.span_set_equal(gold_label.short_answer_span_list,
# pred_label.short_answer_span_list):
# is_correct = True
# break
gold_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in gold_label.short_answer_span_list])
pred_set = IntervalSet([Interval(span.start_token_idx, span.end_token_idx) \
for span in pred_label.short_answer_span_list])
overlap_set = gold_set & pred_set
def calc_len(interval_set):
sum = 0
for span in interval_set:
sum += span.upper_bound - span.lower_bound + 1
return sum
precision = safe_divide(calc_len(overlap_set),
calc_len(pred_set))
recall = safe_divide(calc_len(overlap_set), calc_len(gold_set))
if safe_divide(2 * precision * recall,
precision + recall) > f1:
f1 = safe_divide(2 * precision * recall,
precision + recall)
p = precision
r = recall
elif not gold_has_answer and not pred_has_answer:
f1 = 1
p = 1
r = 1
return gold_has_answer, pred_has_answer, f1, p, r, score
def score_short_answer(gold_label_list, pred_label, threshold=0):
"""Scores a short answer as correct or not.
1) First decide if there is a gold short answer with SHORT_NO_NULL_THRESHOLD.
2) The prediction will get a F1 if:
a. There is a gold short answer.
b. The prediction span *set* match exactly with *one* of the non-null gold
short answer span *set*.
Args:
gold_label_list: A list of NQLabel.
pred_label: A single NQLabel.
Returns:
gold_has_answer, pred_has_answer, f1, score
"""
#print(gold_label_list)
#print(pred_label)
# There is a gold short answer if gold_label_list not empty and non null
# answers is over the threshold (sum over annotators).
gold_has_answer = util.gold_has_short_answer(gold_label_list)
# There is a pred long answer if pred_label is not empty and short answer
# set is not empty.
pred_has_answer = pred_label and (
(not util.is_null_span_list(pred_label.short_answer_span_list,
pred_label.short_score_list, threshold))
or pred_label.yes_no_answer != 'none')
f1 = 0
p = 0
r = 0
# score = pred_label.short_score
# Both sides have short answers, which contains yes/no questions.
if gold_has_answer and pred_has_answer:
if pred_label.yes_no_answer != 'none': # System thinks its y/n questions.
for gold_label in gold_label_list:
if pred_label.yes_no_answer == gold_label.yes_no_answer:
f1 = 1
p = 1
r = 1
break
else:
# 抽取式答案比对
for gold_label in gold_label_list:
gold_set = []
pred_set = []
for span, score in zip(pred_label.short_answer_span_list,
pred_label.short_score_list):
if score >= threshold:
pred_set += [(span.start_token_idx, span.end_token_idx)
]
for span in gold_label.short_answer_span_list:
gold_set += [(span.start_token_idx, span.end_token_idx)]
# 这里对gold_Set 进行处理
#print(gold_set)
def concat_gold_set(gold_set):
def takeFirst(elem):
return elem[0]
gold_set.sort(key=takeFirst)
new_set = []
current_start = gold_set[0][0]
current_end = gold_set[0][1]
for i in range(1, len(gold_set)):
start = gold_set[i][0]
end = gold_set[i][1]
if (start - current_end > 5):
new_set.append([current_start, current_end])
current_end = end
current_start = start
else:
current_end = end
new_set.append([current_start, current_end])
return new_set
gold_set = concat_gold_set(gold_set)
#print(pred_set)
#print(gold_set)
#print("-------------------------------------------")
def count_same(span_list, interval_set):
sum = 0
for span in span_list:
for interval in interval_set:
if span[0] == interval[0] and span[1] == interval[
1]:
sum += 1
break
return sum
correct_interval = count_same(pred_set, gold_set)
precision = safe_divide(correct_interval, len(pred_set))
recall = safe_divide(correct_interval, len(gold_set))
if safe_divide(2 * precision * recall,
precision + recall) > f1:
f1 = safe_divide(2 * precision * recall,
precision + recall)
p = precision
r = recall
elif not gold_has_answer and not pred_has_answer:
f1 = 1
p = 1
r = 1
return gold_has_answer, pred_has_answer, f1, p, r