def remove_exact_duplicates(self, input_list):
unique_qa_list = []
unique_qa_dict = defaultdict(list)
for qa in input_list:
normalized_q = normalize(qa['question'])
unique_qa_dict[normalized_q].append(qa)
# Keep only first occurence for each question
for _, duplicates in unique_qa_dict.items():
unique_qa_list.append(duplicates[0])
unique_qa_list.sort(key=lambda x: x['id'])
return unique_qa_list
def remove_exact_duplicate_questions(self, input_list):
unique_qa_list = []
unique_qa_dict = defaultdict(list)
for qa in input_list:
normalized_q = normalize(qa['question'])
unique_qa_dict[normalized_q].append(qa)
# Keep only first occurence for each question
# Since answering model is seeing same (paragraph, question) input, answer will be same
for _, duplicates in unique_qa_dict.items():
unique_qa_list.append(duplicates[0])
unique_qa_list.sort(key=lambda x: x['id'])
return unique_qa_list
def remove_exact_duplicate_answers(self,
input_list,
question_type="specific"):
"""Within each paragraph, ensure that every GENERAL / SPECIFIC predicted answer is unique."""
unique_qa_list = []
unique_qa_dict = defaultdict(list)
if question_type == "specific":
retain_type = "general"
else:
retain_type = "specific"
for qa in input_list:
# We don't want to be too aggressive about the unanswerable questions since the
# answering module could have made mistakes. Unanswerable questions will be dealt
# with later on in the filtering phase depending on the user settings.
if qa['unanswerable'] is True or retain_type in qa["algorithm"]:
unique_qa_list.append(qa)
continue
normalized_a = normalize(qa['predicted_answer'])
unique_qa_dict[normalized_a].append(qa)
metrics = {
"general_sent": "recall_match",
"specific_sent": "precision_match",
"specific_entity": "recall_match"
}
# Now, sort the duplications according to the precision / recall scores with intended answer
for _, duplicates in unique_qa_dict.items():
duplicates.sort(
key=lambda x: (-x[metrics[x['algorithm']]], x['id']))
unique_qa_list.append(duplicates[0])
unique_qa_list.sort(key=lambda x: x['id'])
return unique_qa_list