def test_evalai_answer_processor(self):
evalai_answer_processor = EvalAIAnswerProcessor()
# Test number
processed = evalai_answer_processor("two")
expected = "2"
self.assertEqual(processed, expected)
# Test article
processed = evalai_answer_processor("a building")
expected = "building"
self.assertEqual(processed, expected)
# Test tokenize
processed = evalai_answer_processor("snow, mountain")
expected = "snow mountain"
self.assertEqual(processed, expected)
# Test contractions
processed = evalai_answer_processor("isnt")
expected = "isn't"
self.assertEqual(processed, expected)
# Test processor
processed = evalai_answer_processor("the two mountain's \t \n ")
expected = "2 mountain 's"
self.assertEqual(processed, expected)
def filter_answers(answers_dset, min_occurence):
"""This will change the answer to preprocessed version
"""
occurence = {}
answer_list = []
evalai_answer_processor = EvalAIAnswerProcessor()
for ans_entry in answers_dset:
gtruth = ans_entry["multiple_choice_answer"]
gtruth = evalai_answer_processor(gtruth)
if gtruth not in occurence:
occurence[gtruth] = set()
occurence[gtruth].add(ans_entry["question_id"])
for answer in occurence.keys():
if len(occurence[answer]) >= min_occurence:
answer_list.append(answer)
print("Num of answers that appear >= %d times: %d" %
(min_occurence, len(answer_list)))
return answer_list
def __init__(self):
super().__init__("vqa_evalai_accuracy")
self.evalai_answer_processor = EvalAIAnswerProcessor()