def evaluate_question_standard(result_file, subset='kptest', version='v1'):
from analysis.eval_vqa_question import QuestionEvaluator
from util import get_dataset_root
vqa_data_root, _ = get_dataset_root()
subset = 'train' if subset == 'train' else 'val'
if version == 'v1':
annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (
vqa_data_root, subset)
question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (
vqa_data_root, subset)
elif version == 'v2':
anno_dir = '/import/vision-ephemeral/fl302/data/VQA2.0'
annotation_file = '%s/v2_mscoco_%s2014_annotations.json' % (anno_dir,
subset)
question_file = '%s/v2_OpenEnded_mscoco_%s2014_questions.json' % (
anno_dir, subset)
else:
raise Exception('unknown version, v1 or v2')
evaluator = QuestionEvaluator(annotation_file, question_file)
evaluator.evaluate(result_file)
evaluator.save_results()
# return evaluator.get_overall_blue4()
return evaluator.get_overall_cider()
def evaluate_question(result_file, subset='kptest'):
from analysis.eval_vqa_question import QuestionEvaluator
from util import get_dataset_root
vqa_data_root, _ = get_dataset_root()
# assert (subset in ['train', 'dev', 'val'])
subset = 'train' if subset == 'train' else 'val'
annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (
vqa_data_root, subset)
question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (
vqa_data_root, subset)
evaluator = QuestionEvaluator(annotation_file, question_file)
evaluator.evaluate(result_file)
evaluator.save_results()
return evaluator.get_overall_blue4()