def evaluate_question_standard(result_file, subset='kptest', version='v1'):
from analysis.eval_vqa_question import QuestionEvaluator
from util import get_dataset_root
vqa_data_root, _ = get_dataset_root()
subset = 'train' if subset == 'train' else 'val'
if version == 'v1':
annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (
vqa_data_root, subset)
question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (
vqa_data_root, subset)
elif version == 'v2':
anno_dir = '/import/vision-ephemeral/fl302/data/VQA2.0'
annotation_file = '%s/v2_mscoco_%s2014_annotations.json' % (anno_dir,
subset)
question_file = '%s/v2_OpenEnded_mscoco_%s2014_questions.json' % (
anno_dir, subset)
else:
raise Exception('unknown version, v1 or v2')
evaluator = QuestionEvaluator(annotation_file, question_file)
evaluator.evaluate(result_file)
evaluator.save_results()
# return evaluator.get_overall_blue4()
return evaluator.get_overall_cider()
def evaluate_question(result_file, subset='kptest'):
from eval_vqa_question import QuestionEvaluator
from util import get_dataset_root
vqa_data_root, _ = get_dataset_root()
# assert (subset in ['train', 'dev', 'val'])
subset = 'train' if subset == 'train' else 'val'
annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (vqa_data_root, subset)
question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root, subset)
evaluator = QuestionEvaluator(annotation_file, question_file)
evaluator.evaluate(result_file)
evaluator.save_results()
return evaluator.get_overall_cider()
def evaluate_ground_truth():
# set ground truth
vqa_data_root, _ = get_dataset_root()
anno_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root,
'val')
gt = VQAQuestion(anno_file)
res = VQAQuestion(anno_file)
# initial test
evaluator = COCOEvalCap(gt, res)
evaluator.evaluate()
results = evaluator.evalImgs
return evaluator.eval['CIDEr']
def evaluate_question(result_file):
from eval_vqa_question import QuestionEvaluator
from util import get_dataset_root
vqa_data_root, _ = get_dataset_root()
base_name = os.path.splitext(result_file)
res_mat_file = '%s_lig.mat' % base_name[0]
subset = 'val'
annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (
vqa_data_root, subset)
question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (
vqa_data_root, subset)
evaluator = QuestionEvaluator(annotation_file, question_file)
evaluator.evaluate(result_file)
evaluator.save_results(eval_res_file=res_mat_file)
return evaluator.get_overall_cider()
def evaluate_oracle(res_file, K=None, eval_multiple=False, split='val'):
def parse_evaluator_scores(_evaluator):
metrics = ['Bleu_4', 'CIDEr']
scores = np.array([_evaluator.eval[m] for m in metrics])
return scores[np.newaxis, :].astype(np.float32)
res_base = VQAQuestion(res_file)
res = subsample_questions(res_base, K)
quest_ids = res.image_ids
# set ground truth
vqa_data_root, _ = get_dataset_root()
anno_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root,
split)
gt = VQAAnnotation(anno_file)
gt.replicate_annotations(quest_ids)
# average test
evaluator = COCOEvalCap(gt, res)
evaluator.setup_scorer(['Bleu', 'CIDEr'])
evaluator.evaluate()
results = evaluator.evalImgs
# dump results
dump_results(results, res_file)
avg_scores = parse_evaluator_scores(evaluator)
# find oracle
matched_inds = find_matched_questions(results, gt.getRealImgIds(), 'BC')
flt_res = FilterVQAQuestion(res_file, matched_inds)
gt_ = VQAAnnotation(anno_file)
gt_.replicate_annotations(flt_res.image_ids)
evaluator = COCOEvalCap(gt_, flt_res)
evaluator.setup_scorer(['Bleu', 'CIDEr'])
evaluator.evaluate()
oracle_scores = parse_evaluator_scores(evaluator)
if eval_multiple:
return avg_scores, oracle_scores # average and mean scores
else:
return oracle_scores.flatten()
import sys
from util import get_dataset_root
dataDir, _ = get_dataset_root()
sys.path.insert(0, '%s/PythonHelperTools/vqaTools' % (dataDir))
sys.path.insert(0, '%s/PythonEvaluationTools/' % (dataDir))
from vqa import VQA
from vqaEvaluation.vqaEval import VQAEval
import matplotlib.pyplot as plt
import skimage.io as io
import json
import random
import os
def write_result_log(model_file, model_type, total_res, per_type_res):
fname = 'result/detail_%s.txt' % model_type
with open(fname, 'a+') as f:
f.write('%s\n' % model_file)
for type in per_type_res:
f.write('%s\t: %0.2f\n' % (type, per_type_res[type]))
f.write('overall\t: %0.2f\n\n' % total_res)
def evaluate_model(resFile, quest_ids, subset='val', version='v1'):
ans_type = None
# set up file names and paths
taskType = 'OpenEnded'
dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType = '%s2014' % subset
if version == 'v1':
import json
import sys
import numpy as np
from util import get_dataset_root
vqa_data_root, toolkit_dir = get_dataset_root()
sys.path.insert(0, toolkit_dir)
# from pycocoevalcap.eval import COCOEvalCap
from pycocoevalcap.coco_eval_toolkit_oracle import COCOEvalCap
import pdb
from time import time
#
# def get_dataset_root():
# import os
# if os.path.exists('/import/vision-ephemeral/fl302/data/VQA'):
# data_root = '/import/vision-ephemeral/fl302/data/VQA'
# toolkit_root = '/homes/fl302/projects/coco-caption'
# else:
# data_root = '/data/fl302/data/VQA'
# toolkit_root = '/data/fl302/data/im2txt/coco-caption'
# return data_root, toolkit_root
class VQAQuestion(object):
def __init__(self, question_file):
anns = load_questions(question_file)
self.image_ids = [info['question_id'] for info in anns]
# self.real_image_ids = [info['image_id'] for info in anns]
self.imgToAnns = {}
for i, info in enumerate(anns):
