def evaluate_question_standard(result_file, subset='kptest', version='v1'): from analysis.eval_vqa_question import QuestionEvaluator from util import get_dataset_root vqa_data_root, _ = get_dataset_root() subset = 'train' if subset == 'train' else 'val' if version == 'v1': annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % ( vqa_data_root, subset) question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % ( vqa_data_root, subset) elif version == 'v2': anno_dir = '/import/vision-ephemeral/fl302/data/VQA2.0' annotation_file = '%s/v2_mscoco_%s2014_annotations.json' % (anno_dir, subset) question_file = '%s/v2_OpenEnded_mscoco_%s2014_questions.json' % ( anno_dir, subset) else: raise Exception('unknown version, v1 or v2') evaluator = QuestionEvaluator(annotation_file, question_file) evaluator.evaluate(result_file) evaluator.save_results() # return evaluator.get_overall_blue4() return evaluator.get_overall_cider()
def evaluate_question(result_file, subset='kptest'): from eval_vqa_question import QuestionEvaluator from util import get_dataset_root vqa_data_root, _ = get_dataset_root() # assert (subset in ['train', 'dev', 'val']) subset = 'train' if subset == 'train' else 'val' annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % (vqa_data_root, subset) question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root, subset) evaluator = QuestionEvaluator(annotation_file, question_file) evaluator.evaluate(result_file) evaluator.save_results() return evaluator.get_overall_cider()
def evaluate_ground_truth(): # set ground truth vqa_data_root, _ = get_dataset_root() anno_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root, 'val') gt = VQAQuestion(anno_file) res = VQAQuestion(anno_file) # initial test evaluator = COCOEvalCap(gt, res) evaluator.evaluate() results = evaluator.evalImgs return evaluator.eval['CIDEr']
def evaluate_question(result_file): from eval_vqa_question import QuestionEvaluator from util import get_dataset_root vqa_data_root, _ = get_dataset_root() base_name = os.path.splitext(result_file) res_mat_file = '%s_lig.mat' % base_name[0] subset = 'val' annotation_file = '%s/Annotations/mscoco_%s2014_annotations.json' % ( vqa_data_root, subset) question_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % ( vqa_data_root, subset) evaluator = QuestionEvaluator(annotation_file, question_file) evaluator.evaluate(result_file) evaluator.save_results(eval_res_file=res_mat_file) return evaluator.get_overall_cider()
def evaluate_oracle(res_file, K=None, eval_multiple=False, split='val'): def parse_evaluator_scores(_evaluator): metrics = ['Bleu_4', 'CIDEr'] scores = np.array([_evaluator.eval[m] for m in metrics]) return scores[np.newaxis, :].astype(np.float32) res_base = VQAQuestion(res_file) res = subsample_questions(res_base, K) quest_ids = res.image_ids # set ground truth vqa_data_root, _ = get_dataset_root() anno_file = '%s/Questions/OpenEnded_mscoco_%s2014_questions.json' % (vqa_data_root, split) gt = VQAAnnotation(anno_file) gt.replicate_annotations(quest_ids) # average test evaluator = COCOEvalCap(gt, res) evaluator.setup_scorer(['Bleu', 'CIDEr']) evaluator.evaluate() results = evaluator.evalImgs # dump results dump_results(results, res_file) avg_scores = parse_evaluator_scores(evaluator) # find oracle matched_inds = find_matched_questions(results, gt.getRealImgIds(), 'BC') flt_res = FilterVQAQuestion(res_file, matched_inds) gt_ = VQAAnnotation(anno_file) gt_.replicate_annotations(flt_res.image_ids) evaluator = COCOEvalCap(gt_, flt_res) evaluator.setup_scorer(['Bleu', 'CIDEr']) evaluator.evaluate() oracle_scores = parse_evaluator_scores(evaluator) if eval_multiple: return avg_scores, oracle_scores # average and mean scores else: return oracle_scores.flatten()
import sys from util import get_dataset_root dataDir, _ = get_dataset_root() sys.path.insert(0, '%s/PythonHelperTools/vqaTools' % (dataDir)) sys.path.insert(0, '%s/PythonEvaluationTools/' % (dataDir)) from vqa import VQA from vqaEvaluation.vqaEval import VQAEval import matplotlib.pyplot as plt import skimage.io as io import json import random import os def write_result_log(model_file, model_type, total_res, per_type_res): fname = 'result/detail_%s.txt' % model_type with open(fname, 'a+') as f: f.write('%s\n' % model_file) for type in per_type_res: f.write('%s\t: %0.2f\n' % (type, per_type_res[type])) f.write('overall\t: %0.2f\n\n' % total_res) def evaluate_model(resFile, quest_ids, subset='val', version='v1'): ans_type = None # set up file names and paths taskType = 'OpenEnded' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = '%s2014' % subset if version == 'v1':
import json import sys import numpy as np from util import get_dataset_root vqa_data_root, toolkit_dir = get_dataset_root() sys.path.insert(0, toolkit_dir) # from pycocoevalcap.eval import COCOEvalCap from pycocoevalcap.coco_eval_toolkit_oracle import COCOEvalCap import pdb from time import time # # def get_dataset_root(): # import os # if os.path.exists('/import/vision-ephemeral/fl302/data/VQA'): # data_root = '/import/vision-ephemeral/fl302/data/VQA' # toolkit_root = '/homes/fl302/projects/coco-caption' # else: # data_root = '/data/fl302/data/VQA' # toolkit_root = '/data/fl302/data/im2txt/coco-caption' # return data_root, toolkit_root class VQAQuestion(object): def __init__(self, question_file): anns = load_questions(question_file) self.image_ids = [info['question_id'] for info in anns] # self.real_image_ids = [info['image_id'] for info in anns] self.imgToAnns = {} for i, info in enumerate(anns):