def extract_type(train=False, val=False, answer_vocab=None):
""" Counting answers under each question type. """
question_type_dict = {}
answer_idx_dict = {}
answers = utils.path_for(answer=True, train=train, val=val)
with open(answers, 'r') as fd:
answers = json.load(fd)
bias = len(answer_vocab) # for unseeing answers
for answer_dict in answers['annotations']:
question_type = answer_dict['question_type']
if question_type not in question_type_dict:
question_type_dict[question_type] = []
answ_idxs = []
for answs in answer_dict['answers']:
answ = data.process_answers(answs['answer'])
answ_idx = answer_vocab.get(answ, bias)
if answ_idx not in answer_idx_dict:
answer_idx_dict[answ_idx] = []
answer_idx_dict[answ_idx].append(question_type) # answer_idx: question_type
if not answ_idx == bias:
answ_idxs.append(answ)
question_type_dict[question_type].extend(answ_idxs) # question_type: answer_idx (needs counting later)
# Count the answers for each question_type
for qt in question_type_dict:
qt_answs = question_type_dict[qt]
question_type_dict[qt] = {'answers': Counter(qt_answs),
'total_answers': len(qt_answs),
'answer_type_num': len(set(qt_answs))}
answer_idx_dict = {ai: Counter(answer_idx_dict[ai]) for ai in answer_idx_dict}
return question_type_dict, answer_idx_dict
def _get_file_(train=False, val=False, test=False, question=False, answer=False): """ Get the correct question or answer file.""" _file = utils.path_for(train=train, val=val, test=test, question=question, answer=answer) with open(_file, 'r') as fd: _object = json.load(fd) return _object
def __init__(self, annotation_file=None, question_file=None):
"""
Constructor of VQA helper class for reading and visualizing questions and answers.
:param annotation_file (str): location of VQA annotation file
:return:
"""
# load dataset
self.dataset = {}
self.questions = {}
self.qa = {}
self.qqa = {}
self.imgToQA = {}
if not annotation_file == None and not question_file == None:
print('loading VQA annotations and questions into memory...')
time_t = datetime.datetime.utcnow()
dataset = json.load(open(annotation_file, 'r'))
questions = json.load(open(question_file, 'r'))
if config.cp_data:
config.cp_data = False
config.qa_path = config.main_path + config.version
config.task = 'OpenEnded'
questions_dummy = json.load(open(utils.path_for(val=True, question=True), 'r'))
dataset_dummy = json.load(open(utils.path_for(val=True, answer=True), 'r'))
questions_dummy['questions'] = questions
dataset_dummy['annotations'] = dataset
questions = questions_dummy
dataset = dataset_dummy
config.cp_data = True
config.qa_path = config.main_path + 'vqa-cp/' + config.version
config.task = 'vqacp'
print(datetime.datetime.utcnow() - time_t)
self.dataset = dataset
self.questions = questions
self.createIndex()
def get_loader(train=False,
val=False,
test=False,
need_test=False,
version='v1'):
""" Returns a data loader for the desired split """
if train and val:
do_val_later = True
val = False
else:
do_val_later = False
split = VQA(utils.path_for(train=train, val=val, test=test, question=True),
utils.path_for(train=train, val=val, test=test, answer=True),
config.preprocessed_path,
answerable_only=train,
dummy_answers=test)
if do_val_later:
val = True
train = False
split += VQA(utils.path_for(train=train,
val=val,
test=test,
question=True),
utils.path_for(train=train,
val=val,
test=test,
answer=True),
config.preprocessed_path,
answerable_only=val,
dummy_answers=test)
loader = data.DataLoader(
split,
batch_size=config.batch_size,
shuffle=train, # only shuffle the data in training
pin_memory=True,
collate_fn=collate_fn,
num_workers=config.data_workers,
)
return loader
def get_loader(split=None, test=False, vocabs=None):
""" Returns a data loader for the desired split """
image_path = config.rcnn_test_path if test else config.rcnn_trainval_path
split = VQA(vocabs, utils.path_for(split), image_path)
loader = torch.utils.data.DataLoader(
split,
batch_size=config.batch_size,
pin_memory=True,
num_workers=config.data_workers,
collate_fn=collate_fn,
)
return loader
def loadRes(self, resFile, quesFile):
"""
Load result file and return a result object.
:param resFile (str) : file name of result file
:return: res (obj) : result api object
"""
res = VQA()
res.questions = json.load(open(quesFile))
if config.cp_data:
config.cp_data = False
config.qa_path = config.main_path + config.version
config.task = 'OpenEnded'
questions_dummy = json.load(open(utils.path_for(val=True, question=True), 'r'))
questions_dummy['questions'] = res.questions
res.questions = questions_dummy
config.cp_data = True
config.qa_path = config.main_path + 'vqa-cp/' + config.version
config.task = 'vqacp'
res.dataset['info'] = copy.deepcopy(self.questions['info'])
res.dataset['task_type'] = copy.deepcopy(self.questions['task_type'])
res.dataset['data_type'] = copy.deepcopy(self.questions['data_type'])
res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype'])
res.dataset['license'] = copy.deepcopy(self.questions['license'])
print('Loading and preparing results... ')
time_t = datetime.datetime.utcnow()
anns = json.load(open(resFile))
assert type(anns) == list, 'results is not an array of objects'
annsQuesIds = [ann['question_id'] for ann in anns]
assert set(annsQuesIds) == set(self.getQuesIds()), \
'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.'
for ann in anns:
quesId = ann['question_id']
if res.dataset['task_type'] == 'Multiple Choice':
assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices'
qaAnn = self.qa[quesId]
ann['image_id'] = qaAnn['image_id']
ann['question_type'] = qaAnn['question_type']
ann['answer_type'] = qaAnn['answer_type']
print('DONE (t=%0.2fs)' % ((datetime.datetime.utcnow() - time_t).total_seconds()))
res.dataset['annotations'] = anns
res.createIndex()
return res
def get_loader(train=False,
val=False,
test=False,
test_split=config.test_split):
""" Returns a data loader for the desired split """
if train and val:
do_val_later = True
val = False
else:
do_val_later = False
split = VQA(
utils.path_for(train=train,
val=val,
test=test,
question=True,
test_split=test_split),
config.preprocessed_trainval_path
if not test else config.preprocessed_test_path,
)
if do_val_later:
val = True
train = False
split += VQA(
utils.path_for(train=train, val=val, test=test, question=True),
config.preprocessed_trainval_path
if not test else config.preprocessed_test_path,
)
loader = torch.utils.data.DataLoader(
split,
batch_size=config.batch_size,
shuffle=train, # only shuffle the data in training
pin_memory=True,
num_workers=config.data_workers,
collate_fn=collate_fn,
)
return loader
def main():
results_path = sys.argv[1] # the result file
annotations_path = utils.path_for(val=True, answer=True)
qta_path = config.question_type_path
with open(results_path, 'r') as fd:
results = json.load(fd)
with open(annotations_path, 'r') as fd:
annotations = json.load(fd)
with open(qta_path, 'r') as fd:
qta = json.load(fd)['qta']
lp_best = 1.0
for k in results:
lp = eval(results[k], annotations, qta)
if lp < lp_best:
lp_best = lp
print("The best LP score is {:.4f}".format(lp_best))
def _load_dataset(cache_path, name, img_id2val):
""" Load entries. img_id2val: dict {img_id -> val} ,
val can be used to retrieve image or features.
"""
train, val, test = False, False, False
if name == 'train':
train = True
elif name == 'val':
val = True
else:
test = True
question_path = utils.path_for(train=train,
val=val,
test=test,
question=True)
questions = json.load(open(question_path, 'r'))
if not config.cp_data:
questions = questions['questions']
questions = sorted(questions, key=lambda x: x['question_id'])
if test: # will be ignored anyway
answers = [{
'image_id': 0,
'question_id': 0,
'question_type': '',
'labels': [],
'scores': []
} for _ in range(len(questions))]
else:
answer_path = os.path.join(cache_path, '{}_target.json'.format(name))
answers = json.load(open(answer_path, 'r'))
answers = sorted(answers, key=lambda x: x['question_id'])
utils.assert_eq(len(questions), len(answers))
entries = []
for question, answer in zip(questions, answers):
if not test:
utils.assert_eq(question['question_id'], answer['question_id'])
utils.assert_eq(question['image_id'], answer['image_id'])
img_id = question['image_id']
entries.append(_create_entry(img_id2val[img_id], question, answer))
return entries
def get_loader(train=False, val=False, test=False):
""" Returns a data loader for the desired split """
if train and val:
do_val_later = True
val = False
else:
do_val_later = False
trainval_path = config.rcnn_trainval_path if config.image_feature == 'rcnn' \
else config.grid_trainval_path
test_path = config.rcnn_test_path if config.image_feature == 'rcnn' \
else config.grid_test_path
split = VQA(
utils.path_for(train=train, val=val, test=test, question=True),
utils.path_for(train=train, val=val, test=test, answer=True),
utils.path_for(train=train, val=val, test=test, knowledge=True),
trainval_path if not test else test_path,
answerable_only=train,
dummy_answers=test,
)
if do_val_later:
val = True
train = False
split += VQA(
utils.path_for(train=train, val=val, test=test, question=True),
utils.path_for(train=train, val=val, test=test, answer=True),
utils.path_for(train=train, val=val, test=test, knowledge=True),
trainval_path if not test else test_path,
answerable_only=val,
dummy_answers=test,
)
loader = torch.utils.data.DataLoader(
split,
batch_size=config.batch_size,
shuffle=train, # only shuffle the data in training
pin_memory=True,
num_workers=config.data_workers,
collate_fn=collate_fn,
)
return loader
import sys
import utils.utils as utils
from vqa_eval.PythonHelperTools.vqaTools.vqa import VQA
from vqa_eval.PythonEvaluationTools.vqaEvaluation.vqaEval import VQAEval
quesFile = utils.path_for(val=True, question=True)
annFile = utils.path_for(val=True, answer=True)
resFile = sys.argv[1]
# create vqa object and vqaRes object
vqa = VQA(annFile, quesFile)
vqaRes = vqa.loadRes(resFile, quesFile)
# create vqaEval object by taking vqa and vqaRes
vqaEval = VQAEval(vqa, vqaRes, n=2) # n is precision of accuracy (number of places after decimal), default is 2
# evaluate results
vqaEval.evaluate()
# print accuracies
print("Overall Accuracy is: {:.2f}\n".format(vqaEval.accuracy['overall']))
print("Per Question Type Accuracy is the following:")
for quesType in vqaEval.accuracy['perQuestionType']:
print("{} : {:.2f}".format(quesType, vqaEval.accuracy['perQuestionType'][quesType]))
print("\n")
print("Per Answer Type Accuracy is the following:")
for ansType in vqaEval.accuracy['perAnswerType']:
print("{} : {:.2f}".format(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
