def extract_type(train=False, val=False, answer_vocab=None): """ Counting answers under each question type. """ question_type_dict = {} answer_idx_dict = {} answers = utils.path_for(answer=True, train=train, val=val) with open(answers, 'r') as fd: answers = json.load(fd) bias = len(answer_vocab) # for unseeing answers for answer_dict in answers['annotations']: question_type = answer_dict['question_type'] if question_type not in question_type_dict: question_type_dict[question_type] = [] answ_idxs = [] for answs in answer_dict['answers']: answ = data.process_answers(answs['answer']) answ_idx = answer_vocab.get(answ, bias) if answ_idx not in answer_idx_dict: answer_idx_dict[answ_idx] = [] answer_idx_dict[answ_idx].append(question_type) # answer_idx: question_type if not answ_idx == bias: answ_idxs.append(answ) question_type_dict[question_type].extend(answ_idxs) # question_type: answer_idx (needs counting later) # Count the answers for each question_type for qt in question_type_dict: qt_answs = question_type_dict[qt] question_type_dict[qt] = {'answers': Counter(qt_answs), 'total_answers': len(qt_answs), 'answer_type_num': len(set(qt_answs))} answer_idx_dict = {ai: Counter(answer_idx_dict[ai]) for ai in answer_idx_dict} return question_type_dict, answer_idx_dict
def _get_file_(train=False, val=False, test=False, question=False, answer=False): """ Get the correct question or answer file.""" _file = utils.path_for(train=train, val=val, test=test, question=question, answer=answer) with open(_file, 'r') as fd: _object = json.load(fd) return _object
def __init__(self, annotation_file=None, question_file=None): """ Constructor of VQA helper class for reading and visualizing questions and answers. :param annotation_file (str): location of VQA annotation file :return: """ # load dataset self.dataset = {} self.questions = {} self.qa = {} self.qqa = {} self.imgToQA = {} if not annotation_file == None and not question_file == None: print('loading VQA annotations and questions into memory...') time_t = datetime.datetime.utcnow() dataset = json.load(open(annotation_file, 'r')) questions = json.load(open(question_file, 'r')) if config.cp_data: config.cp_data = False config.qa_path = config.main_path + config.version config.task = 'OpenEnded' questions_dummy = json.load(open(utils.path_for(val=True, question=True), 'r')) dataset_dummy = json.load(open(utils.path_for(val=True, answer=True), 'r')) questions_dummy['questions'] = questions dataset_dummy['annotations'] = dataset questions = questions_dummy dataset = dataset_dummy config.cp_data = True config.qa_path = config.main_path + 'vqa-cp/' + config.version config.task = 'vqacp' print(datetime.datetime.utcnow() - time_t) self.dataset = dataset self.questions = questions self.createIndex()
def get_loader(train=False, val=False, test=False, need_test=False, version='v1'): """ Returns a data loader for the desired split """ if train and val: do_val_later = True val = False else: do_val_later = False split = VQA(utils.path_for(train=train, val=val, test=test, question=True), utils.path_for(train=train, val=val, test=test, answer=True), config.preprocessed_path, answerable_only=train, dummy_answers=test) if do_val_later: val = True train = False split += VQA(utils.path_for(train=train, val=val, test=test, question=True), utils.path_for(train=train, val=val, test=test, answer=True), config.preprocessed_path, answerable_only=val, dummy_answers=test) loader = data.DataLoader( split, batch_size=config.batch_size, shuffle=train, # only shuffle the data in training pin_memory=True, collate_fn=collate_fn, num_workers=config.data_workers, ) return loader
def get_loader(split=None, test=False, vocabs=None): """ Returns a data loader for the desired split """ image_path = config.rcnn_test_path if test else config.rcnn_trainval_path split = VQA(vocabs, utils.path_for(split), image_path) loader = torch.utils.data.DataLoader( split, batch_size=config.batch_size, pin_memory=True, num_workers=config.data_workers, collate_fn=collate_fn, ) return loader
def loadRes(self, resFile, quesFile): """ Load result file and return a result object. :param resFile (str) : file name of result file :return: res (obj) : result api object """ res = VQA() res.questions = json.load(open(quesFile)) if config.cp_data: config.cp_data = False config.qa_path = config.main_path + config.version config.task = 'OpenEnded' questions_dummy = json.load(open(utils.path_for(val=True, question=True), 'r')) questions_dummy['questions'] = res.questions res.questions = questions_dummy config.cp_data = True config.qa_path = config.main_path + 'vqa-cp/' + config.version config.task = 'vqacp' res.dataset['info'] = copy.deepcopy(self.questions['info']) res.dataset['task_type'] = copy.deepcopy(self.questions['task_type']) res.dataset['data_type'] = copy.deepcopy(self.questions['data_type']) res.dataset['data_subtype'] = copy.deepcopy(self.questions['data_subtype']) res.dataset['license'] = copy.deepcopy(self.questions['license']) print('Loading and preparing results... ') time_t = datetime.datetime.utcnow() anns = json.load(open(resFile)) assert type(anns) == list, 'results is not an array of objects' annsQuesIds = [ann['question_id'] for ann in anns] assert set(annsQuesIds) == set(self.getQuesIds()), \ 'Results do not correspond to current VQA set. Either the results do not have predictions for all question ids in annotation file or there is atleast one question id that does not belong to the question ids in the annotation file.' for ann in anns: quesId = ann['question_id'] if res.dataset['task_type'] == 'Multiple Choice': assert ann['answer'] in self.qqa[quesId]['multiple_choices'], 'predicted answer is not one of the multiple choices' qaAnn = self.qa[quesId] ann['image_id'] = qaAnn['image_id'] ann['question_type'] = qaAnn['question_type'] ann['answer_type'] = qaAnn['answer_type'] print('DONE (t=%0.2fs)' % ((datetime.datetime.utcnow() - time_t).total_seconds())) res.dataset['annotations'] = anns res.createIndex() return res
def get_loader(train=False, val=False, test=False, test_split=config.test_split): """ Returns a data loader for the desired split """ if train and val: do_val_later = True val = False else: do_val_later = False split = VQA( utils.path_for(train=train, val=val, test=test, question=True, test_split=test_split), config.preprocessed_trainval_path if not test else config.preprocessed_test_path, ) if do_val_later: val = True train = False split += VQA( utils.path_for(train=train, val=val, test=test, question=True), config.preprocessed_trainval_path if not test else config.preprocessed_test_path, ) loader = torch.utils.data.DataLoader( split, batch_size=config.batch_size, shuffle=train, # only shuffle the data in training pin_memory=True, num_workers=config.data_workers, collate_fn=collate_fn, ) return loader
def main(): results_path = sys.argv[1] # the result file annotations_path = utils.path_for(val=True, answer=True) qta_path = config.question_type_path with open(results_path, 'r') as fd: results = json.load(fd) with open(annotations_path, 'r') as fd: annotations = json.load(fd) with open(qta_path, 'r') as fd: qta = json.load(fd)['qta'] lp_best = 1.0 for k in results: lp = eval(results[k], annotations, qta) if lp < lp_best: lp_best = lp print("The best LP score is {:.4f}".format(lp_best))
def _load_dataset(cache_path, name, img_id2val): """ Load entries. img_id2val: dict {img_id -> val} , val can be used to retrieve image or features. """ train, val, test = False, False, False if name == 'train': train = True elif name == 'val': val = True else: test = True question_path = utils.path_for(train=train, val=val, test=test, question=True) questions = json.load(open(question_path, 'r')) if not config.cp_data: questions = questions['questions'] questions = sorted(questions, key=lambda x: x['question_id']) if test: # will be ignored anyway answers = [{ 'image_id': 0, 'question_id': 0, 'question_type': '', 'labels': [], 'scores': [] } for _ in range(len(questions))] else: answer_path = os.path.join(cache_path, '{}_target.json'.format(name)) answers = json.load(open(answer_path, 'r')) answers = sorted(answers, key=lambda x: x['question_id']) utils.assert_eq(len(questions), len(answers)) entries = [] for question, answer in zip(questions, answers): if not test: utils.assert_eq(question['question_id'], answer['question_id']) utils.assert_eq(question['image_id'], answer['image_id']) img_id = question['image_id'] entries.append(_create_entry(img_id2val[img_id], question, answer)) return entries
def get_loader(train=False, val=False, test=False): """ Returns a data loader for the desired split """ if train and val: do_val_later = True val = False else: do_val_later = False trainval_path = config.rcnn_trainval_path if config.image_feature == 'rcnn' \ else config.grid_trainval_path test_path = config.rcnn_test_path if config.image_feature == 'rcnn' \ else config.grid_test_path split = VQA( utils.path_for(train=train, val=val, test=test, question=True), utils.path_for(train=train, val=val, test=test, answer=True), utils.path_for(train=train, val=val, test=test, knowledge=True), trainval_path if not test else test_path, answerable_only=train, dummy_answers=test, ) if do_val_later: val = True train = False split += VQA( utils.path_for(train=train, val=val, test=test, question=True), utils.path_for(train=train, val=val, test=test, answer=True), utils.path_for(train=train, val=val, test=test, knowledge=True), trainval_path if not test else test_path, answerable_only=val, dummy_answers=test, ) loader = torch.utils.data.DataLoader( split, batch_size=config.batch_size, shuffle=train, # only shuffle the data in training pin_memory=True, num_workers=config.data_workers, collate_fn=collate_fn, ) return loader
import sys import utils.utils as utils from vqa_eval.PythonHelperTools.vqaTools.vqa import VQA from vqa_eval.PythonEvaluationTools.vqaEvaluation.vqaEval import VQAEval quesFile = utils.path_for(val=True, question=True) annFile = utils.path_for(val=True, answer=True) resFile = sys.argv[1] # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval(vqa, vqaRes, n=2) # n is precision of accuracy (number of places after decimal), default is 2 # evaluate results vqaEval.evaluate() # print accuracies print("Overall Accuracy is: {:.2f}\n".format(vqaEval.accuracy['overall'])) print("Per Question Type Accuracy is the following:") for quesType in vqaEval.accuracy['perQuestionType']: print("{} : {:.2f}".format(quesType, vqaEval.accuracy['perQuestionType'][quesType])) print("\n") print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: print("{} : {:.2f}".format(ansType, vqaEval.accuracy['perAnswerType'][ansType]))