def model_work(image_id, question, encoding=None): #image = np.array(Image.open(f'{DATA_DIR}/{image_idx}.jpg').convert('RGB')) if image_id == None: vqa_object = VQA('mfb', 'eval') image_feat = torch.tensor([encoding]) # (num_objects, 2048) print(image_feat.size()) else: image_idx = image_id vqa_object = VQA('mfb', 'eval') feats = np.load(f'assets/feats/{image_idx}.npz') image_feat = torch.tensor(feats['x'].T) # (num_objects, 2048) ret = vqa_object.inference(question, image_feat) soft_proj = torch.softmax(ret['proj_feat'], dim=-1) values, indices = torch.topk(soft_proj, 5) values, indices = values.squeeze(0), indices.squeeze(0) df = {} df['answers'] = [] df['confidence'] = [] for idx in range(indices.shape[0]): df['answers'].append(vqa_object.ix_to_answer[str(indices[idx].item())]) df['confidence'].append(100 * values[idx].item()) df = pd.DataFrame(df) return df
def __init__(self, image_dir, question_json_file_path, annotation_json_file_path, image_filename_pattern, collate=False, q2i=None, a2i=None, i2a=None, a2i_count=None, img_names=None, img_ids=None, ques_ids=None, method='simple', dataset_type='train', enc_dir=''): print(method) self.image_dir = image_dir self.qjson = question_json_file_path self.ajson = annotation_json_file_path img_prefix = image_filename_pattern.split('{}')[0] self.collate = collate self.q2i = q2i self.a2i = a2i self.i2a = i2a self.a2i_count = a2i_count self.img_ids = img_ids self.ques_ids = ques_ids self.img_names = img_names self.method = method self.vqa = VQA(self.ajson, self.qjson) if self.method == 'simple': self.transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) else: self.transform = transforms.Compose( [transforms.Resize((448, 448)), transforms.ToTensor()]) if not collate: self.img_names = [ f for f in os.listdir(self.image_dir) if '.jpg' in f ] #print(self.img_names) self.img_ids = [] for fname in self.img_names: img_id = fname.split('.')[0].rpartition(img_prefix)[-1] self.img_ids.append(int(img_id)) self.ques_ids = self.vqa.getQuesIds(self.img_ids) self.q2i, self.a2i, self.i2a, self.a2i_count = pre_process_dataset( image_dir, self.qjson, self.ajson, img_prefix) #print("Image names ",self.img_names) self.q2i_len = len(self.q2i) self.a2i_len = len(self.a2i.keys()) self.q2i_keys = self.q2i.keys() self.enc_dir = enc_dir
def main(params): # set up file names and paths taskType = params['task'] dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'val2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) resultPath = params['res_file'].rsplit('/', 1)[0] resultPath = '.' if resultPath == params['res_file'] else resultPath resultType = params['res_file'].rsplit('_', 1)[0].rsplit('/', 1)[-1] fileTypes = ['accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. resFile = params['res_file'] [accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = \ ['%s/%s_%s_%s_%s_%s.json'%(resultPath, taskType, dataType, dataSubType, resultType, fileType) \ for fileType in fileTypes] # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) #n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ vqaEval.evaluate() # print accuracies #print "\n" print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall']), file=sys.stderr) #print "Per Question Type Accuracy is the following:" #for quesType in vqaEval.accuracy['perQuestionType']: # print "%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType]) #print "\n" #print "Per Answer Type Accuracy is the following:" #for ansType in vqaEval.accuracy['perAnswerType']: # print "%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType]) #print "\n" # save evaluation results to ./Results folder print(accuracyFile) json.dump(vqaEval.accuracy, open(accuracyFile, 'w')) json.dump(vqaEval.evalQA, open(evalQAFile, 'w')) json.dump(vqaEval.evalQuesType, open(evalQuesTypeFile, 'w')) json.dump(vqaEval.evalAnsType, open(evalAnsTypeFile, 'w'))
def pre_process_dataset(image_dir, qjson, ajson, img_prefix): print('Preprocessing datatset. \n') vqa = VQA(ajson, qjson) img_names = [f for f in os.listdir(image_dir) if '.jpg' in f] img_names = img_names[:30000] print("length: ", len(img_names)) img_ids = [] for fname in img_names: img_id = fname.split('.')[0].rpartition(img_prefix)[-1] img_ids.append(int(img_id)) print("Done collecting image ids") ques_ids = vqa.getQuesIds(img_ids) q2i = defaultdict(lambda: len(q2i)) pad = q2i["<pad>"] start = q2i["<sos>"] end = q2i["<eos>"] UNK = q2i["<unk>"] a2i_count = {} for ques_id in ques_ids: qa = vqa.loadQA(ques_id)[0] qqa = vqa.loadQQA(ques_id)[0] ques = qqa['question'][:-1] [q2i[x] for x in ques.lower().strip().split(" ")] answers = qa['answers'] for ans in answers: if not ans['answer_confidence'] == 'yes': continue ans = ans['answer'].lower() if ans not in a2i_count: a2i_count[ans] = 1 else: a2i_count[ans] = a2i_count[ans] + 1 print("Done collecting Q/A") a_sort = sorted(a2i_count.items(), key=operator.itemgetter(1), reverse=True) i2a = {} count = 0 a2i = defaultdict(lambda: len(a2i)) for word, _ in a_sort: a2i[word] i2a[a2i[word]] = word count = count + 1 if count == 1000: break print("Done collecting words") return q2i, a2i, i2a, a2i_count
def __init__(self, image_dir, question_json_file_path, annotation_json_file_path, image_filename_pattern): """ Args: image_dir (string): Path to the directory with COCO images question_json_file_path (string): Path to the json file containing the question data annotation_json_file_path (string): Path to the json file containing the annotations mapping images, questions, and answers together image_filename_pattern (string): The pattern the filenames of images in this dataset use (eg "COCO_train2014_{}.jpg") """ self.vqa = VQA(annotation_json_file_path, question_json_file_path) self.idx2key = { i: self.vqa.qa[v]['question_id'] for i, v in enumerate(self.vqa.qa) } self.imgdict = { idt: image_dir + '/' + image_filename_pattern.format(str(idt).zfill(12)) for idt in self.vqa.imgToQA } # self.SetQdict = D2Dict(self.vqa.questions,'questions') # This will prepare the dataset everytime # self.SetAdict = D2Dict(self.vqa.dataset,'annotations') # So, I've saved the files into a pickle object for saving time with open('./supportfiles/QnA.d', 'rb') as f: file = pickle.load(f) self.SetQdict = file['questions'] self.SetAdict = file['annotations']
def __init__(self, summary_writer=None, dataDir='/auto/homes/bat34/VQA', versionType='v2_', taskType='OpenEnded', dataType='mscoco', dataSubType='val2014'): self.writer = summary_writer self.versionType = versionType self.taskType = taskType self.dataType = dataType self.dataSubType = dataSubType self.annFile = '%s/Annotations/%s%s_%s_annotations.json' % ( dataDir, versionType, dataType, dataSubType) self.quesFile = '%s/Questions/%s%s_%s_%s_questions.json' % ( dataDir, versionType, taskType, dataType, dataSubType) self.vqa = VQA(self.annFile, self.quesFile)
def main(): parser = argparse.ArgumentParser() parser.add_argument('-isTrain', type=int, default=1) args = parser.parse_args() nlp = English() #used for conting number of tokens if args.isTrain == 1: annFile = '../data/mscoco_train2014_annotations.json' quesFile = '../data/OpenEnded_mscoco_train2014_questions.json' questions_file = open('../data/preprocessed/questions_train2014.txt', 'w') questions_lengths_file = open( '../data/preprocessed/questions_lengths_train2014.txt', 'w') answers_file = open('../data/preprocessed/answers_train2014.txt', 'w') coco_image_id = open('../data/preprocessed/images_train2014.txt', 'w') trainval = 'training data' else: annFile = '../data/mscoco_val2014_annotations.json' quesFile = '../data/OpenEnded_mscoco_val2014_questions.json' questions_file = open('../data/preprocessed/questions_val2014.txt', 'w') questions_lengths_file = open( '../data/preprocessed/questions_lengths_val2014.txt', 'w') answers_file = open('../data/preprocessed/answers_val2014.txt', 'w') coco_image_id = open('../data/preprocessed/images_val2014.txt', 'w') trainval = 'validation data' #initialize VQA api for QA annotations vqa = VQA(annFile, quesFile) questions = vqa.questions ques = questions['questions'] qa = vqa.qa pbar = progressbar.ProgressBar() print 'Dumping questions,answers, imageIDs, and questions lenghts to text files...' for i, q in pbar(zip(xrange(1, len(ques) + 1), ques)): questions_file.write(q['question'].encode('utf8')) questions_file.write('\n'.encode('utf8')) questions_lengths_file.write( str(len(nlp(q['question']))).encode('utf8')) questions_lengths_file.write('\n'.encode('utf8')) coco_image_id.write(str(q['image_id']).encode('utf8')) coco_image_id.write('\n') if args.isTrain: answers_file.write( getModalAnswer(qa[q['question_id']]['answers']).encode('utf8')) else: answers_file.write( getAllAnswer(qa[q['question_id']]['answers']).encode('utf8')) answers_file.write('\n'.encode('utf8')) print 'completed dumping', trainval
def evaluate(predicted_json_path, ann_path, ques_path): """revised from official evaluation code Args: result_path: predicted result in json format. ann_path: annotation_file path. ques_path: question_file path. result_dir_path: if given, save the evalutation result to the dir path. """ from vqa import VQA from vqaEval import VQAEval vqa = VQA(ann_path, ques_path) result = vqa.loadRes(predicted_json_path, ques_path) vqa_eval = VQAEval(vqa, result, n=2) vqa_eval.evaluate() print("\nOverall Accuracy is: %.02f" % (vqa_eval.accuracy['overall'])) print("Per Question Type Accuracy is the following:") for quesType in vqa_eval.accuracy['perQuestionType']: print("%s: %.02f" % (quesType, vqa_eval.accuracy['perQuestionType'][quesType])) print("Per Answer Type Accuracy is the following:") for ansType in vqa_eval.accuracy['perAnswerType']: print("%s: %.02f" % (ansType, vqa_eval.accuracy['perAnswerType'][ansType])) result_dir_path = predicted_json_path + "_eval" if result_dir_path is not None: if not os.path.exists(result_dir_path): os.makedirs(result_dir_path) json.dump(vqa_eval.accuracy, open(os.path.join(result_dir_path, 'accuracy'), 'w')) json.dump(vqa_eval.evalQA, open(os.path.join(result_dir_path, 'evalQA'), 'w')) json.dump(vqa_eval.evalQuesType, open(os.path.join(result_dir_path, 'evalQuesType'), 'w')) json.dump(vqa_eval.evalAnsType, open(os.path.join(result_dir_path, 'evalAnsType'), 'w'))
def __init__(self, image_dir, question_json_file_path, annotation_json_file_path, image_filename_pattern): self.vqa = VQA(annotation_json_file_path, question_json_file_path) self.idx2key = { i: self.vqa.qa[v]['question_id'] for i, v in enumerate(self.vqa.qa) } self.imgdict = { idt: image_dir + '/' + image_filename_pattern.format(str(idt).zfill(12)) for idt in self.vqa.imgToQA } self.imgkeys = sorted(list(self.imgdict.keys())) pass
class VQA_Evaluator: def __init__(self, summary_writer=None, dataDir='/auto/homes/bat34/VQA', versionType='v2_', taskType='OpenEnded', dataType='mscoco', dataSubType='val2014'): self.writer = summary_writer self.versionType = versionType self.taskType = taskType self.dataType = dataType self.dataSubType = dataSubType self.annFile = '%s/Annotations/%s%s_%s_annotations.json' % ( dataDir, versionType, dataType, dataSubType) self.quesFile = '%s/Questions/%s%s_%s_%s_questions.json' % ( dataDir, versionType, taskType, dataType, dataSubType) self.vqa = VQA(self.annFile, self.quesFile) def evaluate(self, resFile, epoch): vqaRes = self.vqa.loadRes(resFile, self.quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval(self.vqa, vqaRes, n=2) vqaEval.evaluate() print("\n") print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) if self.writer: self.writer.add_scalar('validation/overall_accuracy', vqaEval.accuracy['overall'], epoch) print("Per Question Type Accuracy is the following:") for quesType in vqaEval.accuracy['perQuestionType']: print("%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType])) if self.writer: self.writer.add_scalar( 'validation/%s' % quesType, vqaEval.accuracy['perQuestionType'][quesType], epoch) print("\n") print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) if self.writer: self.writer.add_scalar( 'validation/%s' % ansType, vqaEval.accuracy['perAnswerType'][ansType], epoch) print("\n") return float(vqaEval.accuracy['overall'])
def __init__(self, image_dir, question_json_file_path, annotation_json_file_path, image_filename_pattern): """ Args: image_dir (string): Path to the directory with COCO images question_json_file_path (string): Path to the json file containing the question data annotation_json_file_path (string): Path to the json file containing the annotations mapping images, questions, and answers together image_filename_pattern (string): The pattern the filenames of images in this dataset use (eg "COCO_train2014_{}.jpg") """ self.vqa = VQA(annotation_json_file_path, question_json_file_path) self.idx2key = {i:self.vqa.qa[v]['question_id'] for i,v in enumerate(self.vqa.qa)} if 'val2014' in image_filename_pattern: self.toloadpath = './student_code/supportfiles/ResNetDataVal/feat_' if 'train2014' in image_filename_pattern: self.toloadpath = './student_code/supportfiles/ResNetData/feat_' # self.imgdict = {idt:image_dir+'/'+image_filename_pattern.format(str(idt).zfill(12)) for idt in self.vqa.imgToQA} # self.AnsWords = getAnsWords(path='./student_code/supportfiles/CoAttAns.d') # self.AW2ID = {w:i for i,w in enumerate(self.AnsWords)} # self.W2ID, self.EMB = GetEmbeddings(path='./student_code/supportfiles/GloVe300.d') pass
if s % steps_per_report == 0: print('Steps {} Loss {:.4f}'.format(s, train_loss.result())) self.train_step(self.train_iter.next()) print('Steps {} Loss {:.4f}'.format(steps, train_loss.result())) self.model.save() print('model saved') print('training finished') if __name__ == "__main__": #train_data = VQA(r'D:\documents\coding\Data\coco\v2_mscoco_train2014_annotations.json', #r'D:\documents\coding\Data\coco\v2_OpenEnded_mscoco_train2014_questions.json', #r'D:\documents\coding\Data\coco\train2014\COCO_train2014_{0}.jpg', #r'D:\documents\coding\Data\coco\v2_mscoco_train2014_complementary_pairs.json') train_data = VQA( r'D:\lgy\Document\Python\Data\coco\v2_mscoco_train2014_annotations.json', r'D:\lgy\Document\Python\Data\coco\v2_OpenEnded_mscoco_train2014_questions.json', r'D:\lgy\Document\Python\Data\coco\train2014\COCO_train2014_{0}.jpg') train_iter = VQAIter(train_data, train_data.getQuesIds(ansTypes=['other', 'yes/no']), hp.batch_size, hp.num_chunks) max_qst_len = hp.max_qst_len max_ans_len = hp.max_ans_len model = Transformer(hp.num_layers, hp.d_model, hp.num_heads, hp.dff, max_qst_len + 3, hp.dropout_rate) trainer = Trainer(train_iter, model, 16, max_qst_len, max_ans_len) trainer.train(hp.steps, hp.steps_per_save, hp.steps_per_chunk, hp.steps_per_report)
# MultipleChoice_mscoco_analysis1_second_results_30 imageIDMap = {} resultIDMap = {} print resFile # annReader = open(annFile) # annotations = json.load(annReader) # quesReader = open(quesFile) # questions = json.load(quesReader) resReader = open(resFile) results = json.load(resReader) prevImageID = "" correct = 0.0 total = 0.0 vqa = VQA(annFile, quesFile) for quesDict in results: total += 1 id = int(vqa.qa[quesDict['question_id']]['image_id']) imageIDMap[id] = 0 resultIDMap[id] = 0 for quesDict in results: id = int(vqa.qa[quesDict['question_id']]['image_id']) # print id imageIDMap[id] = imageIDMap[id] + 1 dictLen = len(imageIDMap) for quesDict in results: id = int(vqa.qa[quesDict['question_id']]['image_id'])
from string import punctuation from collections import defaultdict import os import pdb import json DIR = "/home/shashank/data/VQA/dataset/VQAorg" ANNOTATION_TRAIN_PATH = '%s/Annotations/mscoco_train2014_annotations.json' % ( DIR) ANNOTATION_VAL_PATH = '%s/Annotations/mscoco_val2014_annotations.json' % (DIR) QUES_TRAIN_PATH = '%s/Questions/MultipleChoice_mscoco_train2014_questions.json' % ( DIR) QUES_VAL_PATH = '%s/Questions/MultipleChoice_mscoco_val2014_questions.json' % ( DIR) GLOVE_PATH = '%s/WordEmbeddings/glove.6B.100d.txt' % (DIR) vqa_train = VQA(ANNOTATION_TRAIN_PATH, QUES_TRAIN_PATH) vqa_val = VQA(ANNOTATION_VAL_PATH, QUES_VAL_PATH) vocab = {} vocab_size = 0 embedding_dim = 100 def filter_text(text): text = text.lower() text = ''.join([c for c in text if c not in punctuation]) return text def parse_QA(ques_type='yes/no'): """ Returns a list of all questions given the question type
dataset_root_dir = './datasets/' log_format = '{}-log-epoch-{:02}.txt' ckpt_format = 'model-epoch-{:02d}.ckpt' data_loader = get_loader(input_dir=dataset_root_dir, input_vqa_train='train.npy', input_vqa_valid='valid.npy', max_qst_length=30, max_num_ans=10, batch_size=batch_size, num_workers=8) qst_vocab_size = data_loader['train'].dataset.qst_vocab.vocab_size ans_vocab_size = data_loader['train'].dataset.ans_vocab.vocab_size model = VQA(activation, dropout, combination, ans_vocab_size, qst_vocab_size).to(device) optimizer = torch.optim.Adam(model.parameters(), lr=1e-3) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1) for epoch in range(epochs): for phase in ['train', 'valid']: running_loss = 0.0 running_corr_exp = 0 batch_step_size = len(data_loader[phase].dataset) / batch_size if phase == 'train': scheduler.step() model.train() else:
image_feat[0:len(train_image_ids)] = train_image_feature image_feat[len(train_image_ids):] = val_image_feature print 'finished processing features' ################################ # # process the train question # ################################ taskType = 'OpenEnded' dataType = 'mscoco' dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json'%(dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType) imgDir ='%s/Images/%s/' %(dataDir, dataSubType) vqa = VQA(annFile, quesFile) train_question_ids = vqa.getQuesIds() train_image_ids = vqa.getImgIds() train_questions = [] train_answers = [] question_dict_count = dict() answer_dict_count = dict() for idx, q_id in enumerate(train_question_ids): question = vqa.loadQuestion(q_id)[0] question = process_sentence(question) question = question.split() for word in question: question_dict_count[word] = question_dict_count.get(word, 0) + 1 answer = vqa.loadAnswer(q_id)[0] answer_new = [process_answer(ans) for ans in answer]
dataDir = '../../VQA' versionType = 'v2_' # this should be '' when not using VQA v2.0 dataset # 'OpenEnded' only for v2.0. 'OpenEnded' or 'MultipleChoice' for v1.0 taskType = 'OpenEnded' # 'mscoco' only for v1.0. 'mscoco' for real and 'abstract_v002' for abstract for v1.0. dataType = 'mscoco' dataSubType = 'train2014' annFile = '%s/Annotations/%s%s_%s_annotations.json' % (dataDir, versionType, dataType, dataSubType) quesFile = '%s/Questions/%s%s_%s_%s_questions.json' % ( dataDir, versionType, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) # initialize VQA api for QA annotations vqa = VQA(annFile, quesFile) # load and display QA annotations for given question types """ All possible quesTypes for abstract and mscoco has been provided in respective text files in ../QuestionTypes/ folder. """ annIds = vqa.getQuesIds(quesTypes='how many') anns = vqa.loadQA(annIds) randomAnn = random.choice(anns) vqa.showQA([randomAnn]) imgId = randomAnn['image_id'] imgFilename = 'COCO_' + dataSubType + '_' + str(imgId).zfill(12) + '.jpg' if os.path.isfile(imgDir + imgFilename): I = io.imread(imgDir + imgFilename) plt.imshow(I) plt.axis('off')
resultType = 'fake' fileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. # [resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/Results/%s_%s_%s_%s_%s.json'%(dataDir, taskType, dataType, dataSubType, \ # resultType, fileType) for fileType in fileTypes] results_dir = '/home/tanmay/Code/GenVQA/Exp_Results/VQA/QA_explicit_dot_same_lr/answer_classifiers' [resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [ '{}/eval_val_rest_{}.json'.format(results_dir, fileType) for fileType in fileTypes ] # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) #n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ anns = json.load(open(resFile)) quesIds = [ann['question_id'] for ann in anns] vqaEval.evaluate(quesIds)
print 'finished processing features' ################################ # # process the train question # ################################ taskType = 'OpenEnded' dataType = 'mscoco' dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/' % (dataDir, dataSubType) vqa = VQA(annFile, quesFile) train_question_ids = vqa.getQuesIds() train_image_ids = vqa.getImgIds() train_questions = [] train_answers = [] question_dict_count = dict() answer_dict_count = dict() for idx, q_id in enumerate(train_question_ids): question = vqa.qqa[q_id]['question'] question = process_sentence(question) question = question.split() for word in question: question_dict_count[word] = question_dict_count.get(word, 0) + 1 answer = vqa.loadQA(q_id)[0]['answers'][0]['answer'] answer_new = [process_answer(ans) for ans in answer]
annFile ='%s/Val/%s_%s_annotations.json'%(dataDir, dataType, dataSubType) quesFile ='%s/Val/%s_%s_%s_questions.json'%(dataDir, taskType, dataType, dataSubType) imgDir ='%s/val2014/' %(dataDir) resultType ='results.json' fileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # In[3]: [resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['results.json' for fileType in fileTypes] # In[4]: # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # In[5]: # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval(vqa, vqaRes, n=2) #n is precision of accuracy (number of places after decimal), default is 2 # In[6]: # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file
def evaluate_model(resFile, quest_ids, subset='val', version='v1'): ans_type = None # set up file names and paths taskType = 'OpenEnded' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = '%s2014' % subset if version == 'v1': annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % ( dataDir, taskType, dataType, dataSubType) elif version == 'v2': anno_dir = '/import/vision-ephemeral/fl302/data/VQA2.0' annFile = '%s/v2_%s_%s_annotations.json' % (anno_dir, dataType, dataSubType) quesFile = '%s/v2_%s_%s_%s_questions.json' % (anno_dir, taskType, dataType, dataSubType) else: raise Exception('unknown version, v1 or v2') imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) resultType = 'fake' fileTypes = ['accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. [accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = [ '%s/Results/%s_%s_%s_%s_%s.json' % (dataDir, taskType, dataType, dataSubType, \ resultType, fileType) for fileType in fileTypes] # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) # n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ vqaEval.evaluate(quesIds=quest_ids) # print accuracies print "\n" print "Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall']) print "Per Question Type Accuracy is the following:" for quesType in vqaEval.accuracy['perQuestionType']: print "%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType]) print "\n" print "Per Answer Type Accuracy is the following:" for ansType in vqaEval.accuracy['perAnswerType']: print "%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType]) print "\n" if ans_type is None: return vqaEval.accuracy['overall'], vqaEval.accuracy['perAnswerType'] else: return vqaEval.accuracy['overall'], vqaEval.accuracy['perAnswerType'][ ans_type]
def get_vqa(self, vqa_dir, split): if split not in self._instances: que_fname = self.get_que_path(vqa_dir, split) ann_fname = self.get_ann_path(vqa_dir, split) self._instances[split] = VQA(ann_fname, que_fname) return self._instances[split]
import random import os # set up file names and paths dataDir='../' split = 'test' annFile='%s/Annotations/%s.json'%(dataDir, split) imgDir = '%s/Images/' %dataDir # An example result json file has been provided in './Results' folder. resultType ='fake' fileTypes = ['results', 'accuracy', 'captionMetric', 'evalQA', 'evalAnsType', 'answerability'] [resFile, accuracyFile, captionMetricFile, evalQAFile, evalAnsTypeFile, answerabilityFile] = ['%s/Results/%s_%s_%s.json'%(dataDir, split, resultType, fileType) for fileType in fileTypes] # create vqa object and vqaRes object vqa = VQA(annFile) vqaRes = VQA(resFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval(vqa, vqaRes, n=2) #n is precision of accuracy (number of places after decimal), default is 2 """ If you have a list of images on which you would like to evaluate your results, pass it as a list to below function By default it uses all the images in annotation file """ # evaluate VQA results vqaEval.evaluate() # print accuracies
import os import operator import numpy as np from six.moves import cPickle as pickle from collections import defaultdict from vqa import VQA image_dir = "./data/train2014" img_prefix = "COCO_train2014_" qjson = "./data/v2_OpenEnded_mscoco_train2014_questions.json" ajson = "./data/v2_mscoco_train2014_annotations.json" vqa = VQA(ajson, qjson) img_names = [f for f in os.listdir(image_dir) if '.jpg' in f] img_names = img_names[:30000] print("len ",len(img_names)) img_ids = [] for fname in img_names: img_id = fname.split('.')[0].rpartition(img_prefix)[-1] img_ids.append(int(img_id)) ques_ids = vqa.getQuesIds(img_ids) q2i = defaultdict(lambda: len(q2i)) pad = q2i["<pad>"] start = q2i["<sos>"] end = q2i["<eos>"] UNK = q2i["<unk>"]
""" # coding: utf-8 import multiprocessing import sys dataDir = '../../VQA' sys.path.insert(0, '%s/PythonHelperTools/vqaTools' % (dataDir)) from vqa import VQA from vqaEvaluation.vqaEval import VQAEval import os import time annFile = sys.argv[1] quesFile = sys.argv[2] resFile = sys.argv[3] vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) vqaEval = VQAEval(vqa, vqaRes, n=2) all_qids = vqa.getQuesIds() def vqaeval(iter): qid = all_qids[iter] qid_list = [] qid_list.append(qid) vqaEval.evaluate(qid_list) qid_acc_dict = {qid: vqaEval.accuracy['overall']} return qid_acc_dict def reduce_acc(results_list):
dataSubType = args.subtype annFile = '%s/raw/annotations/%s_%s_annotations.json' % (args.dirvqa, dataType, dataSubType) quesFile = '%s/raw/annotations/%s_%s_%s_questions.json' % ( args.dirvqa, taskType, dataType, dataSubType) #imgDir ='/local/cadene/data/raw/%s/%s/' %(dataType, dataSubType) fileTypes = ['results', 'accuracy', 'evalQA', 'evalQuesType', 'evalAnsType'] # An example result json file has been provided in './Results' folder. [resFile, accuracyFile, evalQAFile, evalQuesTypeFile, evalAnsTypeFile] = ['%s/%s_%s_%s_%s_%s.json'%(args.direpoch, taskType, dataType, dataSubType, \ resultType, fileType) for fileType in fileTypes] # create vqa object and vqaRes object vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) #n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ # !!!SPECIFY quesIds!!! # utilise plutot le evaluate.py que j'avais fait pour le code tensorflow
class VqaDataset(Dataset): def __init__(self, image_dir, question_json_file_path, annotation_json_file_path, image_filename_pattern, collate=False, q2i=None, a2i=None, i2a=None, a2i_count=None, img_names=None, img_ids=None, ques_ids=None, method='simple', dataset_type='train', enc_dir=''): print(method) self.image_dir = image_dir self.qjson = question_json_file_path self.ajson = annotation_json_file_path img_prefix = image_filename_pattern.split('{}')[0] self.collate = collate self.q2i = q2i self.a2i = a2i self.i2a = i2a self.a2i_count = a2i_count self.img_ids = img_ids self.ques_ids = ques_ids self.img_names = img_names self.method = method self.vqa = VQA(self.ajson, self.qjson) if self.method == 'simple': self.transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) else: self.transform = transforms.Compose( [transforms.Resize((448, 448)), transforms.ToTensor()]) if not collate: self.img_names = [ f for f in os.listdir(self.image_dir) if '.jpg' in f ] #print(self.img_names) self.img_ids = [] for fname in self.img_names: img_id = fname.split('.')[0].rpartition(img_prefix)[-1] self.img_ids.append(int(img_id)) self.ques_ids = self.vqa.getQuesIds(self.img_ids) self.q2i, self.a2i, self.i2a, self.a2i_count = pre_process_dataset( image_dir, self.qjson, self.ajson, img_prefix) #print("Image names ",self.img_names) self.q2i_len = len(self.q2i) self.a2i_len = len(self.a2i.keys()) self.q2i_keys = self.q2i.keys() self.enc_dir = enc_dir def __len__(self): return len(self.ques_ids) def __getitem__(self, idx): ques_id = self.ques_ids[idx] img_id = self.vqa.getImgIds([ques_id])[0] qa = self.vqa.loadQA(ques_id)[0] qqa = self.vqa.loadQQA(ques_id)[0] img_name = self.img_names[self.img_ids.index(img_id)] if self.method == 'simple': img = default_loader(self.image_dir + '/' + img_name) #imgT = self.transform(img).permute(1, 2, 0) imgT = self.transform(img).float() else: img = default_loader(self.image_dir + '/' + img_name) imgT = self.transform(img).float() ques = qqa['question'][:-1] quesI = [self.q2i["<sos>"]] + [ self.q2i[x.lower()] for x in ques.split(" ") if x.lower() in self.q2i_keys ] + [self.q2i["<eos>"]] if not self.collate: quesI = quesI + [self.q2i["<pad>"]] * (8 - len(quesI)) if self.method == 'simple': quesT = torch.zeros(self.q2i_len).float() for idx in quesI: quesT[idx] = 1 else: quesT = torch.from_numpy(np.array(quesI)).long() answers = qa['answers'] max_count = 0 answer = "" for ans in answers: #if not ans['answer_confidence'] == 'yes': # continue ans = ans['answer'].lower() if ans in self.a2i.keys() and self.a2i_count[ans] > max_count: max_count = self.a2i_count[ans] answer = ans if answer == "": # only for validation gT = torch.from_numpy(np.array([self.a2i_len])).long() else: gT = torch.from_numpy(np.array([self.a2i[answer]])).long() if not self.collate: return {'img': imgT, 'ques': quesT, 'gt': gT} return imgT, quesT, gT, img_name, ques, answer