def validateInternalTestSet(args): from vqaTools.vqaInternal import VQA from vqaTools.vqaEval import VQAEval #config = Attention_LapConfig(load=True, args) config = BOWIMG_GPUConfig(load=True, args=args) restoreModel = config.restoreModel restoreModelPath = config.restoreModelPath print('Running Validation Test on Model') valTestReader = BOWIMGProcessor(config.testAnnotFile, config.rawQnValTestFile, config.testImgFile, config, is_training=False) print('Using BOWIMG model') model = BOWIMGModel(config) model.loadTrainedModel(restoreModel, restoreModelPath) predFile = '{}PredsBOW.csv'.format(restoreModelPath) results, strictAcc = model.runPredict(valTestReader, predFile) model.destruct() valTestReader.destruct() print('predictions made') vqa = VQA(config.testAnnotFileUnresolved, config.originalValQns) vqaRes = vqa.loadRes(results, config.originalValQns) vqaEval = VQAEval(vqa, vqaRes, n=2) vqaEval.evaluate() print('Writing to file..') writeToFile(vqaEval, restoreModelPath, vqa, vqaRes, strictAcc)
def internalValTest(args): import sys sys.path.insert( 0, '/home/jwong/Documents/LinuxWorkspace/Visual-Question-Answering') from vqaTools.vqa import VQA from vqaTools.vqaEval import VQAEval config = BOWIMG_LapConfig(load=False, args=args) annFile = config.originalAnnotVal quesFile = config.valTestQns resFile = 'testResFile.json' vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) vqaEval = VQAEval(vqa, vqaRes, n=2) vqaEval.evaluate() # print accuracies print "\n" print "Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall']) print "Per Question Type Accuracy is the following:" for quesType in vqaEval.accuracy['perQuestionType']: print "%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType]) print "\n" print "Per Answer Type Accuracy is the following:" for ansType in vqaEval.accuracy['perAnswerType']: print "%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType]) print "\n"
def runVal(self, valReader, nEpoch, is_training=True): """Evaluates performance on val set Args: valReader: Returns: metrics: """ accuracies, res, val_losses = [], [], [] correct_predictions, total_predictions = 0., 0. for qnAsWordIDsBatch, seqLens, img_vecs, labels, rawQns, img_ids, qn_ids in \ valReader.getNextBatch(self.config.batch_size): feed = { self.word_ids: qnAsWordIDsBatch, self.sequence_lengths: seqLens, self.img_vecs: img_vecs, self.labels: labels, self.dropout: 1.0 } val_loss, labels_pred = self.sess.run( [self.loss, self.labels_pred], feed_dict=feed) for lab, labPred, qn, img_id, qn_id in zip(labels, labels_pred, rawQns, img_ids, qn_ids): if (lab == labPred): correct_predictions += 1 total_predictions += 1 accuracies.append(lab == labPred) currentPred = {} currentPred['question_id'] = qn_id currentPred['answer'] = self.classToAnsMap[labPred] res.append(currentPred) if not math.isnan(val_loss): val_losses.append(val_loss) epoch_valLoss = np.mean(val_losses) valAcc = np.mean(accuracies) vqaRes = self.vqa.loadRes(res, self.config.originalValQns) vqaEval = VQAEval(self.vqa, vqaRes, n=2) vqaEval.evaluate() return valAcc, correct_predictions, total_predictions, vqaEval.accuracy[ 'overall'], epoch_valLoss
def validateInternalTestSet(args, model=None, restoreModelPath=None): from vqaTools.vqaInternal import VQA from vqaTools.vqaEval import VQAEval #config = Attention_LapConfig(load=True, args) config = Attention_GPUConfig(load=True, args=args) print('Running Validation Test on Model') valTestReader = AttModelInputProcessor(config.testAnnotFile, config.rawQnValTestFile, config.valImgFile, config, is_training=False) if restoreModelPath is None: restoreModel = config.restoreModel restoreModelPath = config.restoreModelPath if model is None: if args.att == 'qn': print('Attention over question and image model') model = QnAttentionModel(config) elif args.att == 'im': print('Attention over image model') model = ImageAttentionModel(config) model.loadTrainedModel(restoreModel, restoreModelPath) predFile = '{}PredsAtt{}.csv'.format(restoreModelPath, args.att) results, strictAcc = model.runPredict(valTestReader, predFile) model.destruct() valTestReader.destruct() print('predictions made') vqa = VQA(config.testAnnotFileUnresolved, config.originalValQns) vqaRes = vqa.loadRes(results, config.originalValQns) vqaEval = VQAEval(vqa, vqaRes, n=2) vqaEval.evaluate() print('Writing to file..') writeToFile(vqaEval, restoreModelPath, vqa, vqaRes, args, strictAcc) print('Internal test complete')
def internalValTest(args): import sys #sys.path.insert(0, '/home/jwong/Documents/LinuxWorkspace/Visual-Question-Answering') from vqaTools.vqaInternal import VQA from vqaTools.vqaEval import VQAEval config = Attention_LapConfig(load=False, args=args) annFile = config.originalAnnotVal quesFile = config.valTestQns resFile = 'testResFile.json' vqa = VQA(annFile, quesFile) vqaRes = vqa.loadRes(resFile, quesFile) vqaEval = VQAEval(vqa, vqaRes, n=2) vqaEval.evaluate() # print accuracies print "\n" print "Overall Accuracy is: %.02f\n" %(vqaEval.accuracy['overall']) print "Per Question Type Accuracy is the following:" for quesType in vqaEval.accuracy['perQuestionType']: print "%s : %.02f" %(quesType, vqaEval.accuracy['perQuestionType'][quesType]) print "\n" print "Per Answer Type Accuracy is the following:" for ansType in vqaEval.accuracy['perAnswerType']: print "%s : %.02f" %(ansType, vqaEval.accuracy['perAnswerType'][ansType]) print "\n" # demo how to use evalQA to retrieve low score result evals = [quesId for quesId in vqaEval.evalQA if vqaEval.evalQA[quesId]<35] #35 is per question percentage accuracy if len(evals) > 0: print('ground truth answers') randomEval = random.choice(evals) # print('RandomEval {}'.format(randomEval)) randomAnn = vqa.loadQA(randomEval) qns, answers = vqa.showQA(randomAnn) print(qns) print(answers) img_ids = vqa.getImgIds(quesIds=[randomEval]) print(img_ids) print '\n' print 'generated answer (accuracy %.02f)'%(vqaEval.evalQA[randomEval]) ann = vqaRes.loadQA(randomEval)[0] print "Answer: %s\n" %(ann['answer']) #imgId = randomAnn[0]['image_id'] #imgFilename = 'COCO_' + dataSubType + '_'+ str(imgId).zfill(12) + '.jpg' #if os.path.isfile(imgDir + imgFilename): # I = io.imread(imgDir + imgFilename) # plt.imshow(I) # plt.axis('off') # plt.show() # plot accuracy for various question types plt.bar(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].values(), align='center') plt.xticks(range(len(vqaEval.accuracy['perQuestionType'])), vqaEval.accuracy['perQuestionType'].keys(), rotation='0',fontsize=10) plt.title('Per Question Type Accuracy', fontsize=10) plt.xlabel('Question Types', fontsize=10) plt.ylabel('Accuracy', fontsize=10) plt.show()