def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'analysis1' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % ( dataDir2, taskType2, dataType2, dataSubType2) imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] # print question questionVector = ld.getBOWVector( question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose=False) temp_dict['answer'] = answerFeatures[predictions[0]] resultsDicts.append(temp_dict) writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json', 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
outputQuestions['task_type'] = taskType outputQuestions['data_type'] = dataType outputQuestions['license'] = {} outputQuestions['data_subtype'] = 'analysis1' outputQuestions['questions'] = [] else: outputQuestionWriter = open(outputQuestionFile) outputAnnotationWriter = open(outputAnnotationFile) # question_json_reader = outputQuestionWriter.read() outputQuestions = json.load(outputQuestionWriter) # annotations_json_reader = outputAnnotationWriter.read() outputAnnotations = json.load(outputAnnotationWriter) glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] while newQuestion != 'no': print '\n' randomAnn = random.choice(anns) origquestion = vqaVal.qqa[randomAnn['question_id']] questionVector = ld.getBOWVector( origquestion['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = randomAnn['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) # temp_dict = {} ansString = randomAnn['multiple_choice_answer'] # temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures)
def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'val2014' # number = '100' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2) resultFile = './../Results/MultipleChoice_mscoco_analysis1_second_results.json' imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] choicesList = vqaVal.qqa[quesID]['multiple_choices'] # print choicesList setChoices = set(choicesList) setAnswers = set(answerFeatures) choiceAndAnswer = list(setChoices.intersection(setAnswers)) choiceIndex = [] for choice in choiceAndAnswer: choiceIndex.append(answerFeatures.index(choice)) #print choiceIndex questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose = False) predict_probaResult = model.predict_proba(x_test,verbose = False) # print "###############Sanity Check############" # print predict_probaResult.size # print predict_probaResult # print predict_probaResult[7] # print predict_probaResult maxPred = 0.0 # print "#######################################" print choiceIndex for item in choiceIndex: print len(choiceIndex), item,answerFeatures[item] for item in choiceIndex: print item,answerFeatures[item],predict_probaResult[0][item] if(maxPred < predict_probaResult[0][item]): maxPred = predict_probaResult[0][item] maxIndex = item print maxPred, maxIndex, answerFeatures[maxIndex] # temp_dict['answer'] = answerFeatures[predictions[0]] temp_dict['answer'] = answerFeatures[maxIndex] resultsDicts.append(temp_dict) writer = open(resultFile, 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
def main(): glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = readGloveData(glove_word_vec_file) vqaTrain = VQA(annFile, quesFile) annotations = vqaTrain.dataset['annotations'] questions = vqaTrain.questions['questions'] answerFeatures = createAnswerFeatures(annotations) # Dumping answer features answer_features_list = open('answer_feature_list.json', 'w') answer_features_list.write(json.dumps(answerFeatures)) # For getting image vectors imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] data = [] X_train = [] Y_train = [] X_test = [] Y_test = [] FILE_INDEX = 0 for question in questions: # quesItem = {} # print question questionVector = getBOWVector( question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = question['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) # quesItem['image_id'] = imgID # quesItem['question'] = question['question'].replace('?', ' ?').split(' ') annotations = vqaTrain.loadQA(ids=[question['question_id']]) for annotation in annotations: ansString = annotation['multiple_choice_answer'] answerVector = getAnswerVector(ansString, answerFeatures) temp_X_train = np.append(imageVector, questionVector) temp_Y_train = answerVector X_train.append(temp_X_train) Y_train.append(temp_Y_train) if len(X_train) >= FILE_LIMIT: train_x_file = open( FILE_PATH + X_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w') train_y_file = open( FILE_PATH + Y_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w') np.save(train_x_file, X_train) np.save(train_y_file, Y_train) X_train = [] Y_train = [] FILE_INDEX = FILE_INDEX + 1 # print len(X_train) # if len(annotations) != 1: # print imgID, " has annotations ", len(annotations) # for ann in annotations: # quesItemCopy = dict(quesItem) # ansString = ann['multiple_choice_answer'] # quesItemCopy['answer'] = ansString # data.append(quesItemCopy) if len(X_train) > 0: train_x_file = open( FILE_PATH + X_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w') train_y_file = open( FILE_PATH + Y_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w') np.save(train_x_file, X_train) np.save(train_y_file, Y_train) X_train = [] Y_train = []
filepath="weights.{epoch:02d}-{val_loss:.2f}.hdf5", save_best_only=True) tfile = '../features/coco_vgg_IDMap.txt' args = {} args['answer_vector_file'] = 'answer_feature_list.json' args['glove_file'] = '../glove/glove.6B.300d.txt' # In[2]: print "Reading GloVE and VGG raw files" glove_word_vec_file = args['glove_file'] word_vec_dict = utilities.readGloveData(glove_word_vec_file) imageDict = cocoImageUtils.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] print "Reading the data and creating features" answer_vector_file = open(args['answer_vector_file'], 'r') answerFeatureVector = json.loads(answer_vector_file.read()) answer_vector_file.close() # In[3]: import sys sys.path.insert(0, './../VQA/PythonHelperTools') from vqaTools.vqa import VQA
def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'analysis1' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2) imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] # print question questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose = False) temp_dict['answer'] = answerFeatures[predictions[0]] resultsDicts.append(temp_dict) writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json', 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
outputQuestions["data_type"] = dataType outputQuestions["license"] = {} outputQuestions["data_subtype"] = "analysis1" outputQuestions["questions"] = [] else: outputQuestionWriter = open(outputQuestionFile) outputAnnotationWriter = open(outputAnnotationFile) # question_json_reader = outputQuestionWriter.read() outputQuestions = json.load(outputQuestionWriter) # annotations_json_reader = outputAnnotationWriter.read() outputAnnotations = json.load(outputAnnotationWriter) glove_word_vec_file = "./../glove/glove.6B.300d.txt" word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat("./../features/coco/vgg_feats.mat")["feats"] while newQuestion != "no": print "\n" randomAnn = random.choice(anns) origquestion = vqaVal.qqa[randomAnn["question_id"]] questionVector = ld.getBOWVector(origquestion["question"].strip().replace("?", " ?").split(), word_vec_dict) imgID = randomAnn["image_id"] imageVector = np.asarray(feats[:, imageDict[imgID]]) # temp_dict = {} ansString = randomAnn["multiple_choice_answer"] # temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector