outputQuestions['info']['date_created'] = '2015-12-11' outputQuestions['task_type'] = taskType outputQuestions['data_type'] = dataType outputQuestions['license'] = {} outputQuestions['data_subtype'] = 'analysis1' outputQuestions['questions'] = [] else: outputQuestionWriter = open(outputQuestionFile) outputAnnotationWriter = open(outputAnnotationFile) # question_json_reader = outputQuestionWriter.read() outputQuestions = json.load(outputQuestionWriter) # annotations_json_reader = outputAnnotationWriter.read() outputAnnotations = json.load(outputAnnotationWriter) glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] while newQuestion != 'no': print '\n' randomAnn = random.choice(anns) origquestion = vqaVal.qqa[randomAnn['question_id']] questionVector = ld.getBOWVector( origquestion['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = randomAnn['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) # temp_dict = {} ansString = randomAnn['multiple_choice_answer'] # temp_dict['question_id'] = quesID
def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'analysis1' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % ( dataDir2, taskType2, dataType2, dataSubType2) imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] # print question questionVector = ld.getBOWVector( question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose=False) temp_dict['answer'] = answerFeatures[predictions[0]] resultsDicts.append(temp_dict) writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json', 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'val2014' # number = '100' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2) resultFile = './../Results/MultipleChoice_mscoco_analysis1_second_results.json' imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] choicesList = vqaVal.qqa[quesID]['multiple_choices'] # print choicesList setChoices = set(choicesList) setAnswers = set(answerFeatures) choiceAndAnswer = list(setChoices.intersection(setAnswers)) choiceIndex = [] for choice in choiceAndAnswer: choiceIndex.append(answerFeatures.index(choice)) #print choiceIndex questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose = False) predict_probaResult = model.predict_proba(x_test,verbose = False) # print "###############Sanity Check############" # print predict_probaResult.size # print predict_probaResult # print predict_probaResult[7] # print predict_probaResult maxPred = 0.0 # print "#######################################" print choiceIndex for item in choiceIndex: print len(choiceIndex), item,answerFeatures[item] for item in choiceIndex: print item,answerFeatures[item],predict_probaResult[0][item] if(maxPred < predict_probaResult[0][item]): maxPred = predict_probaResult[0][item] maxIndex = item print maxPred, maxIndex, answerFeatures[maxIndex] # temp_dict['answer'] = answerFeatures[predictions[0]] temp_dict['answer'] = answerFeatures[maxIndex] resultsDicts.append(temp_dict) writer = open(resultFile, 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
erl = EarlyStopping(monitor='val_loss', patience=3) checkpoint = ModelCheckpoint( filepath="weights.{epoch:02d}-{val_loss:.2f}.hdf5", save_best_only=True) tfile = '../features/coco_vgg_IDMap.txt' args = {} args['answer_vector_file'] = 'answer_feature_list.json' args['glove_file'] = '../glove/glove.6B.300d.txt' # In[2]: print "Reading GloVE and VGG raw files" glove_word_vec_file = args['glove_file'] word_vec_dict = utilities.readGloveData(glove_word_vec_file) imageDict = cocoImageUtils.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] print "Reading the data and creating features" answer_vector_file = open(args['answer_vector_file'], 'r') answerFeatureVector = json.loads(answer_vector_file.read()) answer_vector_file.close() # In[3]: import sys sys.path.insert(0, './../VQA/PythonHelperTools')
def evalResults(): dataDir = './../VQA' taskType = 'MultipleChoice' dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType = 'train2014' annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType) quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType) imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType) vqaTrain = VQA(annFile, quesFile) dummyano = vqaTrain.dataset['annotations'] answerFeatures = ld.createAnswerFeatures(dummyano) dataDir2 = './../VQA' taskType2 = 'MultipleChoice' dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract dataSubType2 = 'analysis1' annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2) quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2) imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2) modelReader = open('./model_definition_100iter.json') json_read = modelReader.read() model = model_from_json(json_read) model.load_weights('./model_weights_100iter.h5py') vqaVal = VQA(annFile2, quesFile2) FILE_INDEX = 0 total = 0.0 correct = 0.0 resultsDicts = [] x_test = [] y_test = [] glove_word_vec_file = './../glove/glove.6B.300d.txt' word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats'] for quesID, annotation in vqaVal.qa.iteritems(): # print quesID # if quesID not in vqaVal.qqa.keys(): # continue question = vqaVal.qqa[quesID] # print question questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict) imgID = annotation['image_id'] imageVector = np.asarray(feats[:, imageDict[imgID]]) temp_dict = {} ansString = annotation['multiple_choice_answer'] temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector) # temp_y_test = answerVector x_test = np.asarray([temp_x_test]) # y_test = np.asarray([temp_y_test]) predictions = model.predict_classes(x_test, verbose = False) temp_dict['answer'] = answerFeatures[predictions[0]] resultsDicts.append(temp_dict) writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json', 'w') json_dump = json.dumps(resultsDicts) writer.write(json_dump)
outputQuestions["task_type"] = taskType outputQuestions["data_type"] = dataType outputQuestions["license"] = {} outputQuestions["data_subtype"] = "analysis1" outputQuestions["questions"] = [] else: outputQuestionWriter = open(outputQuestionFile) outputAnnotationWriter = open(outputAnnotationFile) # question_json_reader = outputQuestionWriter.read() outputQuestions = json.load(outputQuestionWriter) # annotations_json_reader = outputAnnotationWriter.read() outputAnnotations = json.load(outputAnnotationWriter) glove_word_vec_file = "./../glove/glove.6B.300d.txt" word_vec_dict = ld.readGloveData(glove_word_vec_file) imageDict = pramod.generateDictionary(tfile) feats = sio.loadmat("./../features/coco/vgg_feats.mat")["feats"] while newQuestion != "no": print "\n" randomAnn = random.choice(anns) origquestion = vqaVal.qqa[randomAnn["question_id"]] questionVector = ld.getBOWVector(origquestion["question"].strip().replace("?", " ?").split(), word_vec_dict) imgID = randomAnn["image_id"] imageVector = np.asarray(feats[:, imageDict[imgID]]) # temp_dict = {} ansString = randomAnn["multiple_choice_answer"] # temp_dict['question_id'] = quesID # answerVector = ld.getAnswerVector(ansString, answerFeatures) temp_x_test = np.append(imageVector, questionVector)