def evalResults():
dataDir = './../VQA'
taskType = 'MultipleChoice'
dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType = 'train2014'
annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType,
dataSubType)
quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType,
dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType)
vqaTrain = VQA(annFile, quesFile)
dummyano = vqaTrain.dataset['annotations']
answerFeatures = ld.createAnswerFeatures(dummyano)
dataDir2 = './../VQA'
taskType2 = 'MultipleChoice'
dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType2 = 'analysis1'
annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2,
dataSubType2)
quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (
dataDir2, taskType2, dataType2, dataSubType2)
imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2)
modelReader = open('./model_definition_100iter.json')
json_read = modelReader.read()
model = model_from_json(json_read)
model.load_weights('./model_weights_100iter.h5py')
vqaVal = VQA(annFile2, quesFile2)
FILE_INDEX = 0
total = 0.0
correct = 0.0
resultsDicts = []
x_test = []
y_test = []
glove_word_vec_file = './../glove/glove.6B.300d.txt'
word_vec_dict = ld.readGloveData(glove_word_vec_file)
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
for quesID, annotation in vqaVal.qa.iteritems():
# print quesID
# if quesID not in vqaVal.qqa.keys():
# continue
question = vqaVal.qqa[quesID]
# print question
questionVector = ld.getBOWVector(
question['question'].strip().replace('?', ' ?').split(),
word_vec_dict)
imgID = annotation['image_id']
imageVector = np.asarray(feats[:, imageDict[imgID]])
temp_dict = {}
ansString = annotation['multiple_choice_answer']
temp_dict['question_id'] = quesID
# answerVector = ld.getAnswerVector(ansString, answerFeatures)
temp_x_test = np.append(imageVector, questionVector)
# temp_y_test = answerVector
x_test = np.asarray([temp_x_test])
# y_test = np.asarray([temp_y_test])
predictions = model.predict_classes(x_test, verbose=False)
temp_dict['answer'] = answerFeatures[predictions[0]]
resultsDicts.append(temp_dict)
writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json',
'w')
json_dump = json.dumps(resultsDicts)
writer.write(json_dump)
outputQuestions['task_type'] = taskType
outputQuestions['data_type'] = dataType
outputQuestions['license'] = {}
outputQuestions['data_subtype'] = 'analysis1'
outputQuestions['questions'] = []
else:
outputQuestionWriter = open(outputQuestionFile)
outputAnnotationWriter = open(outputAnnotationFile)
# question_json_reader = outputQuestionWriter.read()
outputQuestions = json.load(outputQuestionWriter)
# annotations_json_reader = outputAnnotationWriter.read()
outputAnnotations = json.load(outputAnnotationWriter)
glove_word_vec_file = './../glove/glove.6B.300d.txt'
word_vec_dict = ld.readGloveData(glove_word_vec_file)
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
while newQuestion != 'no':
print '\n'
randomAnn = random.choice(anns)
origquestion = vqaVal.qqa[randomAnn['question_id']]
questionVector = ld.getBOWVector(
origquestion['question'].strip().replace('?', ' ?').split(),
word_vec_dict)
imgID = randomAnn['image_id']
imageVector = np.asarray(feats[:, imageDict[imgID]])
# temp_dict = {}
ansString = randomAnn['multiple_choice_answer']
# temp_dict['question_id'] = quesID
# answerVector = ld.getAnswerVector(ansString, answerFeatures)
def evalResults():
dataDir = './../VQA'
taskType = 'MultipleChoice'
dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType = 'train2014'
annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType)
quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType)
vqaTrain = VQA(annFile, quesFile)
dummyano = vqaTrain.dataset['annotations']
answerFeatures = ld.createAnswerFeatures(dummyano)
dataDir2 = './../VQA'
taskType2 = 'MultipleChoice'
dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType2 = 'val2014'
# number = '100'
annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2)
quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2)
resultFile = './../Results/MultipleChoice_mscoco_analysis1_second_results.json'
imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2)
modelReader = open('./model_definition_100iter.json')
json_read = modelReader.read()
model = model_from_json(json_read)
model.load_weights('./model_weights_100iter.h5py')
vqaVal = VQA(annFile2, quesFile2)
FILE_INDEX = 0
total = 0.0
correct = 0.0
resultsDicts = []
x_test = []
y_test = []
glove_word_vec_file = './../glove/glove.6B.300d.txt'
word_vec_dict = ld.readGloveData(glove_word_vec_file)
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
for quesID, annotation in vqaVal.qa.iteritems():
# print quesID
# if quesID not in vqaVal.qqa.keys():
# continue
question = vqaVal.qqa[quesID]
choicesList = vqaVal.qqa[quesID]['multiple_choices']
# print choicesList
setChoices = set(choicesList)
setAnswers = set(answerFeatures)
choiceAndAnswer = list(setChoices.intersection(setAnswers))
choiceIndex = []
for choice in choiceAndAnswer:
choiceIndex.append(answerFeatures.index(choice))
#print choiceIndex
questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict)
imgID = annotation['image_id']
imageVector = np.asarray(feats[:, imageDict[imgID]])
temp_dict = {}
ansString = annotation['multiple_choice_answer']
temp_dict['question_id'] = quesID
# answerVector = ld.getAnswerVector(ansString, answerFeatures)
temp_x_test = np.append(imageVector, questionVector)
# temp_y_test = answerVector
x_test = np.asarray([temp_x_test])
# y_test = np.asarray([temp_y_test])
predictions = model.predict_classes(x_test, verbose = False)
predict_probaResult = model.predict_proba(x_test,verbose = False)
# print "###############Sanity Check############"
# print predict_probaResult.size
# print predict_probaResult
# print predict_probaResult[7]
# print predict_probaResult
maxPred = 0.0
# print "#######################################"
print choiceIndex
for item in choiceIndex:
print len(choiceIndex), item,answerFeatures[item]
for item in choiceIndex:
print item,answerFeatures[item],predict_probaResult[0][item]
if(maxPred < predict_probaResult[0][item]):
maxPred = predict_probaResult[0][item]
maxIndex = item
print maxPred, maxIndex, answerFeatures[maxIndex]
# temp_dict['answer'] = answerFeatures[predictions[0]]
temp_dict['answer'] = answerFeatures[maxIndex]
resultsDicts.append(temp_dict)
writer = open(resultFile, 'w')
json_dump = json.dumps(resultsDicts)
writer.write(json_dump)
def main():
glove_word_vec_file = './../glove/glove.6B.300d.txt'
word_vec_dict = readGloveData(glove_word_vec_file)
vqaTrain = VQA(annFile, quesFile)
annotations = vqaTrain.dataset['annotations']
questions = vqaTrain.questions['questions']
answerFeatures = createAnswerFeatures(annotations)
# Dumping answer features
answer_features_list = open('answer_feature_list.json', 'w')
answer_features_list.write(json.dumps(answerFeatures))
# For getting image vectors
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
data = []
X_train = []
Y_train = []
X_test = []
Y_test = []
FILE_INDEX = 0
for question in questions:
# quesItem = {}
# print question
questionVector = getBOWVector(
question['question'].strip().replace('?', ' ?').split(),
word_vec_dict)
imgID = question['image_id']
imageVector = np.asarray(feats[:, imageDict[imgID]])
# quesItem['image_id'] = imgID
# quesItem['question'] = question['question'].replace('?', ' ?').split(' ')
annotations = vqaTrain.loadQA(ids=[question['question_id']])
for annotation in annotations:
ansString = annotation['multiple_choice_answer']
answerVector = getAnswerVector(ansString, answerFeatures)
temp_X_train = np.append(imageVector, questionVector)
temp_Y_train = answerVector
X_train.append(temp_X_train)
Y_train.append(temp_Y_train)
if len(X_train) >= FILE_LIMIT:
train_x_file = open(
FILE_PATH + X_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy',
'w')
train_y_file = open(
FILE_PATH + Y_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy',
'w')
np.save(train_x_file, X_train)
np.save(train_y_file, Y_train)
X_train = []
Y_train = []
FILE_INDEX = FILE_INDEX + 1
# print len(X_train)
# if len(annotations) != 1:
# print imgID, " has annotations ", len(annotations)
# for ann in annotations:
# quesItemCopy = dict(quesItem)
# ansString = ann['multiple_choice_answer']
# quesItemCopy['answer'] = ansString
# data.append(quesItemCopy)
if len(X_train) > 0:
train_x_file = open(
FILE_PATH + X_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w')
train_y_file = open(
FILE_PATH + Y_TRAIN_FILE_NAME + str(FILE_INDEX) + '.npy', 'w')
np.save(train_x_file, X_train)
np.save(train_y_file, Y_train)
X_train = []
Y_train = []
filepath="weights.{epoch:02d}-{val_loss:.2f}.hdf5", save_best_only=True)
tfile = '../features/coco_vgg_IDMap.txt'
args = {}
args['answer_vector_file'] = 'answer_feature_list.json'
args['glove_file'] = '../glove/glove.6B.300d.txt'
# In[2]:
print "Reading GloVE and VGG raw files"
glove_word_vec_file = args['glove_file']
word_vec_dict = utilities.readGloveData(glove_word_vec_file)
imageDict = cocoImageUtils.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
print "Reading the data and creating features"
answer_vector_file = open(args['answer_vector_file'], 'r')
answerFeatureVector = json.loads(answer_vector_file.read())
answer_vector_file.close()
# In[3]:
import sys
sys.path.insert(0, './../VQA/PythonHelperTools')
from vqaTools.vqa import VQA
def evalResults():
dataDir = './../VQA'
taskType = 'MultipleChoice'
dataType = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType = 'train2014'
annFile = '%s/Annotations/%s_%s_annotations.json' % (dataDir, dataType, dataSubType)
quesFile = '%s/Questions/%s_%s_%s_questions.json' % (dataDir, taskType, dataType, dataSubType)
imgDir = '%s/Images/%s/%s/' % (dataDir, dataType, dataSubType)
vqaTrain = VQA(annFile, quesFile)
dummyano = vqaTrain.dataset['annotations']
answerFeatures = ld.createAnswerFeatures(dummyano)
dataDir2 = './../VQA'
taskType2 = 'MultipleChoice'
dataType2 = 'mscoco' # 'mscoco' for real and 'abstract_v002' for abstract
dataSubType2 = 'analysis1'
annFile2 = '%s/Annotations/%s_%s_annotations.json' % (dataDir2, dataType2, dataSubType2)
quesFile2 = '%s/Questions/%s_%s_%s_questions.json' % (dataDir2, taskType2, dataType2, dataSubType2)
imgDir2 = '%s/Images/%s/%s/' % (dataDir2, dataType2, dataSubType2)
modelReader = open('./model_definition_100iter.json')
json_read = modelReader.read()
model = model_from_json(json_read)
model.load_weights('./model_weights_100iter.h5py')
vqaVal = VQA(annFile2, quesFile2)
FILE_INDEX = 0
total = 0.0
correct = 0.0
resultsDicts = []
x_test = []
y_test = []
glove_word_vec_file = './../glove/glove.6B.300d.txt'
word_vec_dict = ld.readGloveData(glove_word_vec_file)
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat('./../features/coco/vgg_feats.mat')['feats']
for quesID, annotation in vqaVal.qa.iteritems():
# print quesID
# if quesID not in vqaVal.qqa.keys():
# continue
question = vqaVal.qqa[quesID]
# print question
questionVector = ld.getBOWVector(question['question'].strip().replace('?', ' ?').split(), word_vec_dict)
imgID = annotation['image_id']
imageVector = np.asarray(feats[:, imageDict[imgID]])
temp_dict = {}
ansString = annotation['multiple_choice_answer']
temp_dict['question_id'] = quesID
# answerVector = ld.getAnswerVector(ansString, answerFeatures)
temp_x_test = np.append(imageVector, questionVector)
# temp_y_test = answerVector
x_test = np.asarray([temp_x_test])
# y_test = np.asarray([temp_y_test])
predictions = model.predict_classes(x_test, verbose = False)
temp_dict['answer'] = answerFeatures[predictions[0]]
resultsDicts.append(temp_dict)
writer = open('./../Results/MultipleChoice_mscoco_analysis1_results.json', 'w')
json_dump = json.dumps(resultsDicts)
writer.write(json_dump)
outputQuestions["data_type"] = dataType
outputQuestions["license"] = {}
outputQuestions["data_subtype"] = "analysis1"
outputQuestions["questions"] = []
else:
outputQuestionWriter = open(outputQuestionFile)
outputAnnotationWriter = open(outputAnnotationFile)
# question_json_reader = outputQuestionWriter.read()
outputQuestions = json.load(outputQuestionWriter)
# annotations_json_reader = outputAnnotationWriter.read()
outputAnnotations = json.load(outputAnnotationWriter)
glove_word_vec_file = "./../glove/glove.6B.300d.txt"
word_vec_dict = ld.readGloveData(glove_word_vec_file)
imageDict = pramod.generateDictionary(tfile)
feats = sio.loadmat("./../features/coco/vgg_feats.mat")["feats"]
while newQuestion != "no":
print "\n"
randomAnn = random.choice(anns)
origquestion = vqaVal.qqa[randomAnn["question_id"]]
questionVector = ld.getBOWVector(origquestion["question"].strip().replace("?", " ?").split(), word_vec_dict)
imgID = randomAnn["image_id"]
imageVector = np.asarray(feats[:, imageDict[imgID]])
# temp_dict = {}
ansString = randomAnn["multiple_choice_answer"]
# temp_dict['question_id'] = quesID
# answerVector = ld.getAnswerVector(ansString, answerFeatures)
temp_x_test = np.append(imageVector, questionVector)
# temp_y_test = answerVector
