image_attentions = np.zeros((len(p._original_questions), 4, 4, 2048))
with open('data/attentions_test.pkl', 'rb') as f:
attentions = pickle.load(f)
for i in range(len(p._original_questions)):
image_attentions[i] = attentions[p._original_questions[i]['image_id']]
# Reshape
image_attentions = image_attentions.reshape(
(image_features.shape[0], 16, 2048))
print(image_attentions.shape)
# Normalise features
for i in range(16):
image_attentions[:, i, :] = normalize(image_attentions[:, i, :],
copy=False)
# Predict and ensemble
pred = neuralnet.predict(image_attentions, object_matrix, questions,
'weights/weights_att_soft.hdf5')
pred = pred + neuralnet.predict(image_features, object_matrix, questions,
'weights/weights_baseline_soft.hdf5')
pred = 0.5 * pred
print(pred.shape)
# Produce results
model_evaluator = ProduceResult(p._int_to_answer,
p._answer_to_int,
dataSubType='test-dev2015',
modelName='ensemble')
model_evaluator.produce_results(pred, p._original_questions)
monitor='val_loss',
loss='binary_crossentropy')
# Train network
neuralnet.fit(image_features, questions, answers)
# Load validation set and evaluate prediction on it
pt = PrepareData(
path_images='data_vqa_feat', # Path to image features
subset='val2014', # Desired subset: either train2014 or val2014
taskType=taskType, # 'OpenEnded', 'MultipleChoice', 'all'
cut_data=
data_amount, # Percentage of data to use, 1 = All values, above 1 = 10 samples for debugging
output_path='data')
pt.loadDictionary('data/dictionary.pkl') # Use same dictionary as in training
image_features, questions, _, annotations = pt.load_data()
print("Image features", image_features.shape)
print("Question features", questions.shape)
print("Dictionary size", pt.dic_size)
# Test prediction on validation set
pred = neuralnet.predict_current_state(image_features, questions)
print("=== Results on validation set ===")
model_evaluator = ProduceResult(pt._int_to_answer,
pt._answer_to_int,
dataSubType='val2014',
answer_count=pt._answer_count)
answers = model_evaluator.produce_results(pred, pt._original_questions)
model_evaluator.evaluate(taskType=taskType)