def make_question_vocab(qdic):
"""
Returns a dictionary that maps words to indices.
"""
vdict = {'': 0}
vid = 1
for qid in qdic.keys():
# sequence to list
q_str = qdic[qid]['qstr']
q_list = VQADataProvider.seq_to_list(q_str)
# create dict
for w in q_list:
if not vdict.has_key(w):
vdict[w] = vid
vid += 1
return vdict
def make_question_vocab(qdic):
"""
Returns a dictionary that maps words to indices.
"""
vdict = {'':0}
vid = 1
for qid in qdic.keys():
# sequence to list
q_str = qdic[qid]['qstr']
q_list = VQADataProvider.seq_to_list(q_str)
# create dict
for w in q_list:
if not vdict.has_key(w):
vdict[w] = vid
vid +=1
return vdict
def make_exp_vocab(exp_dic):
"""
Returns a dictionary that maps words to indices.
"""
exp_vdict = {'<EOS>': 0}
exp_vdict[''] = 1
exp_id = 2
for qid in exp_dic.keys():
exp_strs = exp_dic[qid]
for exp_str in exp_strs:
exp_list = VQADataProvider.seq_to_list(exp_str)
for w in exp_list:
if w not in exp_vdict:
exp_vdict[w] = exp_id
exp_id += 1
return exp_vdict
def exec_validation(device_id, mode, it='', visualize=False):
caffe.set_device(device_id)
caffe.set_mode_gpu()
net = caffe.Net('./result/proto_test.prototxt',\
'./result/tmp.caffemodel',\
caffe.TEST)
dp = VQADataProvider(mode=mode,batchsize=64)
total_questions = len(dp.getQuesIds())
epoch = 0
pred_list = []
testloss_list = []
stat_list = []
while epoch == 0:
t_word, t_cont, t_img_feature, t_answer, t_glove_matrix, t_qid_list, t_iid_list, epoch = dp.get_batch_vec()
net.blobs['data'].data[...] = np.transpose(t_word,(1,0))
net.blobs['cont'].data[...] = np.transpose(t_cont,(1,0))
net.blobs['img_feature'].data[...] = t_img_feature
net.blobs['label'].data[...] = t_answer
net.blobs['glove'].data[...] = np.transpose(t_glove_matrix, (1,0,2))
net.forward()
t_pred_list = net.blobs['prediction'].data.argmax(axis=1)
t_pred_str = [dp.vec_to_answer(pred_symbol) for pred_symbol in t_pred_list]
testloss_list.append(net.blobs['loss'].data)
for qid, iid, ans, pred in zip(t_qid_list, t_iid_list, t_answer.tolist(), t_pred_str):
pred_list.append({u'answer':pred, u'question_id': int(dp.getStrippedQuesId(qid))})
if visualize:
q_list = dp.seq_to_list(dp.getQuesStr(qid))
if mode == 'test-dev' or 'test':
ans_str = ''
ans_list = ['']*10
else:
ans_str = dp.vec_to_answer(ans)
ans_list = [ dp.getAnsObj(qid)[i]['answer'] for i in xrange(10)]
stat_list.append({\
'qid' : qid,
'q_list' : q_list,
'iid' : iid,
'answer': ans_str,
'ans_list': ans_list,
'pred' : pred })
percent = 100 * float(len(pred_list)) / total_questions
sys.stdout.write('\r' + ('%.2f' % percent) + '%')
sys.stdout.flush()
mean_testloss = np.array(testloss_list).mean()
if mode == 'val':
valFile = './result/val2015_resfile'
with open(valFile, 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list,mode)
annFile = config.DATA_PATHS['val']['ans_file']
quesFile = config.DATA_PATHS['val']['ques_file']
vqa = VQA(annFile, quesFile)
vqaRes = vqa.loadRes(valFile, quesFile)
vqaEval = VQAEval(vqa, vqaRes, n=2)
vqaEval.evaluate()
acc_overall = vqaEval.accuracy['overall']
acc_perQuestionType = vqaEval.accuracy['perQuestionType']
acc_perAnswerType = vqaEval.accuracy['perAnswerType']
return mean_testloss, acc_overall, acc_perQuestionType, acc_perAnswerType
elif mode == 'test-dev':
filename = './result/vqa_OpenEnded_mscoco_test-dev2015_v3t'+str(it).zfill(8)+'_results'
with open(filename+'.json', 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list,mode)
elif mode == 'test':
filename = './result/vqa_OpenEnded_mscoco_test2015_v3c'+str(it).zfill(8)+'_results'
with open(filename+'.json', 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list,mode)
def exec_validation(device_id, mode, it='', visualize=False):
caffe.set_device(device_id)
caffe.set_mode_gpu()
net = caffe.Net('./result/proto_test.prototxt',\
'./result/tmp.caffemodel',\
caffe.TEST)
dp = VQADataProvider(mode=mode, batchsize=64)
total_questions = len(dp.getQuesIds())
epoch = 0
pred_list = []
testloss_list = []
stat_list = []
while epoch == 0:
t_word, t_cont, t_img_feature, t_answer, t_qid_list, t_iid_list, epoch = dp.get_batch_vec(
)
net.blobs['data'].data[...] = np.transpose(t_word, (1, 0))
net.blobs['cont'].data[...] = np.transpose(t_cont, (1, 0))
net.blobs['img_feature'].data[...] = t_img_feature
net.blobs['label'].data[...] = t_answer
net.forward()
t_pred_list = net.blobs['prediction'].data.argmax(axis=1)
t_pred_str = [
dp.vec_to_answer(pred_symbol) for pred_symbol in t_pred_list
]
testloss_list.append(net.blobs['loss'].data)
for qid, iid, ans, pred in zip(t_qid_list, t_iid_list,
t_answer.tolist(), t_pred_str):
pred_list.append({
'answer': pred,
'question_id': int(dp.getStrippedQuesId(qid))
})
if visualize:
q_list = dp.seq_to_list(dp.getQuesStr(qid))
if mode == 'test-dev' or 'test':
ans_str = ''
ans_list = [''] * 10
else:
ans_str = dp.vec_to_answer(ans)
ans_list = [
dp.getAnsObj(qid)[i]['answer'] for i in range(10)
]
stat_list.append({\
'qid' : qid,
'q_list' : q_list,
'iid' : iid,
'answer': ans_str,
'ans_list': ans_list,
'pred' : pred })
percent = 100 * float(len(pred_list)) / total_questions
sys.stdout.write('\r' + ('%.2f' % percent) + '%')
sys.stdout.flush()
mean_testloss = np.array(testloss_list).mean()
if mode == 'val':
valFile = './result/val2015_resfile'
with open(valFile, 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list, mode)
annFile = config.DATA_PATHS['val']['ans_file']
quesFile = config.DATA_PATHS['val']['ques_file']
vqa = VQA(annFile, quesFile)
vqaRes = vqa.loadRes(valFile, quesFile)
vqaEval = VQAEval(vqa, vqaRes, n=2)
vqaEval.evaluate()
acc_overall = vqaEval.accuracy['overall']
acc_perQuestionType = vqaEval.accuracy['perQuestionType']
acc_perAnswerType = vqaEval.accuracy['perAnswerType']
return mean_testloss, acc_overall, acc_perQuestionType, acc_perAnswerType
elif mode == 'test-dev':
filename = './result/vqa_OpenEnded_mscoco_test-dev2015_v3t' + str(
it).zfill(8) + '_results'
with open(filename + '.json', 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list, mode)
elif mode == 'test':
filename = './result/vqa_OpenEnded_mscoco_test2015_v3c' + str(
it).zfill(8) + '_results'
with open(filename + '.json', 'w') as f:
json.dump(pred_list, f)
if visualize:
visualize_failures(stat_list, mode)
