def test():
import json
import numpy as np
from w2v_answer_encoder import MultiChoiceQuestionManger
model = StateClassifier(input_dim=512, phase='test')
model.build()
prob = model.prob
# Load vocabulary
# to_sentence = SentenceGenerator(trainset='trainval')
# create multiple choice question manger
mc_manager = MultiChoiceQuestionManger(subset='val',
answer_coding='sequence')
sess = tf.Session()
# Load model
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
checkpoint_path = ckpt.model_checkpoint_path
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
# get data
result = []
reader = StateDataPetcher(batch_size=18,
subset='dev',
shuffle=False,
max_epoch=1)
num = reader.num_samples
for itr in range(num):
feat, label, quest_id = reader.pop_batch()
feed_dict = model.fill_feed_dict([feat])
scores = sess.run(prob, feed_dict=feed_dict)
idx = scores.argmax()
# parse question and answer
assert (np.unique(quest_id).size == 1)
quest_id = quest_id[0]
question = mc_manager.get_question(quest_id)
mc_ans = mc_manager.get_candidate_answers(quest_id)
vaq_answer = mc_ans[idx]
real_answer = mc_ans[label.argmax()]
# add result
result.append({u'answer': vaq_answer, u'question_id': quest_id})
# show results
if itr % 100 == 0:
print('============== %d ============' % itr)
print('question id: %d' % quest_id)
print('question\t: %s' % question)
print('answer\t: %s' % real_answer)
print('VAQ answer\t: %s (%0.2f)' % (vaq_answer, scores[idx]))
quest_ids = [res[u'question_id'] for res in result]
# save results
tf.logging.info('Saving results')
res_file = 'result/rescore_state_dev_dev.json'
json.dump(result, open(res_file, 'w'))
from vqa_eval import evaluate_model
acc = evaluate_model(res_file, quest_ids)
print('Over all accuarcy: %0.2f' % acc)
return acc
def test():
from util import unpickle
import json
from inference_utils.question_generator_util import SentenceGenerator
from w2v_answer_encoder import MultiChoiceQuestionManger
config = MLPConfig()
model = SequenceMLP(config, phase='test')
model.build()
prob = model.prob
# Load vocabulary
to_sentence = SentenceGenerator(trainset='trainval')
# create multiple choice question manger
mc_manager = MultiChoiceQuestionManger(subset='trainval',
answer_coding='sequence')
sess = tf.Session()
# Load model
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
checkpoint_path = ckpt.model_checkpoint_path
saver = tf.train.Saver()
saver.restore(sess, checkpoint_path)
# get data
result = []
dataset = unpickle('data/rescore_dev.pkl')
for itr, datum in enumerate(dataset):
seq_index, att_mask, label = _process_datum(datum)
quest_id = datum['quest_id']
quest = seq_index[0].tolist()
feed_dict = model.fill_feed_dict([seq_index, att_mask])
scores = sess.run(prob, feed_dict=feed_dict)
idx = scores.argmax()
# parse question and answer
question = to_sentence.index_to_question([0] + quest)
mc_ans = mc_manager.get_candidate_answers(quest_id)
vaq_answer = mc_ans[idx]
real_answer = mc_ans[label.argmax()]
# add result
result.append({u'answer': vaq_answer, u'question_id': quest_id})
# show results
if itr % 100 == 0:
print('============== %d ============' % itr)
print('question id: %d' % quest_id)
print('question\t: %s' % question)
print('answer\t: %s' % real_answer)
print('VAQ answer\t: %s (%0.2f)' % (vaq_answer, scores[idx]))
quest_ids = [res[u'question_id'] for res in result]
# save results
tf.logging.info('Saving results')
res_file = 'result/rescore_dev_dev.json'
json.dump(result, open(res_file, 'w'))
from vqa_eval import evaluate_model
acc = evaluate_model(res_file, quest_ids)
print('Over all accuarcy: %0.2f' % acc)
