def main(_):
# FLAGS.ckpt_file
ckpt_file = 'model/v1_var_rl_att2_restval_VAQ-VarRL2_r2/model.ckpt-150000'
res_file, mean_vqa_score = ivqa_decoding_beam_search(ckpt_file)
from eval_vqa_question_oracle import evaluate_oracle
evaluate_oracle(res_file)
print('BS mean VQA score: %0.3f' % mean_vqa_score)
def main():
res1 = load_results('vae_ia_rl_attention2_run0')
# res1 = load_results('vae_ia_rl_attention2_run2')
res2 = load_results('vae_rl1')
# res2 = load_results('vae_ia_rl_mlb_r2')
res_file, mean_vqa_score, mean_unk_count = merge_result(res1, res2)
from eval_vqa_question_oracle import evaluate_oracle
evaluate_oracle(res_file)
print('BS mean VQA score: %0.3f' % mean_vqa_score)
print('BS mean #questions: %0.3f' % mean_unk_count)
def test_model(model_path):
with tf.Graph().as_default():
res_file = ivqa_decoding_beam_search(checkpoint_path=model_path,
subset=subset)
if FLAGS.mode == 'full':
cider = evaluate_oracle(res_file, split=target_split)
else:
cider = evaluate_question_standard(res_file)
return float(cider)
def compute_merged_result(method):
res1 = load_results('reference')
num = len(res1)
res2 = load_results(method)
# res2 = load_results('vae_ia_rl_mlb_r2')
res_file, mean_vqa_score, mean_unk_count = merge_result(res1, res2)
from eval_vqa_question_oracle import evaluate_oracle
scores = evaluate_oracle(res_file)
return scores[1], mean_vqa_score
def _procss_worker(ckpt_file, cur_iter):
if cur_iter in backup_exps:
# backup model
print('Copying model')
ckpt_name = os.path.basename(ckpt_file)
_backup_model(ckpt_file, os.path.join(model_backup_dir, ckpt_name))
print('Done')
run_id = backup_exps[cur_iter]
# max_iters = 0
max_iters = 500
else:
run_id = 100
max_iters = 500
# run sampling
res_file, ms, mc = ivqa_decoding_beam_search(ckpt_file, run_id,
max_iters)
# evaluate
res = evaluate_oracle(res_file)
mer_cider, mer_s = compute_merged_result('%s_run%d' %
(FLAGS.method, run_id))
# pdb.set_trace()
res_str = 'O-C: %0.3f, O-B4: %0.3f, m-S: %0.3f, m-C: %0.2f, mer-O-C: %0.3f' % (
res[1], res[0], ms, mc, mer_cider)
return res_str
}
new_results.append(res_i)
scores += scores_i
nums.append(len(scores_i))
scored_qs.append({
'question_id': qid,
'question_inds': qs,
'lm_scores': scores_i
})
maxprob_res_file = res_file = '/data1/fl302/projects/inverse_vqa/result/var_vaq_rand_IVQA-BASIC_lmflt_maxprob.json'
save_json(maxprob_res_file, maxprob_results)
evaluate_question_standard(maxprob_res_file)
sv_file = '/data1/fl302/projects/inverse_vqa/result/var_vaq_rand_IVQA-BASIC_lmscores.json'
save_json(sv_file, scored_qs)
print('Average #questions: %0.2f' % np.mean(nums))
print('Average LM scores: %0.2f' % np.mean(scores))
from eval_vqa_question_oracle import evaluate_oracle
new_res_file = '/data1/fl302/projects/inverse_vqa/result/var_vaq_rand_IVQA-BASIC_lmflt.json'
save_json(new_res_file, new_results)
cider = evaluate_oracle(new_res_file, split='val')
import pdb
pdb.set_trace()
def main(_):
# FLAGS.ckpt_file
res_file, mean_vqa_score = ivqa_decoding_beam_search(None)
from eval_vqa_question_oracle import evaluate_oracle
evaluate_oracle(res_file)
print('BS mean VQA score: %0.3f' % mean_vqa_score)
def test_model(model_path):
with tf.Graph().as_default():
res_file = ivqa_decoding_beam_search(checkpoint_path=model_path,
subset=subset)
cider = evaluate_oracle(res_file)
return cider