def main():
folder = 'mfb_coatt_glove_q%dv%d_%s'%(config.NUM_QUESTION_GLIMPSE, config.NUM_IMG_GLIMPSE,config.TRAIN_DATA_SPLITS)
if not os.path.exists('./%s'%folder):
os.makedirs('./%s'%folder)
question_vocab, answer_vocab = {}, {}
if os.path.exists('./%s/vdict.json'%folder) and os.path.exists('./%s/adict.json'%folder):
print 'restoring vocab'
with open('./%s/vdict.json'%folder,'r') as f:
question_vocab = json.load(f)
with open('./%s/adict.json'%folder,'r') as f:
answer_vocab = json.load(f)
else:
question_vocab, answer_vocab = make_vocab_files()
with open('./%s/vdict.json'%folder,'w') as f:
json.dump(question_vocab, f)
with open('./%s/adict.json'%folder,'w') as f:
json.dump(answer_vocab, f)
print 'question vocab size:', len(question_vocab)
print 'answer vocab size:', len(answer_vocab)
with open('./%s/proto_train.prototxt'%folder, 'w') as f:
f.write(str(mfb_coatt(config.TRAIN_DATA_SPLITS, config.BATCH_SIZE, \
config.MAX_WORDS_IN_QUESTION, len(question_vocab), folder)))
with open('./%s/proto_test.prototxt'%folder, 'w') as f:
f.write(str(mfb_coatt('val', config.VAL_BATCH_SIZE, \
config.MAX_WORDS_IN_QUESTION, len(question_vocab), folder)))
with open('./%s/solver.prototxt'%folder, 'w') as f:
f.write(str(get_solver(folder)))
with open('./%s/auxiliary.json'%folder, 'w') as f:
json.dump(get_auxiliary_json(),f, indent=2)
caffe.set_device(config.TRAIN_GPU_ID)
caffe.set_mode_gpu()
solver = caffe.get_solver('./%s/solver.prototxt'%folder)
train_loss = np.zeros(config.MAX_ITERATIONS+1)
results = []
if config.RESTORE_ITER:
restore_iter = config.RESTORE_ITER
solver.restore('./%s/_iter_%d.solverstate'%(folder,restore_iter))
else:
restore_iter = 0
start = time.clock()
for it in range(restore_iter, config.MAX_ITERATIONS+1):
solver.step(1)
# store the train loss
train_loss[it] = solver.net.blobs['loss'].data
if it % config.PRINT_INTERVAL == 0 and it != 0:
elapsed = (time.clock() - start)
print 'Iteration:', it
c_mean_loss = train_loss[it-config.PRINT_INTERVAL:it].mean()
print 'Train loss:', c_mean_loss, ' Elapsed seconds:', elapsed
start = time.clock()
if it % config.VALIDATE_INTERVAL == 0 and it != restore_iter:
model_name = './%s/tmp.caffemodel'%(folder)
solver.net.save(model_name)
print 'Validating...'
# for test-dev /test set. the json file will be generated under the <folder> file
exec_validation(config.TEST_GPU_ID, 'test-dev', model_name, it=it, folder=folder)
caffe.set_device(config.TRAIN_GPU_ID)
'''
def main():
folder = 'mfh_baseline_%s'%(config.TRAIN_DATA_SPLITS)
if not os.path.exists('./%s'%folder):
os.makedirs('./%s'%folder)
question_vocab, answer_vocab = {}, {}
if os.path.exists('./%s/vdict.json'%folder) and os.path.exists('./%s/adict.json'%folder):
print 'restoring vocab'
with open('./%s/vdict.json'%folder,'r') as f:
question_vocab = json.load(f)
with open('./%s/adict.json'%folder,'r') as f:
answer_vocab = json.load(f)
else:
question_vocab, answer_vocab = make_vocab_files()
with open('./%s/vdict.json'%folder,'w') as f:
json.dump(question_vocab, f)
with open('./%s/adict.json'%folder,'w') as f:
json.dump(answer_vocab, f)
print 'question vocab size:', len(question_vocab)
print 'answer vocab size:', len(answer_vocab)
with open('./%s/proto_train.prototxt'%folder, 'w') as f:
f.write(str(mfh_baseline(config.TRAIN_DATA_SPLITS, config.BATCH_SIZE, \
config.MAX_WORDS_IN_QUESTION, len(question_vocab), folder)))
with open('./%s/proto_test.prototxt'%folder, 'w') as f:
f.write(str(mfh_baseline('val', config.VAL_BATCH_SIZE, \
config.MAX_WORDS_IN_QUESTION, len(question_vocab), folder)))
with open('./%s/solver.prototxt'%folder, 'w') as f:
f.write(str(get_solver(folder)))
with open('./%s/auxiliary.json'%folder, 'w') as f:
json.dump(get_auxiliary_json(),f, indent=2)
caffe.set_device(config.TRAIN_GPU_ID)
caffe.set_mode_gpu()
solver = caffe.get_solver('./%s/solver.prototxt'%folder)
train_loss = np.zeros(config.MAX_ITERATIONS+1)
results = []
if config.RESTORE_ITER:
restore_iter = config.RESTORE_ITER
solver.restore('./%s/_iter_%d.solverstate'%(folder,restore_iter))
else:
restore_iter = 0
start = time.clock()
for it in range(restore_iter, config.MAX_ITERATIONS+1):
solver.step(1)
# store the train loss
train_loss[it] = solver.net.blobs['loss'].data
if it % config.PRINT_INTERVAL == 0 and it != 0:
elapsed = (time.clock() - start)
print 'Iteration:', it
c_mean_loss = train_loss[it-config.PRINT_INTERVAL:it].mean()
print 'Train loss:', c_mean_loss, ' Elapsed seconds:', elapsed
start = time.clock()
if it % config.VALIDATE_INTERVAL == 0 and it != restore_iter:
model_name = './%s/tmp.caffemodel'%(folder)
solver.net.save(model_name)
print 'Validating...'
'''
# for test-dev /test set. the json file will be generated under the <folder> file
exec_validation(config.TEST_GPU_ID, 'test-dev', model_name, it=it, folder=folder)
caffe.set_device(config.TRAIN_GPU_ID)
'''
#for val set. the accuracy will be computed and ploted
test_loss, acc_overall, acc_per_ques, acc_per_ans = exec_validation(config.TEST_GPU_ID, 'val', model_name, it=it, folder=folder)
caffe.set_device(config.TRAIN_GPU_ID)
print 'Test loss:', test_loss
print 'Accuracy:', acc_overall
print 'Test per ans', acc_per_ans
results.append([it, c_mean_loss, test_loss, acc_overall, acc_per_ques, acc_per_ans])
best_result_idx = np.array([x[3] for x in results]).argmax()
print 'Best accuracy of', results[best_result_idx][3], 'was at iteration', results[best_result_idx][0]
drawgraph(results,folder,config.MFB_FACTOR_NUM,config.MFB_OUT_DIM,prefix='mfh_baseline')