def visualise_engine(__C):
# Load parameters
dataset = DatasetLoader(__C).DataSet()
if __C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = __C.CKPT_PATH
else:
path = __C.CKPTS_PATH + \
'/ckpt_' + __C.CKPT_VERSION + \
'/epoch' + str(__C.CKPT_EPOCH) + '.pkl'
print('Loading ckpt from: {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
if __C.N_GPU > 1:
state_dict = ckpt_proc(state_dict)
# Store the prediction list
# qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = ModelLoader(__C).Net(
__C,
pretrained_emb,
token_size,
ans_size,
dataset.token_to_ix
)
net.cuda()
net.eval()
if __C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=__C.DEVICES)
net.load_state_dict(state_dict)
batch_size = 1
dataloader = Data.DataLoader(
dataset,
batch_size=batch_size,
shuffle=True,
num_workers=__C.NUM_WORKERS,
pin_memory=__C.PIN_MEM
)
for step, (
frcn_feat_iter,
grid_feat_iter,
bbox_feat_iter,
ques_ix_iter,
ans_iter, image_id, question, words, target_ans
) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / __C.EVAL_BATCH_SIZE),
), end=' ')
frcn_feat_iter = frcn_feat_iter.cuda()
grid_feat_iter = grid_feat_iter.cuda()
bbox_feat_iter = bbox_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred, img_attention_map, txt_attention_map = net(
frcn_feat_iter,
grid_feat_iter,
bbox_feat_iter,
ques_ix_iter
)
img_attention_map = img_attention_map[:, :, :, 1:]
txt_attention_map = txt_attention_map[:, :, :, 1:len(words) + 1]
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
ans = dataset.ix_to_ans[pred_argmax[0]]
plt.interactive(False)
visualise_img(question['image_filename'][0], question['question'][0], img_attention_map.cpu().data.numpy()[0],
ans, target_ans[0])
visualise_txt(words, txt_attention_map.cpu().data.numpy()[0])
def test_engine(__C, dataset, state_dict=None, validation=False):
# Load parameters
if __C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = __C.CKPT_PATH
else:
path = __C.CKPTS_PATH + \
'/ckpt_' + __C.CKPT_VERSION + \
'/epoch' + str(__C.CKPT_EPOCH) + '.pkl'
# val_ckpt_flag = False
if state_dict is None:
# val_ckpt_flag = True
print('Loading ckpt from: {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
if __C.N_GPU > 1:
state_dict = ckpt_proc(state_dict)
# Store the prediction list
# qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = ModelLoader(__C).Net(__C, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
if __C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=__C.NUM_WORKERS,
pin_memory=__C.PIN_MEM)
for step, (img_iter, frcn_feat_iter, grid_feat_iter, bbox_feat_iter,
ques_ix_iter, ans_iter) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / __C.EVAL_BATCH_SIZE),
),
end=' ')
img_iter = img_iter.cuda()
frcn_feat_iter = frcn_feat_iter.cuda()
grid_feat_iter = grid_feat_iter.cuda()
bbox_feat_iter = bbox_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred = net(img_iter, frcn_feat_iter, grid_feat_iter, bbox_feat_iter,
ques_ix_iter)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != __C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax, (0, __C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append(pred_argmax)
# Save the whole prediction vector
if __C.TEST_SAVE_PRED:
if pred_np.shape[0] != __C.EVAL_BATCH_SIZE:
pred_np = np.pad(pred_np,
((0, __C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
if validation:
if __C.RUN_MODE not in ['train']:
result_eval_file = __C.CACHE_PATH + '/result_run_' + __C.CKPT_VERSION
else:
result_eval_file = __C.CACHE_PATH + '/result_run_' + __C.VERSION
else:
if __C.CKPT_PATH is not None:
result_eval_file = __C.RESULT_PATH + '/result_run_' + __C.CKPT_VERSION
else:
result_eval_file = __C.RESULT_PATH + '/result_run_' + __C.CKPT_VERSION + '_epoch' + str(
__C.CKPT_EPOCH)
if __C.CKPT_PATH is not None:
ensemble_file = __C.PRED_PATH + '/result_run_' + __C.CKPT_VERSION + '.pkl'
else:
ensemble_file = __C.PRED_PATH + '/result_run_' + __C.CKPT_VERSION + '_epoch' + str(
__C.CKPT_EPOCH) + '.pkl'
if __C.RUN_MODE not in ['train']:
log_file = __C.LOG_PATH + '/log_run_' + __C.CKPT_VERSION + '.txt'
else:
log_file = __C.LOG_PATH + '/log_run_' + __C.VERSION + '.txt'
EvalLoader(__C).eval(dataset, ans_ix_list, pred_list, result_eval_file,
ensemble_file, log_file, validation)
def test_engine_demo(__C, dataset, state_dict=None, validation=False):
# Load parameters
if __C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = __C.CKPT_PATH
else:
path = __C.CKPTS_PATH + \
'/ckpt_' + __C.CKPT_VERSION + \
'/epoch' + str(__C.CKPT_EPOCH) + '.pkl'
# val_ckpt_flag = False
if state_dict is None:
# val_ckpt_flag = True
print('Loading ckpt from: {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
if __C.N_GPU > 1:
state_dict = ckpt_proc(state_dict)
# Store the prediction list
# qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = ModelLoader(__C).Net(__C, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
if __C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=__C.NUM_WORKERS,
pin_memory=__C.PIN_MEM)
# --------------------------------------------------------
# Changed from iterative to single input
frcn_feat_iter = torch.tensor(dataset.frcn_feat_iter, dtype=torch.float)
grid_feat_iter = dataset.grid_feat_iter #None
bbox_feat_iter = torch.tensor(dataset.bbox_feat_iter, dtype=torch.float)
ques_ix_iter = torch.tensor(dataset.ques_ix_iter, dtype=torch.long)
ans_iter = dataset.ans_iter #None
frcn_feat_iter = frcn_feat_iter.cuda()
# grid_feat_iter = grid_feat_iter.cuda()
bbox_feat_iter = bbox_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
# --------------------------------------------------------
pred = net(frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != __C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(pred_argmax,
(0, __C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append(pred_argmax)
# Save the whole prediction vector
if __C.TEST_SAVE_PRED:
if pred_np.shape[0] != __C.EVAL_BATCH_SIZE:
pred_np = np.pad(pred_np,
((0, __C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
# --------------------------------------------------------
# Added answer output to console and json file for the demo
# Removed unused validation features
print('The answer is:')
answer = dataset.get_ans(ans_ix_list[0])
print(answer)
# Write to Json file
newjson = dataset.jsonfile
newjson['ans'] = answer
with open('demo/output/output.json', 'w') as f:
json.dump(newjson, f)
# clean up input files
files = glob.glob("demo/input/*")
for f in files:
os.remove(f)