def visualise_engine(__C): # Load parameters dataset = DatasetLoader(__C).DataSet() if __C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = __C.CKPT_PATH else: path = __C.CKPTS_PATH + \ '/ckpt_' + __C.CKPT_VERSION + \ '/epoch' + str(__C.CKPT_EPOCH) + '.pkl' print('Loading ckpt from: {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') if __C.N_GPU > 1: state_dict = ckpt_proc(state_dict) # Store the prediction list # qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = ModelLoader(__C).Net( __C, pretrained_emb, token_size, ans_size, dataset.token_to_ix ) net.cuda() net.eval() if __C.N_GPU > 1: net = nn.DataParallel(net, device_ids=__C.DEVICES) net.load_state_dict(state_dict) batch_size = 1 dataloader = Data.DataLoader( dataset, batch_size=batch_size, shuffle=True, num_workers=__C.NUM_WORKERS, pin_memory=__C.PIN_MEM ) for step, ( frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter, ans_iter, image_id, question, words, target_ans ) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / __C.EVAL_BATCH_SIZE), ), end=' ') frcn_feat_iter = frcn_feat_iter.cuda() grid_feat_iter = grid_feat_iter.cuda() bbox_feat_iter = bbox_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred, img_attention_map, txt_attention_map = net( frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter ) img_attention_map = img_attention_map[:, :, :, 1:] txt_attention_map = txt_attention_map[:, :, :, 1:len(words) + 1] pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) ans = dataset.ix_to_ans[pred_argmax[0]] plt.interactive(False) visualise_img(question['image_filename'][0], question['question'][0], img_attention_map.cpu().data.numpy()[0], ans, target_ans[0]) visualise_txt(words, txt_attention_map.cpu().data.numpy()[0])
def test_engine(__C, dataset, state_dict=None, validation=False): # Load parameters if __C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = __C.CKPT_PATH else: path = __C.CKPTS_PATH + \ '/ckpt_' + __C.CKPT_VERSION + \ '/epoch' + str(__C.CKPT_EPOCH) + '.pkl' # val_ckpt_flag = False if state_dict is None: # val_ckpt_flag = True print('Loading ckpt from: {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') if __C.N_GPU > 1: state_dict = ckpt_proc(state_dict) # Store the prediction list # qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = ModelLoader(__C).Net(__C, pretrained_emb, token_size, ans_size) net.cuda() net.eval() if __C.N_GPU > 1: net = nn.DataParallel(net, device_ids=__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=__C.NUM_WORKERS, pin_memory=__C.PIN_MEM) for step, (img_iter, frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / __C.EVAL_BATCH_SIZE), ), end=' ') img_iter = img_iter.cuda() frcn_feat_iter = frcn_feat_iter.cuda() grid_feat_iter = grid_feat_iter.cuda() bbox_feat_iter = bbox_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred = net(img_iter, frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != __C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, __C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append(pred_argmax) # Save the whole prediction vector if __C.TEST_SAVE_PRED: if pred_np.shape[0] != __C.EVAL_BATCH_SIZE: pred_np = np.pad(pred_np, ((0, __C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) if validation: if __C.RUN_MODE not in ['train']: result_eval_file = __C.CACHE_PATH + '/result_run_' + __C.CKPT_VERSION else: result_eval_file = __C.CACHE_PATH + '/result_run_' + __C.VERSION else: if __C.CKPT_PATH is not None: result_eval_file = __C.RESULT_PATH + '/result_run_' + __C.CKPT_VERSION else: result_eval_file = __C.RESULT_PATH + '/result_run_' + __C.CKPT_VERSION + '_epoch' + str( __C.CKPT_EPOCH) if __C.CKPT_PATH is not None: ensemble_file = __C.PRED_PATH + '/result_run_' + __C.CKPT_VERSION + '.pkl' else: ensemble_file = __C.PRED_PATH + '/result_run_' + __C.CKPT_VERSION + '_epoch' + str( __C.CKPT_EPOCH) + '.pkl' if __C.RUN_MODE not in ['train']: log_file = __C.LOG_PATH + '/log_run_' + __C.CKPT_VERSION + '.txt' else: log_file = __C.LOG_PATH + '/log_run_' + __C.VERSION + '.txt' EvalLoader(__C).eval(dataset, ans_ix_list, pred_list, result_eval_file, ensemble_file, log_file, validation)
def test_engine_demo(__C, dataset, state_dict=None, validation=False): # Load parameters if __C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = __C.CKPT_PATH else: path = __C.CKPTS_PATH + \ '/ckpt_' + __C.CKPT_VERSION + \ '/epoch' + str(__C.CKPT_EPOCH) + '.pkl' # val_ckpt_flag = False if state_dict is None: # val_ckpt_flag = True print('Loading ckpt from: {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') if __C.N_GPU > 1: state_dict = ckpt_proc(state_dict) # Store the prediction list # qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = ModelLoader(__C).Net(__C, pretrained_emb, token_size, ans_size) net.cuda() net.eval() if __C.N_GPU > 1: net = nn.DataParallel(net, device_ids=__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=__C.NUM_WORKERS, pin_memory=__C.PIN_MEM) # -------------------------------------------------------- # Changed from iterative to single input frcn_feat_iter = torch.tensor(dataset.frcn_feat_iter, dtype=torch.float) grid_feat_iter = dataset.grid_feat_iter #None bbox_feat_iter = torch.tensor(dataset.bbox_feat_iter, dtype=torch.float) ques_ix_iter = torch.tensor(dataset.ques_ix_iter, dtype=torch.long) ans_iter = dataset.ans_iter #None frcn_feat_iter = frcn_feat_iter.cuda() # grid_feat_iter = grid_feat_iter.cuda() bbox_feat_iter = bbox_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() # -------------------------------------------------------- pred = net(frcn_feat_iter, grid_feat_iter, bbox_feat_iter, ques_ix_iter) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != __C.EVAL_BATCH_SIZE: pred_argmax = np.pad(pred_argmax, (0, __C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append(pred_argmax) # Save the whole prediction vector if __C.TEST_SAVE_PRED: if pred_np.shape[0] != __C.EVAL_BATCH_SIZE: pred_np = np.pad(pred_np, ((0, __C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) # -------------------------------------------------------- # Added answer output to console and json file for the demo # Removed unused validation features print('The answer is:') answer = dataset.get_ans(ans_ix_list[0]) print(answer) # Write to Json file newjson = dataset.jsonfile newjson['ans'] = answer with open('demo/output/output.json', 'w') as f: json.dump(newjson, f) # clean up input files files = glob.glob("demo/input/*") for f in files: os.remove(f)