def eval(hyper, dataset, state_dict=None, valid=False): # Load parameters if hyper.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = hyper.CKPT_PATH else: path = hyper.CKPTS_PATH + \ 'ckpt_' + hyper.CKPT_VERSION + \ '/epoch' + str(hyper.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') # Store the prediction list qid_list = [ques['question_id'] for ques in dataset.ques_list] q_list = [ques['question'] for ques in dataset.ques_list] im_id_list = [ques['image_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = Net(hyper, pretrained_emb, token_size, ans_size) net.cuda() net.eval() if hyper.N_GPU > 1: net = nn.DataParallel(net, device_ids=hyper.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=hyper.EVAL_BATCH_SIZE, shuffle=False, num_workers=hyper.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / hyper.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred = net(img_feat_iter, ques_ix_iter) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != hyper.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, hyper.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append((pred_argmax)) break #st.write(dataset.ix_to_ans[str([pred_argmax])]) # Save the whole prediction vector if hyper.TEST_SAVE_PRED: if pred_np.shape[0] != hyper.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, hyper.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) old = 0 st.header("Question and Answer") for qix in range(qid_list.__len__()): bbb = int(qid_list[qix]) aaa = dataset.ix_to_ans[str(ans_ix_list[qix])] if old != int(im_id_list[qix]): num_digit = len(str(im_id_list[qix])) name = 'COCO_val2014_' for x in range(0, 12 - num_digit): name = name + '0' image = Image.open('./datasets/coco_extract/images/' + name + str(im_id_list[qix]) + '.jpg') cap = q_list[qix] + " " + aaa # st.image(image) old = int(im_id_list[qix]) q_a = ("(" + str(qix + 1) + ") " + q_list[qix] + " " + aaa) st.subheader(q_a) result = [{ 'answer': aaa, # ix_to_ans(load with json) keys are type of string 'question_id': bbb }] # Write the results to result file if valid: if val_ckpt_flag: result_eval_file = \ hyper.CACHE_PATH + \ 'result_run_' + hyper.CKPT_VERSION + \ '.json' else: result_eval_file = \ hyper.CACHE_PATH + \ 'result_run_' + hyper.VERSION + \ '.json' else: if hyper.CKPT_PATH is not None: result_eval_file = \ hyper.RESULT_PATH + \ 'result_run_' + hyper.CKPT_VERSION + \ '.json' else: result_eval_file = \ hyper.RESULT_PATH + \ 'result_run_' + hyper.CKPT_VERSION + \ '_epoch' + str(hyper.CKPT_EPOCH) + \ '.json' print('Save the result to file: {}'.format(result_eval_file)) json.dump(result, open(result_eval_file, 'w')) # Save the whole prediction vector if hyper.TEST_SAVE_PRED: if hyper.CKPT_PATH is not None: ensemble_file = \ hyper.PRED_PATH + \ 'result_run_' + hyper.CKPT_VERSION + \ '.json' else: ensemble_file = \ hyper.PRED_PATH + \ 'result_run_' + hyper.CKPT_VERSION + \ '_epoch' + str(hyper.CKPT_EPOCH) + \ '.json' print('Save the prediction vector to file: {}'.format(ensemble_file)) pred_list = np.array(pred_list).reshape(-1, ans_size) result_pred = [{ 'pred': pred_list[qix], 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__())] pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1)
def eval(self, dataset, state_dict=None, valid=False): # Load parameters if self.__C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + \ 'ckpt_' + self.__C.CKPT_VERSION + \ '/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') # Store the prediction list qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = Net(self.__C, pretrained_emb, token_size, ans_size) net.cuda() net.eval() if self.__C.N_GPU > 1: net = nn.DataParallel(net, device_ids=self.__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=self.__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / self.__C.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred = net(img_feat_iter, ques_ix_iter) #print(pred) pred_np = pred[0].cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append(pred_argmax) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) result = [ { 'answer': dataset.ix_to_ans[str( ans_ix_list[qix] )], # ix_to_ans(load with json) keys are type of string 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__()) ] # Write the results to result file if valid: if val_ckpt_flag: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.VERSION + \ '.json' else: if self.__C.CKPT_PATH is not None: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print('Save the result to file: {}'.format(result_eval_file)) json.dump(result, open(result_eval_file, 'w')) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if self.__C.CKPT_PATH is not None: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print( 'Save the prediction vector to file: {}'.format(ensemble_file)) pred_list = np.array(pred_list).reshape(-1, ans_size) result_pred = [{ 'pred': pred_list[qix], 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__())] pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1) # Run validation script if valid: # create vqa object and vqaRes object ques_file_path = self.__C.QUESTION_PATH['test'] ans_file_path = self.__C.ANSWER_PATH['test'] vqa = VQA(ans_file_path, ques_file_path) vqaRes = vqa.loadRes(result_eval_file, ques_file_path) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) # n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ vqaEval.evaluate() # print accuracies print("\n") print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) # print("Per Question Type Accuracy is the following:") # for quesType in vqaEval.accuracy['perQuestionType']: # print("%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType])) # print("\n") print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) print("\n") if val_ckpt_flag: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+') else: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+') logfile.write("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) for ansType in vqaEval.accuracy['perAnswerType']: logfile.write( "%s : %.02f " % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) logfile.write("\n\n") logfile.close()
def eval(self, dataset, state_dict=None, valid=False): # 评估模型,传入的数据集是验证集,主要是利用epoch1的检查点.pkl文件进行评估验证集 # 的准确度,从而获取每种答案类型的精确度, # 加载模型参数 if self.__C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + \ 'ckpt_' + self.__C.CKPT_VERSION + \ '/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('加载 ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print("评估模型网络的state_dict:", state_dict) print('完成!') # 存储预测列表 问题id列表,答案列表 qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb # 和train一样调用网络 net = Net(self.__C, pretrained_emb, token_size, ans_size) net.cuda() # 评估 net.eval() if self.__C.N_GPU > 1: net = nn.DataParallel(net, device_ids=self.__C.DEVICES) net.load_state_dict(state_dict) # 数据集加载,此处传入的是验证集, dataloader = Data.DataLoader(dataset, batch_size=self.__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / self.__C.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() # 输出预测的特征向量 pred = net(img_feat_iter, ques_ix_iter) # 转换成cpu的数据的numpy类型 pred_np = pred.cpu().data.numpy() # 沿着一个轴返回最大值:取出一个向量中预测值最大的,最有可能接近真实答案 pred_argmax = np.argmax(pred_np, axis=1) np.savetxt("pre_np.txt", pred_np) np.savetxt("pre_argmax.txt", pred_argmax) # 保存最接近真实答案的索引 if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) # 将这个最接近真实答案的预测答案pre_argmax加入到ans_ix_list中, ans_ix_list.append(pred_argmax) file = open('ans_ix_list.txt', 'w') file.write(str(ans_ix_list)) file.close() # 保存整个预测向量 if self.__C.TEST_SAVE_PRED: if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) result = [ { 'answer': dataset.ix_to_ans[str( ans_ix_list[qix] )], # ix_to_ans(load with json) keys are type of string 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__()) ] # 将结果写入结果文件:问题id:对应预测答案 if valid: if val_ckpt_flag: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.VERSION + \ '.json' else: if self.__C.CKPT_PATH is not None: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print('Save the result to file: {}'.format(result_eval_file)) json.dump(result, open(result_eval_file, 'w')) # 保存整个预测向量 if self.__C.TEST_SAVE_PRED: if self.__C.CKPT_PATH is not None: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print( 'Save the prediction vector to file: {}'.format(ensemble_file)) pred_list = np.array(pred_list).reshape(-1, ans_size) file = open('pred_list.txt', 'w') file.write(str(pred_list)) file.close() result_pred = [{ 'pred': pred_list[qix], 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__())] pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1) # 运行验证脚本 if valid: # 创建vqa对象和vqaRes对象 ques_file_path = self.__C.QUESTION_PATH['train'] ans_file_path = self.__C.ANSWER_PATH['train'] vqa = VQA(ans_file_path, ques_file_path) vqaRes = vqa.loadRes(result_eval_file, ques_file_path) # 通过获取vqa和vqaRes创建vqaEval对象 vqaEval = VQAEval( vqa, vqaRes, n=2 ) # n is precision of accuracy (number of places after decimal), default is 2 # 评估结果 """ 如果您有一个问题id列表,希望对其进行结果评估,请将其作为列表传递给下面的函数 默认情况下,它使用注释文件中的所有问题id """ vqaEval.evaluate() # print accuracies print("\n") #计算全部准确率 print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) # 计算每种答案类型准确率,yes/no,number,other print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) print("\n") # 将评估结果写入log文件 if val_ckpt_flag: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+') else: print('写入日志文件: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+') logfile.write("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) for ansType in vqaEval.accuracy['perAnswerType']: logfile.write( "%s : %.02f " % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) logfile.write("\n\n") logfile.close()
def eval(self, dataset, state_dict=None, valid=False): from core.model.net import Net # Load parameters if self.__C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + \ 'ckpt_' + self.__C.CKPT_VERSION + \ '/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') # Store the prediction list qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = Net( self.__C, pretrained_emb, token_size, ans_size ) net.cuda() net.eval() if self.__C.N_GPU > 1: net = nn.DataParallel(net, device_ids=self.__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader( dataset, batch_size=self.__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True ) for step, ( img_feat_iter, ques_ix_iter, ans_iter ) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / self.__C.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred, recon_loss = net( img_feat_iter, ques_ix_iter ) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1 ) ans_ix_list.append(pred_argmax) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1 ) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) pickle.dump(ans_ix_list, open("ans_ix_list.pkl", "wb+")) result = [{ 'answer': dataset.ix_to_ans[ans_ix_list[qix]], # ix_to_ans(load with json) keys are type of string 'question_id': int(qid_list[qix]) }for qix in range(qid_list.__len__())] ground_truth = dataset.qid_to_ques total = 0 num_correct = 0 sub_num = { 0: { "num":0, "corr":0 }, 1: { "num":0, "corr":0 }, 2: { "num":0, "corr":0 }, 3: { "num":0, "corr":0 } } for result_ in result: grth = ground_truth[str(result_['question_id'])] sub_num[grth['question_type']]['num'] = sub_num[grth['question_type']]['num'] + 1 total += 1 if grth['answer'] == result_['answer']: sub_num[grth['question_type']]['corr'] = sub_num[grth['question_type']]['corr'] + 1 num_correct += 1 test_total = 0 test_corr = 0 for key, value in sub_num.items(): test_total += value['num'] test_corr += value['corr'] assert test_total == total and test_corr == num_correct # Write the results to result file if valid: if val_ckpt_flag: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.VERSION + \ '.json' else: if self.__C.CKPT_PATH is not None: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print('Save the result to file: {}'.format(result_eval_file)) print(result_eval_file) json.dump(result, open(result_eval_file, 'w')) if val_ckpt_flag: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+') ) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+' ) else: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+') ) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+' ) for k, v in sub_num.items(): if k == 0: print("Acc on object is %.2f" % (100.0 * v['corr']/v['num'])) logfile.write("Acc on object is %.2f\n" % (100.0 * v['corr']/v['num'])) if k == 1: print("Acc on number is %.2f" % (100.0 * v['corr']/v['num'])) logfile.write("Acc on number is %.2f\n" % (100.0 * v['corr']/v['num'])) if k == 2: print("Acc on color is %.2f" % (100.0 * v['corr']/v['num'])) logfile.write("Acc on color is %.2f\n" % (100.0 * v['corr']/v['num'])) if k == 3: print("Acc on location is %.2f" % (100.0 * v['corr']/v['num'])) logfile.write("Acc on location is %.2f\n" % (100.0 * v['corr']/v['num'])) print("Total ACC is %.2f" % (100.0*num_correct/total)) logfile.write("Total ACC is %.2f\n" % (100.0*num_correct/total)) logfile.close()
def eval(self, dataset, state_dict=None, valid=False): st.title('VQA Validation') # Load parameters if self.__C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + \ 'ckpt_' + self.__C.CKPT_VERSION + \ '/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') # Store the prediction list qid_list = [ques['question_id'] for ques in dataset.ques_list] q_list = [ques['question'] for ques in dataset.ques_list] im_id_list = [ques['image_id'] for ques in dataset.ques_list] images = [] if st.button('New set of images'): for x in range(0, 10): index = randint(0, len(im_id_list)) num_digit = len(str(index)) name = 'COCO_val2014_' for x in range(0, 12 - num_digit): name = name + '0' image = Image.open(name + '.jpg') images.append(image) pick_img = st.sidebar.radio("Which image?", [x for x in range(1, len(images))]) image_iterator = paginator("Select a sunset page", images) indices_on_page, images_on_page = map(list, zip(*image_iterator)) st.write('result:') st.image(images_on_page, width=200, caption=indices_on_page) # do something with what the user selected here if pick_img: st.write('yass') ans_ix_list = [] pred_list = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = Net(self.__C, pretrained_emb, token_size, ans_size) net.cuda() net.eval() if self.__C.N_GPU > 1: net = nn.DataParallel(net, device_ids=self.__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=self.__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / self.__C.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred = net(img_feat_iter, ques_ix_iter) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append((pred_argmax)) break #st.write(dataset.ix_to_ans[str([pred_argmax])]) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) old = 0 for qix in range(qid_list.__len__()): bbb = int(qid_list[qix]) aaa = dataset.ix_to_ans[str(ans_ix_list[qix])] if old != int(im_id_list[qix]): image = Image.open('/home/akshay/Downloads/val2014/' + 'COCO_val2014_000000' + str(im_id_list[qix]) + '.jpg') cap = q_list[qix] + " " + aaa st.image(image) old = int(im_id_list[qix]) st.write(q_list[qix], " ", aaa) result = [{ 'answer': aaa, # ix_to_ans(load with json) keys are type of string 'question_id': bbb }] # Write the results to result file if valid: if val_ckpt_flag: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.CACHE_PATH + \ 'result_run_' + self.__C.VERSION + \ '.json' else: if self.__C.CKPT_PATH is not None: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: result_eval_file = \ self.__C.RESULT_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print('Save the result to file: {}'.format(result_eval_file)) json.dump(result, open(result_eval_file, 'w')) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if self.__C.CKPT_PATH is not None: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '.json' else: ensemble_file = \ self.__C.PRED_PATH + \ 'result_run_' + self.__C.CKPT_VERSION + \ '_epoch' + str(self.__C.CKPT_EPOCH) + \ '.json' print( 'Save the prediction vector to file: {}'.format(ensemble_file)) pred_list = np.array(pred_list).reshape(-1, ans_size) result_pred = [{ 'pred': pred_list[qix], 'question_id': int(qid_list[qix]) } for qix in range(qid_list.__len__())] pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1) # Run validation script if valid: # create vqa object and vqaRes object ques_file_path = self.__C.QUESTION_PATH['val'] ans_file_path = self.__C.ANSWER_PATH['val'] vqa = VQA(ans_file_path, ques_file_path) vqaRes = vqa.loadRes(result_eval_file, ques_file_path) # create vqaEval object by taking vqa and vqaRes vqaEval = VQAEval( vqa, vqaRes, n=2 ) # n is precision of accuracy (number of places after decimal), default is 2 # evaluate results """ If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function By default it uses all the question ids in annotation file """ vqaEval.evaluate() # print accuracies print("\n") print("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) # print("Per Question Type Accuracy is the following:") # for quesType in vqaEval.accuracy['perQuestionType']: # print("%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType])) # print("\n") print("Per Answer Type Accuracy is the following:") for ansType in vqaEval.accuracy['perAnswerType']: print("%s : %.02f" % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) print("\n") if val_ckpt_flag: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION + '.txt', 'a+') else: print('Write to log file: {}'.format( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+')) logfile = open( self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt', 'a+') logfile.write("Overall Accuracy is: %.02f\n" % (vqaEval.accuracy['overall'])) for ansType in vqaEval.accuracy['perAnswerType']: logfile.write( "%s : %.02f " % (ansType, vqaEval.accuracy['perAnswerType'][ansType])) logfile.write("\n\n") logfile.close()
def eval(self, dataset, state_dict=None, valid=False): # Load parameters if self.__C.CKPT_PATH is not None: print('Warning: you are now using CKPT_PATH args, ' 'CKPT_VERSION and CKPT_EPOCH will not work') path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + \ 'ckpt_' + self.__C.CKPT_VERSION + \ '/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('Finish!') groundtruth_ans_list = dataset.ans_list groundtruth_question_to_ans_dict = {} for _ in groundtruth_ans_list: if _['multiple_choice_answer'] in dataset.ans_to_ix: groundtruth_question_to_ans_dict[ _['question_id']] = dataset.ans_to_ix[ _['multiple_choice_answer']] else: groundtruth_question_to_ans_dict[ _['question_id']] = dataset.ans_to_ix['0'] # Store the prediction list qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] data_size = dataset.data_size # token_size = dataset.token_size ans_size = dataset.ans_size # pretrained_emb = dataset.pretrained_emb net = Net(self.__C, ans_size) net.cuda() net.eval() if self.__C.N_GPU > 1: net = nn.DataParallel(net, device_ids=self.__C.DEVICES) net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=self.__C.EVAL_BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ans_iter, ques_input_idx, ques_attention_mask) in enumerate(dataloader): print("\rEvaluation: [step %4d/%4d]" % ( step, int(data_size / self.__C.EVAL_BATCH_SIZE), ), end=' ') img_feat_iter = img_feat_iter.cuda() ans_iter = ans_iter.cuda() ques_input_idx = ques_input_idx.cuda() ques_attention_mask = ques_attention_mask.cuda() pred = net(img_feat_iter, ques_input_idx.squeeze(1), ques_attention_mask.squeeze(1)) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # Save the answer index if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_argmax = np.pad( pred_argmax, (0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]), mode='constant', constant_values=-1) ans_ix_list.append(pred_argmax) # Save the whole prediction vector if self.__C.TEST_SAVE_PRED: if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE: pred_np = np.pad( pred_np, ((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)), mode='constant', constant_values=-1) pred_list.append(pred_np) print('') ans_ix_list = np.array(ans_ix_list).reshape(-1) result = [ { 'answer': dataset.ix_to_ans[str( ans_ix_list[qix] )], # ix_to_ans(load with json) keys are type of string 'question_id': int(qid_list[qix]), 'answer_id': int(str(ans_ix_list[qix])) } for qix in range(qid_list.__len__()) ] y_true = [] y_pred = [] for _ in result: qid = _['question_id'] if qid in groundtruth_question_to_ans_dict: y_pred.append(_['answer_id']) y_true.append(groundtruth_question_to_ans_dict[qid]) acc = accuracy_score(y_true, y_pred) print('acc :', acc) logfile = open( self.__C.CACHE_PATH + \ 'result_runacc_' + self.__C.CKPT_VERSION + \ '.txt', 'a+' ) logfile.write('acc = ' + str(acc)) logfile.close()
def show(self, dataset, state_dict=None): # Load parameters: if self.__C.CKPT_PATH is not None: print( 'Warning: you are now using CKPT_PATH args, CKPT_VERSION and CKPT_EPOCH will not work' ) path = self.__C.CKPT_PATH else: path = self.__C.CKPTS_PATH + 'ckpt_' + self.__C.CKPT_VERSION + '/epoch' \ + str(self.__C.CKPT_EPOCH) + '.pkl' val_ckpt_flag = False if state_dict is None: val_ckpt_flag = True print('========== Loading ckpt {}'.format(path)) state_dict = torch.load(path)['state_dict'] print('========== Finished!') ques_list = [ques['question'] for ques in dataset.ques_list] qid_list = [ques['question_id'] for ques in dataset.ques_list] ans_ix_list = [] pred_list = [] result = [] data_size = dataset.data_size token_size = dataset.token_size ans_size = dataset.ans_size pretrained_emb = dataset.pretrained_emb net = Net(self.__C, pretrained_emb, token_size, ans_size) net.cuda() net.eval() net.load_state_dict(state_dict) dataloader = Data.DataLoader(dataset, batch_size=self.__C.BATCH_SIZE, shuffle=False, num_workers=self.__C.NUM_WORKERS, pin_memory=True) for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader): img_feat_iter = img_feat_iter.cuda() ques_ix_iter = ques_ix_iter.cuda() pred = net(img_feat_iter, ques_ix_iter) pred_np = pred.cpu().data.numpy() pred_argmax = np.argmax(pred_np, axis=1) # [835] # Save the answer index ans_ix_list.append(pred_argmax) ans_ix_list = np.array(ans_ix_list).reshape(-1) for qix in range(qid_list.__len__()): result.append({ 'question': ques_list[qix], 'answer': dataset.ix_to_ans[str(ans_ix_list[qix])] }) print('========== result') print(result)