def eval(hyper, dataset, state_dict=None, valid=False):
# Load parameters
if hyper.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = hyper.CKPT_PATH
else:
path = hyper.CKPTS_PATH + \
'ckpt_' + hyper.CKPT_VERSION + \
'/epoch' + str(hyper.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
# Store the prediction list
qid_list = [ques['question_id'] for ques in dataset.ques_list]
q_list = [ques['question'] for ques in dataset.ques_list]
im_id_list = [ques['image_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = Net(hyper, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
if hyper.N_GPU > 1:
net = nn.DataParallel(net, device_ids=hyper.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=hyper.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=hyper.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ques_ix_iter, ans_iter) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / hyper.EVAL_BATCH_SIZE),
),
end=' ')
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred = net(img_feat_iter, ques_ix_iter)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != hyper.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax, (0, hyper.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append((pred_argmax))
break
#st.write(dataset.ix_to_ans[str([pred_argmax])])
# Save the whole prediction vector
if hyper.TEST_SAVE_PRED:
if pred_np.shape[0] != hyper.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, hyper.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
old = 0
st.header("Question and Answer")
for qix in range(qid_list.__len__()):
bbb = int(qid_list[qix])
aaa = dataset.ix_to_ans[str(ans_ix_list[qix])]
if old != int(im_id_list[qix]):
num_digit = len(str(im_id_list[qix]))
name = 'COCO_val2014_'
for x in range(0, 12 - num_digit):
name = name + '0'
image = Image.open('./datasets/coco_extract/images/' + name +
str(im_id_list[qix]) + '.jpg')
cap = q_list[qix] + " " + aaa
# st.image(image)
old = int(im_id_list[qix])
q_a = ("(" + str(qix + 1) + ") " + q_list[qix] + " " + aaa)
st.subheader(q_a)
result = [{
'answer':
aaa, # ix_to_ans(load with json) keys are type of string
'question_id': bbb
}]
# Write the results to result file
if valid:
if val_ckpt_flag:
result_eval_file = \
hyper.CACHE_PATH + \
'result_run_' + hyper.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
hyper.CACHE_PATH + \
'result_run_' + hyper.VERSION + \
'.json'
else:
if hyper.CKPT_PATH is not None:
result_eval_file = \
hyper.RESULT_PATH + \
'result_run_' + hyper.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
hyper.RESULT_PATH + \
'result_run_' + hyper.CKPT_VERSION + \
'_epoch' + str(hyper.CKPT_EPOCH) + \
'.json'
print('Save the result to file: {}'.format(result_eval_file))
json.dump(result, open(result_eval_file, 'w'))
# Save the whole prediction vector
if hyper.TEST_SAVE_PRED:
if hyper.CKPT_PATH is not None:
ensemble_file = \
hyper.PRED_PATH + \
'result_run_' + hyper.CKPT_VERSION + \
'.json'
else:
ensemble_file = \
hyper.PRED_PATH + \
'result_run_' + hyper.CKPT_VERSION + \
'_epoch' + str(hyper.CKPT_EPOCH) + \
'.json'
print('Save the prediction vector to file: {}'.format(ensemble_file))
pred_list = np.array(pred_list).reshape(-1, ans_size)
result_pred = [{
'pred': pred_list[qix],
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())]
pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1)
def eval(self, dataset, state_dict=None, valid=False):
# Load parameters
if self.__C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + \
'ckpt_' + self.__C.CKPT_VERSION + \
'/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
# Store the prediction list
qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = Net(self.__C, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
if self.__C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=self.__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=self.__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ques_ix_iter,
ans_iter) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / self.__C.EVAL_BATCH_SIZE),
),
end=' ')
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred = net(img_feat_iter, ques_ix_iter)
#print(pred)
pred_np = pred[0].cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax,
(0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append(pred_argmax)
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
result = [
{
'answer': dataset.ix_to_ans[str(
ans_ix_list[qix]
)], # ix_to_ans(load with json) keys are type of string
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())
]
# Write the results to result file
if valid:
if val_ckpt_flag:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.VERSION + \
'.json'
else:
if self.__C.CKPT_PATH is not None:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print('Save the result to file: {}'.format(result_eval_file))
json.dump(result, open(result_eval_file, 'w'))
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if self.__C.CKPT_PATH is not None:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print(
'Save the prediction vector to file: {}'.format(ensemble_file))
pred_list = np.array(pred_list).reshape(-1, ans_size)
result_pred = [{
'pred': pred_list[qix],
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())]
pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1)
# Run validation script
if valid:
# create vqa object and vqaRes object
ques_file_path = self.__C.QUESTION_PATH['test']
ans_file_path = self.__C.ANSWER_PATH['test']
vqa = VQA(ans_file_path, ques_file_path)
vqaRes = vqa.loadRes(result_eval_file, ques_file_path)
# create vqaEval object by taking vqa and vqaRes
vqaEval = VQAEval(
vqa, vqaRes, n=2
) # n is precision of accuracy (number of places after decimal), default is 2
# evaluate results
"""
If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function
By default it uses all the question ids in annotation file
"""
vqaEval.evaluate()
# print accuracies
print("\n")
print("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
# print("Per Question Type Accuracy is the following:")
# for quesType in vqaEval.accuracy['perQuestionType']:
# print("%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType]))
# print("\n")
print("Per Answer Type Accuracy is the following:")
for ansType in vqaEval.accuracy['perAnswerType']:
print("%s : %.02f" %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
print("\n")
if val_ckpt_flag:
print('Write to log file: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+')
else:
print('Write to log file: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+')
logfile.write("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
for ansType in vqaEval.accuracy['perAnswerType']:
logfile.write(
"%s : %.02f " %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
logfile.write("\n\n")
logfile.close()
def eval(self, dataset, state_dict=None, valid=False):
# 评估模型，传入的数据集是验证集，主要是利用epoch1的检查点.pkl文件进行评估验证集
# 的准确度，从而获取每种答案类型的精确度，
# 加载模型参数
if self.__C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + \
'ckpt_' + self.__C.CKPT_VERSION + \
'/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('加载 ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print("评估模型网络的state_dict:", state_dict)
print('完成!')
# 存储预测列表 问题id列表，答案列表
qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
# 和train一样调用网络
net = Net(self.__C, pretrained_emb, token_size, ans_size)
net.cuda()
# 评估
net.eval()
if self.__C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=self.__C.DEVICES)
net.load_state_dict(state_dict)
# 数据集加载，此处传入的是验证集，
dataloader = Data.DataLoader(dataset,
batch_size=self.__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ques_ix_iter,
ans_iter) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / self.__C.EVAL_BATCH_SIZE),
),
end=' ')
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
# 输出预测的特征向量
pred = net(img_feat_iter, ques_ix_iter)
# 转换成cpu的数据的numpy类型
pred_np = pred.cpu().data.numpy()
# 沿着一个轴返回最大值:取出一个向量中预测值最大的，最有可能接近真实答案
pred_argmax = np.argmax(pred_np, axis=1)
np.savetxt("pre_np.txt", pred_np)
np.savetxt("pre_argmax.txt", pred_argmax)
# 保存最接近真实答案的索引
if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax,
(0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
# 将这个最接近真实答案的预测答案pre_argmax加入到ans_ix_list中，
ans_ix_list.append(pred_argmax)
file = open('ans_ix_list.txt', 'w')
file.write(str(ans_ix_list))
file.close()
# 保存整个预测向量
if self.__C.TEST_SAVE_PRED:
if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
result = [
{
'answer': dataset.ix_to_ans[str(
ans_ix_list[qix]
)], # ix_to_ans(load with json) keys are type of string
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())
]
# 将结果写入结果文件：问题id：对应预测答案
if valid:
if val_ckpt_flag:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.VERSION + \
'.json'
else:
if self.__C.CKPT_PATH is not None:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print('Save the result to file: {}'.format(result_eval_file))
json.dump(result, open(result_eval_file, 'w'))
# 保存整个预测向量
if self.__C.TEST_SAVE_PRED:
if self.__C.CKPT_PATH is not None:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print(
'Save the prediction vector to file: {}'.format(ensemble_file))
pred_list = np.array(pred_list).reshape(-1, ans_size)
file = open('pred_list.txt', 'w')
file.write(str(pred_list))
file.close()
result_pred = [{
'pred': pred_list[qix],
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())]
pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1)
# 运行验证脚本
if valid:
# 创建vqa对象和vqaRes对象
ques_file_path = self.__C.QUESTION_PATH['train']
ans_file_path = self.__C.ANSWER_PATH['train']
vqa = VQA(ans_file_path, ques_file_path)
vqaRes = vqa.loadRes(result_eval_file, ques_file_path)
# 通过获取vqa和vqaRes创建vqaEval对象
vqaEval = VQAEval(
vqa, vqaRes, n=2
) # n is precision of accuracy (number of places after decimal), default is 2
# 评估结果
"""
如果您有一个问题id列表，希望对其进行结果评估，请将其作为列表传递给下面的函数
默认情况下，它使用注释文件中的所有问题id
"""
vqaEval.evaluate()
# print accuracies
print("\n")
#计算全部准确率
print("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
# 计算每种答案类型准确率，yes/no,number,other
print("Per Answer Type Accuracy is the following:")
for ansType in vqaEval.accuracy['perAnswerType']:
print("%s : %.02f" %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
print("\n")
# 将评估结果写入log文件
if val_ckpt_flag:
print('Write to log file: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+')
else:
print('写入日志文件: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+')
logfile.write("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
for ansType in vqaEval.accuracy['perAnswerType']:
logfile.write(
"%s : %.02f " %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
logfile.write("\n\n")
logfile.close()
def eval(self, dataset, state_dict=None, valid=False):
from core.model.net import Net
# Load parameters
if self.__C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + \
'ckpt_' + self.__C.CKPT_VERSION + \
'/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
# Store the prediction list
qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = Net(
self.__C,
pretrained_emb,
token_size,
ans_size
)
net.cuda()
net.eval()
if self.__C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=self.__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(
dataset,
batch_size=self.__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True
)
for step, (
img_feat_iter,
ques_ix_iter,
ans_iter
) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / self.__C.EVAL_BATCH_SIZE),
), end=' ')
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred, recon_loss = net(
img_feat_iter,
ques_ix_iter
)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax,
(0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1
)
ans_ix_list.append(pred_argmax)
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]), (0, 0)),
mode='constant',
constant_values=-1
)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
pickle.dump(ans_ix_list, open("ans_ix_list.pkl", "wb+"))
result = [{
'answer': dataset.ix_to_ans[ans_ix_list[qix]], # ix_to_ans(load with json) keys are type of string
'question_id': int(qid_list[qix])
}for qix in range(qid_list.__len__())]
ground_truth = dataset.qid_to_ques
total = 0
num_correct = 0
sub_num = {
0: {
"num":0,
"corr":0
},
1: {
"num":0,
"corr":0
},
2: {
"num":0,
"corr":0
},
3: {
"num":0,
"corr":0
}
}
for result_ in result:
grth = ground_truth[str(result_['question_id'])]
sub_num[grth['question_type']]['num'] = sub_num[grth['question_type']]['num'] + 1
total += 1
if grth['answer'] == result_['answer']:
sub_num[grth['question_type']]['corr'] = sub_num[grth['question_type']]['corr'] + 1
num_correct += 1
test_total = 0
test_corr = 0
for key, value in sub_num.items():
test_total += value['num']
test_corr += value['corr']
assert test_total == total and test_corr == num_correct
# Write the results to result file
if valid:
if val_ckpt_flag:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.VERSION + \
'.json'
else:
if self.__C.CKPT_PATH is not None:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print('Save the result to file: {}'.format(result_eval_file))
print(result_eval_file)
json.dump(result, open(result_eval_file, 'w'))
if val_ckpt_flag:
print('Write to log file: {}'.format(
self.__C.LOG_PATH +
'log_run_' + self.__C.CKPT_VERSION + '.txt',
'a+')
)
logfile = open(
self.__C.LOG_PATH +
'log_run_' + self.__C.CKPT_VERSION + '.txt',
'a+'
)
else:
print('Write to log file: {}'.format(
self.__C.LOG_PATH +
'log_run_' + self.__C.VERSION + '.txt',
'a+')
)
logfile = open(
self.__C.LOG_PATH +
'log_run_' + self.__C.VERSION + '.txt',
'a+'
)
for k, v in sub_num.items():
if k == 0:
print("Acc on object is %.2f" % (100.0 * v['corr']/v['num']))
logfile.write("Acc on object is %.2f\n" % (100.0 * v['corr']/v['num']))
if k == 1:
print("Acc on number is %.2f" % (100.0 * v['corr']/v['num']))
logfile.write("Acc on number is %.2f\n" % (100.0 * v['corr']/v['num']))
if k == 2:
print("Acc on color is %.2f" % (100.0 * v['corr']/v['num']))
logfile.write("Acc on color is %.2f\n" % (100.0 * v['corr']/v['num']))
if k == 3:
print("Acc on location is %.2f" % (100.0 * v['corr']/v['num']))
logfile.write("Acc on location is %.2f\n" % (100.0 * v['corr']/v['num']))
print("Total ACC is %.2f" % (100.0*num_correct/total))
logfile.write("Total ACC is %.2f\n" % (100.0*num_correct/total))
logfile.close()
def eval(self, dataset, state_dict=None, valid=False):
st.title('VQA Validation')
# Load parameters
if self.__C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + \
'ckpt_' + self.__C.CKPT_VERSION + \
'/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
# Store the prediction list
qid_list = [ques['question_id'] for ques in dataset.ques_list]
q_list = [ques['question'] for ques in dataset.ques_list]
im_id_list = [ques['image_id'] for ques in dataset.ques_list]
images = []
if st.button('New set of images'):
for x in range(0, 10):
index = randint(0, len(im_id_list))
num_digit = len(str(index))
name = 'COCO_val2014_'
for x in range(0, 12 - num_digit):
name = name + '0'
image = Image.open(name + '.jpg')
images.append(image)
pick_img = st.sidebar.radio("Which image?",
[x for x in range(1, len(images))])
image_iterator = paginator("Select a sunset page", images)
indices_on_page, images_on_page = map(list, zip(*image_iterator))
st.write('result:')
st.image(images_on_page, width=200, caption=indices_on_page)
# do something with what the user selected here
if pick_img:
st.write('yass')
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = Net(self.__C, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
if self.__C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=self.__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=self.__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ques_ix_iter,
ans_iter) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / self.__C.EVAL_BATCH_SIZE),
),
end=' ')
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred = net(img_feat_iter, ques_ix_iter)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax,
(0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append((pred_argmax))
break
#st.write(dataset.ix_to_ans[str([pred_argmax])])
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
old = 0
for qix in range(qid_list.__len__()):
bbb = int(qid_list[qix])
aaa = dataset.ix_to_ans[str(ans_ix_list[qix])]
if old != int(im_id_list[qix]):
image = Image.open('/home/akshay/Downloads/val2014/' +
'COCO_val2014_000000' +
str(im_id_list[qix]) + '.jpg')
cap = q_list[qix] + " " + aaa
st.image(image)
old = int(im_id_list[qix])
st.write(q_list[qix], " ", aaa)
result = [{
'answer':
aaa, # ix_to_ans(load with json) keys are type of string
'question_id': bbb
}]
# Write the results to result file
if valid:
if val_ckpt_flag:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.CACHE_PATH + \
'result_run_' + self.__C.VERSION + \
'.json'
else:
if self.__C.CKPT_PATH is not None:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
result_eval_file = \
self.__C.RESULT_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print('Save the result to file: {}'.format(result_eval_file))
json.dump(result, open(result_eval_file, 'w'))
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if self.__C.CKPT_PATH is not None:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'.json'
else:
ensemble_file = \
self.__C.PRED_PATH + \
'result_run_' + self.__C.CKPT_VERSION + \
'_epoch' + str(self.__C.CKPT_EPOCH) + \
'.json'
print(
'Save the prediction vector to file: {}'.format(ensemble_file))
pred_list = np.array(pred_list).reshape(-1, ans_size)
result_pred = [{
'pred': pred_list[qix],
'question_id': int(qid_list[qix])
} for qix in range(qid_list.__len__())]
pickle.dump(result_pred, open(ensemble_file, 'wb+'), protocol=-1)
# Run validation script
if valid:
# create vqa object and vqaRes object
ques_file_path = self.__C.QUESTION_PATH['val']
ans_file_path = self.__C.ANSWER_PATH['val']
vqa = VQA(ans_file_path, ques_file_path)
vqaRes = vqa.loadRes(result_eval_file, ques_file_path)
# create vqaEval object by taking vqa and vqaRes
vqaEval = VQAEval(
vqa, vqaRes, n=2
) # n is precision of accuracy (number of places after decimal), default is 2
# evaluate results
"""
If you have a list of question ids on which you would like to evaluate your results, pass it as a list to below function
By default it uses all the question ids in annotation file
"""
vqaEval.evaluate()
# print accuracies
print("\n")
print("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
# print("Per Question Type Accuracy is the following:")
# for quesType in vqaEval.accuracy['perQuestionType']:
# print("%s : %.02f" % (quesType, vqaEval.accuracy['perQuestionType'][quesType]))
# print("\n")
print("Per Answer Type Accuracy is the following:")
for ansType in vqaEval.accuracy['perAnswerType']:
print("%s : %.02f" %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
print("\n")
if val_ckpt_flag:
print('Write to log file: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.CKPT_VERSION +
'.txt', 'a+')
else:
print('Write to log file: {}'.format(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+'))
logfile = open(
self.__C.LOG_PATH + 'log_run_' + self.__C.VERSION + '.txt',
'a+')
logfile.write("Overall Accuracy is: %.02f\n" %
(vqaEval.accuracy['overall']))
for ansType in vqaEval.accuracy['perAnswerType']:
logfile.write(
"%s : %.02f " %
(ansType, vqaEval.accuracy['perAnswerType'][ansType]))
logfile.write("\n\n")
logfile.close()
def eval(self, dataset, state_dict=None, valid=False):
# Load parameters
if self.__C.CKPT_PATH is not None:
print('Warning: you are now using CKPT_PATH args, '
'CKPT_VERSION and CKPT_EPOCH will not work')
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + \
'ckpt_' + self.__C.CKPT_VERSION + \
'/epoch' + str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('Finish!')
groundtruth_ans_list = dataset.ans_list
groundtruth_question_to_ans_dict = {}
for _ in groundtruth_ans_list:
if _['multiple_choice_answer'] in dataset.ans_to_ix:
groundtruth_question_to_ans_dict[
_['question_id']] = dataset.ans_to_ix[
_['multiple_choice_answer']]
else:
groundtruth_question_to_ans_dict[
_['question_id']] = dataset.ans_to_ix['0']
# Store the prediction list
qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
data_size = dataset.data_size
# token_size = dataset.token_size
ans_size = dataset.ans_size
# pretrained_emb = dataset.pretrained_emb
net = Net(self.__C, ans_size)
net.cuda()
net.eval()
if self.__C.N_GPU > 1:
net = nn.DataParallel(net, device_ids=self.__C.DEVICES)
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=self.__C.EVAL_BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ans_iter, ques_input_idx,
ques_attention_mask) in enumerate(dataloader):
print("\rEvaluation: [step %4d/%4d]" % (
step,
int(data_size / self.__C.EVAL_BATCH_SIZE),
),
end=' ')
img_feat_iter = img_feat_iter.cuda()
ans_iter = ans_iter.cuda()
ques_input_idx = ques_input_idx.cuda()
ques_attention_mask = ques_attention_mask.cuda()
pred = net(img_feat_iter, ques_input_idx.squeeze(1),
ques_attention_mask.squeeze(1))
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1)
# Save the answer index
if pred_argmax.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_argmax = np.pad(
pred_argmax,
(0, self.__C.EVAL_BATCH_SIZE - pred_argmax.shape[0]),
mode='constant',
constant_values=-1)
ans_ix_list.append(pred_argmax)
# Save the whole prediction vector
if self.__C.TEST_SAVE_PRED:
if pred_np.shape[0] != self.__C.EVAL_BATCH_SIZE:
pred_np = np.pad(
pred_np,
((0, self.__C.EVAL_BATCH_SIZE - pred_np.shape[0]),
(0, 0)),
mode='constant',
constant_values=-1)
pred_list.append(pred_np)
print('')
ans_ix_list = np.array(ans_ix_list).reshape(-1)
result = [
{
'answer': dataset.ix_to_ans[str(
ans_ix_list[qix]
)], # ix_to_ans(load with json) keys are type of string
'question_id': int(qid_list[qix]),
'answer_id': int(str(ans_ix_list[qix]))
} for qix in range(qid_list.__len__())
]
y_true = []
y_pred = []
for _ in result:
qid = _['question_id']
if qid in groundtruth_question_to_ans_dict:
y_pred.append(_['answer_id'])
y_true.append(groundtruth_question_to_ans_dict[qid])
acc = accuracy_score(y_true, y_pred)
print('acc :', acc)
logfile = open(
self.__C.CACHE_PATH + \
'result_runacc_' + self.__C.CKPT_VERSION + \
'.txt',
'a+'
)
logfile.write('acc = ' + str(acc))
logfile.close()
def show(self, dataset, state_dict=None):
# Load parameters:
if self.__C.CKPT_PATH is not None:
print(
'Warning: you are now using CKPT_PATH args, CKPT_VERSION and CKPT_EPOCH will not work'
)
path = self.__C.CKPT_PATH
else:
path = self.__C.CKPTS_PATH + 'ckpt_' + self.__C.CKPT_VERSION + '/epoch' \
+ str(self.__C.CKPT_EPOCH) + '.pkl'
val_ckpt_flag = False
if state_dict is None:
val_ckpt_flag = True
print('========== Loading ckpt {}'.format(path))
state_dict = torch.load(path)['state_dict']
print('========== Finished!')
ques_list = [ques['question'] for ques in dataset.ques_list]
qid_list = [ques['question_id'] for ques in dataset.ques_list]
ans_ix_list = []
pred_list = []
result = []
data_size = dataset.data_size
token_size = dataset.token_size
ans_size = dataset.ans_size
pretrained_emb = dataset.pretrained_emb
net = Net(self.__C, pretrained_emb, token_size, ans_size)
net.cuda()
net.eval()
net.load_state_dict(state_dict)
dataloader = Data.DataLoader(dataset,
batch_size=self.__C.BATCH_SIZE,
shuffle=False,
num_workers=self.__C.NUM_WORKERS,
pin_memory=True)
for step, (img_feat_iter, ques_ix_iter,
ans_iter) in enumerate(dataloader):
img_feat_iter = img_feat_iter.cuda()
ques_ix_iter = ques_ix_iter.cuda()
pred = net(img_feat_iter, ques_ix_iter)
pred_np = pred.cpu().data.numpy()
pred_argmax = np.argmax(pred_np, axis=1) # [835]
# Save the answer index
ans_ix_list.append(pred_argmax)
ans_ix_list = np.array(ans_ix_list).reshape(-1)
for qix in range(qid_list.__len__()):
result.append({
'question':
ques_list[qix],
'answer':
dataset.ix_to_ans[str(ans_ix_list[qix])]
})
print('========== result')
print(result)
