def evaluate(model, dataloader, q_dict, c_dict):
score = 0
vq_score = 0
eq_score = 0
print_freq = 50
results = []
for i, (qid, iid, v, q, a, c, l) in enumerate(dataloader):
batch_size = v.size(0)
v = Variable(v, volatile=True).cuda()
q = Variable(q, volatile=True).cuda()
ans_pred, exp_pred = model.evaluate(v, q)
vq_batch_score = compute_score_with_logits(ans_pred[0], a.cuda()).sum()
eq_batch_score = compute_score_with_logits(ans_pred[1], a.cuda()).sum()
batch_score = compute_score_with_logits(ans_pred[0] + ans_pred[1],
a.cuda()).sum()
vq_score += vq_batch_score
eq_score += eq_batch_score
score += batch_score
#batch_score = compute_score_with_logits(ans_pred, a.cuda()).sum()
for j in range(batch_size):
if c[j][0] != 0:
results.append({
'question_id':
qid[j],
'image_id':
iid[j],
'question':
' '.join([
q_dict.idx2word[p] for p in q[j].data.tolist() if p > 0
]),
'answer':
dataloader.dataset.label2ans[ans_pred[0][j].max(0)
[1].data[0]],
'explain_res':
' '.join([
c_dict.idx2word[p] for p in exp_pred[j].data.tolist()
if p > 2
]),
'explain_gt':
' '.join(
[c_dict.idx2word[p] for p in c[j].tolist() if p > 2])
})
if False: #i % print_freq == 0:
print('Batch: [%d/%d]\nQuestion: %s\t Answer Prediction: %s\n'
'Explain GT: %s\nExplain Prediction: %s' %
(i, len(dataloader), results[i * j]['question'],
results[i * j]['answer'], results[i * j]['explain_gt'],
results[i * j]['explain_res']))
vq_score = vq_score / len(dataloader.dataset)
eq_score = eq_score / len(dataloader.dataset)
score = score / len(dataloader.dataset)
scores = {'vq_score': vq_score, 'eq_score': eq_score, 'ens_score': score}
return scores, results
def evaluate(model, dataloader, q_dict, c_dict, gt=True):
score = 0
vq_score = 0
eq_score = 0
print_freq = 50
results = []
for i, (qid, iid, v, q, a, c, l) in enumerate(dataloader):
batch_size = v.size(0)
v = Variable(v, volatile=True).cuda()
q = Variable(q, volatile=True).cuda()
if not gt:
c = None
ans_pred, exp_pred = model.evaluate(v, q, c)
if type(ans_pred) is list:
vq_batch_score = compute_score_with_logits(ans_pred[0],
a.cuda()).sum()
eq_batch_score = compute_score_with_logits(ans_pred[1],
a.cuda()).sum()
vq_score += vq_batch_score
eq_score += eq_batch_score
ans_pred = ans_pred[0] + ans_pred[1]
batch_score = compute_score_with_logits(ans_pred, a.cuda()).sum()
score += batch_score
#batch_score = compute_score_with_logits(ans_pred, a.cuda()).sum()
# for j in range(batch_size):
# if c[j][0] != 0:
# results.append({
# 'question_id': qid[j],
# 'image_id': iid[j],
# 'question': ' '.join([q_dict.idx2word[p] for p in q[j].data.tolist() if p > 0]),
# 'answer': dataloader.dataset.label2ans[ans_pred[0][j].max(0)[1].data[0]],
# 'explain_res': ' '.join([c_dict.idx2word[p] for p in exp_pred[j].data.tolist() if p > 2]),
# 'explain_gt': ' '.join([c_dict.idx2word[p] for p in c[j].tolist() if p > 2])
# })
score = score / len(dataloader.dataset)
if vq_score == 0:
scores = score
else:
vq_score = vq_score / len(dataloader.dataset)
eq_score = eq_score / len(dataloader.dataset)
scores = {
'vq_score': vq_score,
'eq_score': eq_score,
'ens_score': score
}
print('Scores :', scores)
return scores, results
def update_per_batch(self,
model,
answers,
loss,
pred,
curr_size,
logits_key='logits'):
upper_bound = answers.max(1)[0].sum()
self.upper_bound += upper_bound
# self.total_norm += nn.utils.clip_grad_norm_(model.parameters(), 0.25)
# self.count_norm += 1
from train import compute_score_with_logits
batch_score = compute_score_with_logits(pred, answers.data,
logits_key).sum()
self.loss += loss.data * curr_size
self.raw_score += batch_score
self.num_examples += curr_size
def evaluate(model, dataloader, q_dict, c_dict):
score = 0
vq_score = 0
eq_score = 0
print_freq = 50
results = []
for i, (qid, iid, v, q, a, c, l) in enumerate(dataloader):
batch_size = v.size(0)
v = Variable(v, volatile=True).cuda()
q = Variable(q, volatile=True).cuda()
c = Variable(c, volatile=True).cuda()
ans_pred = model.evaluate(v, q, c)
#ans_pred = model(v, q)
c = c.data
batch_score = compute_score_with_logits(ans_pred, a.cuda()).sum()
score += batch_score
for j in range(batch_size):
results.append({
'question_id':
qid[j],
'image_id':
iid[j],
'question':
' '.join(
[q_dict.idx2word[p] for p in q[j].data.tolist() if p > 0]),
'answer_pred':
dataloader.dataset.label2ans[ans_pred[j].max(0)[1].data[0]],
'answer_gt':
dataloader.dataset.label2ans[int(a[j].max(0)[1])],
'explain_gt':
' '.join([c_dict.idx2word[p] for p in c[j].tolist() if p > 2])
})
results = sorted(results, key=lambda x: x['question_id'])
score = score / len(dataloader.dataset)
return score, results
def evaluate(model, dataloader, q_dict, c_dict):
total_others_score = 0
total_yesno_score = 0
total_number_score = 0
total_all_score = 0
vq_score = 0
eq_score = 0
print_freq = 50
results = []
ann_path = os.path.join('data','v2_mscoco_val2014_annotations.json')
anns = json.load(open(ann_path))['annotations']
qid2atype = {a['question_id'] : a['answer_type'] for a in anns}
#qid2qtype = {a['question_id'] : a['question_type'] for a in anns}
atype_map = {'other':0, 'yes/no':1, 'number':2}
others_cnt = 0
yesno_cnt = 0
number_cnt = 0
for i, (qid, iid, v, q, a, c, l) in enumerate(dataloader):
batch_size = v.size(0)
v = Variable(v, volatile=True).cuda()
q = Variable(q, volatile=True).cuda()
ans_pred, exp_pred = model.evaluate(v, q)
atype_list = torch.Tensor([atype_map[qid2atype[id]] for id in qid])
others_idx = (atype_list==0).nonzero().squeeze()
yesno_idx = (atype_list==1).nonzero().squeeze()
number_idx = (atype_list==2).nonzero().squeeze()
others_cnt += len(others_idx)
yesno_cnt += len(yesno_idx)
number_cnt += len(number_idx)
others_score = compute_score_with_logits(ans_pred[others_idx.cuda()], a[others_idx].cuda()).sum()
yesno_score = compute_score_with_logits(ans_pred[yesno_idx.cuda()], a[yesno_idx].cuda()).sum()
number_score = compute_score_with_logits(ans_pred[number_idx.cuda()], a[number_idx].cuda()).sum()
total_others_score += others_score
total_yesno_score += yesno_score
total_number_score += number_score
all_score = compute_score_with_logits(ans_pred, a.cuda()).sum()
total_all_score += all_score
for j in range(batch_size):
results.append({
'question_id': qid[j],
'image_id': iid[j],
'question': ' '.join([q_dict.idx2word[p] for p in q[j].data.tolist() if p > 0]),
'answer_pred': dataloader.dataset.label2ans[ans_pred[j].max(0)[1].data[0]],
'answer_gt': dataloader.dataset.label2ans[int(a[j].max(0)[1])],
'explain_res': ' '.join([c_dict.idx2word[p] for p in exp_pred[j].data.tolist() if p > 2]),
'explain_gt': ' '.join([c_dict.idx2word[p] for p in c[j].tolist() if p > 2])
})
results = sorted(results, key=lambda x: x['question_id'])
total_all_score = total_all_score / len(dataloader.dataset)
total_others_score /= others_cnt
total_yesno_score /= yesno_cnt
total_number_score /= number_cnt
scores = {'all':total_all_score, 'others':total_others_score, 'yes/no':total_yesno_score, 'number':total_number_score}
return scores, results
def evaluate(model, dataloader, model_hps, args, device):
model.eval()
label2ans = dataloader.dataset.label2ans
num_answers = len(label2ans)
relation_type = dataloader.dataset.relation_type
N = len(dataloader.dataset)
results = []
score = 0
pbar = tqdm(total=len(dataloader))
if args.save_logits:
idx = 0
pred_logits = np.zeros((N, num_answers))
gt_logits = np.zeros((N, num_answers))
for i, (v, norm_bb, q, target, qid, _, bb,
spa_adj_matrix, sem_adj_matrix) in enumerate(dataloader):
batch_size = v.size(0)
num_objects = v.size(1)
v = Variable(v).to(device)
norm_bb = Variable(norm_bb).to(device)
q = Variable(q).to(device)
pos_emb, sem_adj_matrix, spa_adj_matrix = prepare_graph_variables(
relation_type, bb, sem_adj_matrix, spa_adj_matrix, num_objects,
model_hps.nongt_dim, model_hps.imp_pos_emb_dim,
model_hps.spa_label_num, model_hps.sem_label_num, device)
pred, att = model(v, norm_bb, q, pos_emb, sem_adj_matrix,
spa_adj_matrix, None)
# Check if target is a placeholder or actual targets
if target.size(-1) == num_answers:
target = Variable(target).to(device)
batch_score = compute_score_with_logits(
pred, target, device).sum()
score += batch_score
if args.save_logits:
gt_logits[idx:batch_size+idx, :] = target.cpu().numpy()
if args.save_logits:
pred_logits[idx:batch_size+idx, :] = pred.cpu().numpy()
idx += batch_size
if args.save_answers:
qid = qid.cpu()
pred = pred.cpu()
current_results = make_json(pred, qid, dataloader)
results.extend(current_results)
pbar.update(1)
score = score / N
results_folder = f"{args.output_folder}/results"
if args.save_logits:
utils.create_dir(results_folder)
save_to = f"{results_folder}/logits_{args.dataset}" +\
f"_{args.split}.npy"
np.save(save_to, pred_logits)
utils.create_dir("./gt_logits")
save_to = f"./gt_logits/{args.dataset}_{args.split}_gt.npy"
if not os.path.exists(save_to):
np.save(save_to, gt_logits)
if args.save_answers:
utils.create_dir(results_folder)
save_to = f"{results_folder}/{args.dataset}_" +\
f"{args.split}.json"
json.dump(results, open(save_to, "w"))
return score
type(tmp)
tmp.keys()
len(tmp)
for i, (v, b, q, a) in enumerate(test_loader):
v = v.cuda()
b = b.cuda()
q = q.long().cuda()
a = a.cuda()
model = model(v, b, q, a)
loss = train.instance_bce_with_logits(pred, a)
loss.backward()
nn.utils.clip_grad_norm_(model.parameters(), 0.25)
optim.step()
optim.zero_grad()
batch_score = train.compute_score_with_logits(pred, a.data).sum()
total_loss += loss.data * v.size(0)
train_score += batch_score
if i % 5000 == 0:
print('{}_iteration_done'.format(i))
total_loss /= len(train_loader.dataset)
train_score = 100 * train_score / len(train_loader.dataset)
model.train(False)
eval_score, bound = train.evaluate(model, eval_loader)
import argparse
import torch
import torch.nn as nn
from torch.utils.data import DataLoader
import numpy as np
def train(model,
train_loader,
eval_loader,
num_epochs,
output,
opt=None,
s_epoch=0):
lr_default = 1e-3 if eval_loader is not None else 7e-4
lr_decay_step = 2
lr_decay_rate = .25
lr_decay_epochs = range(
10, 20, lr_decay_step) if eval_loader is not None else range(
10, 20, lr_decay_step)
gradual_warmup_steps = [
0.5 * lr_default, 1.0 * lr_default, 1.5 * lr_default, 2.0 * lr_default
]
saving_epoch = 3
grad_clip = .25
utils.create_dir(output)
optim = torch.optim.Adamax(filter(lambda p: p.requires_grad, model.parameters()), lr=lr_default) \
if opt is None else opt
logger = utils.Logger(os.path.join(output, 'log.txt'))
best_eval_score = 0
utils.print_model(model, logger)
logger.write('optim: adamax lr=%.4f, decay_step=%d, decay_rate=%.2f, grad_clip=%.2f' % \
(lr_default, lr_decay_step, lr_decay_rate, grad_clip))
for epoch in range(s_epoch, num_epochs):
total_loss = 0
train_score = 0
total_norm = 0
count_norm = 0
t = time.time()
N = 0
if epoch < len(gradual_warmup_steps):
optim.param_groups[0]['lr'] = gradual_warmup_steps[epoch]
logger.write('gradual warmup lr: %.4f' %
optim.param_groups[0]['lr'])
elif epoch in lr_decay_epochs:
optim.param_groups[0]['lr'] *= lr_decay_rate
logger.write('decreased lr: %.4f' % optim.param_groups[0]['lr'])
else:
logger.write('lr: %.4f' % optim.param_groups[0]['lr'])
for i, (v, b, p, e, n, a, idx, types) in enumerate(train_loader):
v = v.cuda()
b = b.cuda()
p = p.cuda()
e = e.cuda()
a = a.cuda()
_, logits = model(v, b, p, e, a)
n_obj = logits.size(2)
logits.squeeze_()
merged_logit = torch.cat(
tuple(logits[j, :, :n[j][0]] for j in range(n.size(0))),
-1).permute(1, 0)
merged_a = torch.cat(
tuple(a[j, :n[j][0], :n_obj] for j in range(n.size(0))), 0)
loss = instance_bce_with_logits(merged_logit, merged_a,
'sum') / v.size(0)
N += n.sum().float()
batch_score = compute_score_with_logits(merged_logit,
merged_a.data).sum()
loss.backward()
total_norm += nn.utils.clip_grad_norm_(model.parameters(),
grad_clip)
count_norm += 1
optim.step()
optim.zero_grad()
total_loss += loss.item() * v.size(0)
train_score += batch_score.item()
total_loss /= N
train_score = 100 * train_score / N
if None != eval_loader:
model.train(False)
eval_score, bound, entropy = evaluate(model, eval_loader)
model.train(True)
logger.write('epoch %d, time: %.2f' % (epoch, time.time() - t))
logger.write('\ttrain_loss: %.2f, norm: %.4f, score: %.2f' %
(total_loss, total_norm / count_norm, train_score))
if eval_loader is not None:
logger.write('\teval score: %.2f/%.2f/%.2f (%.2f)' %
(100 * eval_score[0], 100 * eval_score[1],
100 * eval_score[2], 100 * bound))
eval_score = eval_score[0]
if eval_loader is not None and entropy is not None:
info = ''
for i in range(entropy.size(0)):
info = info + ' %.2f' % entropy[i]
logger.write('\tentropy: ' + info)
if (eval_loader is not None and eval_score > best_eval_score) or (
eval_loader is None and epoch >= saving_epoch):
model_path = os.path.join(output, 'model_epoch%d.pth' % epoch)
utils.save_model(model_path, model, epoch, optim)
if eval_loader is not None:
best_eval_score = eval_score