def validate(loader,
model,
logger,
epoch=0,
print_freq=10,
dual_training=False):
# switch to train mode
model.eval()
meters = logger.reset_meters('val')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
target_question = sample['question']
# To arrange the length of mini-batch by the descending order of question length
new_ids, lengths = process_lengths_sort(target_question)
target_question = Variable(target_question.cuda(async=True),
volatile=True)
input_visual = Variable(sample['visual'].cuda(async=True),
volatile=True)
target_answer = Variable(sample['answer'].cuda(async=True),
volatile=True)
# compute output
generated_a = model(input_visual, target_question, target_answer)
torch.cuda.synchronize()
# Hack for the compatability of reinforce() and DataParallel()
loss_a = F.cross_entropy(generated_a, target_answer)
loss = loss_a
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(generated_a.data,
target_answer.data,
topk=(1, 5, 10))
# bleu_score = calculate_bleu_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word)
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
meters['loss_a'].update(loss_a.data[0], n=batch_size)
# meters['bleu_score'].update(bleu_score, n=batch_size)
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
print('[Val]\tEpoch: [{0}]'
'Time {batch_time.avg:.3f}\t'
'A Loss: {loss_a.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(epoch,
batch_time=meters['batch_time'],
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a']))
logger.log_meters('val', n=epoch)
return meters['acc1'].avg, meters['acc5'].avg, meters['acc10'].avg
def train(loader, model, criterion, optimizer, logger, epoch, print_freq=10):
# switch to train mode
model.train()
meters = logger.reset_meters('train')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(
0) # keys = ['visual', 'question_id', 'question', 'answer']
# question: (B, T), (128, 26)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
input_visual = Variable(sample['visual'])
input_question = Variable(sample['question'])
target_answer = Variable(sample['answer'].cuda(non_blocking=True))
# compute output
output = model(input_visual, input_question)
torch.cuda.synchronize()
loss = criterion(output, target_answer)
meters['loss'].update(loss.item(), n=batch_size)
# measure accuracy
acc1, acc5 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 5))
meters['acc1'].update(acc1.item(), n=batch_size)
meters['acc5'].update(acc5.item(), n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if i % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@5 {acc5.val:.3f} ({acc5.avg:.3f})'.format(
epoch,
i,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
loss=meters['loss'],
acc1=meters['acc1'],
acc5=meters['acc5']))
logger.log_meters('train', n=epoch)
def validate(loader, model, criterion, logger, epoch=0, print_freq=10):
results = []
# switch to evaluate mode
model.eval()
meters = logger.reset_meters('val')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
input_visual = Variable(sample['visual'].cuda(), volatile=True)
input_question = Variable(sample['question'].cuda(), volatile=True)
target_answer = Variable(sample['answer'].cuda(), volatile=True)
# compute output
output = model(input_visual, input_question)
loss = criterion(output, target_answer)
meters['loss'].update(loss.data[0], n=batch_size)
# measure accuracy and record loss
acc1, acc5 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 5))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
# compute predictions for OpenEnded accuracy
_, pred = output.data.cpu().max(1)
pred.squeeze_()
for j in range(batch_size):
results.append({
'question_id': sample['question_id'][j],
'answer': loader.dataset.aid_to_ans[pred[j]]
})
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if i % print_freq == 0:
print('Val: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@5 {acc5.val:.3f} ({acc5.avg:.3f})'.format(
i,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
loss=meters['loss'],
acc1=meters['acc1'],
acc5=meters['acc5']))
print(' * Acc@1 {acc1.avg:.3f} Acc@5 {acc5.avg:.3f}'.format(
acc1=meters['acc1'], acc5=meters['acc1']))
logger.log_meters('val', n=epoch)
return meters['acc1'].avg, results
def evaluate(loader, model, logger, print_freq=10, sampling_num=5):
model.eval()
model.module.set_testing(True, sample_num=sampling_num)
meters = logger.reset_meters('test')
results = []
end = time.time()
blue_score_all = 0
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
input_visual = Variable(sample['visual'].cuda(async=True), volatile=True )
input_answer = Variable(sample['answer'].cuda(async=True), volatile=True)
target_answer = sample['answer']
input_question = Variable(sample['question'].cuda(async=True), volatile=True)
output_answer, g_answers, g_answers_score, generated_q = model(input_visual, input_question, input_answer)
bleu_score = calculate_bleu_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word)
acc1, acc5, acc10 = utils.accuracy(output_answer.cpu().data, target_answer, topk=(1, 5, 10))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
meters['bleu_score'].update(bleu_score, n=batch_size)
g_answers = g_answers.cpu().data
g_answers_score = g_answers_score.cpu().data
for j in range(batch_size):
new_question = generated_q.cpu().data[j].tolist()
new_answer = g_answers[j]
new_answer_score = g_answers_score[j]
sampled_aqa = [[new_question, new_answer, new_answer_score],]
num_result = { 'gt_question': sample['question'][j][1:].tolist(), #sample['question'][j].numpy(),
'gt_answer': sample['answer'][j],
'augmented_qa': sampled_aqa,}
readable_result = {
'gt_question': translate_tokens(sample['question'][j][1:], loader.dataset.wid_to_word),
'gt_answer': loader.dataset.aid_to_ans[sample['answer'][j]],
'augmented_qa': [ [
translate_tokens(item[0], loader.dataset.wid_to_word), # translate question
loader.dataset.aid_to_ans[item[1]], # translate answer
] for item in sampled_aqa],}
results.append({'image': sample['image'][j],
'numeric_result': num_result,
'readable_result': readable_result}, )
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
print('* [Evaluation] Result: Acc@1:{acc1.avg:.3f}\t'
'Acc@5:{acc5.avg:.3f}\tAcc@10:{acc10.avg:.3f}\t'
'Time: {batch_time.avg:.3f}\t'
'BLEU: {bleu_score.avg:.5f}'.format(
acc1=meters['acc1'], acc5=meters['acc5'], acc10=meters['acc10'],
batch_time=meters['batch_time'],
bleu_score=meters['bleu_score']))
model.module.set_testing(False)
return results
def train(loader,
model,
optimizer,
logger,
epoch,
print_freq=10,
dual_training=False,
alternative_train=-1.):
# switch to train mode
model.train()
model.module.set_testing(False)
meters = logger.reset_meters('train')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
target_question = sample['question']
# To arrange the length of mini-batch by the descending order of question length
new_ids, lengths = process_lengths_sort(target_question)
new_ids = Variable(new_ids).detach()
target_question = Variable(target_question.cuda())
input_visual = Variable(sample['visual'].cuda())
target_answer = Variable(sample['answer'].cuda(async=True))
# compute output
output = model(input_visual, target_question, target_answer)
generated_a = output[0]
generated_q = output[1]
additional_loss = output[2].mean()
torch.cuda.synchronize()
# Hack for the compatability of reinforce() and DataParallel()
target_question = pack_padded_sequence(target_question.index_select(
0, new_ids)[:, 1:],
lengths,
batch_first=True)[0]
output = pack_padded_sequence(generated_q.index_select(0, new_ids),
lengths,
batch_first=True)[0]
loss_q = F.cross_entropy(output, target_question)
loss_a = F.cross_entropy(generated_a, target_answer)
if alternative_train > 1. or alternative_train < 0.:
loss = loss_a + loss_q
if dual_training:
loss += additional_loss
else:
if torch.rand(1)[0] > alternative_train:
loss = loss_a
else:
loss = loss_q
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(generated_a.data,
target_answer.data,
topk=(1, 5, 10))
# bleu_score = calculate_bleu_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word)
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
meters['loss_a'].update(loss_a.data[0], n=batch_size)
meters['loss_q'].update(loss_q.data[0], n=batch_size)
meters['dual_loss'].update(additional_loss.data[0], n=batch_size)
# meters['bleu_score'].update(bleu_score, n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if (i + 1) % print_freq == 0:
print(
'[Train]\tEpoch: [{0}][{1}/{2}] '
'Time {batch_time.avg:.3f}\t'
'Data {data_time.avg:.3f}\t'
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}, Dual Loss: {loss_d.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(
epoch,
i + 1,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q'],
loss_d=meters['dual_loss']))
print(
'[Train]\tEpoch: [{0}]'
'Time {batch_time.avg:.3f}\t'
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}, Dual Loss: {loss_d.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(epoch,
batch_time=meters['batch_time'],
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q'],
loss_d=meters['dual_loss']))
logger.log_meters('train', n=epoch)
def train(loader,
model,
criterion,
optimizer,
logger,
epoch,
print_freq=10,
dict=None):
# switch to train mode
model.train()
meters = logger.reset_meters('train')
end = time.time()
results = []
bleu_score = 0
n_sample = 0
for i, sample in enumerate(loader):
pred_dict = {}
gt_dict = {}
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
# input of Bert vs Skip-thoughts
if hasattr(model.module.seq2vec, 'dir_st'):
input_question = Variable(sample['question'])
else:
questions = sample["item_vqa"]["question"]
input_question = torch.zeros([sample['visual'].shape[0], 768])
for j in range(sample['visual'].shape[0]):
input_question[j] = torch.tensor(dict[questions[j]])
input_question = Variable(input_question)
input_visual = Variable(sample['visual'])
target_answer = Variable(sample['answer'].cuda())
# compute output
output, hidden = model(input_visual, input_question)
torch.cuda.synchronize()
loss = criterion(output, target_answer)
meters['loss'].update(loss.item(), n=batch_size)
# measure accuracy
acc1, acc2 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 2))
meters['acc1'].update(acc1.item(), n=batch_size)
meters['acc2'].update(acc2.item(), n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
_, pred = output.data.cpu().max(1)
pred.squeeze_()
for j in range(batch_size):
results.append({
'question_id': sample['question_id'][j],
'answer': loader.dataset.aid_to_ans[pred[j]]
})
pred_dict[sample['question_id'][j]] = loader.dataset.aid_to_ans[
pred[j]]
gt_dict[sample['question_id'][j]] = loader.dataset.aid_to_ans[
target_answer[j]]
# bleu_batch = metrics_utils.compute_bleu_score(pred_dict, gt_dict)
if i % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
# 'Bleu {bleu_batch:.4f} \t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@2 {acc2.val:.3f} ({acc2.avg:.3f})'.format(
epoch,
i,
len(loader),
# bleu_batch=bleu_batch*100,
loss=meters['loss'],
acc1=meters['acc1'],
acc2=meters['acc2']))
logger.log_meters('train', n=epoch)
def validate(loader,
model,
criterion,
logger,
epoch=0,
print_freq=2,
topk=1,
dict=None):
results = []
bleu_score = 0
n_sample = 0
# switch to evaluate mode
model.eval()
meters = logger.reset_meters('val')
end = time.time()
with torch.no_grad():
for i, sample in enumerate(loader):
pred_dict = {}
gt_dict = {}
batch_size = sample['visual'].size(0)
# input of Bert vs Skip-thoughts
if hasattr(model.module.seq2vec, 'dir_st'):
input_question = Variable(sample['question'])
else:
questions = sample["item_vqa"]["question"]
input_question = torch.zeros([sample['visual'].shape[0], 3072])
for j in range(sample['visual'].shape[0]):
input_question[j] = torch.tensor(dict[questions[j]])
input_question = input_question.cuda()
input_visual = sample['visual'].cuda()
target_answer = sample['answer'].cuda()
# compute output
output, hidden = model(input_visual, input_question)
# loss = criterion(output, target_answer)
# meters['loss'].update(loss.item(), n=batch_size)
# measure accuracy and record loss
acc1, acc2 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 2))
meters['acc1'].update(acc1.item(), n=batch_size)
meters['acc2'].update(acc2.item(), n=batch_size)
# compute predictions for OpenEnded accuracy
_, pred = output.data.cpu().max(1)
target_answer = target_answer.data.cpu()
pred.squeeze_()
for j in range(batch_size):
if topk == 1:
item = {
'question_id': sample['question_id'][j],
'answer': loader.dataset.aid_to_ans[pred[j]]
}
else:
item = {'question_id': sample['question_id'][j]}
for topi in range(topk):
item['answer{}'.format(
topi + 1)] = loader.dataset.aid_to_ans[output.topk(
topk)[1][j][topi]]
results.append(item)
pred_dict[sample['question_id']
[j]] = loader.dataset.aid_to_ans[pred[j]]
try:
gt_dict[sample['question_id']
[j]] = loader.dataset.aid_to_ans[target_answer[j]]
except:
gt_dict[sample['question_id']
[j]] = loader.dataset.aid_to_ans[0]
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
bleu_batch = metrics_utils.compute_bleu_score(pred_dict, gt_dict)
if i % print_freq == 0:
print('Val: [{0}/{1}]\t'
'Bleu@ {bleu_batch:.3f} \t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@2 {acc2.val:.3f} ({acc2.avg:.3f})'.format(
i,
len(loader),
bleu_batch=bleu_batch * 100,
acc1=meters['acc1'],
acc2=meters['acc2']))
bleu_score += bleu_batch * batch_size
n_sample += batch_size
bleu_score = bleu_score / n_sample
print(
' * Bleu@ {bleu_score:.3f} Acc@1 {acc1.avg:.3f} Acc@2 {acc2.avg:.3f}'.
format(bleu_score=bleu_score * 100,
acc1=meters['acc1'],
acc2=meters['acc2']))
logger.log_meters('val', n=epoch)
return meters['acc1'].avg, results
def evaluate(
loader: torch.utils.data.DataLoader,
model: torch.nn.Module,
logger: logger.Experiment,
sampling_num: int = 5,
neptune_exp: Optional[neptune.experiments.Experiment] = None,
) -> List[RESULT]:
aid_to_ans = loader.dataset.aid_to_ans + ["UNK"]
model.eval()
model.module.set_testing(True, sample_num=sampling_num)
meters = logger.reset_meters("test")
res_counter = {
"correct@1": 0,
"correct@5": 0,
"correct@10": 0,
"n_sample": 0,
}
results = []
end = time.time()
for sample in loader:
batch_size = sample["visual"].size(0)
input_visual = Variable(sample["visual"].cuda())
input_answer = Variable(sample["answer"].cuda())
target_answer = sample["answer"]
input_question = sample["question"].long().cuda()
output_answer, g_answers, g_answers_score, generated_q = model(
input_visual, input_question, input_answer
)
output_answer_ = output_answer.detach().cpu().numpy()
bleu_score = calculate_bleu_score(
generated_q.cpu().data,
sample["question"],
loader.dataset.wid_to_word,
)
acc1, acc5, acc10 = utils.accuracy(
output_answer.cpu().data, target_answer, topk=(1, 5, 10)
)
correct1, correct5, correct10 = utils.correct_k(
output_answer.cpu().data, target_answer, topk=(1, 5, 10)
)
# accumulate number of correct predictions
meters["acc1"].update(acc1.item(), n=batch_size)
meters["acc5"].update(acc5.item(), n=batch_size)
meters["acc10"].update(acc10.item(), n=batch_size)
meters["bleu_score"].update(bleu_score, n=batch_size)
g_answers = g_answers.cpu().data
g_answers_score = g_answers_score.cpu().data
res_counter["correct@1"] += correct1.item()
res_counter["correct@5"] += correct5.item()
res_counter["correct@10"] += correct10.item()
res_counter["n_sample"] += batch_size
for j in range(batch_size):
new_question = generated_q.cpu().data[j].tolist()
new_answer = g_answers[j]
given_question = input_question.cpu().data[j].tolist()
given_question = translate_tokens(
given_question, loader.dataset.wid_to_word
)
predict_answers = np.flip(np.argsort(output_answer_[j]))[:10]
predict_answers = [
loader.dataset.aid_to_ans[w] for w in predict_answers
]
new_answer_score = g_answers_score[j]
sampled_aqa = [[new_question, new_answer, new_answer_score]]
readable_result = {
"gt_answer": aid_to_ans[sample["answer"][j]],
"augmented_qa": [
[
translate_tokens(
item[0], loader.dataset.wid_to_word
), # translate question
aid_to_ans[item[1]], # translate answer
]
for item in sampled_aqa
],
"given_question": given_question,
"predict_answers": predict_answers,
}
results.append(
{
"image": sample["image"][j],
"readable_result": readable_result,
}
)
# measure elapsed time
meters["batch_time"].update(time.time() - end, n=batch_size)
end = time.time()
print(
"* [Evaluation] Result: Acc@1:{acc1.avg:.3f}\t"
"Acc@5:{acc5.avg:.3f}\tAcc@10:{acc10.avg:.3f}\t"
"Time: {batch_time.avg:.3f}\t"
"BLEU: {bleu_score.avg:.5f}".format(
acc1=meters["acc1"],
acc5=meters["acc5"],
acc10=meters["acc10"],
batch_time=meters["batch_time"],
bleu_score=meters["bleu_score"],
)
)
print(f"{res_counter['correct@1']} / {res_counter['n_sample']}")
if neptune_exp is not None:
neptune_exp.log_metric("Acc@1", meters["acc1"].avg)
neptune_exp.log_metric("Acc@5", meters["acc5"].avg)
neptune_exp.log_metric("Acc@10", meters["acc10"].avg)
neptune_exp.log_metric("N_Correct@1", res_counter["correct@1"])
neptune_exp.log_metric("N_Samples", res_counter["n_sample"])
model.module.set_testing(False)
return results
def validate(loader, model, logger, epoch=0, print_freq=100):
# switch to train mode
model.eval()
meters = logger.reset_meters("val")
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample["visual"].size(0)
# measure data loading time
meters["data_time"].update(time.time() - end, n=batch_size)
target_question = sample["question"]
# To arrange the length of mini-batch by the descending order of question length
new_ids, lengths = process_lengths_sort(target_question)
target_question = Variable(target_question.cuda())
input_visual = Variable(sample["visual"].cuda())
target_answer = Variable(sample["answer"].cuda())
# compute output
output = model(input_visual, target_question, target_answer)
generated_a = output[0]
generated_q = output[1]
additional_loss = output[2].mean()
torch.cuda.synchronize()
# Hack for the compatability of reinforce() and DataParallel()
target_question = pack_padded_sequence(
target_question.index_select(0, new_ids)[:, 1:],
lengths,
batch_first=True,
)[0]
output = pack_padded_sequence(
generated_q.index_select(0, new_ids), lengths, batch_first=True
)[0]
loss_q = F.cross_entropy(output, target_question)
loss_a = F.cross_entropy(generated_a, target_answer)
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(
generated_a.data, target_answer.data, topk=(1, 5, 10)
)
# bleu_score = calculate_bleu_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word)
meters["acc1"].update(acc1.item(), n=batch_size)
meters["acc5"].update(acc5.item(), n=batch_size)
meters["acc10"].update(acc10.item(), n=batch_size)
meters["loss_a"].update(loss_a.data.item(), n=batch_size)
meters["loss_q"].update(loss_q.data.item(), n=batch_size)
meters["dual_loss"].update(additional_loss.data.item(), n=batch_size)
# measure elapsed time
meters["batch_time"].update(time.time() - end, n=batch_size)
# meters['bleu_score'].update(bleu_score, n=batch_size)
end = time.time()
print(
"[Val]\tEpoch: [{0}]"
"Time {batch_time.avg:.3f}\t"
"A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}, Dual Loss: {loss_d.avg:.3f}\t"
"Acc@1 {acc1.avg:.3f}\t"
"Acc@5 {acc5.avg:.3f}\t"
"Acc@10 {acc10.avg:.3f}\t".format(
epoch,
batch_time=meters["batch_time"],
acc1=meters["acc1"],
acc5=meters["acc5"],
acc10=meters["acc10"],
loss_a=meters["loss_a"],
loss_q=meters["loss_q"],
loss_d=meters["dual_loss"],
)
)
logger.log_meters("val", n=epoch)
return (
meters["acc1"].avg,
meters["acc5"].avg,
meters["acc10"].avg,
meters["loss_q"].avg,
)
def evaluate(loader,
model_vqa,
model_vqg,
logger,
print_freq=10,
sampling_num=5):
model_vqa.eval()
model_vqg.eval()
meters = logger.reset_meters('evaluate')
results = []
model_vqg.is_testing = True
end = time.time()
blue_score_all = 0
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
input_visual = Variable(sample['visual'].cuda(async=True),
volatile=True)
target_answer = sample['answer']
input_question = Variable(sample['question'].cuda(async=True),
volatile=True)
# compute output
output_answer = model_vqa(input_visual, input_question)
acc1, acc5, acc10 = utils.accuracy(output_answer.cpu().data,
target_answer,
topk=(1, 5, 10))
meters['acc1'].update(acc1, n=batch_size)
meters['acc5'].update(acc5, n=batch_size)
meters['acc10'].update(acc10, n=batch_size)
g_answers_score, g_answers = torch.topk(F.softmax(output_answer),
sampling_num)
generated_q = []
# answered_a = []
for j in range(sampling_num):
generated_q.append(
model_vqg(prepare_input(input_visual, model_vqg.opt['mode']),
g_answers[:, j].contiguous()))
g_answers = g_answers.cpu().data
g_answers_score = g_answers_score.cpu().data
for j in range(batch_size):
sampled_aqa = []
for k in range(sampling_num):
new_question = generated_q[k].cpu().data[j].tolist()
new_answer = g_answers[j, k]
new_answer_score = g_answers_score[j, k]
sampled_aqa.append(
[new_question, new_answer, new_answer_score])
num_result = {
'gt_question': sample['question'][j]
[1:].tolist(), #sample['question'][j].numpy(),
'gt_answer': sample['answer'][j],
'augmented_qa': sampled_aqa,
}
readable_result = {
'gt_question':
translate_tokens(sample['question'][j][1:],
loader.dataset.wid_to_word),
'gt_answer':
loader.dataset.aid_to_ans[sample['answer'][j]],
'augmented_qa': [
[
translate_tokens(
item[0],
loader.dataset.wid_to_word), # translate question
loader.dataset.aid_to_ans[item[1]], # translate answer
] for item in sampled_aqa
],
}
results.append(
{
'image': sample['image'][j],
'numeric_result': num_result,
'readable_result': readable_result
}, )
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if i % print_freq == 0:
print('Test: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'.format(
i,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time']))
print(' ** Result: Acc@1:{}\tAcc@5:{}\tAcc@10:{}'.format(
meters['acc1'].avg, meters['acc5'].avg, meters['acc10'].avg))
model_vqg.is_testing = False
pdb.set_trace()
model_vqa.train()
model_vqg.train()
return results
def train(loader,
model,
optimizer,
logger,
epoch,
print_freq=10,
dual_training=False,
alternative_train=-1.):
# switch to train mode
model.train()
model.module.set_testing(False)
model.module.use_reinforce = False
meters = logger.reset_meters('train')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
target_question = sample['question']
# To arrange the length of mini-batch by the descending order of question length
lengths = process_lengths(target_question)
target_question = Variable(target_question.cuda())
input_visual = Variable(sample['visual'].cuda())
target_answer = Variable(sample['answer'].cuda(async=True))
# compute output
generated_a, selected_a, generated_q, loss_q = model(
input_visual, target_question, target_answer)
torch.cuda.synchronize()
#pdb.set_trace()
if target_answer.dim() == 2: # use BCE loss
loss_a = sampled_bce_loss(generated_a, target_answer)
vqg_learn = torch.gather(target_answer, 1,
selected_a).type(torch.cuda.FloatTensor)
else:
loss_a = F.cross_entropy(generated_a, target_answer)
vqg_learn = selected_a.eq(target_answer).type(
torch.cuda.FloatTensor)
# loss_q = (loss_q * vqg_learn).sum() / (vqg_learn.sum() + float(np.finfo(np.float32).eps))
loss_q = loss_q.mean()
if alternative_train > 1. or alternative_train < 0.:
loss = loss_a + loss_q
else:
if torch.rand(1)[0] > alternative_train:
loss = loss_a
else:
loss = loss_q
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(generated_a.data,
target_answer.data,
topk=(1, 5, 10))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
meters['loss_a'].update(loss_a.data[0], n=batch_size)
meters['loss_q'].update(loss_q.data[0], n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if (i + 1) % print_freq == 0:
print('[Train]\tEpoch: [{0}][{1}/{2}] '
'Time {batch_time.avg:.3f}\t'
'Data {data_time.avg:.3f}\t'
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(
epoch,
i + 1,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q']))
print('[Train]\tEpoch: [{0}] '
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(epoch,
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q']))
logger.log_meters('train', n=epoch)
def validate(loader,
model,
logger,
epoch=0,
print_freq=10,
dual_training=False):
# switch to train mode
model.eval()
meters = logger.reset_meters('val')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
target_question = sample['question']
# To arrange the length of mini-batch by the descending order of question length
lengths = process_lengths(target_question)
target_question = Variable(target_question.cuda(async=True),
volatile=True)
input_visual = Variable(sample['visual'].cuda(async=True),
volatile=True)
target_answer = Variable(sample['answer'].cuda(async=True),
volatile=True)
# compute output
generated_a, selected_a, generated_q, loss_q = model(
input_visual, target_question, target_answer)
#pdb.set_trace()
if target_answer.dim() == 2: # use BCE loss
loss_a = sampled_bce_loss(generated_a, target_answer)
vqg_learn = torch.gather(target_answer, 1,
selected_a).type(torch.cuda.FloatTensor)
else:
loss_a = F.cross_entropy(generated_a, target_answer)
vqg_learn = selected_a.eq(target_answer).type(
torch.cuda.FloatTensor)
loss_q = (loss_q * vqg_learn).sum() / (vqg_learn.sum() +
float(np.finfo(np.float32).eps))
loss = loss_a + loss_q
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(generated_a.data,
target_answer.data,
topk=(1, 5, 10))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
meters['loss_a'].update(loss_a.data[0], n=batch_size)
meters['loss_q'].update(loss_q.data[0], n=batch_size)
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if (i + 1) % print_freq == 0:
print('[Val]\tEpoch: [{0}][{1}/{2}] '
'Time {batch_time.avg:.3f}\t'
'Data {data_time.avg:.3f}\t'
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(
epoch,
i + 1,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q']))
print('[Val]\tEpoch: [{0}] '
'A Loss: {loss_a.avg:.3f}, Q Loss: {loss_q.avg:.3f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}\t'.format(epoch,
acc1=meters['acc1'],
acc5=meters['acc5'],
acc10=meters['acc10'],
loss_a=meters['loss_a'],
loss_q=meters['loss_q']))
logger.log_meters('val', n=epoch)
return meters['acc1'].avg, meters['acc5'].avg, meters['acc10'].avg
def validate(loader, model, criterion, logger, epoch=0, print_freq=2):
results = []
# switch to evaluate mode
model.eval()
meters = logger.reset_meters('val')
end = time.time()
with torch.no_grad():
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
input_question = Variable(sample['question'])
input_visual = Variable(sample['visual'])
target_answer = Variable(sample['answer'].cuda())
# compute output
try:
output, hidden = model(input_visual, input_question)
except:
output = model(input_visual, input_question)
# measure accuracy and record loss
acc1, acc5 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 5))
meters['acc1'].update(acc1.item(), n=batch_size)
meters['acc5'].update(acc5.item(), n=batch_size)
# compute predictions for OpenEnded accuracy
_, pred = output.data.cpu().max(1)
pred.squeeze_()
for j in range(batch_size):
question_id = sample['question_id'][j]
if isinstance(question_id, torch.Tensor):
question_id = question_id.cpu().numpy().tolist()
answer = loader.dataset.aid_to_ans[pred[j]]
if isinstance(answer, torch.Tensor):
answer = answer.cpu().numpy()
results.append({'question_id': question_id, 'answer': answer})
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if i % print_freq == 0:
print('Val: [{0}/{1}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
# 'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@2 {acc5.val:.3f} ({acc5.avg:.3f})'.format(
i,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
# loss=meters['loss'],
acc1=meters['acc1'],
acc5=meters['acc5']))
print(' * Acc@1 {acc1.avg:.3f} Acc@2 {acc5.avg:.3f}'.format(
acc1=meters['acc1'], acc5=meters['acc5']))
logger.log_meters('val', n=epoch)
return meters['acc1'].avg, results
def train(loader, model, criterion, optimizer, logger, epoch, print_freq=10):
# switch to train mode
model.train()
meters = logger.reset_meters('train')
resnet = torchvision.models.resnet152(pretrained=True)
base_feat = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu,
resnet.maxpool, resnet.layer1, resnet.layer2,
resnet.layer3, resnet.layer4).cuda()
base_feat = nn.DataParallel(base_feat)
end = time.time()
for i, sample in enumerate(loader):
# batch_size = sample['visual'].size(0)
batch_size = sample[0].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
input_rgb = Variable(sample[0]).cuda()
input_thermal = Variable(sample[1]).cuda()
rgb_feat = base_feat(input_rgb)
thermal_feat = base_feat(input_thermal)
# input_visual = Variable(sample['visual'])
# input_question = Variable(sample['question'])
# target_answer = Variable(sample['answer'].cuda(async=True))
# sample_question_float = sample["question"].float()
# summary(model,[sample['visual'],sample_question_float])
# compute output
# output = model(input_visual, input_question)
output = model(rgb_feat, thermal_feat)
torch.cuda.synchronize()
loss = criterion(output, target_answer)
meters['loss'].update(loss.data[0], n=batch_size)
# measure accuracy
acc1, acc5 = utils.accuracy(output.data,
target_answer.data,
topk=(1, 5))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if i % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@5 {acc5.val:.3f} ({acc5.avg:.3f})'.format(
epoch,
i,
len(loader),
batch_time=meters['batch_time'],
data_time=meters['data_time'],
loss=meters['loss'],
acc1=meters['acc1'],
acc5=meters['acc5']))
logger.log_meters('train', n=epoch)
def train(loader, model, criterion, optimizer, logger, epoch, print_freq=10):
# switch to train mode
model.train()
meters = logger.reset_meters('train')
end = time.time()
for i, sample in enumerate(loader):
batch_size = sample['visual'].size(0)
# measure data loading time
meters['data_time'].update(time.time() - end, n=batch_size)
input_visual = Variable(sample['visual']).cuda()
# pdb.set_trace()
if model.module.opt['criterion']['type'] == 'bce':
target_answer = Variable(sample['answer_all'].cuda(async=True))
else: # model.opt['criterion'] == 'bce':
target_answer = Variable(sample['answer'].cuda(async=True))
# compute output
# pdb.set_trace()
output = model(input_visual)
torch.cuda.synchronize()
loss = criterion(output, target_answer)
meters['loss'].update(loss.data[0], n=batch_size)
# measure accuracy
acc1, acc5, acc10 = utils.accuracy(output.data, target_answer.data, topk=(1, 5, 10))
meters['acc1'].update(acc1[0], n=batch_size)
meters['acc5'].update(acc5[0], n=batch_size)
meters['acc10'].update(acc10[0], n=batch_size)
# compute gradient and do SGD step
optimizer.zero_grad()
loss.backward()
torch.cuda.synchronize()
optimizer.step()
torch.cuda.synchronize()
# measure elapsed time
meters['batch_time'].update(time.time() - end, n=batch_size)
end = time.time()
if (i+1) % print_freq == 0:
print('Epoch: [{0}][{1}/{2}]\t'
'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t'
'Data {data_time.val:.3f} ({data_time.avg:.3f})\t'
'Loss {loss.val:.4f} ({loss.avg:.4f})\t'
'Acc@1 {acc1.val:.3f} ({acc1.avg:.3f})\t'
'Acc@5 {acc5.val:.3f} ({acc5.avg:.3f})\t'
'Acc@10 {acc10.val:.3f} ({acc10.avg:.3f})'.format(
epoch, i + 1, len(loader),
batch_time=meters['batch_time'], data_time=meters['data_time'],
loss=meters['loss'], acc1=meters['acc1'], acc5=meters['acc5'],
acc10=meters['acc10']))
print('* [Training] Epoch: [{0}]\t'
'Time {batch_time.avg:.3f}\t'
'Data {data_time.avg:.3f}\t'
'Loss {loss.avg:.4f}\t'
'Acc@1 {acc1.avg:.3f}\t'
'Acc@5 {acc5.avg:.3f}\t'
'Acc@10 {acc10.avg:.3f}'.format(
epoch,
batch_time=meters['batch_time'], data_time=meters['data_time'],
loss=meters['loss'], acc1=meters['acc1'], acc5=meters['acc5'],
acc10=meters['acc10']))
logger.log_meters('train', n=epoch)
