def run(net, loader):
""" Run an epoch over the given loader """
net.eval()
answ = []
idxs = []
accs = []
log_softmax = nn.LogSoftmax().cuda()
for v, q, a, idx, q_len in loader:
var_params = {
'volatile': True,
'requires_grad': False,
}
v = Variable(v.cuda(non_blocking=True), **var_params)
q = Variable(q.cuda(non_blocking=True), **var_params)
a = Variable(a.cuda(non_blocking=True), **var_params)
q_len = Variable(q_len.cuda(non_blocking=True), **var_params)
out = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def train(loader, net, writer, epoch, epochs):
net.train()
device = torch.device(config.device_id)
log_softmax = nn.LogSoftmax(dim=1).to(device)
tq = tqdm(loader,
desc='{} E{:03d}/{:03d}'.format("train", epoch + 1, epochs),
ncols=0)
#
accs = []
i = 0
for q, a, idx in tq:
q = q.to(device)
a = a.to(device)
i = i + 1
out = net(q) # out:batch_size * answer_size
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc = utils.batch_accuracy(out.data, a.data).cpu()
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss.data.mean()), acc=fmt(acc.data.mean()))
accs.append(acc)
if i % 10 == 0:
niter = epoch * len(loader) + i
writer.add_scalar('Train/Loss', loss.data.mean(), niter)
writer.add_scalar('Train/acc', acc.mean(), niter)
accs = torch.cat(accs)
mean_acc = accs.mean()
print("[E{:03d}][mean acc:{:.4f}]".format(epoch + 1, mean_acc))
def run(net, loader, optimizer, tracker, train=False, prefix='', epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax().cuda()
for v, q, a, idx, q_len in tq:
var_params = {
'volatile': not train,
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params)
out = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iterations += 1
else:
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
loss_tracker.append(loss.data[0])
# acc_tracker.append(acc.mean())
for a in acc:
acc_tracker.append(a.item())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
if not train:
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def run(net, loader, edit_set_cmd):
""" Run an epoch over the given loader """
accs = []
answ = []
ss_vc = []
image_ids =[]
ques_ids = []
softmax = nn.Softmax(dim=1).cuda()
for v, q, a, idx, img_id, ques_id, q_len in tqdm(loader): # image, ques to vocab mapped , answer, item (sth to help index shuffled data with), len_val
#ipdb.set_trace()
var_params = {
'volatile': False,
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params) ### len of question
with torch.no_grad():
if config.vis_attention:
out =
else:
out = net(v, q, q_len)
softmax_vc = softmax(out) # torch.size(128,3000)
#ipdb.set_trace() ## check type of softmax_vc- enforce it to torch16 here itself/ alse see what happens when np.16..
acc = utils.batch_accuracy(out.data, a.data).cpu() #torch.Size([128, 1]) official vqa acc for every questions
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1) ### torch.Size([128) !!!! this is the predicted answer id!!!
ipdb.set_trace()
answ.append(answer.view(-1)) # pred_ans_id
ss_vc.append(softmax_vc) # #torch.Size([128, 3000])
accs.append(acc.view(-1)) # official vqa accurcay per question
ques_ids.append(ques_id.view(-1))
if edit_set_cmd:
image_ids.append(img_id)
else:
image_ids.append(img_id.view(-1))
#ipdb.set_trace()
ss_vc = torch.cat(ss_vc, dim=0) ## softmax_vectors
answ = torch.cat(answ, dim=0) ## pred_ans_id
accs = torch.cat(accs, dim=0) ## official vqa accurcay per question
ques_ids = torch.cat(ques_ids, dim=0)
if edit_set_cmd:
image_ids = [item for sublist in image_ids for item in sublist]
else:
image_ids = torch.cat(image_ids, dim=0)
### might be string in edit config case
print('the accuracy is:', torch.mean(accs)) ### mean of entire accuracy vector # tensor(0.6015) for val set
return answ, image_ids, ques_ids, ss_vc
def run(net, loader, optimizer, tracker, train=False, prefix='', epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
epoch_acc = 0
epoch_loss = 0
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax(dim=1).cuda()
for v, q, a, idx, q_len in tq:
var_params = {
'requires_grad': False,
}
v = Variable(v.cuda(), **var_params)
q = Variable(q.cuda(), **var_params)
a = Variable(a.cuda(), **var_params)
q_len = Variable(q_len.cuda(), **var_params)
out = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
epoch_acc += float(acc.float().mean()) * len(q_len) / len(
loader.dataset)
epoch_loss += float(loss * len(q_len) / len(loader.dataset))
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iterations += 1
loss_tracker.append(loss.data.item())
# acc_tracker.append(acc.mean())
for a in acc:
acc_tracker.append(a.item())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
return epoch_loss, epoch_acc
def run(net, loader, optimizer, tracker, train=False, prefix='', epoch=0):
""" Run an epoch over the given loader """
answ = []
idxs = []
accs = []
tracker_class, tracker_params = tracker.MeanMonitor, {}
if train:
net.train()
else:
net.eval()
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax(dim=1).cuda()
for v, q_inputs, q_masks, a, idx in tq:
v = v.to(device, non_blocking=True)
q_inputs = q_inputs.to(device, non_blocking=True)
q_masks = q_masks.to(device, non_blocking=True)
a = a.to(device, non_blocking=True)
out = net(v, q_inputs, q_masks)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iterations += 1
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
loss_tracker.append(loss.item())
for a in acc:
acc_tracker.append(a.item())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def evaluate_ent(model, data):
model.eval()
ent, acc, size = 0, 0, 0
for i, (v, q, a, idx, q_len) in enumerate(data):
if i > 10:
break
v = Variable(v.cuda(async=True))
q = Variable(q.cuda(async=True))
a = Variable(a.cuda(async=True))
q_len = Variable(q_len.cuda(async=True))
out = F.softmax(model(v, q, q_len), 1)
ent += entropy(out).sum().data.cpu()[0]
acc += utils.batch_accuracy(out.data, a.data).mean()
size += 1
return ent / size, acc / size
def evaluate(model, data):
model.eval()
log_softmax = nn.LogSoftmax().cuda()
loss, acc, size = 0, 0, 0
for i, (v, q, a, idx, q_len) in enumerate(data):
if i > 10:
break
v = Variable(v.cuda(async=True))
q = Variable(q.cuda(async=True))
a = Variable(a.cuda(async=True))
q_len = Variable(q_len.cuda(async=True))
out = model(v, q, q_len)
nll = -log_softmax(out)
loss += (nll * a / 10).sum(dim=1).mean().data.cpu()[0]
acc += utils.batch_accuracy(out.data, a.data).mean()
size += 1
return loss / size, acc / size
def run_test(net, loader, optimizer, tracker, prefix='', epoch=0):
""" Run an epoch over the given loader """
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax().cuda()
for v, q, a, idx, q_len in tq:
var_params = {
'volatile': False,
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params))
q = Variable(q.cuda(async=True), **var_params))
a = Variable(a.cuda(async=True), **var_params))
q_len = Variable(q_len.cuda(async=True), **var_params))
out = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
loss_tracker.append(loss.data[0])
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def validate(loader, net, writer, epoch, epochs):
net.eval()
device = torch.device(config.device_id)
log_softmax = nn.LogSoftmax(dim=1).to(device)
tq = tqdm(loader,
desc='{} E{:03d}/{:03d}'.format("val", epoch + 1, epochs),
ncols=0)
#
answ = []
idxs = []
accs = []
i = 0
for q, a, idx in tq:
q = q.to(device)
a = a.to(device)
i = i + 1
with torch.no_grad():
out = net(q) # out:batch_size * answer_size
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss.data.mean()),
acc=fmt(acc.data.mean()))
_, answer = out.data.cpu().max(dim=1)
answ.append(answer)
accs.append(acc)
idxs.append(idx)
if i % 10 == 0:
niter = epoch * len(loader) + i
writer.add_scalar('Val/Loss', loss.data.mean(), niter)
writer.add_scalar('Val/acc', acc.mean(), niter)
mean_acc = (torch.cat(accs, dim=0)).mean()
print("[E{:03d}][mean acc:{:.4f}]".format(epoch + 1, mean_acc))
answ = torch.cat(answ, dim=0).numpy()
accs = torch.cat(accs, dim=0).numpy()
idxs = torch.cat(idxs, dim=0).numpy()
return answ, accs, idxs
def evaluate_hw3():
# set mannual seed
torch.manual_seed(config.seed)
torch.cuda.manual_seed(config.seed)
val_loader = data.get_loader(val=True)
cudnn.benchmark = True
question_keys = val_loader.dataset.vocab['question'].keys()
net = Net(question_keys)
net.load_state_dict(
torch.load('model.pkl', map_location=lambda storage, loc: storage))
net = nn.DataParallel(net).cuda() # Support multiple GPUS
net.eval()
accs = []
for v, q, a, b, idx, v_mask, q_mask, q_len in val_loader:
var_params = {
'requires_grad': False,
}
v = Variable(v.cuda(), **var_params)
q = Variable(q.cuda(), **var_params)
a = Variable(a.cuda(), **var_params)
b = Variable(b.cuda(), **var_params)
q_len = Variable(q_len.cuda(), **var_params)
v_mask = Variable(v_mask.cuda(), **var_params)
q_mask = Variable(q_mask.cuda(), **var_params)
out = net(v, b, q, v_mask, q_mask, q_len)
answer = utils.process_answer(a)
acc = utils.batch_accuracy(out, answer).data.cpu()
accs.append(acc.mean().item())
return sum(accs) / len(accs)
def run(net,
loader,
tracker,
optimizer,
scheduler=None,
loss_criterion=None,
train=False,
prefix='',
epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
top_1_sub, top_1_rel, top_1_obj = [], [], []
top_5_sub, top_5_rel, top_5_obj = [], [], []
top_10_sub, top_10_rel, top_10_obj = [], [], []
gt_sub, gt_rel, gt_obj = [], [], []
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_t_all = tracker.track('{}_loss_all'.format(prefix),
tracker_class(**tracker_params))
loss_t_sub = tracker.track('{}_loss_sub'.format(prefix),
tracker_class(**tracker_params))
loss_t_rel = tracker.track('{}_loss_rel'.format(prefix),
tracker_class(**tracker_params))
loss_t_obj = tracker.track('{}_loss_obj'.format(prefix),
tracker_class(**tracker_params))
acc_t_sub = tracker.track('{}_acc_sub'.format(prefix),
tracker_class(**tracker_params))
acc_t_rel = tracker.track('{}_acc_rel'.format(prefix),
tracker_class(**tracker_params))
acc_t_obj = tracker.track('{}_acc_obj'.format(prefix),
tracker_class(**tracker_params))
unk_t_rel = tracker.track('{}_unk_rel'.format(prefix),
tracker_class(**tracker_params))
unk_t_sub = tracker.track('{}_unk_sub'.format(prefix),
tracker_class(**tracker_params))
unk_t_obj = tracker.track('{}_unk_obj'.format(prefix),
tracker_class(**tracker_params))
# unk_rel, unk_sub, unk_obj = 0, 0, 0
for v, q, rel, rel_sub, rel_rel, rel_obj, q_len in loader:
v = v.cuda(async=True)
q = q.cuda(async=True)
rel_sub = rel_sub.cuda(async=True)
rel_rel = rel_rel.cuda(async=True)
rel_obj = rel_obj.cuda(async=True)
q_len = q_len.cuda(async=True)
sub_prob, rel_prob, obj_prob = net(v, q, rel_sub, rel_rel, rel_obj,
q_len)
# print(sub_prob, rel_prob, obj_prob)
# print(sub_prob.shape, rel_sub.shape)
loss_sub = loss_criterion(sub_prob, rel_sub)
loss_rel = loss_criterion(rel_prob, rel_rel)
loss_obj = loss_criterion(obj_prob, rel_obj)
# loss_sub = -F.log_softmax(sub_prob, dim=1)
# loss_rel = -F.log_softmax(rel_prob, dim=1)
# loss_obj = -F.log_softmax(obj_prob, dim=1)
# loss_sub = (loss_sub * rel_sub / 10).sum(dim=1).mean()
# loss_rel = (loss_rel * rel_rel / 10).sum(dim=1).mean()
# loss_obj = (loss_obj * rel_obj / 10).sum(dim=1).mean()
loss_all = config.lamda_sub * loss_sub + config.lamda_rel * loss_rel + config.lamda_obj * loss_obj
prob_sub_index, acc_sub = utils.batch_accuracy(sub_prob, rel_sub)
unk_t_sub.append(torch.sum(prob_sub_index == 0).item())
prob_rel_index, acc_rel = utils.batch_accuracy(rel_prob, rel_rel)
unk_t_rel.append(torch.sum(prob_rel_index == 0).item())
prob_obj_index, acc_obj = utils.batch_accuracy(obj_prob, rel_obj)
unk_t_obj.append(torch.sum(prob_obj_index == 0).item())
if train:
scheduler.step()
optimizer.zero_grad()
loss_all.backward()
optimizer.step()
else:
# store information about evaluation of this minibatch
top_1_sub.append(sub_prob.topk(1)[1])
top_1_rel.append(rel_prob.topk(1)[1])
top_1_obj.append(obj_prob.topk(1)[1])
top_5_sub.append(sub_prob.topk(5)[1])
top_5_rel.append(rel_prob.topk(5)[1])
top_5_obj.append(obj_prob.topk(5)[1])
top_10_sub.append(sub_prob.topk(10)[1])
top_10_rel.append(rel_prob.topk(10)[1])
top_10_obj.append(obj_prob.topk(10)[1])
gt_sub.append(rel_sub.view(-1, 1))
gt_rel.append(rel_rel.view(-1, 1))
gt_obj.append(rel_obj.view(-1, 1))
loss_t_all.append(loss_all.item())
loss_t_sub.append(loss_sub.item())
loss_t_rel.append(loss_rel.item())
loss_t_obj.append(loss_obj.item())
acc_t_sub.append(acc_sub.mean())
acc_t_rel.append(acc_rel.mean())
acc_t_obj.append(acc_obj.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss_all=fmt(loss_t_all.mean.value),
loss_sub=fmt(loss_t_sub.mean.value),
acc_sub=fmt(acc_t_sub.mean.value),
loss_rel=fmt(loss_t_rel.mean.value),
acc_rel=fmt(acc_t_rel.mean.value),
loss_obj=fmt(loss_t_obj.mean.value),
acc_obj=fmt(acc_t_obj.mean.value),
unk_sub=fmt(unk_t_sub.mean.value),
unk_rel=fmt(unk_t_rel.mean.value),
unk_obj=fmt(unk_t_obj.mean.value))
# print('unk num: ', unk_sub, unk_rel, unk_obj)
if not train:
top_1_sub = torch.cat(top_1_sub, dim=0).cpu().numpy()
top_1_rel = torch.cat(top_1_rel, dim=0).cpu().numpy()
top_1_obj = torch.cat(top_1_obj, dim=0).cpu().numpy()
top_5_sub = torch.cat(top_5_sub, dim=0).cpu().numpy()
top_5_rel = torch.cat(top_5_rel, dim=0).cpu().numpy()
top_5_obj = torch.cat(top_5_obj, dim=0).cpu().numpy()
top_10_sub = torch.cat(top_10_sub, dim=0).cpu().numpy()
top_10_rel = torch.cat(top_10_rel, dim=0).cpu().numpy()
top_10_obj = torch.cat(top_10_obj, dim=0).cpu().numpy()
gt_sub = torch.cat(gt_sub, dim=0).cpu().numpy()
gt_rel = torch.cat(gt_rel, dim=0).cpu().numpy()
gt_obj = torch.cat(gt_obj, dim=0).cpu().numpy()
recall_1 = utils.recall(gt_sub, gt_rel, gt_obj, top_1_sub, top_1_rel,
top_1_obj)
recall_5 = utils.recall(gt_sub, gt_rel, gt_obj, top_5_sub, top_5_rel,
top_5_obj)
recall_10 = utils.recall(gt_sub, gt_rel, gt_obj, top_10_sub,
top_10_rel, top_10_obj)
return recall_1, recall_5, recall_10
def train(model,
epochs,
criterion_type,
optimizer_type,
training_data,
val_data,
print_every,
plot_every,
input_type='sequences',
early_stopping=False,
early_stopping_metric='val_auc',
early_stopping_patience=4,
rate_decay_patience=5,
max_rate_decreases=0,
initial_learning_rate=0.001,
reload_best_on_decay=True,
model_name='best_rnn',
min_auc=0.0,
add_noise=None,
noise_variance=1.0,
noise_scale_term=1.0,
verbose=False,
return_thresh=False):
"""
Trains the given model according to the input arguments.
Args:
models (PyTorch model): The RNN to be trained.
epochs (int): max number of iterations over the training dataset
criterion (PyTorch loss): the loss function to use while backpropagating
to train model
optimizer (PyTorch optimizer): the optimization algorithm to be used
train parameters of model
training_data (PyTorch DatasetFolder): the training dataset that will be
iterated through
val_data (PyTorch DatasetFolder): the validation dataset that will be
used for testing generalization
print_every (int): the number of batches to pass between printing
average loss and accuracy per batch for the batches since the last print
plot_every (int): the number of batches to pass between recording metrics
for plotting
input_type (string): the form that the input items will take
early_stopping (boolean): stop when validation metric of choice doesn't
improve for patience epochs
early_stopping_metric (string): the validation set metric to be
monitored. Can be either 'val_auc' or 'val_loss'.
early_stopping_patience (int): number of epochs with no validation
accuracy imporvement to do before stopping
rate_decay_patience (int): number of epochs without improvement to pass
before reducing learning rate, when max_rate_decreases > 0
max_rate_decreases (int): max number of times to decay learning rate
initial_learning_rate (float): the first and max learning rate used
by the optimizer
model_name (string): the name with which to save the model to the
working directory, which will be model_name.pt
min_auc (float): the min. auc over validation set that must be achieved
before saving the model.
add_noise (string): if not None, then one of 'input_values', 'input_params',
'all_params', or 'weights_only'
noise_variance (float): the variance of the sampling distribution
noise_scale_term (float): amount to scale noise
verbose (boolean): print metrics each epoch if True.
val_thresh (boolean): when True, the function returns the optimal
threshold used to compute metrics over the validation set.
Raises:
ValueError: when input_type not one of 'sequences' or 'hidden_states'. If
'hidden_states' then model_type must also be LSTM
Returns:
if return_thresh:
model (Pytorch model): the trained RNN
training_accs (list): training set accuracy for each fold of CV
validation_accs (list): validation set accuracy for each fold of CV
training_losses (list): training set loss for each fold of CV
training_aucs (list): training set auc roc score for each fold of CV
validation_losses (list): validation set loss for each fold of CV
validation_recalls (list): validation set recall score for each fold of CV
validation_specifs (list): validation set specificity for each fold of CV
validation_aucs (list): validation set auc roc score for each fold of CV
val_thresh (float): the optimal threshold for computing sensitivity and
specificity as computed on the validation set
else val_thresh is not returned
"""
if input_type not in ('sequences', 'hidden_states'):
raise ValueError("Invalid input_form: ", input_type)
model = model.train()
best_auc = min_auc
best_val_loss = 10000
epochs_no_improvement = 0
num_rate_decreases = 0
# hold metrics to be tracked across training
training_losses = []
validation_recalls = []
training_accs = []
training_aucs = []
validation_accs = []
validation_losses = []
validation_specifs = []
validation_aucs = []
# load all validation data onto the GPU
val_loader = get_all_data(val_data, shuffle=False)
# Check and record untrained validation metrics
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = utils.check_metrics(model, val_loader, verbose=False)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
#load all training data onto the GPU
train_loader = get_all_data(training_data, shuffle=True)
# Check and record untrained training set metrics
training_acc, \
_, \
_, \
training_auc, \
training_loss = utils.check_metrics(model, train_loader, verbose=False)
training_accs.append(training_acc)
training_losses.append(training_loss)
training_aucs.append(training_auc)
if verbose:
print("METRICS OF UNTRAINED MODEL")
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC:, ", auc)
print("training AUC: ", training_auc)
learning_rate = initial_learning_rate
if optimizer_type == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
elif optimizer_type == 'SGD':
optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
else:
raise ValueError("ERROR: optimizer " + str(optimizer_type) +
" not supported.")
if criterion_type == 'NLLLoss':
criterion = nn.NLLLoss()
criterion = criterion.to(DEVICE)
else:
raise ValueError("ERROR: criterion " + str(criterion_type) +
" not supported.")
train_loader = get_all_data(training_data, shuffle=True)
for epoch in range(epochs):
# batch-wise metrics to be tracked
training_accuracy = 0.0 # for printing accuracy over training set over epoch w/o recomputing
running_acc = 0.0
running_loss = 0.0
plot_running_acc = 0.0
plot_running_loss = 0.0
num_batches = 0
torch.manual_seed(MANUAL_SEED)
for i, data in enumerate(train_loader, 0):
# get the inputs
inputs, labels = data
model_arch = model.get_architecture()
if model_arch == 'gru' or model_arch == 'lstm':
inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
hidden = model.init_hidden(batch_size=labels.shape[0])
if len(hidden) == 2:
hidden = (hidden[0].to(DEVICE), hidden[1].to(DEVICE))
else:
hidden = hidden.to(DEVICE)
for j in range(MAX_SEQUENCE_LENGTH):
outputs, hidden = model(inputs[:, j].unsqueeze(1).float(), hidden)
else:
inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
outputs = model(inputs.float())
loss = criterion(outputs, labels.squeeze(0).long()).sum()
# zero the parameter gradients
optimizer.zero_grad()
loss.backward()
optimizer.step()
if verbose:
plot_grad_flow(model.named_parameters())
# Update tracked metrics for batch
batch_acc = utils.batch_accuracy(outputs, labels)
training_accuracy += batch_acc
running_loss += loss.item()
running_acc += batch_acc
plot_running_loss += loss.item()
plot_running_acc += batch_acc
if i % print_every == (print_every-1):
print('[epoch: %d, batches: %5d] loss: %.5f | accuracy: %.5f'
% (epoch + 1, i + 1, running_loss/print_every,
running_acc/print_every))
running_loss = 0.0
running_acc = 0.0
if i % plot_every == (plot_every-1):
training_accs.append(plot_running_acc/plot_every)
training_losses.append(plot_running_loss/plot_every)
plot_running_loss = 0.0
plot_running_acc = 0.0
model = model.eval()
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = utils.check_metrics(model, val_loader, verbose=False)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
model = model.train()
training_acc, \
_, \
_, \
training_auc, \
training_loss = utils.check_metrics(model, train_loader, verbose=False)
training_aucs.append(training_auc)
num_batches += 1
# Update tracked metrics for epoch: accuracy, recall, specificity, auc, loss
model = model.eval()
val_acc, \
val_recall, \
val_specif, \
val_auc, \
val_loss = utils.check_metrics(model, val_loader, verbose=False)
model = model.train()
train_acc = training_accuracy/num_batches
if verbose:
print("Training accuracy for epoch: ", train_acc)
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC: ", val_auc)
if early_stopping:
if early_stopping_metric == 'val_loss' and val_loss < best_val_loss:
print("Old best val_loss: ", best_val_loss)
print("New best val_loss: ", val_loss)
print("Validation AUC: ", val_auc)
best_val_loss = val_loss
torch.save(model.state_dict(), './'+ model_name +'.pt')
print("New best model found. Saving now.")
epochs_no_improvement = 0
elif early_stopping_metric == 'val_auc' and val_auc > best_auc:
print("Old best val AUC: ", best_auc)
print("New best val AUC: ", val_auc)
best_auc = val_auc
torch.save(model.state_dict(), './'+ model_name +'.pt')
print("New best model found. Saving now.")
epochs_no_improvement = 0
else:
epochs_no_improvement += 1
if epochs_no_improvement == early_stopping_patience:
print("No decrease in validation loss in %d epochs. Stopping"
" training early." % early_stopping_patience)
break
if ((max_rate_decreases > 0 and epochs_no_improvement > 0) and
(epochs_no_improvement % rate_decay_patience == 0) and
(num_rate_decreases < max_rate_decreases)):
print("No increase in validation AUC score in %d epochs. "
"Reducing learning rate." % rate_decay_patience)
print("Old learning rate:", learning_rate)
learning_rate = learning_rate/2.0
num_rate_decreases += 1
print("New learning rate:", learning_rate)
if reload_best_on_decay:
model.load_state_dict(torch.load('./'+ model_name +'.pt'))
if optimizer_type == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
elif optimizer_type == 'SGD':
optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9, nesterov=True)
else:
raise ValueError("ERROR: optimizer " + str(optimizer_type) +
" not supported.")
model = model.eval()
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = utils.check_metrics(model, val_loader, verbose=False)
model = model.train()
print("Validation AUC: ", auc)
print("Validation Loss: ", val_loss)
print("Epochs without improvement: ", epochs_no_improvement)
if add_noise in ('input_params',
'all_params',
'weights_only') and epoch != (epochs-1):
torch.manual_seed(MANUAL_SEED+epoch)
model = jitter_params(model=model,
params_to_jitter=add_noise,
scale_term=noise_scale_term)
print('Finished Training')
model.load_state_dict(torch.load('./'+ model_name +'.pt'))
model = model.eval()
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss, \
val_thresh = utils.check_metrics(model,
val_loader,
verbose=False,
return_threshold=True)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
model = model.train()
training_acc, \
_, \
_, \
training_auc, \
training_loss = utils.check_metrics(model, train_loader, verbose=False)
training_accs.append(training_acc)
training_losses.append(training_loss)
training_aucs.append(training_auc)
if verbose:
#print("Training accuracy for epoch: ", train_acc)
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC: ", auc)
if return_thresh:
metrics = (training_accs, validation_accs, training_losses,
training_aucs, validation_losses, validation_recalls,
validation_specifs, validation_aucs, val_thresh)
else:
metrics = (training_accs, validation_accs, training_losses,
training_aucs, validation_losses, validation_recalls,
validation_specifs, validation_aucs)
return model, metrics
def run(net, loader, optimizer, tracker, train=False, prefix='', epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
log_softmax = nn.LogSoftmax().to(
"cuda:0" if torch.cuda.is_available() else "cpu")
for v, q, a, idx, q_len in tq:
var_params = {
'requires_grad': False,
}
with torch.set_grad_enabled(train):
v = Variable(
v.to("cuda:0" if torch.cuda.is_available() else "cpu"),
**var_params)
q = Variable(
q.to("cuda:0" if torch.cuda.is_available() else "cpu"),
**var_params)
a = Variable(
a.to("cuda:0" if torch.cuda.is_available() else "cpu"),
**var_params)
q_len = Variable(
q_len.to("cuda:0" if torch.cuda.is_available() else "cpu"),
**var_params)
out = net(v, q, q_len)
nll = -log_softmax(out)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
global total_iterations
update_learning_rate(optimizer, total_iterations)
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iterations += 1
# fig, axarr = plt.subplots(act0.size(0))
else:
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
# if epoch == config.epochs - 1:
# acts = activation['attention'].squeeze()
# actList = [acts[:,x,:,:] for x in range(2)]
# for num in range(2):
# for i in range(actList[num].size(0)):
# # print(act0[idx])
# # axarr[idx].imshow(act0[idx])
# # axarr[idx].set_title("dimension" + str(idx))
# plt.imshow(actList[num][i].cpu())
# plt.title("Modified Attention: Feature Map " + str(num+1))
# plt.savefig("img_karl/img_" + str(idx[i].item())+ "_layer_" + str(num) + ".png")
loss_tracker.append(loss.data.item())
# acc_tracker.append(acc.mean())
for a in acc:
acc_tracker.append(a.item())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
if not train:
answ = list(torch.cat(answ, dim=0))
accs = list(torch.cat(accs, dim=0))
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def run(net, loader, optimizer, scheduler, tracker, train=False, has_answers=True, prefix='', epoch=0):
""" Run an epoch over the given loader """
assert not (train and not has_answers)
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params))
batch_count = 0
batch_max = len(loader)
for v, q, a, b, q_type, idx, q_len in loader:
var_params = {
'volatile': not train,
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
b = Variable(b.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params)
q_type = Variable(q_type.cuda(async=True), **var_params)
if config.use_rl and train:
net.eval()
out, _, _ = net(v, b, q, q_len, q_type)
acc = utils.batch_accuracy(out.data, a.data).cpu()
baseline = []
for i in range(acc.shape[0]):
baseline.append(float(acc[i]))
#float(acc.mean())
net.train()
utils.fix_batchnorm(net)
out, rl_ls, _ = net(v, b, q, q_len, q_type)
acc = utils.batch_accuracy(out.data, a.data).cpu()
current = []
for i in range(acc.shape[0]):
current.append(float(acc[i]))
#float(acc.mean())
#print(baseline - current)
rl_loss = []
assert len(rl_ls) == len(baseline)
for i in range(len(rl_ls)):
rl_loss.append((baseline[i] - current[i]) * rl_ls[i])
#(baseline - current) * sum(rl_ls) / len(rl_ls)
#entropy_loss = sum(entropy_ls) / len(entropy_ls) * 1e-4
loss = sum(rl_loss) #+ entropy_loss
else:
out, _, _ = net(v, b, q, q_len, q_type)
if has_answers:
nll = -F.log_softmax(out, dim=1)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
scheduler.step()
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
if has_answers:
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
if has_answers:
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
if train:
writer.add_scalar('train/loss', loss.item(), epoch * batch_max + batch_count)
writer.add_scalar('train/accu', acc.mean(), epoch * batch_max + batch_count)
#writer.export_scalars_to_json("./log_board.json")
else:
writer.add_scalar('val/loss', loss.item(), epoch * batch_max + batch_count)
writer.add_scalar('val/accu', acc.mean(), epoch * batch_max + batch_count)
#writer.export_scalars_to_json("./log_board.json")
batch_count += 1
if not train:
answ = list(torch.cat(answ, dim=0))
if has_answers:
accs = list(torch.cat(accs, dim=0))
else:
accs = []
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
def main():
# Dataset
print('Creating dataset...')
train_loader = data.get_loader(train=True, batch_size=args.batch_size)
val_loader = data.get_loader(val=True, batch_size=args.batch_size)
# Model
checkpoint = os.path.join(args.checkpoint)
if not os.path.exists(checkpoint):
os.makedirs(checkpoint)
model_path = os.path.join(checkpoint, 'best_model.pt')
print('Loading model...')
print(args.model)
model = get_vqa_model(args.model, train_loader.dataset.num_tokens)
#model = resnet50_CA()
#print(model)
# Test only
print("Number of parameters: ",
sum([param.nelement() for param in model.parameters()]))
if torch.cuda.device_count() > 1:
print("Using", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.cuda()
cudnn.benchmark = True
# optim
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
weight_decay=args.weight_decay)
scheduler = lr_scheduler.MultiStepLR(optimizer, args.schedule, gamma=0.5)
log_softmax = nn.LogSoftmax().cuda()
# Log
log_file = os.path.join(checkpoint, 'log.json')
writer = SummaryWriter(checkpoint)
best_val_acc = -1
# Train and val
for epoch in range(args.num_epochs):
# Train
scheduler.step()
learning_rate = optimizer.param_groups[0]['lr']
print('Start training epoch {}. Learning rate {}'.format(
epoch, learning_rate))
writer.add_scalar('Learning rate', learning_rate, epoch)
model.train()
num_batches = len(train_loader.dataset) // args.batch_size
bar = progressbar.ProgressBar(max_value=num_batches)
running_loss = 0
train_acc = 0
for i, (images, questions, labels, idx,
quest_len) in enumerate(train_loader):
images = Variable(images.cuda())
questions = Variable(questions.cuda())
labels = Variable(labels.cuda())
quest_len = Variable(quest_len.cuda())
optimizer.zero_grad()
outputs = model(images, questions, quest_len)
nll = -log_softmax(outputs)
loss = (nll * labels / 10).sum(dim=1).mean()
train_acc += utils.batch_accuracy(outputs.data, labels.data).sum()
running_loss += loss.data[0]
loss.backward()
optimizer.step()
bar.update(i, force=True)
writer.add_scalar('Training instance loss', loss.data[0],
epoch * num_batches + i)
train_acc = train_acc / len(train_loader.dataset) * 100
train_loss = running_loss / num_batches
print('Training loss %f' % train_loss)
print('Training acc %.2f' % train_acc)
writer.add_scalar('Training loss', train_loss, epoch)
# Validate
model.eval()
val_acc = 0
num_batches = len(val_loader.dataset) // args.batch_size
bar = progressbar.ProgressBar(max_value=num_batches)
running_loss = 0
for i, (images, questions, labels, idx,
quest_len) in enumerate(val_loader):
images = Variable(images.cuda())
questions = Variable(questions.cuda())
labels = Variable(labels.cuda())
quest_len = Variable(quest_len.cuda())
outputs = model(images, questions, quest_len)
nll = -log_softmax(outputs)
loss = (nll * labels / 10).sum(dim=1).mean()
val_acc += utils.batch_accuracy(outputs.data, labels.data).sum()
running_loss += loss.data[0]
bar.update(i, force=True)
val_acc = val_acc / len(val_loader.dataset) * 100
val_loss = running_loss / num_batches
if val_acc > best_val_acc:
best_val_acc = val_acc
if torch.cuda.device_count() > 1:
torch.save(model.module.state_dict(), model_path)
else:
torch.save(model.state_dict(), model_path)
print('Validation loss %f' % (running_loss / num_batches))
print('Validation acc', val_acc)
writer.add_scalar('Validation loss', val_loss, epoch)
writer.add_scalar('Validation acc', val_acc, epoch)
print()
print('Best validation acc %.2f' % best_val_acc)
writer.export_scalars_to_json(log_file)
writer.close()
def run(net,
loader,
optimizer,
scheduler,
tracker,
train=False,
has_answers=True,
prefix='',
epoch=0):
""" Run an epoch over the given loader """
assert not (train and not has_answers)
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
# set learning rate decay policy
if epoch < len(config.gradual_warmup_steps
) and config.schedule_method == 'warm_up':
utils.set_lr(optimizer, config.gradual_warmup_steps[epoch])
utils.print_lr(optimizer, prefix, epoch)
elif (epoch in config.lr_decay_epochs
) and train and config.schedule_method == 'warm_up':
utils.decay_lr(optimizer, config.lr_decay_rate)
utils.print_lr(optimizer, prefix, epoch)
else:
utils.print_lr(optimizer, prefix, epoch)
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
for v, q, a, b, idx, v_mask, q_mask, q_len in loader:
var_params = {
'requires_grad': False,
}
v = Variable(v.cuda(), **var_params)
q = Variable(q.cuda(), **var_params)
a = Variable(a.cuda(), **var_params)
b = Variable(b.cuda(), **var_params)
q_len = Variable(q_len.cuda(), **var_params)
v_mask = Variable(v_mask.cuda(), **var_params)
q_mask = Variable(q_mask.cuda(), **var_params)
out = net(v, b, q, v_mask, q_mask, q_len)
if has_answers:
answer = utils.process_answer(a)
loss = utils.calculate_loss(answer, out, method=config.loss_method)
acc = utils.batch_accuracy(out, answer).data.cpu()
if train:
optimizer.zero_grad()
loss.backward()
# print gradient
if config.print_gradient:
utils.print_grad([(n, p) for n, p in net.named_parameters()
if p.grad is not None])
# clip gradient
clip_grad_norm_(net.parameters(), config.clip_value)
optimizer.step()
if (config.schedule_method == 'batch_decay'):
scheduler.step()
else:
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
if has_answers:
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
if has_answers:
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
if not train:
answ = list(torch.cat(answ, dim=0))
if has_answers:
accs = list(torch.cat(accs, dim=0))
else:
accs = []
idxs = list(torch.cat(idxs, dim=0))
#print('{} E{:03d}:'.format(prefix, epoch), ' Total num: ', len(accs))
#print('{} E{:03d}:'.format(prefix, epoch), ' Average Score: ', float(sum(accs) / len(accs)))
return answ, accs, idxs
def run(net,
loader,
optimizer,
scheduler,
tracker,
train=False,
has_answers=True,
prefix='',
epoch=0):
""" Run an epoch over the given loader """
assert not (train and not has_answers)
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
all_answer = []
idxs = []
accs = []
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
for v, q, a, b, idx, q_len in loader:
var_params = {
'volatile': not train,
'requires_grad': False,
}
# torch.from_numpy([0]).unsqueeze(1), torch.from_numpy([0])
# temp = torch.from_numpy(np.arange(10, dtype=int).reshape(5, 2))
# if b == temp:
# continuext
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
b = Variable(b.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params)
out = net(v, b, q, q_len)
if has_answers:
nll = -F.log_softmax(out, dim=1)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
# optimizer가 lr보다 먼저와야
scheduler.step()
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
# store information about evaluation of this minibatch
all = out.data.cpu()
for a in all:
all_answer.append(a.view(-1))
# print("max",out.data.cpu().max(dim=1))
_, answer = out.data.cpu().max(dim=1) # ???
answ.append(answer.view(-1))
if has_answers:
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
if has_answers:
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
if not train:
answ = list(torch.cat(answ, dim=0))
if has_answers:
accs = list(torch.cat(accs, dim=0))
else:
accs = []
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs, all_answer
def run(net,
loader,
optimizer,
tracker,
train=False,
prefix='',
epoch=0,
dataset=None):
""" Run an epoch over the given loader """
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {
'momentum': 0.99
}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
# answ = [] # edit_vedika FT101
# idxs = []
# accs = []
tq = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
for batch in tq: #for v, q, a, idx, img_id, ques_id, q_len in tq:
v, q, a, idx, img_id, ques_id, q_len = batch
# except image_id- everything is a tensor ## [i[0].dtype for i in [v, ques, ans, idx, ques_id, q_len]]
## [v, ques, ans, idx, ques_id, q_len].dtype = [torch.float32, torch.int64, torch.float32, torch.int64, torch.int64, torch.int64]
if (train and config.orig_edit_equal_batch) or (
train and config.orig_edit_diff_ratio_naive
) or (train and config.orig_edit_diff_ratio_naive_no_edit_ids_repeat):
#edit_v, edit_q, edit_a, edit_idx, edit_img_id, edit_ques_id, edit_q_len = data.get_edit_train_loader(ques_id_batch=ques_id, train=True)
#ipdb.set_trace()
edit_batch = data.get_edit_train_batch(dataset=dataset,
ques_id_batch=ques_id,
item_ids=idx)
#[torch.float32, torch.int64, torch.float32, torch.int64, [torch.int64, torch.int64]]
if edit_batch is not None:
## stack both orig and edit together
v, q, a, idx, ques_id, q_len = [
torch.cat((batch[i], edit_batch[i]), dim=0)
for i in [0, 1, 2, 3, 5, 6]
]
img_id = img_id + edit_batch[4] # edit_img_id = edit_batch[4]
q_len_new = [int(i) for i in q_len]
sorting_order = np.argsort(
q_len_new)[::-1] #as q_len has to be sorted!
#sorted_batch = [[dat[i] for dat in [v, q, a, idx, img_id, ques_id, q_len]] for i in sorting_order] # shape 102
sorted_batch2 = [[
dat[i] for i in sorting_order
] for dat in [v, q, a, idx, img_id, ques_id, q_len]]
v, q, a, idx, ques_id, q_len = [
torch.stack(sorted_batch2[i], dim=0)
for i in [0, 1, 2, 3, 5, 6]
]
img_id = sorted_batch2[4]
# v, q, a, idx, img_id, ques_id, q_len
var_params = {
#'volatile': not train, # taken care for val using: with torch.no_grad():
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params)
if train:
out = net(v, q, q_len)
nll = -log_softmax(
out) ## taking softmax here ## calculating -log(p_pred)
loss_1 = (nll * a / 10).sum(
dim=1
) ### SO THIS COMPLETES CROSS ENTROPY : -p_true* log(p_pred) as 'a/10' does the role of being p_true - ans has avlue 10 where its true
loss = (nll * a / 10).sum(dim=1).mean(
) ## mean of te batch #TODO- divide it into true and edit loss- which are resp being divided by the resp sizes
#abc = torch.Tensor.cpu(ans[13])
# abc[np.where(ans_13!=0)]
#ipdb.set_trace()
if ('data_aug2' in config.model_type
or 'data_aug3' in config.model_type
) and config.edit_loader_type == 'get_edits':
loc2 = {} ## mapping ques_ids to location indices
for key in set(list(ques_id)):
loc2[int(key)] = [
idx for idx, i in enumerate(list(ques_id)) if i == key
]
loc3 = [
key for key in loc2.keys() if len(loc2[key]) > 1
] ## those keys only whihc has two samples: edit and orig in our case
all_true_ids = [
idx for idx, i in enumerate(img_id) if len(i) == 12
]
true_batch_order = [
loc2[key][0]
if len(img_id[loc2[key][0]]) == 12 else loc2[key][1]
for key in loc3
]
edit_batch_order = [
loc2[key][0]
if len(img_id[loc2[key][0]]) == 25 else loc2[key][1]
for key in loc3
] ## TODO only handles one edit case
### just checks to make sure correspondence between true_batch and edit_batch
img_id_true = [
img_id[i] for i in true_batch_order
] #TODO need not save it, can be done on the fly in lines 122-124
img_id_edit = [img_id[i] for i in edit_batch_order]
for i in range(len(true_batch_order)):
assert len(img_id_true[i]) < len(img_id_edit[i])
assert img_id_true[i] == img_id_edit[i][0:12]
assert sorted(set(true_batch_order)
& set(all_true_ids)) == sorted(
set(true_batch_order))
if config.regulate_old_loss: ## TODO right now only for data_aug2 and data_aug3 techniques; for naive ratio mixing- need to do it separately
## divide the loss .... loss = (loss_real/n_real) + (loss_fake/n_fake)
#loss_1_orig = torch.stack([loss_1[i] for i in all_true_ids])
#loss_1_orig_has_edit = torch.stack([loss_1[i] for i in true_batch_order])
#loss_1_edit = torch.stack([loss_1[i] for i in edit_batch_order])
loss_old = loss
loss_orig = torch.stack([loss_1[i]
for i in all_true_ids]).mean()
loss_edit = torch.stack([
loss_1[i] for i in edit_batch_order
]).mean() ## ==loss_1_edit.sum()/num_edit_samples
#loss_e = loss_orig + (len(edit_batch_order)/len(all_true_ids))*loss_edit #TODO tuning parameter now is num_edit_samples/num_true_samples
#loss_5 = loss_orig + 0.5 * loss_edit
# bcd = np.array(torch.Tensor.cpu(loss_1_orig.detach())) #TODO loss_track implementation to save both loss_old, loss_orig, loss_edit
if config.enforce_consistency: # ony for data_aug3 strategy
softmax_out = just_softmax(out)
softmax_out = Variable(softmax_out.cuda(async=True),
**var_params)
softmax_orig = torch.stack([
softmax_out[i] for i in true_batch_order
]) # for orig softmax out
softmax_edit = torch.stack([
softmax_out[i] for i in edit_batch_order
]) ### for edit take- -log_softmax
nll_orig = torch.stack([nll[i] for i in true_batch_order])
nll_edit = torch.stack([nll[i] for i in edit_batch_order])
# consistency_loss_CE = consistency_criterion_CE(softmax_edit, softmax_orig) #RuntimeError: Expected object of scalar type Long but got scalar type Float for argument #2 'target'
consistency_loss_edit_orig = (nll_edit * softmax_orig).sum(
dim=1).mean() # softmax(orig)* log_softmax(edit)
consistency_loss_orig_edit = (nll_orig * softmax_edit).sum(
dim=1).mean() # softmax(edit)* log_softmax(orig)
if config.old_CE:
consistency_loss_CE = (consistency_loss_edit_orig +
consistency_loss_orig_edit) / 2
else:
consistency_loss_CE = consistency_loss_edit_orig
consistency_loss_KL = consistency_criterion_KL(
softmax_orig, softmax_edit)
consistency_loss_MSE = consistency_criterion_MSE(
softmax_orig, softmax_edit)
if config.regulate_old_loss:
loss = loss_orig + (
config.lam_edit_loss * loss_edit
) + (config.lam_CE * consistency_loss_CE) + (
config.lam_KL * abs(consistency_loss_KL)) + (
config.lam_MSE * consistency_loss_MSE)
else:
loss_vqa = loss
loss = loss_vqa + (
config.lam_CE * consistency_loss_CE) + (
config.lam_KL * abs(consistency_loss_KL)) + (
config.lam_MSE * consistency_loss_MSE)
# loss_track = {
# 'loss': loss.item(),
# 'loss_vqa': loss_vqa.item(),
# 'consistency batch loss_CE': consistency_loss_CE.item(), #if config.enforce_consistency else 0.0,
# 'consistency batch loss_MSE': consistency_loss_MSE.item(),
# 'consistency batch loss_KL': consistency_loss_KL.item() }
acc = utils.batch_accuracy(out.data, a.data).cpu()
global total_iterations
update_learning_rate(optimizer, total_iterations)
#ipdb.set_trace()
optimizer.zero_grad()
loss.backward()
optimizer.step()
total_iterations += 1
else:
with torch.no_grad():
out = net(v, q, q_len)
nll = -log_softmax(out) ## taking softmax here
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(
out.data,
a.data).cpu() ### taking care of volatile=True for val
# # store information about evaluation of this minibatch # edit_vedika FT101
# _, answer = out.data.cpu().max(dim=1)
# answ.append(answer.view(-1))
# accs.append(acc.view(-1))
# idxs.append(idx.view(-1).clone())
# if config.enforce_consistency and train: # TODO
## File "train.py", line 192, in run
#tq.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
##TypeError: unsupported format string passed to dict.__format__
# loss_tracker.append(loss_track) #data[0])
loss_tracker.append(loss.item())
for a in acc:
acc_tracker.append(a.item())
fmt = '{:.4f}'.format
tq.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
def train(model,
epochs,
criterion_type,
optimizer_type,
train_loader,
val_loader,
print_every,
plot_every,
early_stopping=False,
early_stopping_metric='val_auc',
early_stopping_patience=4,
rate_decay=False,
rate_decay_patience=5,
initial_learning_rate=0.001,
model_name='best_rnn',
min_auc=0.0,
verbose=False,
return_thresh=False):
"""
Trains the given model according to the input arguments.
Args:
models (PyTorch model): The RNN to be trained.
epochs (int): max number of iterations over the training dataset
criterion (PyTorch loss): the loss function to use while backpropagating
to train model
optimizer (PyTorch optimizer): the optimization algorithm to be used
train parameters of model
train_loader (PyTorch DataLoader): the training dataloader that will be
iterated through
val_loader (PyTorch DataLoader): the validation dataloader that will be
used for testing generalization
print_every (int): the number of batches to pass between printing
average loss per batch for the batches since the last print
model_name (string): the name with which to save the model to the
working directory, which will be model_name.pt
early_stopping (boolean): stop when validation metric of choice doesn't
improve for patience epochs
early_stopping_metric (string): the validation set metric to be
monitored. Can be either 'val_auc' or 'val_loss'.
patience (int): number of epochs with no validation accuracy imporvement
to do before stopping
min_auc (float): the min. auc over validation set that must be achieved
before saving the model.
verbose (boolean): print metrics each epoch if True.
val_thresh (boolean): when True, the function returns the optimal
threshold used to compute metrics over the validation set.
Returns:
if return_thresh:
model (Pytorch model): the trained RNN
training_accs (list)
validation_accs (list)
training_losses (list)
training_aucs (list)
validation_losses (list)
validation_recalls (list)
validation_specifs (list)
validation_aucs (list)
val_thresh (float)
else val_thresh is not returned
"""
model = model.train()
best_auc = min_auc
best_val_loss = 10000
epochs_no_improvement = 0
# hold metrics to be tracked across training
training_losses = []
validation_recalls = []
training_accs = []
training_aucs = []
validation_accs = []
validation_losses = []
validation_specifs = []
validation_aucs = []
# Check untrained metrics
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = check_metrics(model, val_loader, verbose=False)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
training_acc, \
training_recall, \
training_specif, \
training_auc, \
training_loss = check_metrics(model, train_loader, verbose=False)
training_accs.append(training_acc)
training_losses.append(training_loss)
training_aucs.append(training_auc)
if verbose:
print("METRICS OF UNTRAINED MODEL")
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC:, ", auc)
print("training AUC: ", training_auc)
learning_rate = initial_learning_rate
if optimizer_type == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
else:
print("ERROR: optimizer " + str(optimizer_type) + " not supported.")
return
if criterion_type == 'NLLLoss':
criterion = nn.NLLLoss()
criterion = criterion.to(DEVICE)
else:
print("ERROR: criterion " + str(criterion_type) + " not supported.")
return
for epoch in range(epochs):
# batch-wise metrics to be tracked
training_accuracy = 0.0 # for printing accuracy over training set over epoch w/o recomputing
running_acc = 0.0
running_loss = 0.0
plot_running_acc = 0.0
plot_running_loss = 0.0
num_batches = 0
torch.manual_seed(MANUAL_SEED)
for i, data in enumerate(train_loader, 0):
# get the inputs
inputs, labels = data
inputs, labels = inputs.to(DEVICE), labels.to(DEVICE)
hidden = model.init_hidden(batch_size=labels.shape[0])
# zero the parameter gradients
optimizer.zero_grad()
# forward + backward + optimize
for j in range(MAX_SEQUENCE_LENGTH):
outputs, hidden = model(inputs[:, j].unsqueeze(1).float(),
hidden)
loss = criterion(outputs, labels.squeeze(0).long()).sum()
loss.backward()
if verbose:
plot_grad_flow(model.named_parameters())
optimizer.step()
# Update tracked metrics for batch
batch_acc = batch_accuracy(outputs, labels)
training_accuracy += batch_acc
running_loss += loss.item()
running_acc += batch_acc
plot_running_loss += loss.item()
plot_running_acc += batch_acc
if i % print_every == (print_every - 1):
print('[epoch: %d, batches: %5d] loss: %.5f | accuracy: %.5f' %
(epoch + 1, i + 1, running_loss / print_every,
running_acc / print_every))
running_loss = 0.0
running_acc = 0.0
if i % plot_every == (plot_every - 1):
training_accs.append(plot_running_acc / plot_every)
training_losses.append(plot_running_loss / plot_every)
plot_running_loss = 0.0
plot_running_acc = 0.0
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = check_metrics(model, val_loader, verbose=False)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
training_acc, \
training_recall, \
training_specif, \
training_auc, \
training_loss = check_metrics(model, train_loader, verbose=False)
training_aucs.append(training_auc)
num_batches += 1
# Update tracked metrics for epoch: accuracy, recall, specificity, auc, loss
val_acc, \
val_recall, \
val_specif, \
val_auc, \
val_loss = check_metrics(model, val_loader, verbose=False)
train_acc = training_accuracy / num_batches
if verbose:
print("Training accuracy for epoch: ", train_acc)
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC: ", val_auc)
if early_stopping:
if early_stopping_metric == 'val_loss' and val_loss < best_val_loss:
print("Old best val_loss: ", best_val_loss)
print("New best val_loss: ", val_loss)
print("Validation AUC: ", val_auc)
best_val_loss = val_loss
torch.save(model.state_dict(), './' + model_name + '.pt')
print("New best model found. Saving now.")
epochs_no_improvement = 0
elif early_stopping_metric == 'val_auc' and val_auc > best_auc:
print("Old best val AUC: ", best_auc)
print("New best val AUC: ", val_auc)
best_auc = val_auc
torch.save(model.state_dict(), './' + model_name + '.pt')
print("New best model found. Saving now.")
epochs_no_improvement = 0
else:
epochs_no_improvement += 1
if epochs_no_improvement == early_stopping_patience:
print("No decrease in validation loss in %d epochs. Stopping" +
"training early." % early_stopping_patience)
break
if rate_decay and epochs_no_improvement > 0 and \
(epochs_no_improvement % rate_decay_patience == 0):
print("No increase in validation AUC score in %d epochs. " +
"Reducing learning rate." % rate_decay_patience)
print("Old learning rate:", learning_rate)
learning_rate = learning_rate / 2.0
print("New learning rate:", learning_rate)
model.load_state_dict(torch.load('./' + model_name + '.pt'))
if optimizer_type == 'Adam':
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
else:
print("ERROR: optimizer " + str(optimizer_type) +
" not supported.")
return
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss = check_metrics(model, val_loader, verbose=False)
print("Validation AUC: ", auc)
print("Validation Loss: ", val_loss)
print("Epochs without improvement: ", epochs_no_improvement)
print('Finished Training')
model.load_state_dict(torch.load('./' + model_name + '.pt'))
val_acc, \
val_recall, \
val_specif, \
auc, \
val_loss, \
val_thresh = check_metrics(model, val_loader, verbose=False, return_threshold=True)
validation_accs.append(val_acc)
validation_losses.append(val_loss)
validation_recalls.append(val_recall)
validation_specifs.append(val_specif)
validation_aucs.append(auc)
training_acc, \
training_recall, \
training_specif, \
training_auc, \
training_loss = check_metrics(model, train_loader, verbose=False)
training_accs.append(training_acc)
training_losses.append(training_loss)
training_aucs.append(training_auc)
if verbose:
#print("Training accuracy for epoch: ", train_acc)
print("validation accuracy: ", val_acc)
print("validation loss: ", val_loss)
print("validation recall: ", val_recall)
print("validation specificity: ", val_specif)
print("validation AUC: ", auc)
if return_thresh:
metrics = (training_accs, validation_accs, training_losses,
training_aucs, validation_losses, validation_recalls,
validation_specifs, validation_aucs, val_thresh)
else:
metrics = (training_accs, validation_accs, training_losses,
training_aucs, validation_losses, validation_recalls,
validation_specifs, validation_aucs)
return model, metrics
def run(net,
loader,
optimizer,
scheduler,
tracker,
train=False,
prefix='',
epoch=0):
""" Run an epoch over the given loader """
if train:
net.train()
# tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
# set learning rate decay policy
if epoch < len(config.gradual_warmup_steps
) and config.schedule_method == 'warm_up':
utils.set_lr(optimizer, config.gradual_warmup_steps[epoch])
elif (epoch in config.lr_decay_epochs
) and train and config.schedule_method == 'warm_up':
utils.decay_lr(optimizer, config.lr_decay_rate)
utils.print_lr(optimizer, prefix, epoch)
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix),
tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix),
tracker_class(**tracker_params))
for v, q, a, b, idx, v_mask, q_mask, q_len in loader:
var_params = {
'requires_grad': False,
}
v = Variable(v.cuda(), **var_params)
q = Variable(q.cuda(), **var_params)
a = Variable(a.cuda(), **var_params)
b = Variable(b.cuda(), **var_params)
q_len = Variable(q_len.cuda(), **var_params)
v_mask = Variable(v_mask.cuda(), **var_params)
q_mask = Variable(q_mask.cuda(), **var_params)
out = net(v, b, q, v_mask, q_mask, q_len)
answer = utils.process_answer(a)
loss = utils.calculate_loss(answer, out, method=config.loss_method)
acc = utils.batch_accuracy(out, answer).data.cpu()
if train:
optimizer.zero_grad()
loss.backward()
# clip gradient
clip_grad_norm_(net.parameters(), config.clip_value)
optimizer.step()
if config.schedule_method == 'batch_decay':
scheduler.step()
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss=fmt(loss_tracker.mean.value),
acc=fmt(acc_tracker.mean.value))
return acc_tracker.mean.value, loss_tracker.mean.value
def run(net, loader, optimizer, scheduler, tracker, train=False, has_answers=True, prefix='', epoch=0):
""" Run an epoch over the given loader """
assert not (train and not has_answers)
if train:
net.train()
tracker_class, tracker_params = tracker.MovingMeanMonitor, {'momentum': 0.99}
else:
net.eval()
tracker_class, tracker_params = tracker.MeanMonitor, {}
answ = []
idxs = []
accs = []
loader = tqdm(loader, desc='{} E{:03d}'.format(prefix, epoch), ncols=0)
loss_tracker = tracker.track('{}_loss'.format(prefix), tracker_class(**tracker_params))
acc_tracker = tracker.track('{}_acc'.format(prefix), tracker_class(**tracker_params))
for v, q, a, b, idx, v_mask, q_mask, _ in loader:
var_params = {'requires_grad': False}
v = Variable(v.cuda(), **var_params)
q = Variable(q.cuda(), **var_params)
a = Variable(a.cuda(), **var_params)
b = Variable(b.cuda(), **var_params)
v_mask = Variable(v_mask.cuda(), **var_params)
q_mask = Variable(q_mask.cuda(), **var_params)
out = net(v, b, q, v_mask, q_mask)
if has_answers:
nll = -F.log_softmax(out, dim=1)
loss = (nll * a / 10).sum(dim=1).mean()
acc = utils.batch_accuracy(out.data, a.data).cpu()
if train:
scheduler.step()
optimizer.zero_grad()
loss.backward()
# clip gradient
for p in net.parameters():
if p.requires_grad:
clip_grad_value_(p, 0.25)
optimizer.step()
else:
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1)
answ.append(answer.view(-1))
if has_answers:
accs.append(acc.view(-1))
idxs.append(idx.view(-1).clone())
if has_answers:
loss_tracker.append(loss.item())
acc_tracker.append(acc.mean())
fmt = '{:.4f}'.format
loader.set_postfix(loss=fmt(loss_tracker.mean.value), acc=fmt(acc_tracker.mean.value))
if not train:
answ = list(torch.cat(answ, dim=0))
if has_answers:
accs = list(torch.cat(accs, dim=0))
else:
accs = []
idxs = list(torch.cat(idxs, dim=0))
return answ, accs, idxs
batch_loss = 0
train_accs = []
print(datetime.now())
model.train()
for v,q,a,item,q_len in training:
q = Variable(q.cuda(async=True),**var_params)
a = Variable(a.cuda(async=True),**var_params)
v = Variable(v.cuda(async=True),**var_params)
q_len = Variable(q_len.cuda(async=True), **var_params)
o = model(q,v,q_len,var_params)
optimizer.zero_grad()
loss =(-o*(a/10)).sum(dim=1).mean() # F.nll_loss(o,a)
loss.backward()
optimizer.step()
batch_loss += loss.data[0]
acc = utils.batch_accuracy(o.data,a.data).cpu()
train_accs.append(acc.view(-1))
train_acc= torch.cat(train_accs,dim=0).mean()
print("epoch %s, loss %s, accuracy %s" %(str(i),str(batch_loss/config.batch_size),str(train_acc)))
if (i+1)%config.val_interval ==0:
val_accs = []
model.eval()
for v,q,a,item,q_len in val:
q = Variable(q.cuda(async=True),**val_params)
a = Variable(a.cuda(async=True),**val_params)
v = Variable(v.cuda(async=True),**val_params)
q_len = Variable(q_len.cuda(async=True), **val_params)
o = model(q,v,q_len,val_params)
acc = utils.batch_accuracy(o.data,a.data).cpu()
val_accs.append(acc.view(-1))
val_acc=torch.cat(val_accs,dim=0).mean()
def run(net, loader, edit_set_cmd, model_name):
""" Run an epoch over the given loader """
answ = []
accs = []
ss_vc = []
image_ids =[]
ques_ids = []
softmax = nn.Softmax(dim=1).cuda()
for v, q, a, idx, img_id, ques_id, q_len in tqdm(loader): # image, ques to vocab mapped , answer, item (sth to help index shuffled data with), len_val
#ipdb.set_trace()
var_params = {
'volatile': False,
'requires_grad': False,
}
v = Variable(v.cuda(async=True), **var_params)
q = Variable(q.cuda(async=True), **var_params)
a = Variable(a.cuda(async=True), **var_params)
q_len = Variable(q_len.cuda(async=True), **var_params) ### len of question
with torch.no_grad():
out = net(v, q, q_len)
softmax_vc = softmax(out) # torch.size(128,3000)
#ipdb.set_trace() ## check type of softmax_vc- enforce it to torch16 here itself/ alse see what happens when np.16..
acc = utils.batch_accuracy(out.data, a.data).cpu() #torch.Size([128, 1]) official vqa acc for every questions
# store information about evaluation of this minibatch
_, answer = out.data.cpu().max(dim=1) ### torch.Size([128) !!!! this is the predicted answer id!!!
answ.append(answer.view(-1)) # pred_ans_id
ss_vc.append(softmax_vc) # #torch.Size([128, 3000])
accs.append(acc.view(-1)) # official vqa accurcay per question
ques_ids.append(ques_id.view(-1))
if config.vis_attention:
output_qids_answers = []
if config.fintuned_model_test:
model_name = 'finetuned_' + model_name
if edit_set_cmd:
saaa_vqa_ans_q_id = '/BS/vedika3/nobackup/pytorch-vqa/cvpr_rebuttal_' + model_name + '_edit_vqa_ans_q_id.pickle'
print(img_id)
ipdb.set_trace()
output_qids_answers += [
{'ans_id': p, 'ques_id': qid, 'accuracy': acc}
for p, qid, acc in zip(answ, ques_ids, accs)]
else:
saaa_vqa_ans_q_id = '/BS/vedika3/nobackup/pytorch-vqa/cvpr_rebuttal_' + model_name + '_orig_vqa_ans_q_id.pickle'
print(img_id)
ipdb.set_trace()
output_qids_answers += [
{'ans_id': p, 'ques_id': qid, 'accuracy': acc}
for p, qid, acc in zip(answ, ques_ids, accs)]
with open(saaa_vqa_ans_q_id, 'wb') as f:
pickle.dump(output_qids_answers, f, pickle.HIGHEST_PROTOCOL)
exit()
if edit_set_cmd:
image_ids.append(img_id)
else:
image_ids.append(img_id.view(-1))
ss_vc = torch.cat(ss_vc, dim=0) ## softmax_vectors
answ = torch.cat(answ, dim=0) ## pred_ans_id
accs = torch.cat(accs, dim=0) ## official vqa accurcay per question
ques_ids = torch.cat(ques_ids, dim=0)
if edit_set_cmd:
image_ids = [item for sublist in image_ids for item in sublist]
else:
image_ids = torch.cat(image_ids, dim=0)
### might be string in edit config case
print('the accuracy is:', torch.mean(accs)) ### mean of entire accuracy vector # tensor(0.6015) for val set
return answ, image_ids, ques_ids, ss_vc
