def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, labels) in enumerate(dev_loader):
#print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]])
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
labels = torch.autograd.Variable(labels)
role_predict = model(img)
top1.add_point_noun_single_role(role_predict, labels)
top5.add_point_noun_single_role(role_predict, labels)
del role_predict, img, labels
#break
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file = False):
model.eval()
print ('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
t1 = time.time()
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict = model(img, verb)
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, labels
#break
print('eval, time: %.2f' % ( time.time()-t1))
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, verb_img_feat, verb,
labels) in enumerate(dev_loader):
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb_img_feat = torch.autograd.Variable(verb_img_feat.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb_img_feat = torch.autograd.Variable(verb_img_feat)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
verb_predict, role_pred_topk = model.forward_eval(
img, verb_img_feat)
top1.add_point_eval5_log_sorted(img_id, verb_predict, verb,
role_pred_topk, labels)
top5.add_point_eval5_log_sorted(img_id, verb_predict, verb,
role_pred_topk, labels)
del verb_predict, role_pred_topk, img, verb, labels
#break
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file = False):
model.eval()
print ('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
print("{}/{} batches\r".format(i+1,mx)),
#print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]])
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
labels = torch.autograd.Variable(labels.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
labels = torch.autograd.Variable(labels)
verb = torch.autograd.Variable(verb)
verb_predict, role_predict = model(img)
top1.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels)
top5.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels)
del verb_predict, img, verb
#break
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, verb_pred, verb) in enumerate(dev_loader):
#print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]])
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb_pred = torch.autograd.Variable(verb_pred.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
verb_pred = torch.autograd.Variable(verb_pred)
verb = torch.autograd.Variable(verb)
verb_predict = model(img, verb_pred)
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
del verb_predict, img, verb
#break
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file = False):
model.eval()
print ('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, labels, verb_role_impact) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx))
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
verb_role_impact = torch.autograd.Variable(verb_role_impact.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
verb_role_impact = torch.autograd.Variable(verb_role_impact)
role_predict = model(img, verb, verb_role_impact)
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, labels
#break
return top1, top5, 0
def eval_output(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
#img_id_list = ['feeding_266.jpg', 'packing_252.jpg','igniting_123.jpg','assembling_263.jpg', 'lathering_25.jpg', 'sharpening_232.jpg', 'nipping_51.jpg']
verb_name = 'sharpening'
tot_score = []
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
#print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]])
show_att = False
if verb_name in img_id[0]:
print('handling ', img_id[0])
show_att = True
else:
continue
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict, mean_score = model.forward_vis(img, verb, show_att)
tot_score.append(mean_score)
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, labels
#break
a = torch.sum(torch.stack(tot_score).squeeze(), 0)
print(a)
#print(tot_score)
return top1, top5, 0
def eval_output(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
img_id_list = [
'vaulting_90.jpg', 'dialing_213.jpg', 'stirring_68.jpg',
'ailing_175.jpg', 'unloading_97.jpg', 'clenching_115.jpg',
'imitating_44.jpg', 'assembling_230.jpg', 'wrapping_101.jpg',
'descending_105.jpg', 'stuffing_202.jpg', 'dining_67.jpg',
'tattooing_72.jpg', 'fetching_83.jpg', 'giving_46.jpg',
'striking_202.jpg', 'tickling_86.jpg', 'brewing_110.jpg',
'taping_142.jpg', 'fording_40.jpg'
]
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
#print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]])
show_att = False
if img_id[0] in img_id_list:
print('handling ', img_id[0])
show_att = True
else:
continue
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict = model.forward_vis(img, verb, show_att)
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, labels
#break
return top1, top5, 0
def eval_output(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
img_id_list = ['pawing_37.jpg']
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
show_att = False
if img_id[0] in img_id_list:
print('handling ', img_id[0])
show_att = True
else:
continue
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict = model.forward_vis(img, verb, show_att)
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, labels
#break
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
print('evaluating model...')
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
for i, (img_id, img, img_feat, verb, labels) in enumerate(dev_loader):
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
img_feat = torch.autograd.Variable(img_feat.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
img_feat = torch.autograd.Variable(img_feat)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict, verb_pred, verb_hidden_rep = model(
img, img_feat, verb)
if write_to_file:
top1.add_point_both_wrong(verb_pred, verb, role_predict,
labels)
top5.add_point_both_wrong(verb_pred, verb, role_predict,
labels)
else:
top1.add_point_both_wrong(verb_pred, verb, role_predict,
labels)
top5.add_point_both_wrong(verb_pred, verb, role_predict,
labels)
del role_predict, img, verb, labels
#break
return top1, top5, 0
def train(model, train_loader, dev_loader, optimizer, scheduler, max_epoch, model_dir, encoder, gpu_mode, clip_norm, model_name, model_saving_name, eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
if gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model
#pmodel = model
all = count_parameters(model)
cnn = count_parameters(model.convnet)
print('model parameters - all, cnn, base ', all, cnn)
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
for epoch in range(max_epoch):
t = time.time()
for i, (_, img, verb, labels) in enumerate(train_loader):
total_steps += 1
#min_t = time.time()
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
role_predict = pmodel(img, verb)
loss = model.calculate_loss(verb, role_predict, labels)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
optimizer.step()
optimizer.zero_grad()
train_loss += loss.item()
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
#print('after minibatch ', i, time.time() - min_t)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results_nouns()
top5_a = top5.get_average_results_nouns()
print ("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}"
.format(total_steps-1,epoch,i, utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a,"{:.2f}","5-"), loss.item(),
train_loss / ((total_steps-1)%eval_frequency) ))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print ('Dev {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(model.state_dict(), model_dir + "/{}_{}.model".format( model_name, model_saving_name))
print ('New best model saved! {0}'.format(max_score))
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
del role_predict, loss, img, verb, labels
print('epoch %d, time: %.2f' % (epoch, time.time()-t))
print('Epoch ', epoch, ' completed!')
scheduler.step()
def train(model,
train_loader,
dev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
model_name,
model_saving_name,
eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
'''if gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
for epoch in range(max_epoch):
for i, (img_id, img, verb_pred, verb) in enumerate(train_loader):
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb_pred = torch.autograd.Variable(verb_pred.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
verb_pred = torch.autograd.Variable(verb_pred)
verb = torch.autograd.Variable(verb)
verb_predict = pmodel(img, verb_pred)
loss = model.calculate_verb_loss(verb_predict, verb)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
optimizer.step()
optimizer.zero_grad()
train_loss += loss.item()
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results()
top5_a = top5.get_average_results()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder,
gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils.format_dict(top1_avg, '{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(
model.state_dict(), model_dir +
"/{}_{}.model".format(model_name, model_saving_name))
print('New best model saved! {0}'.format(max_score))
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3)
del verb_predict, loss, img, verb
print('Epoch ', epoch, ' completed!')
scheduler.step()