def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, labels) in enumerate(dev_loader):
#prit("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
t0 = time.time()
t1 = time.time()
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
if write_to_file:
verb_predict, _, agent_names, place_names, agent_10, place_10 = model.forward_eval(
img_id, img, verb, labels)
else:
verb_predict, _ = model(img_id, img, verb, labels)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
if write_to_file:
top1.add_point_verb_only_eval_eval(img_id, verb_predict, verb,
agent_names, place_names,
agent_10, place_10)
top5.add_point_verb_only_eval_eval(img_id, verb_predict, verb,
agent_names, place_names,
agent_10, place_10)
else:
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
del img, verb_predict, verb, labels
#break
#return top1, top5, val_loss/mx
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, frcnn_feat, verb, roles, role_q, q_len,
labels) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
roles = torch.autograd.Variable(roles.cuda())
frcnn_feat = torch.autograd.Variable(frcnn_feat.cuda())
role_q = torch.autograd.Variable(role_q.cuda())
q_len = torch.autograd.Variable(q_len.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
frcnn_feat = torch.autograd.Variable(frcnn_feat)
role_q = torch.autograd.Variable(role_q)
q_len = torch.autograd.Variable(q_len)
roles = torch.autograd.Variable(roles)
labels = torch.autograd.Variable(labels)
role_predict = model(frcnn_feat, verb, role_q)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
if write_to_file:
top1.add_point_noun_log(img_id, verb, role_predict, labels)
top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_noun(verb, role_predict, labels)
top5.add_point_noun(verb, role_predict, labels)
del role_predict, img, verb, roles, labels
#break
#return top1, top5, val_loss/mx
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file = False):
model.eval()
val_loss = 0
print ('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, True)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (_id, img, verb) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
verb_predict = model.forward_eval(_id, img)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_multi_verb(_id, verb_predict, verb)
top5.add_point_multi_verb(_id, verb_predict, verb)
del verb_predict, img, verb
#break
#return top1, top5, val_loss/mx
all = top1.all_res
logit_info = model.logits
#print('roles :', role_dict)
#fail_val_all = top1.value_all_dict
#pass_val_dict = top1.vall_all_correct
with open('all_pred.json', 'w') as fp:
json.dump(all, fp, indent=4)
with open('all_logits.json', 'w') as fp:
json.dump(logit_info, fp, indent=4)
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, True)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, verbq, roles,
labels) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
roles = torch.autograd.Variable(roles.cuda())
verbq = torch.autograd.Variable(verbq.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
verbq = torch.autograd.Variable(verbq)
roles = torch.autograd.Variable(roles)
labels = torch.autograd.Variable(labels)
verb_predict = model(img, verbq)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
del verb_predict, img, verb, roles, labels
#break
#return top1, top5, val_loss/mx
pass_list = top1.pass_list
with open('passverb_gtagent.txt', 'w') as filehandle:
for listitem in pass_list:
filehandle.write('{}\n'.format(listitem))
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, questions, labels) in enumerate(dev_loader):
print("{}/{} batches\r".format(i + 1, mx)),
t0 = time.time()
t1 = time.time()
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
questions = torch.autograd.Variable(questions.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
questions = torch.autograd.Variable(questions)
labels = torch.autograd.Variable(labels)
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
verb_predict, role_predict = model(img_id, img, verb, labels)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_eval5_log_sorted(img_id, verb_predict, verb,
role_predict, labels)
top5.add_point_eval5_log_sorted(img_id, verb_predict, verb,
role_predict, labels)
del img, verb_predict, verb, labels
#break
#return top1, top5, val_loss/mx
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, v_img, r_img, verb, labels) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
v_img = torch.autograd.Variable(v_img.cuda())
r_img = torch.autograd.Variable(r_img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
v_img = torch.autograd.Variable(v_img)
r_img = torch.autograd.Variable(r_img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
verb_predict = model.forward_eval_gttemplte_predplace(
img_id, v_img, r_img, verb, labels)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
if write_to_file:
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
else:
top1.add_point_verb_only_eval(img_id, verb_predict, verb)
top5.add_point_verb_only_eval(img_id, verb_predict, verb)
del v_img, r_img, verb_predict, verb, labels
#break
#return top1, top5, val_loss/mx
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (_id, img, verb) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
verb_predict = model.forward_eval(img)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_verb_beamdirect(_id, verb_predict, verb)
top5.add_point_verb_beamdirect(_id, verb_predict, verb)
del verb_predict, img, verb
#break
#return top1, top5, val_loss/mx
'''topkissue1 = top1.topk_issue
topkissue5 = top5.topk_issue
with open('topkissue1.json', 'w') as fp:
json.dump(topkissue1, fp, indent=4)
with open('topkissue5.json', 'w') as fp:
json.dump(topkissue5, fp, indent=4)'''
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (id, img, verb, labels) in enumerate(dev_loader):
#prit("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
#print('handling batch :', id)
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
labels = torch.autograd.Variable(labels)
verb_predict, role_predict = model(img)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_eval5_log_sorted(id, verb_predict, verb,
role_predict, labels)
top5.add_point_eval5_log_sorted(id, verb_predict, verb,
role_predict, labels)
del img, verb
#break
#return top1, top5, val_loss/mx
return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
#top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, verbq, roleq, roles,
ans) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
verbq = torch.autograd.Variable(verbq.cuda())
roleq = torch.autograd.Variable(roleq.cuda())
roles = torch.autograd.Variable(roles.cuda())
ans = torch.autograd.Variable(ans.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
verbq = torch.autograd.Variable(verbq)
roleq = torch.autograd.Variable(roleq)
roles = torch.autograd.Variable(roles)
ans = torch.autograd.Variable(ans)
loss_res, verb_predict, label_predict = model(
img, verbq, roleq, None, None)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
if write_to_file:
top1.add_point_eval5_log(img_id, verb_predict, verb,
label_predict, ans)
#top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_eval5_log(img_id, verb_predict, verb,
label_predict, ans)
#top5.add_point_noun(verb, role_predict, labels)
del verb_predict, label_predict, img, verbq, roleq, verb, ans
#break
#return top1, top5, val_loss/mx
return top1, None, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, write_to_file)
#top5 = imsitu_scorer(encoder, 5, 3)
with torch.no_grad():
mx = len(dev_loader)
for i, (img_id, img, verb, roles, role_q, q_len,
labels) in enumerate(dev_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
t0 = time.time()
t1 = time.time()
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
roles = torch.autograd.Variable(roles.cuda())
role_q = torch.autograd.Variable(role_q.cuda())
q_len = torch.autograd.Variable(q_len.cuda())
verb = torch.autograd.Variable(verb.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
role_q = torch.autograd.Variable(role_q)
q_len = torch.autograd.Variable(q_len)
roles = torch.autograd.Variable(roles)
labels = torch.autograd.Variable(labels)
verb_predict, label_predict = model(img, None)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
if write_to_file:
top1.add_point_eval5_log(img_id, verb_predict, verb,
label_predict, labels)
#top5.add_point_noun_log(img_id, verb, role_predict, labels)
else:
top1.add_point_eval5_log(img_id, verb_predict, verb,
label_predict, labels)
#top5.add_point_noun(verb, role_predict, labels)
del verb_predict, label_predict, img, verb, labels
#break
#return top1, top5, val_loss/mx
return top1, None, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False):
model.eval()
val_loss = 0
print('evaluating model...')
top1 = imsitu_scorer(encoder, 1, 3, True)
with torch.no_grad():
mx = len(dev_loader)
for i, (img, rot) in enumerate(dev_loader):
#print("{}/{} batches\r".format(i+1,mx)) ,
'''im_data = torch.squeeze(im_data,0)
im_info = torch.squeeze(im_info,0)
gt_boxes = torch.squeeze(gt_boxes,0)
num_boxes = torch.squeeze(num_boxes,0)
verb = torch.squeeze(verb,0)
roles = torch.squeeze(roles,0)
labels = torch.squeeze(labels,0)'''
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
rot = torch.autograd.Variable(rot.cuda())
else:
img = torch.autograd.Variable(img)
rot = torch.autograd.Variable(rot)
rot_predict = model(img)
'''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels)
val_loss += loss.item()'''
top1.add_point_rot_only(rot_predict, rot)
del img
#break
#return top1, top5, val_loss/mx
return top1, None, 0
def train(model,
train_loader,
dev_loader,
traindev_loader,
optimizer,
scheduler,
max_epoch,
model_dir,
encoder,
gpu_mode,
clip_norm,
lr_max,
model_name,
model_saving_name,
args,
eval_frequency=4000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
time_all = time.time()
'''if gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
'''if scheduler.get_lr()[0] < lr_max:
scheduler.step()'''
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (_, img, verb, questions, labels) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
t0 = time.time()
t1 = time.time()
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
questions = torch.autograd.Variable(questions.cuda())
labels = torch.autograd.Variable(labels.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
questions = torch.autograd.Variable(questions)
labels = torch.autograd.Variable(labels)
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
place_predict, loss1 = pmodel(img, questions, labels, verb)
#verb_predict, rol1pred, role_predict = pmodel.forward_eval5(img)
#print ("forward time = {}".format(time.time() - t1))
t1 = time.time()
loss = loss1
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
#loss = model.calculate_loss(verb, role_predict, labels, args)
#loss = model.calculate_eval_loss_new(verb_predict, verb, rol1pred, labels, args)
#loss = loss_ * random.random() #try random loss
#print ("loss time = {}".format(time.time() - t1))
t1 = time.time()
#print('current loss = ', loss)
loss.backward()
#print ("backward time = {}".format(time.time() - t1))
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
'''for param in filter(lambda p: p.requires_grad,model.parameters()):
print(param.grad.data.sum())'''
#start debugger
#import pdb; pdb.set_trace()
optimizer.step()
optimizer.zero_grad()
'''print('grad check :')
for f in model.parameters():
print('data is')
print(f.data)
print('grad is')
print(f.grad)'''
train_loss += loss.item()
#top1.add_point_eval5(verb_predict, verb, role_predict, labels)
#top5.add_point_eval5(verb_predict, verb, role_predict, labels)
top1.add_point_agent_only(place_predict, labels)
top5.add_point_agent_only(place_predict, labels)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results_nouns()
top5_a = top5.get_average_results_nouns()
print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}".
format(total_steps - 1, epoch, i,
utils_imsitu.format_dict(top1_a, "{:.2f}", "1-"),
utils_imsitu.format_dict(top5_a, "{:.2f}", "5-"),
loss.item(), train_loss /
((total_steps - 1) % eval_frequency)))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder,
gpu_mode)
model.train()
top1_avg = top1.get_average_results_nouns()
top5_avg = top5.get_average_results_nouns()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
#avg_score /= 8
avg_score = top1_avg["value*"]
print('Dev {} average :{:.2f} {} {}'.format(
total_steps - 1, avg_score * 100,
utils_imsitu.format_dict(top1_avg, '{:.2f}', '1-'),
utils_imsitu.format_dict(top5_avg, '{:.2f}', '5-')))
#print('Dev loss :', val_loss)
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(
model.state_dict(),
model_dir + "/{}_place_ftall{}.model".format(
model_name, model_saving_name))
print('New best model saved! {0}'.format(max_score))
#eval on the trainset
'''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))'''
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
del place_predict, loss, img, verb, labels
#break
print('Epoch ', epoch, ' completed!')
scheduler.step()
def train(model, train_loader, dev_loader, traindev_loader, optimizer, scheduler, max_epoch, model_dir, encoder, gpu_mode, clip_norm, lr_max, model_name, args,eval_frequency=1000):
model.train()
train_loss = 0
total_steps = 0
print_freq = 400
dev_score_list = []
'''if model.gpu_mode >= 0 :
ngpus = 2
device_array = [i for i in range(0,ngpus)]
pmodel = torch.nn.DataParallel(model, device_ids=device_array)
else:
pmodel = model'''
pmodel = model
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
'''print('init param data check :')
for f in model.parameters():
if f.requires_grad:
print(f.data.size())'''
for epoch in range(max_epoch):
#print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size())
#sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1
mx = len(train_loader)
for i, (_id, img, verb) in enumerate(train_loader):
#print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) ,
total_steps += 1
if gpu_mode >= 0:
img = torch.autograd.Variable(img.cuda())
verb = torch.autograd.Variable(verb.cuda())
else:
img = torch.autograd.Variable(img)
verb = torch.autograd.Variable(verb)
#optimizer.zero_grad()
'''print('all inputs')
print(img)
print('=========================================================================')
print(verb)
print('=========================================================================')
print(roles)
print('=========================================================================')
print(labels)'''
verb_predict = pmodel(img, _id)
'''g = make_dot(verb_predict, model.state_dict())
g.view()'''
loss = model.calculate_loss(verb_predict, verb)
#print('current loss = ', loss)
loss.backward()
# torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm)
optimizer.step()
optimizer.zero_grad()
train_loss += loss.item()
top1.add_point_multi_verb(_id, verb_predict, verb)
top5.add_point_multi_verb(_id, verb_predict, verb)
if total_steps % print_freq == 0:
top1_a = top1.get_average_results()
top5_a = top5.get_average_results()
print ("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}"
.format(total_steps-1,epoch,i, utils.format_dict(top1_a, "{:.2f}", "1-"),
utils.format_dict(top5_a,"{:.2f}","5-"), loss.item(),
train_loss / ((total_steps-1)%eval_frequency) ))
if total_steps % eval_frequency == 0:
top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
#todo : top 5
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
#avg_score = top1_avg["value*"]
avg_score /= 2
print ('Dev {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))
#print('Dev loss :', val_loss)
dev_score_list.append(avg_score)
max_score = max(dev_score_list)
if max_score == dev_score_list[-1]:
torch.save(model.state_dict(), model_dir + "/{}_agent2verbq_diffeval_test.model".format(model_name))
print ('New best model saved! {0}'.format(max_score))
#eval on the trainset
'''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode)
model.train()
top1_avg = top1.get_average_results()
top5_avg = top5.get_average_results()
avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \
top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"]
avg_score /= 8
print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100,
utils.format_dict(top1_avg,'{:.2f}', '1-'),
utils.format_dict(top5_avg, '{:.2f}', '5-')))'''
print('current train loss', train_loss)
train_loss = 0
top1 = imsitu_scorer(encoder, 1, 3)
top5 = imsitu_scorer(encoder, 5, 3)
del loss, img
#break
print('Epoch ', epoch, ' completed!')
scheduler.step()
