def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, labels) in enumerate(dev_loader): #prit("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) , t0 = time.time() t1 = time.time() if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) labels = torch.autograd.Variable(labels) '''print('all inputs') print(img) print('=========================================================================') print(verb) print('=========================================================================') print(roles) print('=========================================================================') print(labels)''' if write_to_file: verb_predict, _, agent_names, place_names, agent_10, place_10 = model.forward_eval( img_id, img, verb, labels) else: verb_predict, _ = model(img_id, img, verb, labels) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' if write_to_file: top1.add_point_verb_only_eval_eval(img_id, verb_predict, verb, agent_names, place_names, agent_10, place_10) top5.add_point_verb_only_eval_eval(img_id, verb_predict, verb, agent_names, place_names, agent_10, place_10) else: top1.add_point_verb_only_eval(img_id, verb_predict, verb) top5.add_point_verb_only_eval(img_id, verb_predict, verb) del img, verb_predict, verb, labels #break #return top1, top5, val_loss/mx return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, frcnn_feat, verb, roles, role_q, q_len, labels) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) roles = torch.autograd.Variable(roles.cuda()) frcnn_feat = torch.autograd.Variable(frcnn_feat.cuda()) role_q = torch.autograd.Variable(role_q.cuda()) q_len = torch.autograd.Variable(q_len.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) frcnn_feat = torch.autograd.Variable(frcnn_feat) role_q = torch.autograd.Variable(role_q) q_len = torch.autograd.Variable(q_len) roles = torch.autograd.Variable(roles) labels = torch.autograd.Variable(labels) role_predict = model(frcnn_feat, verb, role_q) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' if write_to_file: top1.add_point_noun_log(img_id, verb, role_predict, labels) top5.add_point_noun_log(img_id, verb, role_predict, labels) else: top1.add_point_noun(verb, role_predict, labels) top5.add_point_noun(verb, role_predict, labels) del role_predict, img, verb, roles, labels #break #return top1, top5, val_loss/mx return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file = False): model.eval() val_loss = 0 print ('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, True) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (_id, img, verb) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) verb_predict = model.forward_eval(_id, img) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_multi_verb(_id, verb_predict, verb) top5.add_point_multi_verb(_id, verb_predict, verb) del verb_predict, img, verb #break #return top1, top5, val_loss/mx all = top1.all_res logit_info = model.logits #print('roles :', role_dict) #fail_val_all = top1.value_all_dict #pass_val_dict = top1.vall_all_correct with open('all_pred.json', 'w') as fp: json.dump(all, fp, indent=4) with open('all_logits.json', 'w') as fp: json.dump(logit_info, fp, indent=4) return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, True) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, verbq, roles, labels) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) roles = torch.autograd.Variable(roles.cuda()) verbq = torch.autograd.Variable(verbq.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) verbq = torch.autograd.Variable(verbq) roles = torch.autograd.Variable(roles) labels = torch.autograd.Variable(labels) verb_predict = model(img, verbq) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_verb_only_eval(img_id, verb_predict, verb) top5.add_point_verb_only_eval(img_id, verb_predict, verb) del verb_predict, img, verb, roles, labels #break #return top1, top5, val_loss/mx pass_list = top1.pass_list with open('passverb_gtagent.txt', 'w') as filehandle: for listitem in pass_list: filehandle.write('{}\n'.format(listitem)) return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, questions, labels) in enumerate(dev_loader): print("{}/{} batches\r".format(i + 1, mx)), t0 = time.time() t1 = time.time() if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) questions = torch.autograd.Variable(questions.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) questions = torch.autograd.Variable(questions) labels = torch.autograd.Variable(labels) '''print('all inputs') print(img) print('=========================================================================') print(verb) print('=========================================================================') print(roles) print('=========================================================================') print(labels)''' verb_predict, role_predict = model(img_id, img, verb, labels) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels) top5.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels) del img, verb_predict, verb, labels #break #return top1, top5, val_loss/mx return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, v_img, r_img, verb, labels) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: v_img = torch.autograd.Variable(v_img.cuda()) r_img = torch.autograd.Variable(r_img.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: v_img = torch.autograd.Variable(v_img) r_img = torch.autograd.Variable(r_img) verb = torch.autograd.Variable(verb) labels = torch.autograd.Variable(labels) verb_predict = model.forward_eval_gttemplte_predplace( img_id, v_img, r_img, verb, labels) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' if write_to_file: top1.add_point_verb_only_eval(img_id, verb_predict, verb) top5.add_point_verb_only_eval(img_id, verb_predict, verb) else: top1.add_point_verb_only_eval(img_id, verb_predict, verb) top5.add_point_verb_only_eval(img_id, verb_predict, verb) del v_img, r_img, verb_predict, verb, labels #break #return top1, top5, val_loss/mx return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (_id, img, verb) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) verb_predict = model.forward_eval(img) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_verb_beamdirect(_id, verb_predict, verb) top5.add_point_verb_beamdirect(_id, verb_predict, verb) del verb_predict, img, verb #break #return top1, top5, val_loss/mx '''topkissue1 = top1.topk_issue topkissue5 = top5.topk_issue with open('topkissue1.json', 'w') as fp: json.dump(topkissue1, fp, indent=4) with open('topkissue5.json', 'w') as fp: json.dump(topkissue5, fp, indent=4)''' return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (id, img, verb, labels) in enumerate(dev_loader): #prit("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' #print('handling batch :', id) if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) labels = torch.autograd.Variable(labels) verb_predict, role_predict = model(img) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_eval5_log_sorted(id, verb_predict, verb, role_predict, labels) top5.add_point_eval5_log_sorted(id, verb_predict, verb, role_predict, labels) del img, verb #break #return top1, top5, val_loss/mx return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) #top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, verbq, roleq, roles, ans) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) verbq = torch.autograd.Variable(verbq.cuda()) roleq = torch.autograd.Variable(roleq.cuda()) roles = torch.autograd.Variable(roles.cuda()) ans = torch.autograd.Variable(ans.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) verbq = torch.autograd.Variable(verbq) roleq = torch.autograd.Variable(roleq) roles = torch.autograd.Variable(roles) ans = torch.autograd.Variable(ans) loss_res, verb_predict, label_predict = model( img, verbq, roleq, None, None) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' if write_to_file: top1.add_point_eval5_log(img_id, verb_predict, verb, label_predict, ans) #top5.add_point_noun_log(img_id, verb, role_predict, labels) else: top1.add_point_eval5_log(img_id, verb_predict, verb, label_predict, ans) #top5.add_point_noun(verb, role_predict, labels) del verb_predict, label_predict, img, verbq, roleq, verb, ans #break #return top1, top5, val_loss/mx return top1, None, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, write_to_file) #top5 = imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, roles, role_q, q_len, labels) in enumerate(dev_loader): #print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) , t0 = time.time() t1 = time.time() if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) roles = torch.autograd.Variable(roles.cuda()) role_q = torch.autograd.Variable(role_q.cuda()) q_len = torch.autograd.Variable(q_len.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) role_q = torch.autograd.Variable(role_q) q_len = torch.autograd.Variable(q_len) roles = torch.autograd.Variable(roles) labels = torch.autograd.Variable(labels) verb_predict, label_predict = model(img, None) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' if write_to_file: top1.add_point_eval5_log(img_id, verb_predict, verb, label_predict, labels) #top5.add_point_noun_log(img_id, verb, role_predict, labels) else: top1.add_point_eval5_log(img_id, verb_predict, verb, label_predict, labels) #top5.add_point_noun(verb, role_predict, labels) del verb_predict, label_predict, img, verb, labels #break #return top1, top5, val_loss/mx return top1, None, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() val_loss = 0 print('evaluating model...') top1 = imsitu_scorer(encoder, 1, 3, True) with torch.no_grad(): mx = len(dev_loader) for i, (img, rot) in enumerate(dev_loader): #print("{}/{} batches\r".format(i+1,mx)) , '''im_data = torch.squeeze(im_data,0) im_info = torch.squeeze(im_info,0) gt_boxes = torch.squeeze(gt_boxes,0) num_boxes = torch.squeeze(num_boxes,0) verb = torch.squeeze(verb,0) roles = torch.squeeze(roles,0) labels = torch.squeeze(labels,0)''' if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) rot = torch.autograd.Variable(rot.cuda()) else: img = torch.autograd.Variable(img) rot = torch.autograd.Variable(rot) rot_predict = model(img) '''loss = model.calculate_eval_loss(verb_predict, verb, role_predict, labels) val_loss += loss.item()''' top1.add_point_rot_only(rot_predict, rot) del img #break #return top1, top5, val_loss/mx return top1, None, 0
def train(model, train_loader, dev_loader, traindev_loader, optimizer, scheduler, max_epoch, model_dir, encoder, gpu_mode, clip_norm, lr_max, model_name, model_saving_name, args, eval_frequency=4000): model.train() train_loss = 0 total_steps = 0 print_freq = 400 dev_score_list = [] time_all = time.time() '''if gpu_mode >= 0 : ngpus = 2 device_array = [i for i in range(0,ngpus)] pmodel = torch.nn.DataParallel(model, device_ids=device_array) else: pmodel = model''' pmodel = model '''if scheduler.get_lr()[0] < lr_max: scheduler.step()''' top1 = imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer(encoder, 5, 3) '''print('init param data check :') for f in model.parameters(): if f.requires_grad: print(f.data.size())''' for epoch in range(max_epoch): #print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size()) #sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1 mx = len(train_loader) for i, (_, img, verb, questions, labels) in enumerate(train_loader): #print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) , t0 = time.time() t1 = time.time() total_steps += 1 if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) questions = torch.autograd.Variable(questions.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) questions = torch.autograd.Variable(questions) labels = torch.autograd.Variable(labels) '''print('all inputs') print(img) print('=========================================================================') print(verb) print('=========================================================================') print(roles) print('=========================================================================') print(labels)''' place_predict, loss1 = pmodel(img, questions, labels, verb) #verb_predict, rol1pred, role_predict = pmodel.forward_eval5(img) #print ("forward time = {}".format(time.time() - t1)) t1 = time.time() loss = loss1 '''g = make_dot(verb_predict, model.state_dict()) g.view()''' #loss = model.calculate_loss(verb, role_predict, labels, args) #loss = model.calculate_eval_loss_new(verb_predict, verb, rol1pred, labels, args) #loss = loss_ * random.random() #try random loss #print ("loss time = {}".format(time.time() - t1)) t1 = time.time() #print('current loss = ', loss) loss.backward() #print ("backward time = {}".format(time.time() - t1)) torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm) '''for param in filter(lambda p: p.requires_grad,model.parameters()): print(param.grad.data.sum())''' #start debugger #import pdb; pdb.set_trace() optimizer.step() optimizer.zero_grad() '''print('grad check :') for f in model.parameters(): print('data is') print(f.data) print('grad is') print(f.grad)''' train_loss += loss.item() #top1.add_point_eval5(verb_predict, verb, role_predict, labels) #top5.add_point_eval5(verb_predict, verb, role_predict, labels) top1.add_point_agent_only(place_predict, labels) top5.add_point_agent_only(place_predict, labels) if total_steps % print_freq == 0: top1_a = top1.get_average_results_nouns() top5_a = top5.get_average_results_nouns() print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}". format(total_steps - 1, epoch, i, utils_imsitu.format_dict(top1_a, "{:.2f}", "1-"), utils_imsitu.format_dict(top5_a, "{:.2f}", "5-"), loss.item(), train_loss / ((total_steps - 1) % eval_frequency))) if total_steps % eval_frequency == 0: top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode) model.train() top1_avg = top1.get_average_results_nouns() top5_avg = top5.get_average_results_nouns() avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \ top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"] #avg_score /= 8 avg_score = top1_avg["value*"] print('Dev {} average :{:.2f} {} {}'.format( total_steps - 1, avg_score * 100, utils_imsitu.format_dict(top1_avg, '{:.2f}', '1-'), utils_imsitu.format_dict(top5_avg, '{:.2f}', '5-'))) #print('Dev loss :', val_loss) dev_score_list.append(avg_score) max_score = max(dev_score_list) if max_score == dev_score_list[-1]: torch.save( model.state_dict(), model_dir + "/{}_place_ftall{}.model".format( model_name, model_saving_name)) print('New best model saved! {0}'.format(max_score)) #eval on the trainset '''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode) model.train() top1_avg = top1.get_average_results() top5_avg = top5.get_average_results() avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \ top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"] avg_score /= 8 print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100, utils.format_dict(top1_avg,'{:.2f}', '1-'), utils.format_dict(top5_avg, '{:.2f}', '5-')))''' print('current train loss', train_loss) train_loss = 0 top1 = imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer(encoder, 5, 3) del place_predict, loss, img, verb, labels #break print('Epoch ', epoch, ' completed!') scheduler.step()
def train(model, train_loader, dev_loader, traindev_loader, optimizer, scheduler, max_epoch, model_dir, encoder, gpu_mode, clip_norm, lr_max, model_name, args,eval_frequency=1000): model.train() train_loss = 0 total_steps = 0 print_freq = 400 dev_score_list = [] '''if model.gpu_mode >= 0 : ngpus = 2 device_array = [i for i in range(0,ngpus)] pmodel = torch.nn.DataParallel(model, device_ids=device_array) else: pmodel = model''' pmodel = model top1 = imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer(encoder, 5, 3) '''print('init param data check :') for f in model.parameters(): if f.requires_grad: print(f.data.size())''' for epoch in range(max_epoch): #print('current sample : ', i, img.size(), verb.size(), roles.size(), labels.size()) #sizes batch_size*3*height*width, batch*504*1, batch*6*190*1, batch*3*6*lebale_count*1 mx = len(train_loader) for i, (_id, img, verb) in enumerate(train_loader): #print("epoch{}-{}/{} batches\r".format(epoch,i+1,mx)) , total_steps += 1 if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) #optimizer.zero_grad() '''print('all inputs') print(img) print('=========================================================================') print(verb) print('=========================================================================') print(roles) print('=========================================================================') print(labels)''' verb_predict = pmodel(img, _id) '''g = make_dot(verb_predict, model.state_dict()) g.view()''' loss = model.calculate_loss(verb_predict, verb) #print('current loss = ', loss) loss.backward() # torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm) optimizer.step() optimizer.zero_grad() train_loss += loss.item() top1.add_point_multi_verb(_id, verb_predict, verb) top5.add_point_multi_verb(_id, verb_predict, verb) if total_steps % print_freq == 0: top1_a = top1.get_average_results() top5_a = top5.get_average_results() print ("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}" .format(total_steps-1,epoch,i, utils.format_dict(top1_a, "{:.2f}", "1-"), utils.format_dict(top5_a,"{:.2f}","5-"), loss.item(), train_loss / ((total_steps-1)%eval_frequency) )) if total_steps % eval_frequency == 0: top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode) model.train() top1_avg = top1.get_average_results() top5_avg = top5.get_average_results() #todo : top 5 avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \ top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"] #avg_score = top1_avg["value*"] avg_score /= 2 print ('Dev {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100, utils.format_dict(top1_avg,'{:.2f}', '1-'), utils.format_dict(top5_avg, '{:.2f}', '5-'))) #print('Dev loss :', val_loss) dev_score_list.append(avg_score) max_score = max(dev_score_list) if max_score == dev_score_list[-1]: torch.save(model.state_dict(), model_dir + "/{}_agent2verbq_diffeval_test.model".format(model_name)) print ('New best model saved! {0}'.format(max_score)) #eval on the trainset '''top1, top5, val_loss = eval(model, traindev_loader, encoder, gpu_mode) model.train() top1_avg = top1.get_average_results() top5_avg = top5.get_average_results() avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \ top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"] avg_score /= 8 print ('TRAINDEV {} average :{:.2f} {} {}'.format(total_steps-1, avg_score*100, utils.format_dict(top1_avg,'{:.2f}', '1-'), utils.format_dict(top5_avg, '{:.2f}', '5-')))''' print('current train loss', train_loss) train_loss = 0 top1 = imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer(encoder, 5, 3) del loss, img #break print('Epoch ', epoch, ' completed!') scheduler.step()