def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() print('evaluating model...') top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) with torch.no_grad(): mx = len(dev_loader) for i, (img_id, img, verb, labels) in enumerate(dev_loader): print("{}/{} batches\r".format(i + 1, mx)), #print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]]) if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) labels = torch.autograd.Variable(labels.cuda()) verb = torch.autograd.Variable(verb.cuda()) else: img = torch.autograd.Variable(img) labels = torch.autograd.Variable(labels) verb = torch.autograd.Variable(verb) verb_predict, role_predict = model(img) top1.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels) top5.add_point_eval5_log_sorted(img_id, verb_predict, verb, role_predict, labels) del verb_predict, img, verb return top1, top5, 0
def eval(model, dev_loader, encoder, gpu_mode, write_to_file=False): model.eval() print('evaluating model...') top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3, write_to_file) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) with torch.no_grad(): for i, (img_id, img, verb, labels) in enumerate(dev_loader): #print(img_id[0], encoder.verb2_role_dict[encoder.verb_list[verb[0]]]) if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) labels = torch.autograd.Variable(labels) role_predict = model(img, verb) top1.add_point_noun(verb, role_predict, labels) top5.add_point_noun(verb, role_predict, labels) del role_predict, img, verb, labels return top1, top5, 0
def train(model, train_loader, dev_loader, optimizer, scheduler, max_epoch, model_dir, encoder, gpu_mode, clip_norm, model_name, model_saving_name, eval_frequency=4000): model.train() train_loss = 0 total_steps = 0 print_freq = 400 dev_score_list = [] if gpu_mode >= 0: ngpus = 2 device_array = [i for i in range(0, ngpus)] pmodel = torch.nn.DataParallel(model, device_ids=device_array) else: pmodel = model top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) for epoch in range(max_epoch): for i, (_, img, verb, labels) in enumerate(train_loader): total_steps += 1 if gpu_mode >= 0: img = torch.autograd.Variable(img.cuda()) verb = torch.autograd.Variable(verb.cuda()) labels = torch.autograd.Variable(labels.cuda()) else: img = torch.autograd.Variable(img) verb = torch.autograd.Variable(verb) labels = torch.autograd.Variable(labels) role_predict = pmodel(img, verb) loss = model.calculate_loss(verb, role_predict, labels) loss.backward() torch.nn.utils.clip_grad_norm_(model.parameters(), clip_norm) optimizer.step() optimizer.zero_grad() train_loss += loss.item() top1.add_point_noun(verb, role_predict, labels) top5.add_point_noun(verb, role_predict, labels) if total_steps % print_freq == 0: top1_a = top1.get_average_results_nouns() top5_a = top5.get_average_results_nouns() print("{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}". format(total_steps - 1, epoch, i, utils.format_dict(top1_a, "{:.2f}", "1-"), utils.format_dict(top5_a, "{:.2f}", "5-"), loss.item(), train_loss / ((total_steps - 1) % eval_frequency))) if total_steps % eval_frequency == 0: top1, top5, val_loss = eval(model, dev_loader, encoder, gpu_mode) model.train() top1_avg = top1.get_average_results_nouns() top5_avg = top5.get_average_results_nouns() avg_score = top1_avg["verb"] + top1_avg["value"] + top1_avg["value-all"] + top5_avg["verb"] + \ top5_avg["value"] + top5_avg["value-all"] + top5_avg["value*"] + top5_avg["value-all*"] avg_score /= 8 print('Dev {} average :{:.2f} {} {}'.format( total_steps - 1, avg_score * 100, utils.format_dict(top1_avg, '{:.2f}', '1-'), utils.format_dict(top5_avg, '{:.2f}', '5-'))) dev_score_list.append(avg_score) max_score = max(dev_score_list) if max_score == dev_score_list[-1]: torch.save( model.state_dict(), model_dir + "/{}_{}.model".format(model_name, model_saving_name)) print('New best model saved! {0}'.format(max_score)) print('current train loss', train_loss) train_loss = 0 top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) del role_predict, loss, img, verb, labels print('Epoch ', epoch, ' completed!') scheduler.step()
def eval(model, loader, encoder, logging=False): model.eval() verbloss = 0 nounsloss = 0 gtloss = 0 top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) with torch.no_grad(): for __, (_, img, verb, nouns) in enumerate(loader): if torch.cuda.is_available(): img = img.cuda() verb = verb.cuda() nouns = nouns.cuda() with autocast(): # automix precision stuff pred_verb, pred_nouns, pred_gt_nouns = model(img, verb) top1.add_point_both(pred_verb, verb, pred_nouns, nouns, pred_gt_nouns) top5.add_point_both(pred_verb, verb, pred_nouns, nouns, pred_gt_nouns) if torch.cuda.is_available(): vl = model.module.verb_loss(pred_verb, verb) nl = model.module.nouns_loss(pred_nouns, nouns) gtl = model.module.nouns_loss(pred_gt_nouns, nouns) else: vl = model.verb_loss(pred_verb, verb) nl = model.nouns_loss(pred_nouns, nouns) gtl = model.nouns_loss(pred_gt_nouns, nouns) verbloss += vl.item() nounsloss += nl.item() gtloss += gtl.item() verbloss /= len(loader) nounsloss /= len(loader) gtloss /= len(loader) val_losses = { 'verb_loss': verbloss, 'nouns_loss': nounsloss, 'gt_loss': gtloss } #print scores avg_score = 0 if logging is True: top1_a = top1.get_average_results_both() top5_a = top5.get_average_results_both() avg_score = top1_a['verb'] + top1_a['value'] + top1_a['value-all'] + \ top5_a['verb'] + top5_a['value'] + top5_a['value-all'] + \ top1_a['gt-value'] + top1_a['gt-value-all'] avg_score /= 8 avg_score = avg_score * 100 print('val losses = [v: {:.2f}, n: {:.2f}, gt: {:.2f}]'.format( val_losses['verb_loss'], val_losses['nouns_loss'], val_losses['gt_loss'])) gt = {key: top1_a[key] for key in ['gt-value', 'gt-value-all']} one_val = {key: top1_a[key] for key in ['verb', 'value', 'value-all']} print('{}\n{}\n{}, mean = {:.2f}\n'.format( utils.format_dict(one_val, '{:.2f}', '1-'), utils.format_dict(top5_a, '{:.2f}', '5-'), utils.format_dict(gt, '{:.2f}', ''), avg_score)) return top1, top5, val_losses, avg_score
def train(model, train_loader, dev_loader, optimizer, max_epoch, encoder, model_saving_name, folder, checkpoint=None): model.train() avg_scores = [] verb_losses = [] nouns_losses = [] val_avg_scores = [] val_verb_losses = [] val_nouns_losses = [] epoch = 0 # if checkpoint resume stuffs if checkpoint is not None: epoch = checkpoint['epoch'] avg_scores = checkpoint['avg_scores'] verb_losses = checkpoint['verb_losses'] nouns_losses = checkpoint['nouns_losses'] val_avg_scores = checkpoint['val_avg_scores'] val_verb_losses = checkpoint['val_verb_losses'] val_nouns_losses = checkpoint['val_nouns_losses'] if torch.cuda.is_available(): model.module.load_state_dict(checkpoint['model_state_dict']) else: model.load_state_dict(checkpoint['model_state_dict']) optimizer.load_state_dict(checkpoint['optimizer_state_dict']) #mix precision stuff scaler = GradScaler() for e in range(epoch, max_epoch): verb_loss_accum = 0 nouns_loss_accum = 0 gt_nouns_loss_accum = 0 print('Epoch-{}, lr: {:.4f}'.format(e, optimizer.param_groups[0]['lr'])) top1 = imsitu_scorer.imsitu_scorer(encoder, 1, 3) top5 = imsitu_scorer.imsitu_scorer(encoder, 5, 3) for i, (_, img, verb, nouns) in enumerate(train_loader): if torch.cuda.is_available(): img = img.cuda() verb = verb.cuda() nouns = nouns.cuda() optimizer.zero_grad() with autocast(): #mix precision stuff pred_verb, pred_nouns, pred_gt_nouns = model(img, verb) #predict and calculate lossess if torch.cuda.is_available(): verb_loss = model.module.verb_loss(pred_verb, verb) nouns_loss = model.module.nouns_loss(pred_nouns, nouns) gt_nouns_loss = model.module.nouns_loss( pred_gt_nouns, nouns) else: verb_loss = model.verb_loss(pred_verb, verb) nouns_loss = model.nouns_loss(pred_nouns, nouns) gt_nouns_loss = model.nouns_loss(pred_gt_nouns, nouns) loss = verb_loss + nouns_loss # backpropagate errors and stuffs scaler.scale(loss).backward() scaler.unscale_(optimizer) torch.nn.utils.clip_grad_norm_(model.parameters(), 1) scaler.step(optimizer) scaler.update() top1.add_point_both(pred_verb, verb, pred_nouns, nouns, pred_gt_nouns) top5.add_point_both(pred_verb, verb, pred_nouns, nouns, pred_gt_nouns) verb_loss_accum += verb_loss.item() nouns_loss_accum += nouns_loss.item() gt_nouns_loss_accum += gt_nouns_loss.item() #fin epoch #epoch accuracy mean top1_a = top1.get_average_results_both() top5_a = top5.get_average_results_both() avg_score = top1_a['verb'] + top1_a['value'] + top1_a['value-all'] + \ top5_a['verb'] + top5_a['value'] + top5_a['value-all'] + \ top1_a['gt-value'] + top1_a['gt-value-all'] avg_score /= 8 avg_score *= 100 avg_scores.append(avg_score) #epoch lossess verb_loss_mean = verb_loss_accum / len(train_loader) nouns_loss_mean = nouns_loss_accum / len(train_loader) gt_nouns_loss_mean = gt_nouns_loss_accum / len(train_loader) verb_losses.append(verb_loss_mean) nouns_losses.append(nouns_loss_mean) #print stuffs print('training losses = [v: {:.2f}, n: {:.2f}, gt: {:.2f}]'.format( verb_loss_mean, nouns_loss_mean, gt_nouns_loss_mean)) gt = {key: top1_a[key] for key in ['gt-value', 'gt-value-all']} one_val = {key: top1_a[key] for key in ['verb', 'value', 'value-all']} print('{}\n{}\n{}, mean = {:.2f}\n{}'.format( utils.format_dict(one_val, '{:.2f}', '1-'), utils.format_dict(top5_a, '{:.2f}', '5-'), utils.format_dict(gt, '{:.2f}', ''), avg_score, '-' * 50)) # evaluating top1, top5, val_losses, val_avg_score = eval(model, dev_loader, encoder, logging=True) model.train() #val mean scores val_avg_scores.append(val_avg_score) val_verb_losses.append(val_losses['verb_loss']) val_nouns_losses.append(val_losses['nouns_loss']) plt.plot(verb_losses, label='verb losses') plt.plot(nouns_losses, label='nouns losses') plt.plot(avg_scores, label='accuracy mean') plt.plot(val_verb_losses, '-.', label='val verb losses') plt.plot(val_nouns_losses, '-.', label='val nouns losses') plt.plot(val_avg_scores, '-.', label='val accuracy mean') plt.grid() plt.legend() plt.savefig(pjoin(folder, model_saving_name + '.png')) plt.clf() #always saving but no need if it's not the best score checkpoint = { 'epoch': e + 1, 'avg_scores': avg_scores, 'verb_losses': verb_losses, 'nouns_losses': nouns_losses, 'val_avg_scores': val_avg_scores, 'val_verb_losses': val_verb_losses, 'val_nouns_losses': val_nouns_losses, 'model_state_dict': model.state_dict(), 'optimizer_state_dict': optimizer.state_dict() } if torch.cuda.is_available(): checkpoint.update({'model_state_dict': model.module.state_dict()}) torch.save(checkpoint, pjoin(folder, model_saving_name))