# ----- saving the checkpoint ----- # if iters % args.save_every == 0: args.logger.info('save (back-up) checkpoints at iter={}'.format(iters)) with torch.cuda.device(args.gpu): torch.save(best.model.state_dict(), '{}_iter={}.pt'.format(args.model_name, iters)) torch.save([iters, best.opt.state_dict()], '{}_iter={}.pt.states'.format(args.model_name, iters)) # ----- meta-validation ----- # if iters % args.eval_every == 0: progressbar.close() dev_iters = iters weights = model.save_fast_weights() lang_U = Us[0] fast_weights = weights self_opt = torch.optim.Adam([ p for p in model.get_parameters(type=args.finetune_params) if p.requires_grad ], betas=(0.9, 0.98), eps=1e-9) corpus_bleu = -1 outputs_data = valid_model(args, model, dev_real, dev_metrics,
self_opt.step() info = ' Inner-loop[{}]: loss={:.3f}, lr={:.8f}, batch_size={}'.format( data_name, export(loss_inner), self_opt.param_groups[0]['lr'], bs_inner) if use_prog_bar: progressbar.update(1) progressbar.set_description(info) if use_prog_bar: progressbar.close() return model.save_fast_weights() # ----- meta-validation ----- # dev_iters = iters weights = model.save_fast_weights() self_opt = torch.optim.Adam([ p for p in model.get_parameters(type=args.finetune_params) if p.requires_grad ], betas=(0.9, 0.98), eps=1e-9) corpus_bleu = -1 # training start.. best = Best(max, 'corpus_bleu', 'i', model=model, opt=self_opt, path=args.model_name,
info = ' Inner-loop[{}]: loss={:.3f}, lr={:.8f}, batch_size={}'.format(data_name, export(loss_inner), self_opt.param_groups[0]['lr'], bs_inner) if use_prog_bar: progressbar.update(1) progressbar.set_description(info) if use_prog_bar: progressbar.close() return model.save_fast_weights() # ----- meta-validation ----- # dev_iters = iters weights = model.save_fast_weights() fast_weights = weights self_opt = torch.optim.Adam([p for p in model.get_parameters(type='fast') if p.requires_grad], betas=(0.9, 0.98), eps=1e-9) corpus_bleu = -1 # training start.. best = Best(max, 'corpus_bleu', 'i', model=model, opt=self_opt, path=args.model_name, gpu=args.gpu) dev_metrics = Metrics('dev', 'loss', 'gleu') outputs_data = valid_model(args, model, dev_real, dev_metrics, print_out=True) corpus_bleu0 = outputs_data['corpus_bleu'] if args.tensorboard and (not args.debug): writer.add_scalar('dev/BLEU_corpus_', outputs_data['corpus_bleu'], dev_iters)
# ----- saving the checkpoint ----- # if iters % args.save_every == 0: args.logger.info('save (back-up) checkpoints at iter={}'.format(iters)) with torch.cuda.device(args.gpu): torch.save(best.model.state_dict(), '{}_iter={}.pt'.format(args.model_name, iters)) torch.save([iters, best.opt.state_dict()], '{}_iter={}.pt.states'.format(args.model_name, iters)) # ----- meta-validation ----- # if iters % args.eval_every == 0: dev_iters = iters with torch.cuda.device(args.gpu): weights = copy.deepcopy( model.save_fast_weights()) # --- initial params outs = copy.deepcopy(model.encoder.out.state_dict()) fast_weights = weights self_opt = torch.optim.Adam( [p for p in model.get_parameters(type='fast') if p.requires_grad], betas=(0.9, 0.98), eps=1e-9) corpus_bleu = -1 corpus_gleu = -1 outputs_data = valid_model(args, model, dev_real, dev_metrics, print_out=True)
# ----- saving the checkpoint ----- # if iters % args.save_every == 0: args.logger.info('save (back-up) checkpoints at iter={}'.format(iters)) with torch.cuda.device(args.gpu): torch.save(best.model.state_dict(), '{}_iter={}.pt'.format(args.model_name, iters)) torch.save([iters, best.opt.state_dict()], '{}_iter={}.pt.states'.format(args.model_name, iters)) # ----- meta-validation ----- # if iters % args.eval_every == 0: dev_iters = iters with torch.cuda.device(args.gpu): weights = copy.deepcopy( model.save_fast_weights()) # --- initial params outs = copy.deepcopy(model.encoder.out.state_dict()) fast_weights = weights self_opt = torch.optim.Adam( [p for p in model.get_parameters(type='full') if p.requires_grad], betas=(0.9, 0.98), eps=1e-9) corpus_bleu = -1 corpus_gleu = -1 outputs_data = valid_model(args, model, dev_real, dev_metrics, print_out=True)