# ----- saving the checkpoint ----- #
if iters % args.save_every == 0:
args.logger.info('save (back-up) checkpoints at iter={}'.format(iters))
with torch.cuda.device(args.gpu):
torch.save(best.model.state_dict(),
'{}_iter={}.pt'.format(args.model_name, iters))
torch.save([iters, best.opt.state_dict()],
'{}_iter={}.pt.states'.format(args.model_name, iters))
# ----- meta-validation ----- #
if iters % args.eval_every == 0:
progressbar.close()
dev_iters = iters
weights = model.save_fast_weights()
lang_U = Us[0]
fast_weights = weights
self_opt = torch.optim.Adam([
p for p in model.get_parameters(type=args.finetune_params)
if p.requires_grad
],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
outputs_data = valid_model(args,
model,
dev_real,
dev_metrics,
self_opt.step()
info = ' Inner-loop[{}]: loss={:.3f}, lr={:.8f}, batch_size={}'.format(
data_name, export(loss_inner), self_opt.param_groups[0]['lr'],
bs_inner)
if use_prog_bar:
progressbar.update(1)
progressbar.set_description(info)
if use_prog_bar:
progressbar.close()
return model.save_fast_weights()
# ----- meta-validation ----- #
dev_iters = iters
weights = model.save_fast_weights()
self_opt = torch.optim.Adam([
p for p in model.get_parameters(type=args.finetune_params)
if p.requires_grad
],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
# training start..
best = Best(max,
'corpus_bleu',
'i',
model=model,
opt=self_opt,
path=args.model_name,
info = ' Inner-loop[{}]: loss={:.3f}, lr={:.8f}, batch_size={}'.format(data_name, export(loss_inner), self_opt.param_groups[0]['lr'], bs_inner)
if use_prog_bar:
progressbar.update(1)
progressbar.set_description(info)
if use_prog_bar:
progressbar.close()
return model.save_fast_weights()
# ----- meta-validation ----- #
dev_iters = iters
weights = model.save_fast_weights()
fast_weights = weights
self_opt = torch.optim.Adam([p for p in model.get_parameters(type='fast') if p.requires_grad], betas=(0.9, 0.98), eps=1e-9)
corpus_bleu = -1
# training start..
best = Best(max, 'corpus_bleu', 'i', model=model, opt=self_opt, path=args.model_name, gpu=args.gpu)
dev_metrics = Metrics('dev', 'loss', 'gleu')
outputs_data = valid_model(args, model, dev_real, dev_metrics, print_out=True)
corpus_bleu0 = outputs_data['corpus_bleu']
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/BLEU_corpus_', outputs_data['corpus_bleu'], dev_iters)
# ----- saving the checkpoint ----- #
if iters % args.save_every == 0:
args.logger.info('save (back-up) checkpoints at iter={}'.format(iters))
with torch.cuda.device(args.gpu):
torch.save(best.model.state_dict(),
'{}_iter={}.pt'.format(args.model_name, iters))
torch.save([iters, best.opt.state_dict()],
'{}_iter={}.pt.states'.format(args.model_name, iters))
# ----- meta-validation ----- #
if iters % args.eval_every == 0:
dev_iters = iters
with torch.cuda.device(args.gpu):
weights = copy.deepcopy(
model.save_fast_weights()) # --- initial params
outs = copy.deepcopy(model.encoder.out.state_dict())
fast_weights = weights
self_opt = torch.optim.Adam(
[p for p in model.get_parameters(type='fast') if p.requires_grad],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
corpus_gleu = -1
outputs_data = valid_model(args,
model,
dev_real,
dev_metrics,
print_out=True)
# ----- saving the checkpoint ----- #
if iters % args.save_every == 0:
args.logger.info('save (back-up) checkpoints at iter={}'.format(iters))
with torch.cuda.device(args.gpu):
torch.save(best.model.state_dict(),
'{}_iter={}.pt'.format(args.model_name, iters))
torch.save([iters, best.opt.state_dict()],
'{}_iter={}.pt.states'.format(args.model_name, iters))
# ----- meta-validation ----- #
if iters % args.eval_every == 0:
dev_iters = iters
with torch.cuda.device(args.gpu):
weights = copy.deepcopy(
model.save_fast_weights()) # --- initial params
outs = copy.deepcopy(model.encoder.out.state_dict())
fast_weights = weights
self_opt = torch.optim.Adam(
[p for p in model.get_parameters(type='full') if p.requires_grad],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
corpus_gleu = -1
outputs_data = valid_model(args,
model,
dev_real,
dev_metrics,
print_out=True)