dev_iters = iters
weights = model.save_fast_weights()
lang_U = Us[0]
fast_weights = weights
self_opt = torch.optim.Adam([
p for p in model.get_parameters(type=args.finetune_params)
if p.requires_grad
],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
outputs_data = valid_model(args,
model,
dev_real,
dev_metrics,
print_out=False,
U=lang_U)
corpus_bleu0 = outputs_data['corpus_bleu']
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/BLEU_corpus_', outputs_data['corpus_bleu'],
dev_iters)
for j in range(args.valid_epochs):
args.logger.info("Fine-tuning epoch: {}".format(j))
dev_metrics.reset()
inner_loop(args, (train_real, args.src),
model,
None,
#
# ----------------------------------------------------------------------------------------------------------------- #
# if resume training
if (args.load_from is not None) and (args.resume):
with torch.cuda.device(args.gpu): # very important.
offset, opt_states = torch.load(
args.resume_dir + '/' + args.load_from + '.pt.states',
map_location=lambda storage, loc: storage.cuda())
else:
offset = 0
print('offset {}'.format(offset))
if args.support_size == 0:
dev_out = valid_model(args, model, dev_real, print_out=False, beam=4)
tst_out = valid_model(args, model, test_real, print_out=False, beam=4)
sys.exit(1)
# ---- updates ------ #
iters = 100000 # offset
eposides = 0
tokens = 0
time0 = time.time()
def get_learning_rate(i, lr0=0.1, disable=False):
if not disable:
return lr0 * 10 / math.sqrt(args.d_model) * min(
1 / math.sqrt(i), i / (args.warmup * math.sqrt(args.warmup)))
return 0.00002
if iters % args.eval_every == 0:
progressbar.close()
dev_iters = iters
weights = model.save_fast_weights()
fast_weights = weights
self_opt = torch.optim.Adam(
[p for p in model.get_parameters(type='fast') if p.requires_grad],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
outputs_data = valid_model(args,
model,
dev_real,
dev_metrics,
print_out=True)
corpus_bleu0 = outputs_data['corpus_bleu']
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/BLEU_corpus_', outputs_data['corpus_bleu'],
dev_iters)
for j in range(args.valid_epochs):
args.logger.info("Fine-tuning epoch: {}".format(j))
dev_metrics.reset()
inner_loop(args, (train_real, "ro"),
model,
None,
[p for p in model.get_parameters(type='fast') if p.requires_grad],
betas=(0.9, 0.98),
eps=1e-9)
corpus_bleu = -1
# training start..
best = Best(max,
'corpus_bleu',
'i',
model=model,
opt=self_opt,
path=args.model_name,
gpu=args.gpu)
dev_metrics = Metrics('dev', 'loss', 'gleu')
outputs_data = valid_model(args, model, dev_real, dev_metrics, print_out=True)
corpus_bleu0 = outputs_data['corpus_bleu']
fast_weights = [(weights, corpus_bleu0)]
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/BLEU_corpus_', outputs_data['corpus_bleu'],
dev_iters)
for j in range(args.valid_epochs):
args.logger.info("Fine-tuning epoch: {}".format(j))
dev_metrics.reset()
inner_loop(args, (train_real, "ro"),
model,
None,
dev_iters,