if args.tensorboard and (not args.debug):
writer.add_scalar('dev/Loss', dev_metrics.loss, dev_iters)
writer.add_scalar('dev/BLEU_corpus_',
outputs_data['corpus_bleu'], dev_iters)
if outputs_data['corpus_bleu'] > corpus_bleu:
corpus_bleu = outputs_data['corpus_bleu']
args.logger.info('model:' + args.prefix + args.hp_str + "\n")
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/zero_shot_BLEU', corpus_bleu0, iters)
writer.add_scalar('dev/fine_tune_BLEU', corpus_bleu, iters)
args.logger.info('validation done.\n')
model.load_fast_weights(weights) # --- comming back to normal
# -- restart the progressbar --
progressbar = tqdm(total=args.eval_every, desc='start training')
if not args.debug:
best.accumulate(corpus_bleu, iters)
args.logger.info(
'the best model is achieved at {}, corpus BLEU={}'.format(
best.i, best.corpus_bleu))
# ----- meta-training ------- #
model.train()
if iters > args.maximum_steps:
args.logger.info('reach the maximum updating steps.')
break
logger.info(f'Starting with HPARAMS: {hp_str}')
model_name = args.models_dir + '/' + 'eval.' + args.prefix + hp_str
# build the model
model = UniversalTransformer(SRC, TRG, args)
# logger.info(str(model))
if args.load_from == '0':
args.load_from = None
if args.load_from is not None:
with torch.cuda.device(args.gpu):
# model.load_state_dict(torch.load(args.models_dir + '/' + args.load_from + '.pt',
# map_location=lambda storage, loc: storage.cuda())) # load the pretrained models.
model.load_fast_weights(torch.load(
args.resume_dir + '/' + args.load_from + '.pt',
map_location=lambda storage, loc: storage.cuda()),
type='meta') # load the pretrained models.
# use cuda
if args.gpu > -1:
model.cuda(args.gpu)
# additional information
args.__dict__.update({
'model_name': model_name,
'hp_str': hp_str,
'logger': logger
})
# tensorboard writer
if args.tensorboard and (not args.debug):
# ----------------------------------------------------------------------------------------------------------------- #
assert args.load_from is not None, 'load the model'
steps = 10000
while True:
load_from = args.load_from + 'iter={}'.format(steps)
if not os.path.exists(args.models_dir + '/' + load_from + '.pt'):
break
logger.info(load_from)
with torch.cuda.device(args.gpu):
# model.load_state_dict(torch.load(args.models_dir + '/' + args.load_from + '.pt',
# map_location=lambda storage, loc: storage.cuda())) # load the pretrained models.
model.load_fast_weights(torch.load(
args.models_dir + '/' + load_from + '.pt',
map_location=lambda storage, loc: storage.cuda()),
type='meta') # load the pretrained models.
# if resume training
with torch.cuda.device(args.gpu): # very important.
offset, opt_states = torch.load(
args.models_dir + '/' + load_from + '.pt.states',
map_location=lambda storage, loc: storage.cuda())
print('offset {}'.format(offset))
# ---- updates ------ #
iters = offset
eposides = 0
tokens = 0
time0 = time.time()
outputs_data['corpus_bleu'], dev_iters)
if outputs_data['corpus_gleu'] > corpus_gleu:
corpus_gleu = outputs_data['corpus_gleu']
if outputs_data['corpus_bleu'] > corpus_bleu:
corpus_bleu = outputs_data['corpus_bleu']
args.logger.info('model:' + args.prefix + args.hp_str + "\n")
if args.tensorboard and (not args.debug):
writer.add_scalar('dev/zero_shot_BLEU', corpus_bleu0, iters)
writer.add_scalar('dev/fine_tune_BLEU', corpus_bleu, iters)
args.logger.info('validation done.\n')
model.load_fast_weights(weights) # --- comming back to normal
model.encoder.out.load_state_dict(outs) # --- comming back to normal
if not args.debug:
best.accumulate(corpus_bleu, corpus_gleu, iters)
args.logger.info(
'the best model is achieved at {}, corpus GLEU={}, corpus BLEU={}'
.format(best.i, best.corpus_gleu, best.corpus_bleu))
# ----- meta-training ------- #
model.train()
if iters > args.maximum_steps:
args.logger.info('reach the maximum updating steps.')
break
# ----- inner-loop ------