}
}))
optimizer = Optimizer(model.parameters(),
args.optim,
args.learning_rate,
args.max_grad_norm,
lr_decay=args.learning_rate_decay,
start_decay_at=args.start_decay_at)
criterion = make_criterion(len(src_dict), src_dict.get_pad())
print('* number of parameters: %d' % model.n_params())
print(model)
if args.gpu:
model.cuda(), criterion.cuda()
trainer = EncoderDecoderTrainer(model, {
'train': train,
'valid': valid
}, criterion, optimizer)
trainer.add_loggers(StdLogger())
trainer.add_loggers(VisdomLogger(env='encdec'))
hook = make_encdec_hook(args.target, args.gpu, beam=args.beam)
num_checkpoints = len(train) // (args.checkpoint * args.hooks_per_epoch)
trainer.add_hook(hook, num_checkpoints=num_checkpoints)
trainer.train(args.epochs, args.checkpoint, shuffle=True, gpu=args.gpu)
start_decay_at=args.start_decay_at,
lr_decay=args.lr_decay,
on_lr_update=on_lr_update)
class VAETrainer(Trainer):
def on_batch_end(self, epoch, batch, loss):
# reset kl weight
total_batches = len(self.datasets['train'])
self.model.kl_weight = self.model.kl_schedule(batch +
total_batches *
epoch)
losses = [{
'loss': 'log-loss'
}, {
'loss': 'kl',
'format': lambda loss: loss
}]
trainer = VAETrainer(model, {
'train': train,
'valid': valid,
'test': test
},
optimizer,
losses=losses)
trainer.add_loggers(
StdLogger(), VisdomLogger(env='vae', losses=('rec', 'kl'), max_y=600))
trainer.train(args.epochs, args.checkpoints, shuffle=True)
model.parameters(), args.optim, args.learning_rate, args.max_grad_norm,
lr_decay=args.learning_rate_decay, start_decay_at=args.start_decay_at)
criterion = make_criterion(len(src_dict), src_dict.get_pad())
model.apply(u.make_initializer(
rnn={'type': 'orthogonal', 'args': {'gain': 1.0}}))
print('* number of parameters: %d' % model.n_params())
print(model)
if args.gpu:
model.cuda(), criterion.cuda()
trainer = EncoderDecoderTrainer(
model, {'train': train, 'valid': valid}, criterion, optimizer)
# target = args.target.split() if args.target else None
# hook = make_encdec_hook(args.target, args.gpu)
# Logging
if args.csv != 'empty':
trainer.add_loggers(notesLogger(args=args, model=model, save_path=args.csv))
if args.logging:
trainer.add_loggers(StdLogger(args=args, model=model))
if args.visdom:
trainer.add_loggers(StdLogger(), VisdomLogger(env='encdec'))
num_checkpoints = len(train) // (args.checkpoint * args.hooks_per_epoch)
# trainer.add_hook(hook, num_checkpoints=num_checkpoints)
trainer.train(args.epochs, args.checkpoint, shuffle=True, gpu=args.gpu)
criterion, optim)
# hooks
early_stopping = None
if args.early_stopping > 0:
early_stopping = EarlyStopping(args.early_stopping)
model_check_hook = make_lm_check_hook(
d, method=args.decoding_method, temperature=args.temperature,
max_seq_len=args.max_seq_len, seed_text=args.seed, gpu=args.gpu,
early_stopping=early_stopping)
num_checkpoints = len(train) // (args.checkpoint * args.hooks_per_epoch)
trainer.add_hook(model_check_hook, num_checkpoints=num_checkpoints)
# loggers
visdom_logger = VisdomLogger(
log_checkpoints=args.log_checkpoints, title=args.prefix, env='lm',
server='http://' + args.visdom_server)
trainer.add_loggers(StdLogger(), visdom_logger)
trainer.train(args.epochs, args.checkpoint, gpu=args.gpu)
if args.save:
test_ppl = trainer.validate_model(test=True)
print("Test perplexity: %g" % test_ppl)
if args.save:
f = '{prefix}.{cell}.{layers}l.{hid_dim}h.{emb_dim}e.{bptt}b.{ppl}'
fname = f.format(ppl="%.2f" % test_ppl, **vars(args))
if os.path.isfile(fname):
answer = input("File [%s] exists. Overwrite? (y/n): " % fname)
if answer.lower() not in ("y", "yes"):
print("Goodbye!")
# model.encoder.register_backward_hook(u.log_grad)
if args.load_embeddings:
weight = load_embeddings(
train.d['src'].vocab, args.flavor, args.suffix, '~/data/word_embeddings')
model.init_embeddings(weight)
criterion = vae_criterion(vocab, train.d['src'].get_pad())
if args.gpu:
model.cuda(), criterion.cuda()
def on_lr_update(old_lr, new_lr):
trainer.log("info", "Resetting lr [%g -> %g]" % (old_lr, new_lr))
optimizer = Optimizer(
model.parameters(), args.optim, lr=args.learning_rate,
max_norm=args.max_norm, weight_decay=args.weight_decay,
# SGD-only
start_decay_at=args.start_decay_at, lr_decay=args.lr_decay,
on_lr_update=on_lr_update)
trainer = VAETrainer(
model, datasets, criterion, optimizer, inflection_point=args.inflection_point)
trainer.add_loggers(
StdLogger(),
VisdomLogger(env='vae_gender', losses=('rec', 'kl'), max_y=600))
# trainer.add_hook(kl_weight_hook)
trainer.train(args.epochs, args.checkpoints, shuffle=True, gpu=args.gpu)
{'src': src_dict, 'trg': src_dict},
batch_size=args.batch_size, gpu=args.gpu,
fitted=True)
test = PairedDataset(
[s for t in datasets for s in datasets[t]['test'].data['src']],
[s for t in datasets for s in datasets[t]['test'].data['trg']],
{'src': src_dict, 'trg': src_dict},
batch_size=args.batch_size, gpu=args.gpu,
fitted=True)
stdlogger = StdLogger(outputfile='.multitarget.log')
trainer = EncoderDecoderTrainer(
model, {'train': train, 'valid': valid, 'test': test},
criterion, optimizer)
trainer.add_loggers(
stdlogger, VisdomLogger(env='multitarget', title='general'))
num_checkpoints = \
max(len(train) // (args.checkpoint * args.hooks_per_epoch), 1)
trainer.add_hook(hook, num_checkpoints=num_checkpoints)
trainer.log('info', ' * vocabulary size. %d' % len(src_dict))
trainer.log('info', ' * maximum batch size. %d' % args.batch_size)
trainer.log('info', ' * number of train batches. %d' % len(train))
trainer.log('info', ' * number of parameters. %d' % model.n_params())
trainer.log('info', str(model))
trainer.log('info', "**********************")
trainer.log('info', "Training general model")
trainer.train(args.epochs, args.checkpoint, shuffle=True, gpu=args.gpu)
# train decoders
for target in datasets:
m.parameters(), args.optim, lr=args.lr, max_norm=args.max_norm,
lr_decay=args.lr_decay, start_decay_at=args.start_decay_at,
decay_every=args.decay_every)
# create trainer
trainer = Trainer(m, {"train": train, "test": test, "valid": valid}, optim)
# hooks
early_stopping = None
if args.early_stopping > 0:
early_stopping = EarlyStopping(10, patience=args.early_stopping)
model_hook = u.make_lm_hook(
d, method=args.decoding_method, temperature=args.temperature,
max_seq_len=args.max_seq_len, gpu=args.gpu,
early_stopping=early_stopping)
trainer.add_hook(model_hook, hooks_per_epoch=args.hooks_per_epoch)
# loggers
trainer.add_loggers(StdLogger())
if args.visdom:
visdom_logger = VisdomLogger(
log_checkpoints=args.log_checkpoints, env='lm',
server='http://' + args.visdom_host)
trainer.add_loggers(visdom_logger)
(best_model, valid_loss), test_loss = trainer.train(
args.epochs, args.checkpoint)
if args.save:
u.save_checkpoint(args.save_path, best_model, d, vars(args), ppl=test_loss)