def test_loading_optimizer(self, train_func, sgd, optimizer, load_function,
checkpoint):
load_function.returnvalue = checkpoint
mock_model = mock.Mock()
mock_model.params.returnvalue = True
n_epoches = 2
trainer = SupervisedTrainer(batch_size=16)
trainer.train(mock_model,
self.dataset,
n_epoches,
resume=True,
checkpoint_path='dummy',
optimizer='sgd')
self.assertFalse(
sgd.called,
"Failed to not call Optimizer() when optimizer should be loaded from checkpoint"
)
trainer.train(mock_model,
self.dataset,
n_epoches,
resume=False,
checkpoint_path='dummy',
optimizer='sgd')
sgd.assert_called()
return
def train_pcfg_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
opt.file_name = generate_filename_from_options(opt)
# Seed
torch.manual_seed(opt.seed)
torch.cuda.manual_seed_all(opt.seed)
# Prepare data
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
seq2seq, _, output_vocab = initialize_model(opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
# timestamp for tensorboard run
now = datetime.datetime.now()
timestamp = now.strftime("%m-%d_%H-%M-%S")
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(pad, eos)
run_folder = 'runs/' + opt.file_name + '/' + timestamp
model_folder = 'models/' + opt.file_name
trainer = SupervisedTrainer(expt_dir=model_folder)
checkpoint_path = os.path.join(
model_folder, opt.load_checkpoint) if opt.resume_training else None
early_stop = EarlyStoppingCallback(patience=100)
reduce_lr = ReduceLRonPlateauCallback(factor=0.5, patience=50)
# custom callbacks to log to tensorboard and do early stopping
custom_cbs = [TensorboardCallback(run_folder), early_stop, reduce_lr]
# Train
seq2seq, logs = trainer.train(seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every,
custom_callbacks=custom_cbs)
if opt.write_logs:
logs.write_to_file(run_folder + '/logs')
def train_lookup_model():
parser = init_argparser()
opt = parser.parse_args()
default_settings = opt.default_settings
# Add machine-task to path
# gets the lookupt task from tasks
T = get_task("lookup", is_mini=True)
print("Got Task")
parameters = T.default_params[default_settings]
train_path = T.train_path
valid_path = T.valid_path
test_paths = T.test_paths
# # Prepare logging and data set
init_logging(parameters)
src, tgt, train, dev, monitor_data = prepare_iters(
parameters, train_path, test_paths, valid_path,
parameters['batch_size'])
# Prepare model
seq2seq, output_vocab = initialize_model(parameters, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
# Prepare training
losses = [NLLLoss(ignore_index=pad)]
for loss in losses:
loss.to(device)
loss_weights = [1.]
metrics = [SequenceAccuracy(ignore_index=pad)]
trainer = SupervisedTrainer(expt_dir='../models')
# Train
print("Training")
seq2seq, _ = trainer.train(seq2seq,
train,
num_epochs=20,
dev_data=dev,
monitor_data=monitor_data,
optimizer='adam',
checkpoint_path='../models',
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=10,
print_every=10)
def train_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
# Prepare logging and data set
init_logging(opt)
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
if opt.load_checkpoint is not None:
seq2seq, input_vocab, output_vocab = load_model_from_checkpoint(
opt, src, tgt)
else:
seq2seq, input_vocab, output_vocab = initialize_model(
opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
sos = tgt.SYM_EOS
unk = tgt.unk_token
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(
opt, pad, unk, sos, eos, input_vocab, output_vocab)
checkpoint_path = os.path.join(
opt.output_dir, opt.load_checkpoint) if opt.resume_training else None
trainer = SupervisedTrainer(expt_dir=opt.output_dir)
# Train
seq2seq, logs = trainer.train(
seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every)
if opt.write_logs:
output_path = os.path.join(opt.output_dir, opt.write_logs)
logs.write_to_file(output_path)
def train_pcfg_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
opt.file_name = generate_filename_from_options(opt)
# Prepare data
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
seq2seq, input_vocab, output_vocab = initialize_model(opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
sos = tgt.SYM_EOS
unk = tgt.unk_token
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(pad, eos)
run_folder = 'runs/' + opt.file_name
trainer = SupervisedTrainer(expt_dir=run_folder + '/models')
checkpoint_path = os.path.join(
run_folder +
'/models', opt.load_checkpoint) if opt.resume_training else None
# custom callback to log to tensorboard
custom_cb = [TensorboardCallback(run_folder)]
# Train
seq2seq, logs = trainer.train(
seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every)
if opt.write_logs:
logs.write_to_file(run_folder + '/logs')
checkpoint_path = os.path.join(opt.output_dir,
opt.load_checkpoint) if opt.resume else None
# create trainer
t = SupervisedTrainer(loss=losses,
metrics=metrics,
loss_weights=loss_weights,
batch_size=opt.batch_size,
eval_batch_size=opt.eval_batch_size,
checkpoint_every=opt.save_every,
print_every=opt.print_every,
expt_dir=opt.output_dir)
seq2seq, logs = t.train(seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume=opt.resume,
checkpoint_path=checkpoint_path)
if opt.write_logs:
output_path = os.path.join(opt.output_dir, opt.write_logs)
logs.write_to_file(output_path)
# evaluator = Evaluator(loss=loss, batch_size=opt.batch_size)
# dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
NUM_EPOCHS = 10
HIDDEN_SIZE = 128
init_logging()
# Get data
train_iter, valid_iter, test_iters, src, tgt = get_iters()
# Prepare model
baseline_seq2seq = get_baseline_model(src, tgt, HIDDEN_SIZE)
baseline_seq2seq.to(device)
# Prepare training
pad = tgt.vocab.stoi[tgt.pad_token]
losses = [NLLLoss(ignore_index=pad).to(device)]
metrics = [SequenceAccuracy(ignore_index=pad)]
trainer = SupervisedTrainer(expt_dir='runs/models/baseline')
# Train
logging.info("Training")
seq2seq, logs = trainer.train(baseline_seq2seq,
train_iter,
dev_data=valid_iter,
monitor_data=test_iters,
num_epochs=NUM_EPOCHS,
optimizer='adam',
checkpoint_path='runs/models/baseline',
losses=losses,
metrics=metrics,
checkpoint_every=100,
print_every=100)
output_unk_symbol=tgt.unk_token))
checkpoint_path = os.path.join(opt.output_dir,
opt.load_checkpoint) if opt.resume else None
# create trainer
expt_dir = os.path.join(opt.output_dir, experiment.get_key())
t = SupervisedTrainer(expt_dir=expt_dir)
seq2seq, logs = t.train(model=seq2seq,
data=train,
dev_data=dev,
monitor_data=monitor_data,
num_epochs=opt.epochs,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every,
random_seed=opt.random_seed,
custom_callbacks=[CometLogger(experiment)])
if opt.write_logs:
output_path = os.path.join(opt.output_dir, opt.write_logs)
logs.write_to_file(output_path)