def train_pcfg_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
opt.file_name = generate_filename_from_options(opt)
# Seed
torch.manual_seed(opt.seed)
torch.cuda.manual_seed_all(opt.seed)
# Prepare data
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
seq2seq, _, output_vocab = initialize_model(opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
# timestamp for tensorboard run
now = datetime.datetime.now()
timestamp = now.strftime("%m-%d_%H-%M-%S")
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(pad, eos)
run_folder = 'runs/' + opt.file_name + '/' + timestamp
model_folder = 'models/' + opt.file_name
trainer = SupervisedTrainer(expt_dir=model_folder)
checkpoint_path = os.path.join(
model_folder, opt.load_checkpoint) if opt.resume_training else None
early_stop = EarlyStoppingCallback(patience=100)
reduce_lr = ReduceLRonPlateauCallback(factor=0.5, patience=50)
# custom callbacks to log to tensorboard and do early stopping
custom_cbs = [TensorboardCallback(run_folder), early_stop, reduce_lr]
# Train
seq2seq, logs = trainer.train(seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every,
custom_callbacks=custom_cbs)
if opt.write_logs:
logs.write_to_file(run_folder + '/logs')
def test_resume_from_multiple_of_epoches(self, mock_evaluator, mock_checkpoint, mock_func):
mock_model = mock.Mock()
mock_optim = mock.Mock()
trainer = SupervisedTrainer(batch_size=16)
trainer.optimizer = mock_optim
n_epoches = 1
start_epoch = 1
step = 7
trainer._train_epoches(self.dataset, mock_model, n_epoches, start_epoch, step, dev_data=self.dataset)
def test_batch_num_when_resuming(self, mock_evaluator, mock_checkpoint, mock_func):
mock_model = mock.Mock()
mock_optim = mock.Mock()
trainer = SupervisedTrainer(batch_size=16)
trainer.optimizer = mock_optim
n_epoches = 1
start_epoch = 1
steps_per_epoch = 7
step = 3
trainer._train_epoches(self.dataset, mock_model, n_epoches, start_epoch, step)
self.assertEqual(steps_per_epoch - step, mock_func.call_count)
def train_lookup_model():
parser = init_argparser()
opt = parser.parse_args()
default_settings = opt.default_settings
# Add machine-task to path
# gets the lookupt task from tasks
T = get_task("lookup", is_mini=True)
print("Got Task")
parameters = T.default_params[default_settings]
train_path = T.train_path
valid_path = T.valid_path
test_paths = T.test_paths
# # Prepare logging and data set
init_logging(parameters)
src, tgt, train, dev, monitor_data = prepare_iters(
parameters, train_path, test_paths, valid_path,
parameters['batch_size'])
# Prepare model
seq2seq, output_vocab = initialize_model(parameters, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
# Prepare training
losses = [NLLLoss(ignore_index=pad)]
for loss in losses:
loss.to(device)
loss_weights = [1.]
metrics = [SequenceAccuracy(ignore_index=pad)]
trainer = SupervisedTrainer(expt_dir='../models')
# Train
print("Training")
seq2seq, _ = trainer.train(seq2seq,
train,
num_epochs=20,
dev_data=dev,
monitor_data=monitor_data,
optimizer='adam',
checkpoint_path='../models',
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=10,
print_every=10)
def train_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
# Prepare logging and data set
init_logging(opt)
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
if opt.load_checkpoint is not None:
seq2seq, input_vocab, output_vocab = load_model_from_checkpoint(
opt, src, tgt)
else:
seq2seq, input_vocab, output_vocab = initialize_model(
opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
sos = tgt.SYM_EOS
unk = tgt.unk_token
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(
opt, pad, unk, sos, eos, input_vocab, output_vocab)
checkpoint_path = os.path.join(
opt.output_dir, opt.load_checkpoint) if opt.resume_training else None
trainer = SupervisedTrainer(expt_dir=opt.output_dir)
# Train
seq2seq, logs = trainer.train(
seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every)
if opt.write_logs:
output_path = os.path.join(opt.output_dir, opt.write_logs)
logs.write_to_file(output_path)
def train_pcfg_model():
# Create command line argument parser and validate chosen options
parser = init_argparser()
opt = parser.parse_args()
opt = validate_options(parser, opt)
opt.file_name = generate_filename_from_options(opt)
# Prepare data
src, tgt, train, dev, monitor_data = prepare_iters(opt)
# Prepare model
seq2seq, input_vocab, output_vocab = initialize_model(opt, src, tgt, train)
pad = output_vocab.stoi[tgt.pad_token]
eos = tgt.eos_id
sos = tgt.SYM_EOS
unk = tgt.unk_token
# Prepare training
losses, loss_weights, metrics = prepare_losses_and_metrics(pad, eos)
run_folder = 'runs/' + opt.file_name
trainer = SupervisedTrainer(expt_dir=run_folder + '/models')
checkpoint_path = os.path.join(
run_folder +
'/models', opt.load_checkpoint) if opt.resume_training else None
# custom callback to log to tensorboard
custom_cb = [TensorboardCallback(run_folder)]
# Train
seq2seq, logs = trainer.train(
seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume_training,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
print_every=opt.print_every)
if opt.write_logs:
logs.write_to_file(run_folder + '/logs')
def test_loading_optimizer(self, train_func, sgd, optimizer, load_function,
checkpoint):
load_function.returnvalue = checkpoint
mock_model = mock.Mock()
mock_model.params.returnvalue = True
n_epoches = 2
trainer = SupervisedTrainer(batch_size=16)
trainer.train(mock_model,
self.dataset,
n_epoches,
resume=True,
checkpoint_path='dummy',
optimizer='sgd')
self.assertFalse(
sgd.called,
"Failed to not call Optimizer() when optimizer should be loaded from checkpoint"
)
trainer.train(mock_model,
self.dataset,
n_epoches,
resume=False,
checkpoint_path='dummy',
optimizer='sgd')
sgd.assert_called()
return
def test_resume_from_multiple_of_epoches(self, mock_evaluator, mock_checkpoint, mock_func):
mock_optim = mock.Mock()
trainer = SupervisedTrainer()
trainer.model = mock.Mock()
trainer.optimizer = mock.Mock()
callbacks = CallbackContainer(trainer)
n_epoches = 1
start_epoch = 1
step = 7
trainer.set_local_parameters(123, [], [], [], 1, 1)
trainer._train_epoches(
self.data_iterator, n_epoches, start_epoch, step, callbacks, self.data_iterator)
def test_batch_num_when_resuming(self, mock_evaluator, mock_checkpoint, mock_func):
trainer = SupervisedTrainer()
trainer.model = mock.Mock()
trainer.optimizer = mock.Mock()
callbacks = CallbackContainer(trainer)
n_epoches = 1
start_epoch = 1
steps_per_epoch = len(self.data_iterator)
step = 3
trainer.set_local_parameters(123, [], [], [], 1, 1)
trainer._train_epoches(self.data_iterator, n_epoches,
start_epoch, step, callbacks, self.data_iterator)
self.assertEqual(steps_per_epoch - step, mock_func.call_count)
metrics = [
WordAccuracy(ignore_index=pad),
SequenceAccuracy(ignore_index=pad),
FinalTargetAccuracy(ignore_index=pad, eos_id=tgt.eos_id)
]
# Since we need the actual tokens to determine k-grammar accuracy,
# we also provide the input and output vocab and relevant special symbols
# metrics.append(SymbolRewritingAccuracy(
# input_vocab=input_vocab,
# output_vocab=output_vocab,
# use_output_eos=output_eos_used,
# input_pad_symbol=src.pad_token,
# output_sos_symbol=tgt.SYM_SOS,
# output_pad_symbol=tgt.pad_token,
# output_eos_symbol=tgt.SYM_EOS,
# output_unk_symbol=tgt.unk_token))
data_func = SupervisedTrainer.get_batch_data
#################################################################################
# Evaluate model on test set
evaluator = Evaluator(batch_size=opt.batch_size, loss=losses, metrics=metrics)
losses, metrics = evaluator.evaluate(model=seq2seq,
data=test,
get_batch_data=data_func)
total_loss, log_msg, _ = SupervisedTrainer.get_losses(losses, metrics, 0)
logging.info(log_msg)
# output_vocab=output_vocab,
# use_output_eos=use_output_eos,
# input_pad_symbol=src.pad_token,
# output_sos_symbol=tgt.SYM_SOS,
# output_pad_symbol=tgt.pad_token,
# output_eos_symbol=tgt.SYM_EOS,
# output_unk_symbol=tgt.unk_token))
checkpoint_path = os.path.join(opt.output_dir,
opt.load_checkpoint) if opt.resume else None
# create trainer
t = SupervisedTrainer(loss=losses,
metrics=metrics,
loss_weights=loss_weights,
batch_size=opt.batch_size,
eval_batch_size=opt.eval_batch_size,
checkpoint_every=opt.save_every,
print_every=opt.print_every,
expt_dir=opt.output_dir)
seq2seq, logs = t.train(seq2seq,
train,
num_epochs=opt.epochs,
dev_data=dev,
monitor_data=monitor_data,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume=opt.resume,
checkpoint_path=checkpoint_path)
def create_trainer(opt, losses, loss_weights, metrics):
return SupervisedTrainer(loss=losses, metrics=metrics, loss_weights=loss_weights, batch_size=opt.batch_size,
eval_batch_size=opt.eval_batch_size, checkpoint_every=opt.save_every,
print_every=opt.print_every, expt_dir=opt.output_dir)
NUM_EPOCHS = 10
HIDDEN_SIZE = 128
init_logging()
# Get data
train_iter, valid_iter, test_iters, src, tgt = get_iters()
# Prepare model
baseline_seq2seq = get_baseline_model(src, tgt, HIDDEN_SIZE)
baseline_seq2seq.to(device)
# Prepare training
pad = tgt.vocab.stoi[tgt.pad_token]
losses = [NLLLoss(ignore_index=pad).to(device)]
metrics = [SequenceAccuracy(ignore_index=pad)]
trainer = SupervisedTrainer(expt_dir='runs/models/baseline')
# Train
logging.info("Training")
seq2seq, logs = trainer.train(baseline_seq2seq,
train_iter,
dev_data=valid_iter,
monitor_data=test_iters,
num_epochs=NUM_EPOCHS,
optimizer='adam',
checkpoint_path='runs/models/baseline',
losses=losses,
metrics=metrics,
checkpoint_every=100,
print_every=100)
if 'sym_rwr_acc' in opt.metrics:
metrics.append(
SymbolRewritingAccuracy(input_vocab=input_vocab,
output_vocab=output_vocab,
use_output_eos=use_output_eos,
output_sos_symbol=tgt.SYM_SOS,
output_pad_symbol=tgt.pad_token,
output_eos_symbol=tgt.SYM_EOS,
output_unk_symbol=tgt.unk_token))
checkpoint_path = os.path.join(opt.output_dir,
opt.load_checkpoint) if opt.resume else None
# create trainer
expt_dir = os.path.join(opt.output_dir, experiment.get_key())
t = SupervisedTrainer(expt_dir=expt_dir)
seq2seq, logs = t.train(model=seq2seq,
data=train,
dev_data=dev,
monitor_data=monitor_data,
num_epochs=opt.epochs,
optimizer=opt.optim,
teacher_forcing_ratio=opt.teacher_forcing_ratio,
learning_rate=opt.lr,
resume_training=opt.resume,
checkpoint_path=checkpoint_path,
losses=losses,
metrics=metrics,
loss_weights=loss_weights,
checkpoint_every=opt.save_every,
@mock.patch('seq2seq.trainer.SupervisedTrainer._train_batch', return_value=[])
@mock.patch('seq2seq.util.checkpoint.Checkpoint.save')
@mock.patch('seq2seq.evaluator.Evaluator.evaluate', return_value=([],[]))
def test_batch_num_when_resuming(self, mock_evaluator, mock_checkpoint, mock_func):
=======
@mock.patch('machine.trainer.SupervisedTrainer._train_batch',
return_value=[])
@mock.patch('machine.util.checkpoint.Checkpoint.save')
@mock.patch('machine.evaluator.Evaluator.evaluate', return_value=([], []))
def test_batch_num_when_resuming(
self, mock_evaluator, mock_checkpoint, mock_func):
>>>>>>> upstream/master
mock_model = mock.Mock()
mock_optim = mock.Mock()
trainer = SupervisedTrainer(batch_size=16)
trainer.optimizer = mock_optim
n_epoches = 1
start_epoch = 1
steps_per_epoch = 7
step = 3
trainer._train_epoches(self.dataset, mock_model,
n_epoches, start_epoch, step)
self.assertEqual(steps_per_epoch - step, mock_func.call_count)
@mock.patch('machine.trainer.SupervisedTrainer._train_batch',
return_value=0)
@mock.patch('machine.util.checkpoint.Checkpoint.save')
@mock.patch('machine.evaluator.Evaluator.evaluate', return_value=([], []))
def test_resume_from_multiple_of_epoches(
self, mock_evaluator, mock_checkpoint, mock_func):