def __init__(self,
expt_dir='experiment',
loss=[NLLLoss()],
loss_weights=None,
metrics=[],
batch_size=64,
eval_batch_size=128,
random_seed=None,
checkpoint_every=100,
print_every=100):
self._trainer = "Simple Trainer"
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
k = NLLLoss()
self.loss = loss
self.metrics = metrics
self.loss_weights = loss_weights or len(loss) * [1.]
self.evaluator = Evaluator(loss=self.loss,
metrics=self.metrics,
batch_size=eval_batch_size)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
if not os.path.isabs(expt_dir):
expt_dir = os.path.join(os.getcwd(), expt_dir)
self.expt_dir = expt_dir
if not os.path.exists(self.expt_dir):
os.makedirs(self.expt_dir)
self.batch_size = batch_size
self.logger = logging.getLogger(__name__)
def set_local_parameters(self, random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every):
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.losses = losses
self.metrics = metrics
self.loss_weights = loss_weights or len(losses)*[1.]
self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.logger = logging.getLogger(__name__)
self._stop_training = False
def test_set_eval_mode(self, mock_eval, mock_call):
""" Make sure that evaluation is done in evaluation mode. """
mock_mgr = MagicMock()
mock_mgr.attach_mock(mock_eval, 'eval')
mock_mgr.attach_mock(mock_call, 'call')
evaluator = Evaluator(batch_size=64)
with patch('machine.evaluator.evaluator.torch.stack', return_value=None), \
patch('machine.metrics.WordAccuracy.eval_batch', return_value=None), \
patch('machine.metrics.WordAccuracy.eval_batch', return_value=None), \
patch('machine.loss.NLLLoss.eval_batch', return_value=None):
evaluator.evaluate(self.seq2seq, self.dataset, trainer.get_batch_data)
num_batches = int(math.ceil(len(self.dataset) / evaluator.batch_size))
expected_calls = [call.eval()] + num_batches * [call.call(ANY, ANY, ANY)]
self.assertEquals(expected_calls, mock_mgr.mock_calls)
metrics = [
WordAccuracy(ignore_index=pad),
SequenceAccuracy(ignore_index=pad),
FinalTargetAccuracy(ignore_index=pad, eos_id=tgt.eos_id)
]
# Since we need the actual tokens to determine k-grammar accuracy,
# we also provide the input and output vocab and relevant special symbols
# metrics.append(SymbolRewritingAccuracy(
# input_vocab=input_vocab,
# output_vocab=output_vocab,
# use_output_eos=output_eos_used,
# input_pad_symbol=src.pad_token,
# output_sos_symbol=tgt.SYM_SOS,
# output_pad_symbol=tgt.pad_token,
# output_eos_symbol=tgt.SYM_EOS,
# output_unk_symbol=tgt.unk_token))
data_func = SupervisedTrainer.get_batch_data
#################################################################################
# Evaluate model on test set
evaluator = Evaluator(batch_size=opt.batch_size, loss=losses, metrics=metrics)
losses, metrics = evaluator.evaluate(model=seq2seq,
data=test,
get_batch_data=data_func)
total_loss, log_msg, _ = SupervisedTrainer.get_losses(losses, metrics, 0)
logging.info(log_msg)
def len_filter(example):
return len(example.src) <= max_len and len(example.tgt) <= max_len
# generate test set
test = torchtext.data.TabularDataset(path=opt.test_data,
format='tsv',
fields=[('src', src), ('tgt', tgt)],
filter_pred=len_filter)
# Prepare loss
weight = torch.ones(len(output_vocab))
pad = output_vocab.stoi[tgt.pad_token]
loss = NLLLoss(pad)
metrics = [WordAccuracy(pad), SequenceAccuracy(pad)]
if torch.cuda.is_available():
loss.cuda()
#################################################################################
# Evaluate model on test set
evaluator = Evaluator(loss=[loss], metrics=metrics, batch_size=opt.batch_size)
losses, metrics = evaluator.evaluate(seq2seq, test,
SupervisedTrainer.get_batch_data)
print([
"{}: {:6f}".format(type(metric).__name__, metric.get_val())
for metric in metrics
])
class SupervisedTrainer(object):
""" The SupervisedTrainer class helps in setting up a training framework in a
supervised setting.
Args:
expt_dir (optional, str): experiment Directory to store details of the experiment,
by default it makes a folder in the current directory to store the details (default: `experiment`).
"""
def __init__(self, expt_dir='experiment'):
self._trainer = "Simple Trainer"
if not os.path.isabs(expt_dir):
expt_dir = os.path.join(os.getcwd(), expt_dir)
self.expt_dir = expt_dir
if not os.path.exists(self.expt_dir):
os.makedirs(self.expt_dir)
def set_local_parameters(self, random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every):
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.losses = losses
self.metrics = metrics
self.loss_weights = loss_weights or len(losses)*[1.]
self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.logger = logging.getLogger(__name__)
self._stop_training = False
def _train_batch(self, input_variable, input_lengths, target_variable, teacher_forcing_ratio):
loss = self.losses
# Forward propagation
decoder_outputs, decoder_hidden, other = self.model(
input_variable, input_lengths, target_variable, teacher_forcing_ratio=teacher_forcing_ratio)
losses = self.evaluator.compute_batch_loss(
decoder_outputs, decoder_hidden, other, target_variable)
# Backward propagation
for i, loss in enumerate(losses, 0):
loss.scale_loss(self.loss_weights[i])
loss.backward(retain_graph=True)
self.optimizer.step()
self.model.zero_grad()
return losses
def _train_epoches(self, data, n_epochs,
start_epoch, start_step,
callbacks,
dev_data, monitor_data=[],
teacher_forcing_ratio=0):
steps_per_epoch = len(data)
total_steps = steps_per_epoch * n_epochs
# give start information to callbacks
callbacks.set_info(start_step, start_epoch,
steps_per_epoch,
total_steps)
# set data as attribute to trainer
self.train_data = data
self.val_data = dev_data
self.monitor_data = monitor_data
# ########################################
# This is used to resume training from same place in dataset
# after loading from checkpoint
s = start_step
if start_epoch > 1:
s -= (start_epoch - 1) * steps_per_epoch
########################################
# Call all callbacks
callbacks.on_train_begin()
for epoch in range(start_epoch, n_epochs + 1):
callbacks.on_epoch_begin(epoch)
self.model.train()
for batch in data:
# Skip over the batches that are below start step
if epoch == start_epoch and s > 0:
s -= 1
continue
callbacks.on_batch_begin(batch)
input_variables, input_lengths, target_variables = self.get_batch_data(
batch)
self.batch_losses = self._train_batch(input_variables,
input_lengths,
target_variables,
teacher_forcing_ratio)
callbacks.on_batch_end(batch)
callbacks.on_epoch_end(epoch)
# Stop training early if flag _stop_training is True
if self._stop_training:
break
logs = callbacks.on_train_end()
return logs
def train(self, model, data,
dev_data,
num_epochs=5,
resume_training=False,
monitor_data={},
optimizer=None,
teacher_forcing_ratio=0,
custom_callbacks=[],
learning_rate=0.001,
checkpoint_path=None,
top_k=5,
losses=[NLLLoss()],
loss_weights=None,
metrics=[],
random_seed=None,
checkpoint_every=100,
print_every=100):
""" Run training for a given model.
Args:
model (machine.models): model to run training on, if `resume=True`, it would be
overwritten by the model loaded from the latest checkpoint.
data (torchtext.data.Iterator: torchtext iterator object to train on
num_epochs (int, optional): number of epochs to run (default 5)
resume_training(bool, optional): resume training with the latest checkpoint up until the number of epochs (default False)
dev_data (torchtext.data.Iterator): dev/validation set iterator
Note: must not pass in the train iterator here as this gets evaluated during training (in between batches)
If you want to evaluate on the full train during training then make two iterators and pass the second one here
monitor_data (list of torchtext.data.Iterator, optional): list of iterators to test on (default None)
Note: must not pass in the train iterator here as this gets evaluated during training (in between batches)
If you want to evaluate on the full train during training then make two iterators and pass the second one here
optimizer (machine.optim.Optimizer, optional): optimizer for training
(default: Optimizer(pytorch.optim.Adam, max_grad_norm=5))
teacher_forcing_ratio (float, optional): teaching forcing ratio (default 0)
custom_callbacks (list, optional): list of custom call backs (see utils.callbacks.callback for base class)
learing_rate (float, optional): learning rate used by the optimizer (default 0.001)
checkpoint_path (str, optional): path to load checkpoint from in case training should be resumed
top_k (int): how many models should be stored during training
loss (list, optional): list of machine.loss.Loss objects for training (default: [machine.loss.NLLLoss])
metrics (list, optional): list of machine.metric.metric objects to be computed during evaluation
checkpoint_every (int, optional): number of epochs to checkpoint after, (default: 100)
print_every (int, optional): number of iterations to print after, (default: 100)
Returns:
model (machine.models): trained model.
"""
self.set_local_parameters(random_seed, losses, metrics,
loss_weights, checkpoint_every, print_every)
# If training is set to resume
if resume_training:
resume_checkpoint = Checkpoint.load(checkpoint_path)
model = resume_checkpoint.model
self.model = model
self.optimizer = resume_checkpoint.optimizer
# A walk around to set optimizing parameters properly
resume_optim = self.optimizer.optimizer
defaults = resume_optim.param_groups[0]
defaults.pop('params', None)
defaults.pop('initial_lr', None)
self.optimizer.optimizer = resume_optim.__class__(
self.model.parameters(), **defaults)
start_epoch = resume_checkpoint.epoch
step = resume_checkpoint.step
else:
start_epoch = 1
step = 0
self.model = model
def get_optim(optim_name):
optims = {'adam': optim.Adam, 'adagrad': optim.Adagrad,
'adadelta': optim.Adadelta, 'adamax': optim.Adamax,
'rmsprop': optim.RMSprop, 'sgd': optim.SGD,
None: optim.Adam}
return optims[optim_name]
self.optimizer = Optimizer(get_optim(optimizer)(self.model.parameters(),
lr=learning_rate),
max_grad_norm=5)
self.logger.info("Optimizer: %s, Scheduler: %s" %
(self.optimizer.optimizer, self.optimizer.scheduler))
callbacks = CallbackContainer(self,
[Logger(),
ModelCheckpoint(top_k=top_k),
History()] + custom_callbacks)
logs = self._train_epoches(data, num_epochs,
start_epoch, step, dev_data=dev_data,
monitor_data=monitor_data,
callbacks=callbacks,
teacher_forcing_ratio=teacher_forcing_ratio)
return self.model, logs
@staticmethod
def get_batch_data(batch):
# TODO Maybe move this method / or make optional - this is seq2seq specific
input_variables, input_lengths = getattr(batch, machine.src_field_name)
target_variables = {'decoder_output': getattr(batch, machine.tgt_field_name),
'encoder_input': input_variables} # The k-grammar metric needs to have access to the inputs
return input_variables, input_lengths, target_variables
loss.to(device)
metrics = [
WordAccuracy(ignore_index=pad),
SequenceAccuracy(ignore_index=pad),
FinalTargetAccuracy(ignore_index=pad, eos_id=tgt.eos_id)
]
# Since we need the actual tokens to determine k-grammar accuracy,
# we also provide the input and output vocab and relevant special symbols
# metrics.append(SymbolRewritingAccuracy(
# input_vocab=input_vocab,
# output_vocab=output_vocab,
# use_output_eos=output_eos_used,
# input_pad_symbol=src.pad_token,
# output_sos_symbol=tgt.SYM_SOS,
# output_pad_symbol=tgt.pad_token,
# output_eos_symbol=tgt.SYM_EOS,
# output_unk_symbol=tgt.unk_token))
data_func = SupervisedTrainer.get_batch_data
##########################################################################
# Evaluate model on test set
evaluator = Evaluator(loss=losses, metrics=metrics)
losses, metrics = evaluator.evaluate(seq2seq, test_iterator, data_func)
total_loss, log_msg, _ = Callback.get_losses(losses, metrics, 0)
logging.info(log_msg)
class SupervisedTrainer(object):
""" The SupervisedTrainer class helps in setting up a training framework in a
supervised setting.
Args:
expt_dir (optional, str): experiment Directory to store details of the experiment,
by default it makes a folder in the current directory to store the details (default: `experiment`).
loss (list, optional): list of machine.loss.Loss objects for training (default: [machine.loss.NLLLoss])
metrics (list, optional): list of machine.metric.metric objects to be computed during evaluation
batch_size (int, optional): batch size for experiment, (default: 64)
checkpoint_every (int, optional): number of epochs to checkpoint after, (default: 100)
print_every (int, optional): number of iterations to print after, (default: 100)
"""
def __init__(self,
expt_dir='experiment',
loss=[NLLLoss()],
loss_weights=None,
metrics=[],
batch_size=64,
eval_batch_size=128,
random_seed=None,
checkpoint_every=100,
print_every=100):
self._trainer = "Simple Trainer"
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
k = NLLLoss()
self.loss = loss
self.metrics = metrics
self.loss_weights = loss_weights or len(loss) * [1.]
self.evaluator = Evaluator(loss=self.loss,
metrics=self.metrics,
batch_size=eval_batch_size)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
if not os.path.isabs(expt_dir):
expt_dir = os.path.join(os.getcwd(), expt_dir)
self.expt_dir = expt_dir
if not os.path.exists(self.expt_dir):
os.makedirs(self.expt_dir)
self.batch_size = batch_size
self.logger = logging.getLogger(__name__)
def _train_batch(self, input_variable, input_lengths, target_variable,
model, teacher_forcing_ratio):
loss = self.loss
# Forward propagation
decoder_outputs, decoder_hidden, other = model(
input_variable,
input_lengths,
target_variable,
teacher_forcing_ratio=teacher_forcing_ratio)
losses = self.evaluator.compute_batch_loss(decoder_outputs,
decoder_hidden, other,
target_variable)
# Backward propagation
for i, loss in enumerate(losses, 0):
loss.scale_loss(self.loss_weights[i])
loss.backward(retain_graph=True)
self.optimizer.step()
model.zero_grad()
return losses
def _train_epoches(self,
data,
model,
n_epochs,
start_epoch,
start_step,
dev_data=None,
monitor_data=[],
teacher_forcing_ratio=0,
top_k=5):
log = self.logger
print_loss_total = defaultdict(float) # Reset every print_every
epoch_loss_total = defaultdict(float) # Reset every epoch
epoch_loss_avg = defaultdict(float)
print_loss_avg = defaultdict(float)
iterator_device = torch.cuda.current_device(
) if torch.cuda.is_available() else -1
batch_iterator = torchtext.data.BucketIterator(
dataset=data,
batch_size=self.batch_size,
sort=False,
sort_within_batch=True,
sort_key=lambda x: len(x.src),
device=iterator_device,
repeat=False)
steps_per_epoch = len(batch_iterator)
total_steps = steps_per_epoch * n_epochs
step = start_step
step_elapsed = 0
# store initial model to be sure at least one model is stored
val_data = dev_data or data
losses, metrics = self.evaluator.evaluate(model, val_data,
self.get_batch_data)
total_loss, log_msg, model_name = self.get_losses(
losses, metrics, step)
log.info(log_msg)
logs = Log()
loss_best = top_k * [total_loss]
best_checkpoints = top_k * [None]
best_checkpoints[0] = model_name
Checkpoint(
model=model,
optimizer=self.optimizer,
epoch=start_epoch,
step=start_step,
input_vocab=data.fields[machine.src_field_name].vocab,
output_vocab=data.fields[machine.tgt_field_name].vocab).save(
self.expt_dir, name=model_name)
for epoch in range(start_epoch, n_epochs + 1):
log.info("Epoch: %d, Step: %d" % (epoch, step))
batch_generator = batch_iterator.__iter__()
# consuming seen batches from previous training
for _ in range((epoch - 1) * steps_per_epoch, step):
next(batch_generator)
model.train(True)
for batch in batch_generator:
step += 1
step_elapsed += 1
input_variables, input_lengths, target_variables = self.get_batch_data(
batch)
losses = self._train_batch(input_variables,
input_lengths.tolist(),
target_variables, model,
teacher_forcing_ratio)
# Record average loss
for loss in losses:
name = loss.log_name
print_loss_total[name] += loss.get_loss()
epoch_loss_total[name] += loss.get_loss()
# print log info according to print_every parm
if step % self.print_every == 0 and step_elapsed > self.print_every:
for loss in losses:
name = loss.log_name
print_loss_avg[
name] = print_loss_total[name] / self.print_every
print_loss_total[name] = 0
m_logs = {}
train_losses, train_metrics = self.evaluator.evaluate(
model, data, self.get_batch_data)
train_loss, train_log_msg, model_name = self.get_losses(
train_losses, train_metrics, step)
logs.write_to_log('Train', train_losses, train_metrics,
step)
logs.update_step(step)
m_logs['Train'] = train_log_msg
# compute vals for all monitored sets
for m_data in monitor_data:
losses, metrics = self.evaluator.evaluate(
model, monitor_data[m_data], self.get_batch_data)
total_loss, log_msg, model_name = self.get_losses(
losses, metrics, step)
m_logs[m_data] = log_msg
logs.write_to_log(m_data, losses, metrics, step)
all_losses = ' '.join([
'%s:\t %s\n' % (os.path.basename(name), m_logs[name])
for name in m_logs
])
log_msg = 'Progress %d%%, %s' % (step / total_steps * 100,
all_losses)
log.info(log_msg)
# check if new model should be saved
if step % self.checkpoint_every == 0 or step == total_steps:
# compute dev loss
losses, metrics = self.evaluator.evaluate(
model, val_data, self.get_batch_data)
total_loss, log_msg, model_name = self.get_losses(
losses, metrics, step)
max_eval_loss = max(loss_best)
if total_loss < max_eval_loss:
index_max = loss_best.index(max_eval_loss)
# rm prev model
if best_checkpoints[index_max] is not None:
shutil.rmtree(
os.path.join(self.expt_dir,
best_checkpoints[index_max]))
best_checkpoints[index_max] = model_name
loss_best[index_max] = total_loss
# save model
Checkpoint(model=model,
optimizer=self.optimizer,
epoch=epoch,
step=step,
input_vocab=data.fields[
machine.src_field_name].vocab,
output_vocab=data.fields[
machine.tgt_field_name].vocab).save(
self.expt_dir, name=model_name)
if step_elapsed == 0: continue
for loss in losses:
epoch_loss_avg[
loss.log_name] = epoch_loss_total[loss.log_name] / min(
steps_per_epoch, step - start_step)
epoch_loss_total[loss.log_name] = 0
if dev_data is not None:
losses, metrics = self.evaluator.evaluate(
model, dev_data, self.get_batch_data)
loss_total, log_, model_name = self.get_losses(
losses, metrics, step)
self.optimizer.update(loss_total,
epoch) # TODO check if this makes sense!
log_msg += ", Dev set: " + log_
model.train(mode=True)
else:
self.optimizer.update(epoch_loss_avg,
epoch) # TODO check if this makes sense!
log.info(log_msg)
return logs
def train(self,
model,
data,
num_epochs=5,
resume=False,
dev_data=None,
monitor_data={},
optimizer=None,
teacher_forcing_ratio=0,
learning_rate=0.001,
checkpoint_path=None,
top_k=5):
""" Run training for a given model.
Args:
model (machine.models): model to run training on, if `resume=True`, it would be
overwritten by the model loaded from the latest checkpoint.
data (machine.dataset.dataset.Dataset): dataset object to train on
num_epochs (int, optional): number of epochs to run (default 5)
resume(bool, optional): resume training with the latest checkpoint, (default False)
dev_data (machine.dataset.dataset.Dataset, optional): dev Dataset (default None)
optimizer (machine.optim.Optimizer, optional): optimizer for training
(default: Optimizer(pytorch.optim.Adam, max_grad_norm=5))
teacher_forcing_ratio (float, optional): teaching forcing ratio (default 0)
learing_rate (float, optional): learning rate used by the optimizer (default 0.001)
checkpoint_path (str, optional): path to load checkpoint from in case training should be resumed
top_k (int): how many models should be stored during training
Returns:
model (machine.models): trained model.
"""
# If training is set to resume
if resume:
resume_checkpoint = Checkpoint.load(checkpoint_path)
model = resume_checkpoint.model
self.optimizer = resume_checkpoint.optimizer
# A walk around to set optimizing parameters properly
resume_optim = self.optimizer.optimizer
defaults = resume_optim.param_groups[0]
defaults.pop('params', None)
defaults.pop('initial_lr', None)
self.optimizer.optimizer = resume_optim.__class__(
model.parameters(), **defaults)
start_epoch = resume_checkpoint.epoch
step = resume_checkpoint.step
else:
start_epoch = 1
step = 0
def get_optim(optim_name):
optims = {
'adam': optim.Adam,
'adagrad': optim.Adagrad,
'adadelta': optim.Adadelta,
'adamax': optim.Adamax,
'rmsprop': optim.RMSprop,
'sgd': optim.SGD,
None: optim.Adam
}
return optims[optim_name]
self.optimizer = Optimizer(get_optim(optimizer)(model.parameters(),
lr=learning_rate),
max_grad_norm=5)
self.logger.info("Optimizer: %s, Scheduler: %s" %
(self.optimizer.optimizer, self.optimizer.scheduler))
logs = self._train_epoches(data,
model,
num_epochs,
start_epoch,
step,
dev_data=dev_data,
monitor_data=monitor_data,
teacher_forcing_ratio=teacher_forcing_ratio,
top_k=top_k)
return model, logs
@staticmethod
def get_batch_data(batch):
input_variables, input_lengths = getattr(batch, machine.src_field_name)
target_variables = {
'decoder_output': getattr(batch, machine.tgt_field_name),
'encoder_input': input_variables
} # The k-grammar metric needs to have access to the inputs
# If available, also get provided attentive guidance data
if hasattr(batch, machine.attn_field_name):
attention_target = getattr(batch, machine.attn_field_name)
target_variables['attention_target'] = attention_target
return input_variables, input_lengths, target_variables
@staticmethod
def get_losses(losses, metrics, step):
total_loss = 0
model_name = ''
log_msg = ''
for metric in metrics:
val = metric.get_val()
log_msg += '%s %.4f ' % (metric.name, val)
model_name += '%s_%.2f_' % (metric.log_name, val)
for loss in losses:
val = loss.get_loss()
log_msg += '%s %.4f ' % (loss.name, val)
model_name += '%s_%.2f_' % (loss.log_name, val)
total_loss += val
model_name += 's%d' % step
return total_loss, log_msg, model_name
metrics = [
WordAccuracy(ignore_index=pad),
SequenceAccuracy(ignore_index=pad),
FinalTargetAccuracy(ignore_index=pad, eos_id=tgt.eos_id)
]
# Since we need the actual tokens to determine k-grammar accuracy,
# we also provide the input and output vocab and relevant special symbols
# metrics.append(SymbolRewritingAccuracy(
# input_vocab=input_vocab,
# output_vocab=output_vocab,
# use_output_eos=output_eos_used,
# input_pad_symbol=src.pad_token,
# output_sos_symbol=tgt.SYM_SOS,
# output_pad_symbol=tgt.pad_token,
# output_eos_symbol=tgt.SYM_EOS,
# output_unk_symbol=tgt.unk_token))
data_func = SupervisedTrainer.get_batch_data
#################################################################################
# Evaluate model on test set
evaluator = Evaluator(loss=losses, metrics=metrics)
losses, metrics = evaluator.evaluate(model=seq2seq,
data_iterator=test,
get_batch_data=data_func)
total_loss, log_msg, _ = Callback.get_losses(losses, metrics, 0)
logging.info(log_msg)