def fit(model, ts, vs, es, **kwargs):
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir'))
conll_output = kwargs.get('conll_output', None)
txts = kwargs.get('txts', None)
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
#validation_improvement_fn = kwargs.get('validation_improvement', None)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
#if validation_improvement_fn is not None:
# validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved)
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test')
return test_metrics
def fit(model, ts, vs, es, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
model_file = get_model_file('lm', 'pytorch', kwargs.get('basedir'))
best_metric = 10000
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es, reporting_fns, phase='Test')
return test_metrics
def fit(model, ts, vs, es=None, **kwargs):
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file(kwargs, 'seq2seq', 'pytorch')
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', basic_reporting))
print('reporting', reporting_fns)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(Seq2SeqTrainerPyTorch, model, **kwargs)
min_metric = 10000
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif test_metrics[early_stopping_metric] < min_metric:
last_improved = epoch
min_metric = test_metrics[early_stopping_metric]
print('New min %.3f' % min_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on min_metric %.3f at epoch %d' % (min_metric, last_improved))
if es is not None:
model.load(model_file)
trainer = Seq2SeqTrainerPyTorch(model, **kwargs)
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, epochs=5, do_early_stopping=True,
early_stopping_metric='bleu', **kwargs):
patience = int(kwargs.get('patience', epochs))
after_train_fn = kwargs.get('after_train_fn', None)
model_file = get_model_file('seq2seq', 'dy', kwargs.get('basedir'))
trainer = create_trainer(model, **kwargs)
best_metric = 0
if do_early_stopping:
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
logger.info("Doing early stopping on [%s] with patience [%d]", early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info("New best %.3f", best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info("Stopping due to persistent failures to improve")
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = model.load(model_file)
test_metrics = trainer.test(es, reporting_fns, phase='Test')
return test_metrics
def fit(model, ts, vs, es=None, **kwargs):
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('seq2seq', 'pytorch', kwargs.get('basedir'))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'bleu')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
model.load(model_file)
trainer = Seq2SeqTrainerPyTorch(model, **kwargs)
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, epochs=5, do_early_stopping=True,
early_stopping_metric='bleu', **kwargs):
patience = int(kwargs.get('patience', epochs))
after_train_fn = kwargs.get('after_train_fn', None)
model_file = get_model_file('seq2seq', 'dy', kwargs.get('basedir'))
trainer = create_trainer(model, **kwargs)
best_metric = 0
if do_early_stopping:
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
print("Doing early stopping on [{}] with patience [{}]".format(early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
print("New best {:.3f}".format(best_metric))
model.save(model_file)
elif (epoch - last_improved) > patience:
print("Stopping due to persistent failures to improve")
break
if do_early_stopping is True:
print('Best performance on {}: {:.3f} at epoch {}'.format(early_stopping_metric, best_metric, last_improved))
if es is not None:
print('Reloading best checkpoint')
model = model.load(model_file)
trainer.test(es, reporting_fns, phase='Test')
def fit(model_params, _, ts, vs, **kwargs):
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
epochs = kwargs.get('epochs', 2)
TRAIN_FLAG()
trainer = create_trainer(model_params, **kwargs)
test_metrics = trainer.test(vs, reporting_fns, phase="Test-Before", dataset=False)
for epoch in range(epochs):
trainer.train(ts, reporting_fns, dataset=False)
test_metrics = trainer.test(vs, reporting_fns, phase="Test", dataset=False)
trainer.checkpoint()
trainer.model.save(model_file)
def fit(model_params, _, ts, vs=None, **kwargs):
"""Calibrate a model with temperature scaling"""
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
batchsz = kwargs['batchsz']
lengths_key = model_params.get('lengths_key')
## First, make tf.datasets for ts, vs and es
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md
# effective_batch_sz = args.batchsz*args.gpus
train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts, lengths_key))
train_dataset = train_dataset.batch(batchsz, drop_remainder=False)
train_dataset = train_dataset.repeat(2)
train_dataset = train_dataset.prefetch(NUM_PREFETCH)
valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs, lengths_key))
valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False)
valid_dataset = valid_dataset.repeat(2)
valid_dataset = valid_dataset.prefetch(NUM_PREFETCH)
iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset),
tf.compat.v1.data.get_output_shapes(train_dataset))
features, y = iter.get_next()
# Add features to the model params
model_params.update(features)
model_params['y'] = tf.one_hot(tf.reshape(y, [-1]), len(model_params['labels']))
# create the initialisation operations
train_init_op = iter.make_initializer(train_dataset)
valid_init_op = iter.make_initializer(valid_dataset)
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
TRAIN_FLAG()
trainer = create_trainer(model_params, **kwargs)
last_improved = 0
trainer.sess.run(train_init_op)
trainer.train(ts, reporting_fns)
trainer.sess.run(valid_init_op)
test_metrics = trainer.test(vs, reporting_fns, phase='Test')
trainer.checkpoint()
trainer.model.save(model_file)
def fit(model, ts, vs, es, epochs=20, do_early_stopping=True, early_stopping_metric='acc', **kwargs):
autobatchsz = kwargs.get('autobatchsz', 1)
verbose = kwargs.get('verbose', {'print': kwargs.get('verbose_print', False), 'file': kwargs.get('verbose_file', None)})
model_file = get_model_file('classify', 'dynet', kwargs.get('basedir'))
best_metric = 0
if do_early_stopping:
patience = kwargs.get('patience', epochs)
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns)
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = model.load(model_file)
test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose)
return test_metrics
def fit(model_params, ts, vs, es, **kwargs):
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir'))
conll_output = kwargs.get('conll_output', None)
txts = kwargs.get('txts', None)
num_loader_workers = int(kwargs.get('num_loader_workers', 0))
pin_memory = bool(kwargs.get('pin_memory', True))
if not isinstance(ts, DataLoader):
ts = DataLoader(ts,
num_workers=num_loader_workers,
batch_size=None,
pin_memory=pin_memory)
if not isinstance(vs, DataLoader):
vs = DataLoader(vs, batch_size=None, pin_memory=pin_memory)
if es and not isinstance(es, DataLoader):
es = DataLoader(es, batch_size=None, pin_memory=pin_memory)
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(model_params, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(trainer.model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es,
reporting_fns,
conll_output=conll_output,
txts=txts,
phase='Test')
return test_metrics
def fit(model, ts, vs, es, **kwargs):
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir'))
conll_output = kwargs.get('conll_output', None)
txts = kwargs.get('txts', None)
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
#validation_improvement_fn = kwargs.get('validation_improvement', None)
after_train_fn = kwargs.get('after_train_fn', None)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
#if validation_improvement_fn is not None:
# validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved)
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es,
reporting_fns,
conll_output=conll_output,
txts=txts,
phase='Test')
return test_metrics
def train(model, ts, vs, es=None, **kwargs):
"""
Train a classifier using TensorFlow
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* Additional arguments are supported, see :func:`baseline.tf.optimize` for full list
:return:
"""
n = int(kwargs.get('test_epochs', 5))
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' %
(early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
model.sess.run(tf.global_variables_initializer())
model.set_saver(tf.train.Saver())
max_metric = 0
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if epoch > 0 and epoch % n == 0 and epoch < epochs - 1:
print(color('Running test', Colors.GREEN))
trainer.test(es, reporting_fns, phase='Test')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif test_metrics[early_stopping_metric] > max_metric:
last_improved = epoch
max_metric = test_metrics[early_stopping_metric]
print('New max %.3f' % max_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print(
color('Stopping due to persistent failures to improve',
Colors.RED))
break
if do_early_stopping is True:
print('Best performance on max_metric %.3f at epoch %d' %
(max_metric, last_improved))
if es is not None:
print(color('Reloading best checkpoint', Colors.GREEN))
trainer.recover_last_checkpoint()
trainer.test(es, reporting_fns, phase='Test')
def fit(model_params, ts, vs, es, **kwargs):
"""
Train a classifier using TensorFlow with a `feed_dict`. This
is the previous default behavior for training. To use this, you need to pass
`fit_func: feed_dict` in your MEAD config
:param model_params: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *nsteps* (`int`) -- If we should report every n-steps, this should be passed
* *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e
* *clip* (`int`) -- If we are doing gradient clipping, what value to use
* *optim* (`str`) -- The name of the optimizer we are using
* *lr* (`float`) -- The learning rate we are using
* *mom* (`float`) -- If we are using SGD, what value to use for momentum
* *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9`
* *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999`
* *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8
:return: None
"""
epochs = int(kwargs.get('epochs', 5))
patience = int(kwargs.get('patience', epochs))
conll_output = kwargs.get('conll_output', None)
span_type = kwargs.get('span_type', 'iob')
txts = kwargs.get('txts', None)
model_file = get_model_file('tagger', 'tf', kwargs.get('basedir'))
TRAIN_FLAG()
trainer = create_trainer(model_params, **kwargs)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = bool(kwargs.get('verbose', False))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' %
(early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
print('New best %.3f' % best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on %s: %.3f at epoch %d' %
(early_stopping_metric, best_metric, last_improved))
if es is not None:
trainer.recover_last_checkpoint()
# What to do about overloading this??
evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose)
timer = Timer()
test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts)
duration = timer.elapsed()
for reporting in reporting_fns:
reporting(test_metrics, 0, 'Test')
trainer.log.debug({'phase': 'Test', 'time': duration})
def fit_datasets(model_params, ts, vs, es=None, **kwargs):
"""
Train an language model using TensorFlow with `tf.dataset`. This
is the default behavior for training.
:param model_params: The model (or parameters to create the model) to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *nsteps* (`int`) -- If we should report every n-steps, this should be passed
* *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e
* *clip* (`int`) -- If we are doing gradient clipping, what value to use
* *optim* (`str`) -- The name of the optimizer we are using
* *lr* (`float`) -- The learning rate we are using
* *mom* (`float`) -- If we are using SGD, what value to use for momentum
* *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9`
* *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999`
* *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8
:return: None
"""
epochs = int(kwargs.get('epochs', 5))
patience = int(kwargs.get('patience', epochs))
model_file = get_model_file('lm', 'tf', kwargs.get('basedir'))
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
batchsz = kwargs['batchsz']
test_batchsz = kwargs.get('test_batchsz', batchsz)
tgt_key = model_params.get('tgt_key')
train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts))
train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ)
train_dataset = train_dataset.batch(batchsz, drop_remainder=False)
train_dataset = train_dataset.repeat(epochs + 1)
train_dataset = train_dataset.prefetch(NUM_PREFETCH)
valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs))
valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False)
valid_dataset = valid_dataset.repeat(epochs + 1)
valid_dataset = valid_dataset.prefetch(NUM_PREFETCH)
iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset),
tf.compat.v1.data.get_output_shapes(train_dataset))
features, tgt = iter.get_next()
# Add features to the model params
model_params.update(features)
model_params.update({'y': tgt})
# create the initialization operations
train_init_op = iter.make_initializer(train_dataset)
valid_init_op = iter.make_initializer(valid_dataset)
TRAIN_FLAG()
trainer = create_trainer(model_params, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.sess.run(train_init_op)
trainer.train(ts, reporting_fns)
trainer.sess.run(valid_init_op)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
print('New best %.3f' % best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved))
if es is not None:
print('Reloading best checkpoint')
trainer.recover_last_checkpoint()
test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es))
test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False)
test_dataset = test_dataset.repeat(epochs + 1)
test_dataset = test_dataset.prefetch(NUM_PREFETCH)
test_init_op = iter.make_initializer(test_dataset)
trainer.sess.run(test_init_op)
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
model_file = get_model_file('lm', 'tf', kwargs.get('basedir'))
after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(model, **kwargs)
init = tf.global_variables_initializer()
feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()}
model.sess.run(init, feed_dict)
saver = tf.train.Saver()
model.set_saver(saver)
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
best_metric = 1000
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
trainer.recover_last_checkpoint()
test_metrics = trainer.test(es, reporting_fns, phase='Test')
return test_metrics
def fit(model, ts, vs, es, **kwargs):
"""
Train a classifier using PyTorch
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs: See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *optim* --
Optimizer to use, defaults to `sgd`
* *eta, lr* (``float``) --
Learning rate, defaults to 0.01
* *mom* (``float``) --
Momentum (SGD only), defaults to 0.9 if optim is `sgd`
:return:
"""
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)})
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'pytorch', kwargs.get('basedir'))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('eatly_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
trainer = create_trainer(model, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns)
if do_early_stopping is False:
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose)
return test_metrics
def fit(model, ts, vs, es, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
conll_output = kwargs.get('conll_output', None)
span_type = kwargs.get('span_type', 'iob')
txts = kwargs.get('txts', None)
model_file = get_model_file('tagger', 'tf', kwargs.get('basedir'))
after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()}
init = tf.global_variables_initializer()
model.sess.run(init, feed_dict)
saver = tf.train.Saver()
model.save_using(saver)
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = bool(kwargs.get('verbose', False))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
trainer.recover_last_checkpoint()
# What to do about overloading this??
evaluator = TaggerEvaluatorTf(model, span_type, verbose)
start = time.time()
test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts)
duration = time.time() - start
for reporting in reporting_fns:
reporting(test_metrics, 0, 'Test')
trainer.log.debug({'phase': 'Test', 'time': duration})
return test_metrics
def fit(model, ts, vs, es, **kwargs):
num_iters = kwargs.get('num_iters', 5)
kwargs['warmup_steps'] = num_iters * len(ts)
kwargs['lr'] = kwargs.get('max_lr', 10.)
trainer = create_trainer(model, **kwargs)
lrs = []
losses = []
use_val = kwargs.get('use_val', False)
beta = kwargs.get('smooth_beta', 0.05)
log = kwargs.get('log_scale', True)
best_loss = six.MAXSIZE
diverge_threshold = kwargs.get('diverge_threshold', 5)
stop = False
i = 0
be = kwargs.get('backend', 'tf')
if be == 'tf':
import tensorflow as tf
tables = tf.tables_initializer()
model.sess.run(tables)
model.sess.run(tf.global_variables_initializer())
model.set_saver(tf.train.Saver())
for _ in range(num_iters):
if stop:
break
for batch in ts:
i += 1
train_metrics = trainer.train([batch], [])
if use_val:
val_metrics = trainer.test(vs, [])
loss = val_metrics['avg_loss']
else:
loss = train_metrics['avg_loss']
if losses and beta > 0:
loss = beta * loss + (1 - beta) * losses[-1]
loss /= (1 - beta ** i)
losses.append(loss)
if be == 'tf':
lrs.append(model.sess.run("OptimizeLoss/lr:0"))
else:
lrs.append(trainer.optimizer.current_lr)
if loss < best_loss:
best_loss = loss
if loss > diverge_threshold * best_loss:
print("Stopping Early")
stop = True
break
plt.plot(lrs, losses)
if log:
plt.xscale('log')
plt.xlabel('Learning Rate (log scale)')
else:
plt.xlabel('Learning Rate')
if use_val:
plt.ylabel('Average Validation Loss')
else:
plt.ylabel('Per Batch Training Loss')
plt.show()
def fit(model_params, ts, vs, es=None, **kwargs):
"""
Train an language model using TensorFlow with a `feed_dict`.
:param model_params: The model (or parameters to create the model) to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
* *epochs* (``int``) -- how many epochs. Default to 5
* *outfile* -- Model output file
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *nsteps* (`int`) -- If we should report every n-steps, this should be passed
* *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e
* *clip* (`int`) -- If we are doing gradient clipping, what value to use
* *optim* (`str`) -- The name of the optimizer we are using
* *lr* (`float`) -- The learning rate we are using
* *mom* (`float`) -- If we are using SGD, what value to use for momentum
* *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9`
* *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999`
* *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8
* *after_train_fn* (`func`) -- A callback to fire after ever epoch of training
:return: None
"""
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
model_file = get_model_file('lm', 'tf', kwargs.get('basedir'))
after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(model_params, **kwargs)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
best_metric = 1000
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(trainer.model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
print('New best %.3f' % best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved))
if es is not None:
trainer.recover_last_checkpoint()
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, **kwargs):
"""
Train a classifier using TensorFlow
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* Additional arguments are supported, see :func:`baseline.tf.optimize` for full list
:return:
"""
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)})
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
ema = True if kwargs.get('ema_decay') is not None else False
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()}
model.sess.run(tf.global_variables_initializer(), feed_dict)
model.set_saver(tf.train.Saver())
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
trainer.recover_last_checkpoint()
test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose)
return test_metrics
def train(model, ts, vs, es=None, **kwargs):
"""
Train a classifier using TensorFlow
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* Additional arguments are supported, see :func:`baseline.tf.optimize` for full list
:return:
"""
n = int(kwargs.get('test_epochs', 5))
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
model.sess.run(tf.global_variables_initializer())
model.set_saver(tf.train.Saver())
max_metric = 0
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if epoch > 0 and epoch % n == 0 and epoch < epochs - 1:
print(color('Running test', Colors.GREEN))
trainer.test(es, reporting_fns, phase='Test')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif test_metrics[early_stopping_metric] > max_metric:
last_improved = epoch
max_metric = test_metrics[early_stopping_metric]
print('New max %.3f' % max_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print(color('Stopping due to persistent failures to improve', Colors.RED))
break
if do_early_stopping is True:
print('Best performance on max_metric %.3f at epoch %d' % (max_metric, last_improved))
if es is not None:
print(color('Reloading best checkpoint', Colors.GREEN))
trainer.recover_last_checkpoint()
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
conll_output = kwargs.get('conll_output', None)
#TODO: ???? Really?
txts = kwargs.get('txts', None)
model_file = get_model_file(kwargs, 'tagger', 'tf')
after_train_fn = kwargs[
'after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(TaggerTrainerTf, model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
init = tf.global_variables_initializer()
model.sess.run(init)
saver = tf.train.Saver()
model.save_using(saver)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' %
(early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', basic_reporting))
print('reporting', reporting_fns)
max_metric = 0
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
model.save(model_file)
elif test_metrics[early_stopping_metric] > max_metric:
last_improved = epoch
max_metric = test_metrics[early_stopping_metric]
print('New max %.3f' % max_metric)
trainer.checkpoint()
model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on max_metric %.3f at epoch %d' %
(max_metric, last_improved))
if es is not None:
trainer.recover_last_checkpoint()
# What to do about overloading this??
evaluator = TaggerEvaluatorTf(model)
test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts)
for reporting in reporting_fns:
reporting(test_metrics, 0, 'Test')
def fit(model_params, ts, vs, es, **kwargs):
"""
Train a classifier using PyTorch
:param model_params: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs: See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *optim* --
Optimizer to use, defaults to `sgd`
* *eta, lr* (``float``) --
Learning rate, defaults to 0.01
* *mom* (``float``) --
Momentum (SGD only), defaults to 0.9 if optim is `sgd`
:return:
"""
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = kwargs.get(
'verbose', {
'console': kwargs.get('verbose_console', False),
'file': kwargs.get('verbose_file', None)
})
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'pytorch', kwargs.get('basedir'))
output = kwargs.get('output')
txts = kwargs.get('txts')
num_loader_workers = int(kwargs.get('num_loader_workers', 0))
pin_memory = bool(kwargs.get('pin_memory', True))
if not isinstance(ts, DataLoader):
ts = DataLoader(ts,
num_workers=num_loader_workers,
batch_size=None,
pin_memory=pin_memory)
if not isinstance(vs, DataLoader):
vs = DataLoader(vs, batch_size=None, pin_memory=pin_memory)
if es and not isinstance(es, DataLoader):
es = DataLoader(es, batch_size=None, pin_memory=pin_memory)
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
trainer = create_trainer(model_params, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns)
if do_early_stopping is False:
trainer.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(model, **kwargs)
test_metrics = trainer.test(es,
reporting_fns,
phase='Test',
verbose=verbose,
output=output,
txts=txts)
return test_metrics
def fit(model, ts, vs, es, **kwargs):
num_iters = kwargs.get('num_iters', 5)
kwargs['warmup_steps'] = num_iters * len(ts)
kwargs['lr'] = kwargs.get('max_lr', 10.)
trainer = create_trainer(model, **kwargs)
lrs = []
losses = []
use_val = kwargs.get('use_val', False)
beta = kwargs.get('smooth_beta', 0.05)
log = kwargs.get('log_scale', True)
best_loss = six.MAXSIZE
diverge_threshold = kwargs.get('diverge_threshold', 5)
stop = False
i = 0
be = kwargs.get('backend', 'tf')
if be == 'tf':
import tensorflow as tf
tables = tf.compat.v1.tables_initializer()
model.sess.run(tables)
model.sess.run(tf.compat.v1.global_variables_initializer())
model.set_saver(tf.compat.v1.train.Saver())
for _ in range(num_iters):
if stop:
break
for batch in ts:
i += 1
train_metrics = trainer.train([batch], [])
if use_val:
val_metrics = trainer.test(vs, [])
loss = val_metrics['avg_loss']
else:
loss = train_metrics['avg_loss']
if losses and beta > 0:
loss = beta * loss + (1 - beta) * losses[-1]
loss /= (1 - beta ** i)
losses.append(loss)
if be == 'tf':
lrs.append(model.sess.run("OptimizeLoss/lr:0"))
else:
lrs.append(trainer.optimizer.current_lr)
if loss < best_loss:
best_loss = loss
if loss > diverge_threshold * best_loss:
print("Stopping Early")
stop = True
break
plt.plot(lrs, losses)
if log:
plt.xscale('log')
plt.xlabel('Learning Rate (log scale)')
else:
plt.xlabel('Learning Rate')
if use_val:
plt.ylabel('Average Validation Loss')
else:
plt.ylabel('Per Batch Training Loss')
plt.show()
def fit(model, ts, vs, es=None, **kwargs):
"""
Train a classifier using TensorFlow
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* Additional arguments are supported, see :func:`baseline.tf.optimize` for full list
:return:
"""
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = kwargs.get(
'verbose', {
'console': kwargs.get('verbose_console', False),
'file': kwargs.get('verbose_file', None)
})
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'tf', kwargs.get('basedir'))
ema = True if kwargs.get('ema_decay') is not None else False
output = kwargs.get('output')
txts = kwargs.get('txts')
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
m = model.replicas[0] if hasattr(model, 'replicas') else model
feed_dict = {
k: v
for e in m.embeddings.values() for k, v in e.get_feed_dict().items()
}
model.sess.run(tf.global_variables_initializer(), feed_dict)
model.set_saver(tf.train.Saver())
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
trainer.recover_last_checkpoint()
test_metrics = trainer.test(es,
reporting_fns,
phase='Test',
verbose=verbose,
output=output,
txts=txts)
return test_metrics
def fit_datasets(model_params, ts, vs, es=None, **kwargs):
"""
Train a tagger using TensorFlow with `tf.dataset`. This
is the default behavior for training.
:param model_params: The model (or parameters to create the model) to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *nsteps* (`int`) -- If we should report every n-steps, this should be passed
* *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e
* *clip* (`int`) -- If we are doing gradient clipping, what value to use
* *optim* (`str`) -- The name of the optimizer we are using
* *lr* (`float`) -- The learning rate we are using
* *mom* (`float`) -- If we are using SGD, what value to use for momentum
* *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9`
* *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999`
* *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8
:return: None
"""
conll_output = kwargs.get('conll_output', None)
span_type = kwargs.get('span_type', 'iob')
txts = kwargs.get('txts', None)
model_file = get_model_file('tagger', 'tf', kwargs.get('basedir'))
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)})
epochs = int(kwargs.get('epochs', 20))
batchsz = kwargs['batchsz']
## First, make tf.datasets for ts, vs and es
# https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md
# effective_batch_sz = args.batchsz*args.gpus
test_batchsz = kwargs.get('test_batchsz', batchsz)
# This is a little awkward:
lengths_key = model_params.get('lengths_key')
train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts, lengths_key))
train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ)
train_dataset = train_dataset.batch(batchsz, drop_remainder=False)
train_dataset = train_dataset.repeat(epochs + 1)
train_dataset = train_dataset.prefetch(NUM_PREFETCH)
valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs, lengths_key))
valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False)
valid_dataset = valid_dataset.repeat(epochs + 1)
valid_dataset = valid_dataset.prefetch(NUM_PREFETCH)
iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset),
tf.compat.v1.data.get_output_shapes(train_dataset))
features, y = iter.get_next()
# Add features to the model params
model_params.update(features)
model_params['y'] = y
# create the initialisation operations
train_init_op = iter.make_initializer(train_dataset)
valid_init_op = iter.make_initializer(valid_dataset)
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', []))
print('reporting', reporting_fns)
TRAIN_FLAG()
trainer = create_trainer(model_params, **kwargs)
last_improved = 0
for epoch in range(epochs):
trainer.sess.run(train_init_op)
trainer.train(ts, reporting_fns)
trainer.sess.run(valid_init_op)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
print('New best %.3f' % best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved))
if es is not None:
print('Reloading best checkpoint')
trainer.recover_last_checkpoint()
test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es, lengths_key))
test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False)
test_dataset = test_dataset.repeat(epochs + 1)
test_dataset = test_dataset.prefetch(NUM_PREFETCH)
test_init_op = iter.make_initializer(test_dataset)
trainer.sess.run(test_init_op)
# What to do about overloading this??
evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose)
start = time.time()
test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts)
duration = time.time() - start
for reporting in reporting_fns:
reporting(test_metrics, 0, 'Test')
trainer.log.debug({'phase': 'Test', 'time': duration})
def fit(model, ts, vs, es, **kwargs):
"""
Train a classifier using PyTorch
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs: See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model.pyth
* *patience* --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *optim* --
Optimizer to use, defaults to `sgd`
* *eta, lr* (``float``) --
Learning rate, defaults to 0.01
* *mom* (``float``) --
Momentum (SGD only), defaults to 0.9 if optim is `sgd`
:return:
"""
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file(kwargs, 'classify', 'pytorch')
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
patience = kwargs.get('patience', epochs)
print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience))
reporting_fns = listify(kwargs.get('reporting', basic_reporting))
print('reporting', reporting_fns)
trainer = create_trainer(ClassifyTrainerPyTorch, model, **kwargs)
max_metric = 0
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns)
if do_early_stopping is False:
model.save(model_file)
elif test_metrics[early_stopping_metric] > max_metric:
last_improved = epoch
max_metric = test_metrics[early_stopping_metric]
print('New max %.3f' % max_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
print('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
print('Best performance on max_metric %.3f at epoch %d' % (max_metric, last_improved))
if es is not None:
print('Reloading best checkpoint')
model = torch.load(model_file)
trainer = create_trainer(ClassifyTrainerPyTorch, model, **kwargs)
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
conll_output = kwargs.get('conll_output', None)
span_type = kwargs.get('span_type', 'iob')
txts = kwargs.get('txts', None)
model_file = get_model_file('tagger', 'tf', kwargs.get('basedir'))
after_train_fn = kwargs[
'after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(model, **kwargs)
tables = tf.tables_initializer()
model.sess.run(tables)
init = tf.global_variables_initializer()
model.sess.run(init)
saver = tf.train.Saver()
model.save_using(saver)
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
verbose = bool(kwargs.get('verbose', False))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
trainer.recover_last_checkpoint()
# What to do about overloading this??
evaluator = TaggerEvaluatorTf(model, span_type, verbose)
start = time.time()
test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts)
duration = time.time() - start
for reporting in reporting_fns:
reporting(test_metrics, 0, 'Test')
trainer.log.debug({'phase': 'Test', 'time': duration})
def fit(model, ts, vs, es=None, **kwargs):
"""
Train a classifier using Keras
:param model: The model to train
:param ts: A training data set
:param vs: A validation data set
:param es: A test data set, can be None
:param kwargs:
See below
:Keyword Arguments:
* *do_early_stopping* (``bool``) --
Stop after evaluation data is no longer improving. Defaults to True
* *epochs* (``int``) -- how many epochs. Default to 20
* *outfile* -- Model output file, defaults to classifier-model-keras
* *patience* (``int``) --
How many epochs where evaluation is no longer improving before we give up
* *reporting* --
Callbacks which may be used on reporting updates
* *optim* --
What optimizer to use. Defaults to `adam`
:return:
"""
trainer = create_trainer(model, **kwargs)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
epochs = int(kwargs.get('epochs', 20))
model_file = get_model_file('classify', 'keras', kwargs.get('basedir'))
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'acc')
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
max_metric = 0
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
test_metrics = trainer.test(vs, reporting_fns)
if do_early_stopping is False:
model.save(model_file)
elif test_metrics[early_stopping_metric] > max_metric:
last_improved = epoch
max_metric = test_metrics[early_stopping_metric]
logger.info('New max %.3f', max_metric)
model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on max_metric %.3f at epoch %d', max_metric, last_improved)
if es is not None:
logger.info('Reloading best checkpoint')
model = model.load(model_file)
trainer = ClassifyTrainerKeras(model, **kwargs)
trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, **kwargs):
epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5
patience = int(kwargs['patience']) if 'patience' in kwargs else epochs
model_file = get_model_file('seq2seq', 'tf', kwargs.get('basedir'))
after_train_fn = kwargs[
'after_train_fn'] if 'after_train_fn' in kwargs else None
trainer = create_trainer(model, **kwargs)
feed_dict = {
k: v
for e in model.src_embeddings.values()
for k, v in e.get_feed_dict().items()
}
feed_dict.update(model.tgt_embedding.get_feed_dict())
init = tf.global_variables_initializer()
model.sess.run(init, feed_dict)
saver = tf.train.Saver()
trainer.prepare(saver)
checkpoint = kwargs.get('checkpoint')
if checkpoint is not None:
latest = tf.train.latest_checkpoint(checkpoint)
print('Reloading ' + latest)
model.saver.restore(model.sess, latest)
do_early_stopping = bool(kwargs.get('do_early_stopping', True))
best_metric = 0
if do_early_stopping:
early_stopping_metric = kwargs.get('early_stopping_metric', 'bleu')
early_stopping_cmp, best_metric = get_metric_cmp(
early_stopping_metric, kwargs.get('early_stopping_cmp'))
patience = kwargs.get('patience', epochs)
logger.info('Doing early stopping on [%s] with patience [%d]',
early_stopping_metric, patience)
reporting_fns = listify(kwargs.get('reporting', []))
logger.info('reporting %s', reporting_fns)
last_improved = 0
for epoch in range(epochs):
trainer.train(ts, reporting_fns)
if after_train_fn is not None:
after_train_fn(model)
test_metrics = trainer.test(vs, reporting_fns, phase='Valid')
if do_early_stopping is False:
trainer.checkpoint()
trainer.model.save(model_file)
elif early_stopping_cmp(test_metrics[early_stopping_metric],
best_metric):
last_improved = epoch
best_metric = test_metrics[early_stopping_metric]
logger.info('New best %.3f', best_metric)
trainer.checkpoint()
trainer.model.save(model_file)
elif (epoch - last_improved) > patience:
logger.info('Stopping due to persistent failures to improve')
break
if do_early_stopping is True:
logger.info('Best performance on %s: %.3f at epoch %d',
early_stopping_metric, best_metric, last_improved)
if es is not None:
trainer.recover_last_checkpoint()
test_metrics = trainer.test(es, reporting_fns, phase='Test')
return test_metrics