def fit(model, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir')) conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) #validation_improvement_fn = kwargs.get('validation_improvement', None) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): #if validation_improvement_fn is not None: # validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved) last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test') return test_metrics
def fit(model, ts, vs, es, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs do_early_stopping = bool(kwargs.get('do_early_stopping', True)) model_file = get_model_file('lm', 'pytorch', kwargs.get('basedir')) best_metric = 10000 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) trainer.test(es, reporting_fns, phase='Test')
def fit(model, ts, vs, es=None, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('seq2seq', 'pytorch', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'bleu') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: model.load(model_file) trainer = Seq2SeqTrainerPyTorch(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def fit(model, ts, vs, es=None, epochs=5, do_early_stopping=True, early_stopping_metric='bleu', **kwargs): patience = int(kwargs.get('patience', epochs)) after_train_fn = kwargs.get('after_train_fn', None) model_file = get_model_file('seq2seq', 'dy', kwargs.get('basedir')) trainer = create_trainer(model, **kwargs) best_metric = 0 if do_early_stopping: early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) logger.info("Doing early stopping on [%s] with patience [%d]", early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info("New best %.3f", best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info("Stopping due to persistent failures to improve") break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def __init__(self, store: Store, metric: str, user_cmp: Optional[str] = None, **kwargs) -> None: """ :param store: The odin job db :param metric: The metric used to compare models :param user_cmp: A user supplied comparison operator :param kwargs: absorb kwargs from other exports """ super().__init__() self.store = store self.metric = metric cmp, self.default_value = get_metric_cmp(self.metric, user_cmp) self.reverse = cmp in {gt, ge} self.best_value = 0
def fit(model, ts, vs, es=None, epochs=5, do_early_stopping=True, early_stopping_metric='avg_loss', **kwargs): patience = int(kwargs.get('patience', epochs)) after_train_fn = kwargs.get('after_train_fn', None) model_file = get_model_file('lm', 'dy', kwargs.get('basedir')) trainer = create_trainer(model, **kwargs) best_metric = 10000 if do_early_stopping: early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) print("Doing early stopping on [{}] with patience [{}]".format(early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print("New best {:.3f}".format(best_metric)) model.save(model_file) elif (epoch - last_improved) > patience: print("Stopping due to persistent failures to improve") break if do_early_stopping is True: print('Best performance on {}: {:.3f} at epoch {}'.format(early_stopping_metric, best_metric, last_improved)) if es is not None: print('Reloading best checkpoint') model = model.load(model_file) trainer.test(es, reporting_fns, phase='Test')
def __init__(self, user_cmp: str = None) -> None: xpctl = MagicMock() dataset = 'test' def build_mock(dataset): def _build(_): m = MagicMock() m.summary = {dataset: dataset} return m return _build xpctl.task_summary.side_effect = build_mock(dataset) super().__init__(xpctl, "test", dataset, "f1") self.comparator: Callable[[float, float], bool] = None self.comparator, self.best_value = get_metric_cmp( self.metric, user_cmp)
def __init__(self, api: XpctlApi, task: str, dataset: str, metric: str, user_cmp: str = None, **kwargs) -> None: """Construct a best model in batch :param api: An XPCTL client api object :param task: A string name for the task :param dataset: A string key for the dataset :param metric: The metric to evaluate :param user_cmp: A comparator function :param kwargs: """ super().__init__(api, task, dataset=None, metric=metric) self.cmp: Callable[[float, float], bool] = None self.cmp, self.best_value = get_metric_cmp(self.metric, user_cmp)
def fit(model, ts, vs, es, epochs=20, do_early_stopping=True, early_stopping_metric='acc', **kwargs): autobatchsz = kwargs.get('autobatchsz', 1) verbose = kwargs.get('verbose', {'print': kwargs.get('verbose_print', False), 'file': kwargs.get('verbose_file', None)}) model_file = get_model_file('classify', 'dynet', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: patience = kwargs.get('patience', epochs) early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = model.load(model_file) test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics
def fit(model, ts, vs, es, **kwargs): """ Train a classifier using PyTorch :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *optim* -- Optimizer to use, defaults to `sgd` * *eta, lr* (``float``) -- Learning rate, defaults to 0.01 * *mom* (``float``) -- Momentum (SGD only), defaults to 0.9 if optim is `sgd` :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'pytorch', kwargs.get('basedir')) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('eatly_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics
def fit(model, ts, vs, es=None, **kwargs): """ Train a classifier using TensorFlow :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * Additional arguments are supported, see :func:`baseline.tf.optimize` for full list :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get( 'verbose', { 'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None) }) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) ema = True if kwargs.get('ema_decay') is not None else False output = kwargs.get('output') txts = kwargs.get('txts') best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) m = model.replicas[0] if hasattr(model, 'replicas') else model feed_dict = { k: v for e in m.embeddings.values() for k, v in e.get_feed_dict().items() } model.sess.run(tf.global_variables_initializer(), feed_dict) model.set_saver(tf.train.Saver()) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose, output=output, txts=txts) return test_metrics
def fit(model, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir')) conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) #validation_improvement_fn = kwargs.get('validation_improvement', None) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): #if validation_improvement_fn is not None: # validation_improvement_fn(early_stopping_metric, test_metrics, epoch, max_metric, last_improved) last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test') return test_metrics
def fit(model, ts, vs, es=None, **kwargs): """ Train a classifier using TensorFlow :param model: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * Additional arguments are supported, see :func:`baseline.tf.optimize` for full list :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'tf', kwargs.get('basedir')) ema = True if kwargs.get('ema_decay') is not None else False best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} model.sess.run(tf.global_variables_initializer(), feed_dict) model.set_saver(tf.train.Saver()) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose) return test_metrics
def fit(model_params, ts, vs, es, **kwargs): """ Train a classifier using PyTorch :param model_params: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after eval data is not improving. Default to True * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *optim* -- Optimizer to use, defaults to `sgd` * *eta, lr* (``float``) -- Learning rate, defaults to 0.01 * *mom* (``float``) -- Momentum (SGD only), defaults to 0.9 if optim is `sgd` :return: """ do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get( 'verbose', { 'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None) }) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('classify', 'pytorch', kwargs.get('basedir')) output = kwargs.get('output') txts = kwargs.get('txts') num_loader_workers = int(kwargs.get('num_loader_workers', 0)) pin_memory = bool(kwargs.get('pin_memory', True)) if not isinstance(ts, DataLoader): ts = DataLoader(ts, num_workers=num_loader_workers, batch_size=None, pin_memory=pin_memory) if not isinstance(vs, DataLoader): vs = DataLoader(vs, batch_size=None, pin_memory=pin_memory) if es and not isinstance(es, DataLoader): es = DataLoader(es, batch_size=None, pin_memory=pin_memory) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) trainer = create_trainer(model_params, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns) if do_early_stopping is False: trainer.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, phase='Test', verbose=verbose, output=output, txts=txts) return test_metrics
def fit(model_params, ts, vs, es, **kwargs): """ Train a classifier using TensorFlow with a `feed_dict`. This is the previous default behavior for training. To use this, you need to pass `fit_func: feed_dict` in your MEAD config :param model_params: The model to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ epochs = int(kwargs.get('epochs', 5)) patience = int(kwargs.get('patience', epochs)) conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = bool(kwargs.get('verbose', False)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: trainer.recover_last_checkpoint() # What to do about overloading this?? evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose) timer = Timer() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = timer.elapsed() for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration})
def fit_eager_distributed(model_params, ts, vs, es=None, **kwargs): """ Train an language model using TensorFlow with `tf.dataset`. This is the default behavior for training. :param model_params: The model (or parameters to create the model) to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ epochs = int(kwargs.get('epochs', 5)) patience = int(kwargs.get('patience', epochs)) model_file = get_model_file('lm', 'tf', kwargs.get('basedir')) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) batchsz = kwargs['batchsz'] test_batchsz = kwargs.get('test_batchsz', batchsz) tgt_key = model_params.get('tgt_key') train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts)) train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ) train_dataset = train_dataset.batch(batchsz, drop_remainder=True) train_dataset = train_dataset.prefetch(NUM_PREFETCH) valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs)) valid_dataset = valid_dataset.batch(batchsz, drop_remainder=True) valid_dataset = valid_dataset.prefetch(NUM_PREFETCH) trainer = LanguageModelTrainerDistributedTf(model_params, **kwargs) train_dataset = trainer.distribute(train_dataset) valid_dataset = trainer.distribute(valid_dataset) last_improved = 0 SET_TRAIN_FLAG(True) for epoch in range(epochs): trainer.train(train_dataset, reporting_fns, steps=len(ts)) test_metrics = trainer.test(valid_dataset, reporting_fns, phase='Valid', steps=len(vs)) if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: print('Reloading best checkpoint') trainer.recover_last_checkpoint() trainer.strategy = tf.distribute.OneDeviceStrategy('/device:GPU:0') test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es)) test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False) test_dataset = test_dataset.prefetch(NUM_PREFETCH) test_dataset = trainer.distribute(test_dataset) trainer.test(test_dataset, reporting_fns, phase='Test', steps=len(es))
def fit_datasets(model_params, ts, vs, es=None, **kwargs): """ Train a tagger using TensorFlow with `tf.dataset`. This is the default behavior for training. :param model_params: The model (or parameters to create the model) to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 20 * *outfile* -- Model output file, defaults to classifier-model.pyth * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 :return: None """ conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = kwargs.get('verbose', {'console': kwargs.get('verbose_console', False), 'file': kwargs.get('verbose_file', None)}) epochs = int(kwargs.get('epochs', 20)) batchsz = kwargs['batchsz'] ## First, make tf.datasets for ts, vs and es # https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/distribute/README.md # effective_batch_sz = args.batchsz*args.gpus test_batchsz = kwargs.get('test_batchsz', batchsz) # This is a little awkward: lengths_key = model_params.get('lengths_key') train_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(ts, lengths_key)) train_dataset = train_dataset.shuffle(buffer_size=SHUF_BUF_SZ) train_dataset = train_dataset.batch(batchsz, drop_remainder=False) train_dataset = train_dataset.repeat(epochs + 1) train_dataset = train_dataset.prefetch(NUM_PREFETCH) valid_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(vs, lengths_key)) valid_dataset = valid_dataset.batch(batchsz, drop_remainder=False) valid_dataset = valid_dataset.repeat(epochs + 1) valid_dataset = valid_dataset.prefetch(NUM_PREFETCH) iter = tf.compat.v1.data.Iterator.from_structure(tf.compat.v1.data.get_output_types(train_dataset), tf.compat.v1.data.get_output_shapes(train_dataset)) features, y = iter.get_next() # Add features to the model params model_params.update(features) model_params['y'] = y # create the initialisation operations train_init_op = iter.make_initializer(train_dataset) valid_init_op = iter.make_initializer(valid_dataset) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) TRAIN_FLAG() trainer = create_trainer(model_params, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.sess.run(train_init_op) trainer.train(ts, reporting_fns) trainer.sess.run(valid_init_op) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: print('Reloading best checkpoint') trainer.recover_last_checkpoint() test_dataset = tf.data.Dataset.from_tensor_slices(to_tensors(es, lengths_key)) test_dataset = test_dataset.batch(test_batchsz, drop_remainder=False) test_dataset = test_dataset.repeat(epochs + 1) test_dataset = test_dataset.prefetch(NUM_PREFETCH) test_init_op = iter.make_initializer(test_dataset) trainer.sess.run(test_init_op) # What to do about overloading this?? evaluator = TaggerEvaluatorTf(trainer.model, span_type, verbose) start = time.time() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = time.time() - start for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration})
def fit(model_params, ts, vs, es=None, **kwargs): """ Train an language model using TensorFlow with a `feed_dict`. :param model_params: The model (or parameters to create the model) to train :param ts: A training data set :param vs: A validation data set :param es: A test data set, can be None :param kwargs: See below :Keyword Arguments: * *do_early_stopping* (``bool``) -- Stop after evaluation data is no longer improving. Defaults to True * *verbose* (`dict`) A dictionary containing `console` boolean and `file` name if on * *epochs* (``int``) -- how many epochs. Default to 5 * *outfile* -- Model output file * *patience* -- How many epochs where evaluation is no longer improving before we give up * *reporting* -- Callbacks which may be used on reporting updates * *nsteps* (`int`) -- If we should report every n-steps, this should be passed * *ema_decay* (`float`) -- If we are doing an exponential moving average, what decay to us4e * *clip* (`int`) -- If we are doing gradient clipping, what value to use * *optim* (`str`) -- The name of the optimizer we are using * *lr* (`float`) -- The learning rate we are using * *mom* (`float`) -- If we are using SGD, what value to use for momentum * *beta1* (`float`) -- Adam-specific hyper-param, defaults to `0.9` * *beta2* (`float`) -- Adam-specific hyper-param, defaults to `0.999` * *epsilon* (`float`) -- Adam-specific hyper-param, defaults to `1e-8 * *after_train_fn* (`func`) -- A callback to fire after ever epoch of training :return: None """ epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs model_file = get_model_file('lm', 'tf', kwargs.get('basedir')) after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model_params, **kwargs) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 1000 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) print('Doing early stopping on [%s] with patience [%d]' % (early_stopping_metric, patience)) reporting_fns = listify(kwargs.get('reporting', [])) print('reporting', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(trainer.model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] print('New best %.3f' % best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: print('Stopping due to persistent failures to improve') break if do_early_stopping is True: print('Best performance on %s: %.3f at epoch %d' % (early_stopping_metric, best_metric, last_improved)) if es is not None: trainer.recover_last_checkpoint() trainer.test(es, reporting_fns, phase='Test')
def fit(model_params, ts, vs, es, **kwargs): do_early_stopping = bool(kwargs.get('do_early_stopping', True)) epochs = int(kwargs.get('epochs', 20)) model_file = get_model_file('tagger', 'pytorch', kwargs.get('basedir')) conll_output = kwargs.get('conll_output', None) txts = kwargs.get('txts', None) num_loader_workers = int(kwargs.get('num_loader_workers', 0)) pin_memory = bool(kwargs.get('pin_memory', True)) if not isinstance(ts, DataLoader): ts = DataLoader(ts, num_workers=num_loader_workers, batch_size=None, pin_memory=pin_memory) if not isinstance(vs, DataLoader): vs = DataLoader(vs, batch_size=None, pin_memory=pin_memory) if es and not isinstance(es, DataLoader): es = DataLoader(es, batch_size=None, pin_memory=pin_memory) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) after_train_fn = kwargs.get('after_train_fn', None) trainer = create_trainer(model_params, **kwargs) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(trainer.model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: logger.info('Reloading best checkpoint') model = torch.load(model_file) trainer = create_trainer(model, **kwargs) test_metrics = trainer.test(es, reporting_fns, conll_output=conll_output, txts=txts, phase='Test') return test_metrics
def fit(model, ts, vs, es=None, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs model_file = get_model_file('lm', 'tf', kwargs.get('basedir')) after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) init = tf.global_variables_initializer() feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} model.sess.run(init, feed_dict) saver = tf.train.Saver() model.set_saver(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 1000 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'avg_loss') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics
def fit(model, ts, vs, es, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) after_train_fn = kwargs['after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) feed_dict = {k: v for e in model.embeddings.values() for k, v in e.get_feed_dict().items()} init = tf.global_variables_initializer() model.sess.run(init, feed_dict) saver = tf.train.Saver() model.save_using(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = bool(kwargs.get('verbose', False)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp(early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() # What to do about overloading this?? evaluator = TaggerEvaluatorTf(model, span_type, verbose) start = time.time() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = time.time() - start for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration}) return test_metrics
def fit(model, ts, vs, es, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs conll_output = kwargs.get('conll_output', None) span_type = kwargs.get('span_type', 'iob') txts = kwargs.get('txts', None) model_file = get_model_file('tagger', 'tf', kwargs.get('basedir')) after_train_fn = kwargs[ 'after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) tables = tf.tables_initializer() model.sess.run(tables) init = tf.global_variables_initializer() model.sess.run(init) saver = tf.train.Saver() model.save_using(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) verbose = bool(kwargs.get('verbose', False)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'acc') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() # What to do about overloading this?? evaluator = TaggerEvaluatorTf(model, span_type, verbose) start = time.time() test_metrics = evaluator.test(es, conll_output=conll_output, txts=txts) duration = time.time() - start for reporting in reporting_fns: reporting(test_metrics, 0, 'Test') trainer.log.debug({'phase': 'Test', 'time': duration})
def fit(model, ts, vs, es=None, **kwargs): epochs = int(kwargs['epochs']) if 'epochs' in kwargs else 5 patience = int(kwargs['patience']) if 'patience' in kwargs else epochs model_file = get_model_file('seq2seq', 'tf', kwargs.get('basedir')) after_train_fn = kwargs[ 'after_train_fn'] if 'after_train_fn' in kwargs else None trainer = create_trainer(model, **kwargs) feed_dict = { k: v for e in model.src_embeddings.values() for k, v in e.get_feed_dict().items() } feed_dict.update(model.tgt_embedding.get_feed_dict()) init = tf.global_variables_initializer() model.sess.run(init, feed_dict) saver = tf.train.Saver() trainer.prepare(saver) checkpoint = kwargs.get('checkpoint') if checkpoint is not None: latest = tf.train.latest_checkpoint(checkpoint) print('Reloading ' + latest) model.saver.restore(model.sess, latest) do_early_stopping = bool(kwargs.get('do_early_stopping', True)) best_metric = 0 if do_early_stopping: early_stopping_metric = kwargs.get('early_stopping_metric', 'bleu') early_stopping_cmp, best_metric = get_metric_cmp( early_stopping_metric, kwargs.get('early_stopping_cmp')) patience = kwargs.get('patience', epochs) logger.info('Doing early stopping on [%s] with patience [%d]', early_stopping_metric, patience) reporting_fns = listify(kwargs.get('reporting', [])) logger.info('reporting %s', reporting_fns) last_improved = 0 for epoch in range(epochs): trainer.train(ts, reporting_fns) if after_train_fn is not None: after_train_fn(model) test_metrics = trainer.test(vs, reporting_fns, phase='Valid') if do_early_stopping is False: trainer.checkpoint() trainer.model.save(model_file) elif early_stopping_cmp(test_metrics[early_stopping_metric], best_metric): last_improved = epoch best_metric = test_metrics[early_stopping_metric] logger.info('New best %.3f', best_metric) trainer.checkpoint() trainer.model.save(model_file) elif (epoch - last_improved) > patience: logger.info('Stopping due to persistent failures to improve') break if do_early_stopping is True: logger.info('Best performance on %s: %.3f at epoch %d', early_stopping_metric, best_metric, last_improved) if es is not None: trainer.recover_last_checkpoint() test_metrics = trainer.test(es, reporting_fns, phase='Test') return test_metrics