def _test_loggers_pickle(tmpdir, monkeypatch, logger_class): """Verify that pickling trainer with logger works.""" _patch_comet_atexit(monkeypatch) logger_args = _get_logger_args(logger_class, tmpdir) logger = logger_class(**logger_args) # this can cause pickle error if the experiment object is not picklable # the logger needs to remove it from the state before pickle _ = logger.experiment # logger also has to avoid adding un-picklable attributes to self in .save logger.log_metrics({"a": 1}) logger.save() # test pickling loggers pickle.dumps(logger) trainer = Trainer(max_epochs=1, logger=logger) pkl_bytes = pickle.dumps(trainer) trainer2 = pickle.loads(pkl_bytes) trainer2.logger.log_metrics({"acc": 1.0}) # make sure we restored properly assert trainer2.logger.name == logger.name assert trainer2.logger.save_dir == logger.save_dir
def test_loggers_pickle(neptune, tmpdir, monkeypatch, logger_class): """Verify that pickling trainer with logger works.""" _patch_comet_atexit(monkeypatch) logger_args = _get_logger_args(logger_class, tmpdir) logger = logger_class(**logger_args) # this can cause pickle error if the experiment object is not picklable # the logger needs to remove it from the state before pickle _ = logger.experiment # test pickling loggers pickle.dumps(logger) trainer = Trainer( max_epochs=1, logger=logger, ) pkl_bytes = pickle.dumps(trainer) trainer2 = pickle.loads(pkl_bytes) trainer2.logger.log_metrics({'acc': 1.0}) # make sure we restord properly assert trainer2.logger.name == logger.name assert trainer2.logger.save_dir == logger.save_dir
def test_loggers_fit_test_all(tmpdir, monkeypatch): """Verify that basic functionality of all loggers.""" _test_loggers_fit_test(tmpdir, TensorBoardLogger) with mock.patch("pytorch_lightning.loggers.comet.comet_ml"), mock.patch( "pytorch_lightning.loggers.comet.CometOfflineExperiment"): _patch_comet_atexit(monkeypatch) _test_loggers_fit_test(tmpdir, CometLogger) with mock.patch("pytorch_lightning.loggers.mlflow.mlflow"), mock.patch( "pytorch_lightning.loggers.mlflow.MlflowClient"): _test_loggers_fit_test(tmpdir, MLFlowLogger) with mock.patch("pytorch_lightning.loggers.neptune.neptune", new_callable=create_neptune_mock): _test_loggers_fit_test(tmpdir, NeptuneLogger) with mock.patch("pytorch_lightning.loggers.test_tube.Experiment"): _test_loggers_fit_test(tmpdir, TestTubeLogger) with mock.patch("pytorch_lightning.loggers.wandb.wandb") as wandb: wandb.run = None wandb.init().step = 0 _test_loggers_fit_test(tmpdir, WandbLogger)
def test_logger_created_on_rank_zero_only(tmpdir, monkeypatch, logger_class): """ Test that loggers get replaced by dummy loggers on global rank > 0""" _patch_comet_atexit(monkeypatch) try: _test_logger_created_on_rank_zero_only(tmpdir, logger_class) except (ImportError, ModuleNotFoundError): pytest.xfail(f"multi-process test requires {logger_class.__class__} dependencies to be installed.")
def test_loggers_pickle_all(tmpdir, monkeypatch, logger_class): """ Test that the logger objects can be pickled. This test only makes sense if the packages are installed. """ _patch_comet_atexit(monkeypatch) try: _test_loggers_pickle(tmpdir, monkeypatch, logger_class) except (ImportError, ModuleNotFoundError): pytest.xfail(f"pickle test requires {logger_class.__class__} dependencies to be installed.")
def test_logger_with_prefix_all(tmpdir, monkeypatch): """Test that prefix is added at the beginning of the metric keys.""" prefix = "tmp" # Comet with mock.patch("pytorch_lightning.loggers.comet.comet_ml"), mock.patch( "pytorch_lightning.loggers.comet.CometOfflineExperiment"): _patch_comet_atexit(monkeypatch) logger = _instantiate_logger(CometLogger, save_dir=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log_metrics.assert_called_once_with( {"tmp-test": 1.0}, epoch=None, step=0) # MLflow with mock.patch("pytorch_lightning.loggers.mlflow.mlflow"), mock.patch( "pytorch_lightning.loggers.mlflow.MlflowClient"): logger = _instantiate_logger(MLFlowLogger, save_dir=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log_metric.assert_called_once_with( ANY, "tmp-test", 1.0, ANY, 0) # Neptune with mock.patch("pytorch_lightning.loggers.neptune.neptune"): logger = _instantiate_logger(NeptuneLogger, api_key="test", project="project", save_dir=tmpdir, prefix=prefix) assert logger.experiment.__getitem__.call_count == 2 logger.log_metrics({"test": 1.0}, step=0) assert logger.experiment.__getitem__.call_count == 3 logger.experiment.__getitem__.assert_called_with("tmp/test") logger.experiment.__getitem__().log.assert_called_once_with(1.0) # TensorBoard with mock.patch("pytorch_lightning.loggers.tensorboard.SummaryWriter"): logger = _instantiate_logger(TensorBoardLogger, save_dir=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.add_scalar.assert_called_once_with( "tmp-test", 1.0, 0) # WandB with mock.patch("pytorch_lightning.loggers.wandb.wandb") as wandb: logger = _instantiate_logger(WandbLogger, save_dir=tmpdir, prefix=prefix) wandb.run = None wandb.init().step = 0 logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log.assert_called_once_with({ "tmp-test": 1.0, "trainer/global_step": 0 })
def test_loggers_save_dir_and_weights_save_path_all(tmpdir, monkeypatch, logger_class): """Test the combinations of save_dir, weights_save_path and default_root_dir.""" with contextlib.ExitStack() as stack: for mgr in LOGGER_CTX_MANAGERS: stack.enter_context(mgr) _patch_comet_atexit(monkeypatch) _test_loggers_save_dir_and_weights_save_path(tmpdir, CometLogger)
def test_loggers_save_dir_and_weights_save_path(wandb, tmpdir, monkeypatch, logger_class): """ Test the combinations of save_dir, weights_save_path and default_root_dir. """ _patch_comet_atexit(monkeypatch) class TestLogger(logger_class): # for this test it does not matter what these attributes are # so we standardize them to make testing easier @property def version(self): return 'version' @property def name(self): return 'name' model = EvalModelTemplate() trainer_args = dict( default_root_dir=tmpdir, max_steps=1, ) # no weights_save_path given save_dir = tmpdir / 'logs' weights_save_path = None logger = TestLogger(**_get_logger_args(TestLogger, save_dir)) trainer = Trainer(**trainer_args, logger=logger, weights_save_path=weights_save_path) trainer.fit(model) assert trainer.weights_save_path == trainer.default_root_dir assert trainer.checkpoint_callback.dirpath == os.path.join( logger.save_dir, 'name', 'version', 'checkpoints') assert trainer.default_root_dir == tmpdir # with weights_save_path given, the logger path and checkpoint path should be different save_dir = tmpdir / 'logs' weights_save_path = tmpdir / 'weights' logger = TestLogger(**_get_logger_args(TestLogger, save_dir)) trainer = Trainer(**trainer_args, logger=logger, weights_save_path=weights_save_path) trainer.fit(model) assert trainer.weights_save_path == weights_save_path assert trainer.logger.save_dir == save_dir assert trainer.checkpoint_callback.dirpath == weights_save_path / 'name' / 'version' / 'checkpoints' assert trainer.default_root_dir == tmpdir # no logger given weights_save_path = tmpdir / 'weights' trainer = Trainer(**trainer_args, logger=False, weights_save_path=weights_save_path) trainer.fit(model) assert trainer.weights_save_path == weights_save_path assert trainer.checkpoint_callback.dirpath == weights_save_path / 'checkpoints' assert trainer.default_root_dir == tmpdir
def test_logger_created_on_rank_zero_only(tmpdir, monkeypatch, logger_class): """Test that loggers get replaced by dummy loggers on global rank > 0.""" _patch_comet_atexit(monkeypatch) try: if logger_class is TestTubeLogger: with pytest.deprecated_call(match="TestTubeLogger is deprecated since v1.5"): _test_logger_created_on_rank_zero_only(tmpdir, logger_class) else: _test_logger_created_on_rank_zero_only(tmpdir, logger_class) except (ImportError, ModuleNotFoundError): pytest.xfail(f"multi-process test requires {logger_class.__class__} dependencies to be installed.")
def test_loggers_pickle_all(tmpdir, monkeypatch, logger_class): """Test that the logger objects can be pickled. This test only makes sense if the packages are installed. """ _patch_comet_atexit(monkeypatch) try: if logger_class is TestTubeLogger: with pytest.deprecated_call(match="TestTubeLogger is deprecated since v1.5"): _test_loggers_pickle(tmpdir, monkeypatch, logger_class) else: _test_loggers_pickle(tmpdir, monkeypatch, logger_class) except (ImportError, ModuleNotFoundError): pytest.xfail(f"pickle test requires {logger_class.__class__} dependencies to be installed.")
def test_loggers_fit_test(wandb, neptune, tmpdir, monkeypatch, logger_class): """Verify that basic functionality of all loggers.""" os.environ['PL_DEV_DEBUG'] = '0' _patch_comet_atexit(monkeypatch) model = EvalModelTemplate() class StoreHistoryLogger(logger_class): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.history = [] def log_metrics(self, metrics, step): super().log_metrics(metrics, step) self.history.append((step, metrics)) logger_args = _get_logger_args(logger_class, tmpdir) logger = StoreHistoryLogger(**logger_args) if logger_class == WandbLogger: # required mocks for Trainer logger.experiment.id = 'foo' logger.experiment.project_name.return_value = 'bar' trainer = Trainer( max_epochs=1, logger=logger, limit_train_batches=0.2, limit_val_batches=0.5, fast_dev_run=True, default_root_dir=tmpdir, ) trainer.fit(model) trainer.test() log_metric_names = [(s, sorted(m.keys())) for s, m in logger.history] if logger_class == TensorBoardLogger: expected = [(0, ['hp_metric']), (0, ['epoch', 'train_some_val']), (0, ['early_stop_on', 'epoch', 'val_acc']), (0, ['hp_metric']), (1, ['epoch', 'test_acc', 'test_loss'])] assert log_metric_names == expected else: expected = [(0, ['epoch', 'train_some_val']), (0, ['early_stop_on', 'epoch', 'val_acc']), (1, ['epoch', 'test_acc', 'test_loss'])] assert log_metric_names == expected
def test_logger_with_prefix_all(tmpdir, monkeypatch): """ Test that prefix is added at the beginning of the metric keys. """ prefix = 'tmp' # Comet with mock.patch('pytorch_lightning.loggers.comet.comet_ml'), \ mock.patch('pytorch_lightning.loggers.comet.CometOfflineExperiment'): _patch_comet_atexit(monkeypatch) logger = _instantiate_logger(CometLogger, save_idr=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log_metrics.assert_called_once_with({"tmp-test": 1.0}, epoch=None, step=0) # MLflow with mock.patch('pytorch_lightning.loggers.mlflow.mlflow'), \ mock.patch('pytorch_lightning.loggers.mlflow.MlflowClient'): logger = _instantiate_logger(MLFlowLogger, save_idr=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log_metric.assert_called_once_with(ANY, "tmp-test", 1.0, ANY, 0) # Neptune with mock.patch('pytorch_lightning.loggers.neptune.neptune'): logger = _instantiate_logger(NeptuneLogger, save_idr=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log_metric.assert_called_once_with("tmp-test", x=0, y=1.0) # TensorBoard with mock.patch('pytorch_lightning.loggers.tensorboard.SummaryWriter'): logger = _instantiate_logger(TensorBoardLogger, save_idr=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.add_scalar.assert_called_once_with("tmp-test", 1.0, 0) # TestTube with mock.patch('pytorch_lightning.loggers.test_tube.Experiment'): logger = _instantiate_logger(TestTubeLogger, save_idr=tmpdir, prefix=prefix) logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log.assert_called_once_with({"tmp-test": 1.0}, global_step=0) # WandB with mock.patch('pytorch_lightning.loggers.wandb.wandb') as wandb: logger = _instantiate_logger(WandbLogger, save_idr=tmpdir, prefix=prefix) wandb.run = None wandb.init().step = 0 logger.log_metrics({"test": 1.0}, step=0) logger.experiment.log.assert_called_once_with({'tmp-test': 1.0}, step=0)
def test_logger_created_on_rank_zero_only(neptune, tmpdir, monkeypatch, logger_class): """ Test that loggers get replaced by dummy loggers on global rank > 0""" _patch_comet_atexit(monkeypatch) logger_args = _get_logger_args(logger_class, tmpdir) logger = logger_class(**logger_args) model = EvalModelTemplate() trainer = Trainer( logger=logger, default_root_dir=tmpdir, distributed_backend='ddp_cpu', num_processes=2, max_steps=1, checkpoint_callback=True, callbacks=[RankZeroLoggerCheck()], ) result = trainer.fit(model) assert result == 1
def test_loggers_save_dir_and_weights_save_path_all(tmpdir, monkeypatch): """Test the combinations of save_dir, weights_save_path and default_root_dir.""" _test_loggers_save_dir_and_weights_save_path(tmpdir, TensorBoardLogger) with mock.patch("pytorch_lightning.loggers.comet.comet_ml"), mock.patch( "pytorch_lightning.loggers.comet.CometOfflineExperiment"): _patch_comet_atexit(monkeypatch) _test_loggers_save_dir_and_weights_save_path(tmpdir, CometLogger) with mock.patch("pytorch_lightning.loggers.mlflow.mlflow"), mock.patch( "pytorch_lightning.loggers.mlflow.MlflowClient"): _test_loggers_save_dir_and_weights_save_path(tmpdir, MLFlowLogger) with mock.patch("pytorch_lightning.loggers.test_tube.Experiment"): _test_loggers_save_dir_and_weights_save_path(tmpdir, TestTubeLogger) with mock.patch("pytorch_lightning.loggers.wandb.wandb"): _test_loggers_save_dir_and_weights_save_path(tmpdir, WandbLogger)
def test_loggers_fit_test_all(tmpdir, monkeypatch): """ Verify that basic functionality of all loggers. """ _test_loggers_fit_test(tmpdir, TensorBoardLogger) with mock.patch('pytorch_lightning.loggers.comet.comet_ml'), \ mock.patch('pytorch_lightning.loggers.comet.CometOfflineExperiment'): _patch_comet_atexit(monkeypatch) _test_loggers_fit_test(tmpdir, CometLogger) with mock.patch('pytorch_lightning.loggers.mlflow.mlflow'), \ mock.patch('pytorch_lightning.loggers.mlflow.MlflowClient'): _test_loggers_fit_test(tmpdir, MLFlowLogger) with mock.patch('pytorch_lightning.loggers.neptune.neptune'): _test_loggers_fit_test(tmpdir, NeptuneLogger) with mock.patch('pytorch_lightning.loggers.test_tube.Experiment'): _test_loggers_fit_test(tmpdir, TestTubeLogger) with mock.patch('pytorch_lightning.loggers.wandb.wandb'): _test_loggers_fit_test(tmpdir, WandbLogger)