def test_running_test_after_fitting(tmpdir): """Verify test() on fitted model.""" class ModelTrainValTest(BoringModel): def validation_step(self, *args, **kwargs): output = super().validation_step(*args, **kwargs) self.log('val_loss', output['x']) return output def test_step(self, *args, **kwargs): output = super().test_step(*args, **kwargs) self.log('test_loss', output['y']) return output model = ModelTrainValTest() # logger file to get meta logger = tutils.get_default_logger(tmpdir) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) # fit model trainer = Trainer( default_root_dir=tmpdir, progress_bar_refresh_rate=0, max_epochs=2, limit_train_batches=0.4, limit_val_batches=0.2, limit_test_batches=0.2, callbacks=[checkpoint], logger=logger, ) trainer.fit(model) assert trainer.state.finished, f"Training failed with {trainer.state}" trainer.test() # test we have good test accuracy tutils.assert_ok_model_acc(trainer, key='test_loss', thr=0.5)
def test_running_test_no_val(tmpdir): """Verify `test()` works on a model with no `val_dataloader`. It performs train and test only """ class ModelTrainTest(BoringModel): def val_dataloader(self): pass def test_step(self, *args, **kwargs): output = super().test_step(*args, **kwargs) self.log("test_loss", output["y"]) return output model = ModelTrainTest() # logger file to get meta logger = tutils.get_default_logger(tmpdir) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) # fit model trainer = Trainer( default_root_dir=tmpdir, enable_progress_bar=False, max_epochs=1, limit_train_batches=0.4, limit_val_batches=0.2, limit_test_batches=0.2, callbacks=[checkpoint], logger=logger, ) trainer.fit(model) assert trainer.state.finished, f"Training failed with {trainer.state}" trainer.test() # test we have good test accuracy tutils.assert_ok_model_acc(trainer, key="test_loss")
def test_running_test_pretrained_model_cpu(tmpdir): """Verify test() on pretrained model.""" tutils.reset_seed() dm = ClassifDataModule() model = ClassificationModel() # logger file to get meta logger = tutils.get_default_logger(tmpdir) # logger file to get weights checkpoint = tutils.init_checkpoint_callback(logger) trainer_options = dict( progress_bar_refresh_rate=0, max_epochs=2, limit_train_batches=2, limit_val_batches=2, limit_test_batches=2, callbacks=[checkpoint], logger=logger, default_root_dir=tmpdir, ) # fit model trainer = Trainer(**trainer_options) trainer.fit(model, datamodule=dm) # correct result and ok accuracy assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}" pretrained_model = ClassificationModel.load_from_checkpoint( trainer.checkpoint_callback.best_model_path) new_trainer = Trainer(**trainer_options) new_trainer.test(pretrained_model, datamodule=dm) # test we have good test accuracy tutils.assert_ok_model_acc(new_trainer, key='test_acc', thr=0.45)