def test_running_test_after_fitting(tmpdir):
"""Verify test() on fitted model."""
class ModelTrainValTest(BoringModel):
def validation_step(self, *args, **kwargs):
output = super().validation_step(*args, **kwargs)
self.log('val_loss', output['x'])
return output
def test_step(self, *args, **kwargs):
output = super().test_step(*args, **kwargs)
self.log('test_loss', output['y'])
return output
model = ModelTrainValTest()
# logger file to get meta
logger = tutils.get_default_logger(tmpdir)
# logger file to get weights
checkpoint = tutils.init_checkpoint_callback(logger)
# fit model
trainer = Trainer(
default_root_dir=tmpdir,
progress_bar_refresh_rate=0,
max_epochs=2,
limit_train_batches=0.4,
limit_val_batches=0.2,
limit_test_batches=0.2,
callbacks=[checkpoint],
logger=logger,
)
trainer.fit(model)
assert trainer.state.finished, f"Training failed with {trainer.state}"
trainer.test()
# test we have good test accuracy
tutils.assert_ok_model_acc(trainer, key='test_loss', thr=0.5)
def test_running_test_no_val(tmpdir):
"""Verify `test()` works on a model with no `val_dataloader`.
It performs train and test only
"""
class ModelTrainTest(BoringModel):
def val_dataloader(self):
pass
def test_step(self, *args, **kwargs):
output = super().test_step(*args, **kwargs)
self.log("test_loss", output["y"])
return output
model = ModelTrainTest()
# logger file to get meta
logger = tutils.get_default_logger(tmpdir)
# logger file to get weights
checkpoint = tutils.init_checkpoint_callback(logger)
# fit model
trainer = Trainer(
default_root_dir=tmpdir,
enable_progress_bar=False,
max_epochs=1,
limit_train_batches=0.4,
limit_val_batches=0.2,
limit_test_batches=0.2,
callbacks=[checkpoint],
logger=logger,
)
trainer.fit(model)
assert trainer.state.finished, f"Training failed with {trainer.state}"
trainer.test()
# test we have good test accuracy
tutils.assert_ok_model_acc(trainer, key="test_loss")
def test_running_test_pretrained_model_cpu(tmpdir):
"""Verify test() on pretrained model."""
tutils.reset_seed()
dm = ClassifDataModule()
model = ClassificationModel()
# logger file to get meta
logger = tutils.get_default_logger(tmpdir)
# logger file to get weights
checkpoint = tutils.init_checkpoint_callback(logger)
trainer_options = dict(
progress_bar_refresh_rate=0,
max_epochs=2,
limit_train_batches=2,
limit_val_batches=2,
limit_test_batches=2,
callbacks=[checkpoint],
logger=logger,
default_root_dir=tmpdir,
)
# fit model
trainer = Trainer(**trainer_options)
trainer.fit(model, datamodule=dm)
# correct result and ok accuracy
assert trainer.state == TrainerState.FINISHED, f"Training failed with {trainer.state}"
pretrained_model = ClassificationModel.load_from_checkpoint(
trainer.checkpoint_callback.best_model_path)
new_trainer = Trainer(**trainer_options)
new_trainer.test(pretrained_model, datamodule=dm)
# test we have good test accuracy
tutils.assert_ok_model_acc(new_trainer, key='test_acc', thr=0.45)
