def test_result_obj_on_tpu(tmpdir):
seed_everything(1234)
batches = 5
epochs = 2
model = EvalModelTemplate()
model.training_step = model.training_step_result_obj
model.training_step_end = None
model.training_epoch_end = None
model.validation_step = model.validation_step_result_obj
model.validation_step_end = None
model.validation_epoch_end = None
model.test_step = model.test_step_result_obj
model.test_step_end = None
model.test_epoch_end = None
trainer_options = dict(default_root_dir=tmpdir,
max_epochs=epochs,
callbacks=[EarlyStopping()],
log_every_n_steps=2,
limit_train_batches=batches,
weights_summary=None,
tpu_cores=8)
tpipes.run_model_test(trainer_options, model, on_gpu=False, with_hpc=False)
def test_result_obj_on_tpu(tmpdir):
seed_everything(1234)
os.environ['PL_DEV_DEBUG'] = '1'
batches = 5
epochs = 2
model = EvalModelTemplate()
model.training_step = model.training_step_result_obj
model.training_step_end = None
model.training_epoch_end = None
model.validation_step = model.validation_step_result_obj
model.validation_step_end = None
model.validation_epoch_end = None
model.test_step = model.test_step_result_obj
model.test_step_end = None
model.test_epoch_end = None
trainer_options = dict(default_root_dir=tmpdir,
max_epochs=epochs,
early_stop_callback=True,
row_log_interval=2,
limit_train_batches=batches,
weights_summary=None,
tpu_cores=8)
tpipes.run_model_test(trainer_options, model, on_gpu=False, with_hpc=False)
def test_result_obj_predictions_ddp_spawn(tmpdir):
seed_everything(4321)
distributed_backend = 'ddp_spawn'
option = 0
import os
os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'
dm = TrialMNISTDataModule(tmpdir)
prediction_file = Path('predictions.pt')
model = EvalModelTemplate(learning_rate=0.005)
model.test_option = option
model.prediction_file = prediction_file.as_posix()
model.test_step = model.test_step_result_preds
model.test_step_end = None
model.test_epoch_end = None
model.test_end = None
prediction_files = [
Path('predictions_rank_0.pt'),
Path('predictions_rank_1.pt')
]
for prediction_file in prediction_files:
if prediction_file.exists():
prediction_file.unlink()
trainer = Trainer(default_root_dir=tmpdir,
max_epochs=3,
weights_summary=None,
deterministic=True,
distributed_backend=distributed_backend,
gpus=[0, 1])
# Prediction file shouldn't exist yet because we haven't done anything
# assert not model.prediction_file.exists()
result = trainer.fit(model, dm)
assert result == 1
result = trainer.test(datamodule=dm)
result = result[0]
assert result['test_loss'] < 0.6
assert result['test_acc'] > 0.8
dm.setup('test')
# check prediction file now exists and is of expected length
size = 0
for prediction_file in prediction_files:
assert prediction_file.exists()
predictions = torch.load(prediction_file)
size += len(predictions)
assert size == len(dm.mnist_test)
def test_val_step_full_loop_result_dp(tmpdir):
# TODO: finish the full train, val, test loop with dp
os.environ['PL_DEV_DEBUG'] = '1'
batches = 10
epochs = 3
model = EvalModelTemplate()
model.training_step = model.training_step_full_loop_result_obj_dp
model.training_step_end = model.training_step_end_full_loop_result_obj_dp
model.training_epoch_end = model.training_epoch_end_full_loop_result_obj_dp
model.validation_step = model.eval_step_full_loop_result_obj_dp
model.validation_step_end = model.eval_step_end_full_loop_result_obj_dp
model.validation_epoch_end = model.eval_epoch_end_full_loop_result_obj_dp
model.test_step = model.eval_step_full_loop_result_obj_dp
model.test_step_end = model.eval_step_end_full_loop_result_obj_dp
model.test_epoch_end = model.eval_epoch_end_full_loop_result_obj_dp
trainer = Trainer(
default_root_dir=tmpdir,
distributed_backend='dp',
gpus=[0, 1],
max_epochs=epochs,
early_stop_callback=True,
log_every_n_steps=2,
limit_train_batches=batches,
weights_summary=None,
)
trainer.fit(model)
results = trainer.test()
# assert we returned all metrics requested
assert len(results) == 1
results = results[0]
assert 'test_epoch_end_metric' in results
# make sure we saw all the correct keys along all paths
seen_keys = set()
for metric in trainer.dev_debugger.logged_metrics:
seen_keys.update(metric.keys())
assert 'train_step_metric' in seen_keys
assert 'train_step_end_metric' in seen_keys
assert 'train_epoch_end_metric_epoch' in seen_keys
assert 'validation_step_metric_step/epoch_0' in seen_keys
assert 'validation_step_metric_epoch' in seen_keys
assert 'validation_step_end_metric' in seen_keys
assert 'validation_epoch_end_metric' in seen_keys
assert 'test_step_metric_step/epoch_2' in seen_keys
assert 'test_step_metric_epoch' in seen_keys
assert 'test_step_end_metric' in seen_keys
assert 'test_epoch_end_metric' in seen_keys
def test_full_loop_result_cpu(tmpdir):
seed_everything(1234)
os.environ['PL_DEV_DEBUG'] = '1'
batches = 10
epochs = 2
model = EvalModelTemplate()
model.training_step = model.training_step_result_obj
model.training_step_end = None
model.training_epoch_end = None
model.validation_step = model.validation_step_result_obj
model.validation_step_end = None
model.validation_epoch_end = None
model.test_step = model.test_step_result_obj
model.test_step_end = None
model.test_epoch_end = None
trainer = Trainer(
default_root_dir=tmpdir,
max_epochs=epochs,
early_stop_callback=True,
log_every_n_steps=2,
limit_train_batches=batches,
weights_summary=None,
)
trainer.fit(model)
results = trainer.test()
# assert we returned all metrics requested
assert len(results) == 1
results = results[0]
assert results['test_loss'] < 0.3
assert results['test_acc'] > 0.9
assert len(results) == 2
assert 'early_stop_on' not in results
assert 'checkpoint_on' not in results
results2 = trainer.test()[0]
for k, v in results.items():
assert results2[k] == v
def test_result_obj_predictions(tmpdir, test_option, do_train, gpus):
tutils.reset_seed()
dm = TrialMNISTDataModule(tmpdir)
prediction_file = Path(tmpdir) / 'predictions.pt'
model = EvalModelTemplate()
model.test_option = test_option
model.prediction_file = prediction_file.as_posix()
model.test_step = model.test_step_result_preds
model.test_step_end = None
model.test_epoch_end = None
model.test_end = None
if prediction_file.exists():
prediction_file.unlink()
trainer = Trainer(
default_root_dir=tmpdir,
max_epochs=3,
weights_summary=None,
deterministic=True,
gpus=gpus
)
# Prediction file shouldn't exist yet because we haven't done anything
assert not prediction_file.exists()
if do_train:
result = trainer.fit(model, dm)
assert result == 1
result = trainer.test(datamodule=dm)
result = result[0]
assert result['test_loss'] < 0.6
assert result['test_acc'] > 0.8
else:
result = trainer.test(model, datamodule=dm)
# check prediction file now exists and is of expected length
assert prediction_file.exists()
predictions = torch.load(prediction_file)
assert len(predictions) == len(dm.mnist_test)
def test_dm_reload_dataloaders_every_epoch(tmpdir):
"""Test datamodule, where trainer argument
reload_dataloaders_every_epoch is set to True/False"""
dm = CustomMNISTDataModule(tmpdir)
model = EvalModelTemplate()
model.validation_step = None
model.validation_step_end = None
model.validation_epoch_end = None
model.test_step = None
model.test_step_end = None
model.test_epoch_end = None
trainer = Trainer(
default_root_dir=tmpdir,
max_epochs=2,
limit_train_batches=0.01,
reload_dataloaders_every_epoch=True,
)
trainer.fit(model, dm)
def test_train_loop_only(tmpdir):
dm = TrialMNISTDataModule(tmpdir)
model = EvalModelTemplate()
model.validation_step = None
model.validation_step_end = None
model.validation_epoch_end = None
model.test_step = None
model.test_step_end = None
model.test_epoch_end = None
trainer = Trainer(
default_root_dir=tmpdir,
max_epochs=3,
weights_summary=None,
)
# fit model
result = trainer.fit(model, dm)
assert result == 1
assert trainer.logger_connector.callback_metrics['loss'] < 0.6
def test_testpass_overrides(tmpdir):
# todo: check duplicated tests against trainer_checks
hparams = EvalModelTemplate.get_default_hparams()
# Misconfig when neither test_step or test_end is implemented
with pytest.raises(MisconfigurationException, match='.*not implement `test_dataloader`.*'):
model = EvalModelTemplate(hparams)
model.test_dataloader = LightningModule.test_dataloader
Trainer().test(model)
# Misconfig when neither test_step or test_end is implemented
with pytest.raises(MisconfigurationException):
model = EvalModelTemplate(hparams)
model.test_step = LightningModule.test_step
Trainer().test(model)
# No exceptions when one or both of test_step or test_end are implemented
model = EvalModelTemplate(hparams)
model.test_step_end = LightningModule.test_step_end
Trainer().test(model)
model = EvalModelTemplate(hparams)
Trainer().test(model)
