def test_default_logger_callbacks_cpu_model(tmpdir):
"""Test each of the trainer options."""
trainer_options = dict(
default_root_dir=tmpdir,
max_epochs=1,
gradient_clip_val=1.0,
overfit_batches=0.20,
progress_bar_refresh_rate=0,
limit_train_batches=0.01,
limit_val_batches=0.01,
)
model = EvalModelTemplate()
tpipes.run_model_test_without_loggers(trainer_options, model)
# test freeze on cpu
model.freeze()
model.unfreeze()
def test_default_logger_callbacks_cpu_model(tmpdir):
"""Test each of the trainer options."""
trainer_options = dict(
default_root_dir=tmpdir,
max_epochs=1,
gradient_clip_val=1.0,
overfit_pct=0.20,
progress_bar_refresh_rate=0,
train_percent_check=0.01,
val_percent_check=0.01,
)
model = EvalModelTemplate(tutils.get_default_hparams())
tutils.run_model_test_without_loggers(trainer_options, model)
# test freeze on cpu
model.freeze()
model.unfreeze()
def test_early_stopping_cpu_model(tmpdir):
"""Test each of the trainer options."""
stopping = EarlyStopping(monitor='val_loss', min_delta=0.1)
trainer_options = dict(
default_root_dir=tmpdir,
early_stop_callback=stopping,
gradient_clip_val=1.0,
overfit_pct=0.20,
track_grad_norm=2,
train_percent_check=0.1,
val_percent_check=0.1,
)
model = EvalModelTemplate()
tutils.run_model_test(trainer_options, model, on_gpu=False)
# test freeze on cpu
model.freeze()
model.unfreeze()
def test_early_stopping_cpu_model(tmpdir):
"""Test each of the trainer options."""
stopping = EarlyStopping(monitor='early_stop_on', min_delta=0.1)
trainer_options = dict(
default_root_dir=tmpdir,
early_stop_callback=stopping,
max_epochs=2,
gradient_clip_val=1.0,
overfit_batches=0.20,
track_grad_norm=2,
limit_train_batches=0.1,
limit_val_batches=0.1,
)
model = EvalModelTemplate()
tpipes.run_model_test(trainer_options, model, on_gpu=False)
# test freeze on cpu
model.freeze()
model.unfreeze()
def test_dp_resume(tmpdir):
"""Make sure DP continues training correctly."""
hparams = EvalModelTemplate.get_default_hparams()
model = EvalModelTemplate(**hparams)
trainer_options = dict(
max_epochs=1,
gpus=2,
distributed_backend='dp',
)
# get logger
logger = tutils.get_default_logger(tmpdir)
# exp file to get weights
# logger file to get weights
checkpoint = tutils.init_checkpoint_callback(logger)
# add these to the trainer options
trainer_options['logger'] = logger
trainer_options['checkpoint_callback'] = checkpoint
# fit model
trainer = Trainer(**trainer_options)
trainer.is_slurm_managing_tasks = True
result = trainer.fit(model)
# track epoch before saving. Increment since we finished the current epoch, don't want to rerun
real_global_epoch = trainer.current_epoch + 1
# correct result and ok accuracy
assert result == 1, 'amp + dp model failed to complete'
# ---------------------------
# HPC LOAD/SAVE
# ---------------------------
# save
trainer.hpc_save(tmpdir, logger)
# init new trainer
new_logger = tutils.get_default_logger(tmpdir, version=logger.version)
trainer_options['logger'] = new_logger
trainer_options['checkpoint_callback'] = ModelCheckpoint(tmpdir)
trainer_options['limit_train_batches'] = 0.5
trainer_options['limit_val_batches'] = 0.2
trainer_options['max_epochs'] = 1
new_trainer = Trainer(**trainer_options)
# set the epoch start hook so we can predict before the model does the full training
def assert_good_acc():
assert new_trainer.current_epoch == real_global_epoch and new_trainer.current_epoch > 0
# if model and state loaded correctly, predictions will be good even though we
# haven't trained with the new loaded model
dp_model = new_trainer.model
dp_model.eval()
dataloader = trainer.train_dataloader
tutils.run_prediction(dataloader, dp_model, dp=True)
# new model
model = EvalModelTemplate(**hparams)
model.on_train_start = assert_good_acc
# fit new model which should load hpc weights
new_trainer.fit(model)
# test freeze on gpu
model.freeze()
model.unfreeze()
def test_model_freeze_unfreeze():
model = EvalModelTemplate()
model.freeze()
model.unfreeze()
def test_model_freeze_unfreeze():
model = EvalModelTemplate(tutils.get_default_hparams())
model.freeze()
model.unfreeze()
