def test_optimizer_return_options():
trainer = Trainer()
model, hparams = get_model()
# single optimizer
opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
opt_b = torch.optim.SGD(model.parameters(), lr=0.002)
optim, lr_sched = trainer.init_optimizers(opt_a)
assert len(optim) == 1 and len(lr_sched) == 0
# opt tuple
opts = (opt_a, opt_b)
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
assert len(lr_sched) == 0
# opt list
opts = [opt_a, opt_b]
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
assert len(lr_sched) == 0
# opt tuple of lists
opts = ([opt_a], ['lr_scheduler'])
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 1 and len(lr_sched) == 1
assert optim[0] == opts[0][0] and lr_sched[0] == 'lr_scheduler'
def test_optimizer_return_options():
tutils.reset_seed()
trainer = Trainer()
model, hparams = tutils.get_model()
# single optimizer
opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
opt_b = torch.optim.SGD(model.parameters(), lr=0.002)
optim, lr_sched = trainer.init_optimizers(opt_a)
assert len(optim) == 1 and len(lr_sched) == 0
# opt tuple
opts = (opt_a, opt_b)
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
assert len(lr_sched) == 0
# opt list
opts = [opt_a, opt_b]
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 2 and optim[0] == opts[0] and optim[1] == opts[1]
assert len(lr_sched) == 0
# opt tuple of lists
scheduler = torch.optim.lr_scheduler.StepLR(opt_a, 10)
opts = ([opt_a], [scheduler])
optim, lr_sched = trainer.init_optimizers(opts)
assert len(optim) == 1 and len(lr_sched) == 1
assert optim[0] == opts[0][0] and \
lr_sched[0] == dict(scheduler=scheduler, interval='epoch',
frequency=1, reduce_on_plateau=False,
monitor='val_loss')
def test_v1_8_0_trainer_optimizers_mixin():
trainer = Trainer()
model = BoringModel()
trainer.strategy.connect(model)
trainer.lightning_module.trainer = trainer
with pytest.deprecated_call(
match=r"`TrainerOptimizersMixin.init_optimizers` was deprecated in v1.6 and will be removed in v1.8."
):
trainer.init_optimizers(model)
with pytest.deprecated_call(
match=r"`TrainerOptimizersMixin.convert_to_lightning_optimizers` was deprecated in v1.6 and will be removed in "
"v1.8."
):
trainer.convert_to_lightning_optimizers()
def test_none_optimizer_warning():
trainer = Trainer()
model = EvalModelTemplate()
model.configure_optimizers = model.configure_optimizers__empty
with pytest.warns(UserWarning, match='will run with no optimizer'):
_, __, ___ = trainer.init_optimizers(model)
def test_none_optimizer_warning():
trainer = Trainer()
model = EvalModelTemplate(tutils.get_default_hparams())
model.configure_optimizers = lambda: None
with pytest.warns(UserWarning, match='will run with no optimizer'):
_, __, ___ = trainer.init_optimizers(model)
def test_none_optimizer_warning():
tutils.reset_seed()
trainer = Trainer()
model, hparams = tutils.get_default_model()
model.configure_optimizers = lambda: None
with pytest.warns(UserWarning, match='will run with no optimizer'):
_, __, ___ = trainer.init_optimizers(model)
def run_model_test(trainer_options,
model: LightningModule,
data: LightningDataModule = None,
on_gpu: bool = True,
version=None,
with_hpc: bool = True,
min_acc: float = 0.25):
reset_seed()
save_dir = trainer_options['default_root_dir']
# logger file to get meta
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)
trainer = Trainer(**trainer_options)
initial_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
trainer.fit(model, datamodule=data)
post_train_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
assert trainer.state.finished, f"Training failed with {trainer.state}"
# Check that the model is actually changed post-training
change_ratio = torch.norm(initial_values - post_train_values)
assert change_ratio > 0.1, f"the model is changed of {change_ratio}"
# test model loading
pretrained_model = load_model_from_checkpoint(
logger, trainer.checkpoint_callback.best_model_path, type(model))
# test new model accuracy
test_loaders = model.test_dataloader(
) if not data else data.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
if not isinstance(model, BoringModel):
for dataloader in test_loaders:
run_prediction_eval_model_template(model,
dataloader,
min_acc=min_acc)
if with_hpc:
if trainer._distrib_type in (DistributedType.DDP,
DistributedType.DDP_SPAWN,
DistributedType.DDP2):
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = \
trainer.init_optimizers(pretrained_model)
# test HPC saving
trainer.checkpoint_connector.hpc_save(save_dir, logger)
# test HPC loading
checkpoint_path = trainer.checkpoint_connector.get_max_ckpt_path_from_folder(
save_dir)
trainer.checkpoint_connector.restore(checkpoint_path)
def run_model_test(trainer_options,
model,
on_gpu: bool = True,
version=None,
with_hpc: bool = True):
reset_seed()
save_dir = trainer_options['default_root_dir']
# logger file to get meta
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)
if 'checkpoint_callback' not in trainer_options:
trainer_options.update(checkpoint_callback=True)
trainer = Trainer(**trainer_options)
initial_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
result = trainer.fit(model)
post_train_values = torch.tensor(
[torch.sum(torch.abs(x)) for x in model.parameters()])
assert result == 1, 'trainer failed'
# Check that the model is actually changed post-training
assert torch.norm(initial_values - post_train_values) > 0.1
# test model loading
pretrained_model = load_model_from_checkpoint(
logger, trainer.checkpoint_callback.best_model_path)
# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
for dataloader in test_loaders:
run_prediction(dataloader, pretrained_model)
if with_hpc:
if trainer.use_ddp or trainer.use_ddp2:
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = \
trainer.init_optimizers(pretrained_model)
# test HPC loading / saving
trainer.checkpoint_connector.hpc_save(save_dir, logger)
trainer.checkpoint_connector.hpc_load(save_dir, on_gpu=on_gpu)
def test_optimizer_return_options():
tutils.reset_seed()
trainer = Trainer()
model, hparams = tutils.get_default_model()
# single optimizer
opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
opt_b = torch.optim.SGD(model.parameters(), lr=0.002)
scheduler_a = torch.optim.lr_scheduler.StepLR(opt_a, 10)
scheduler_b = torch.optim.lr_scheduler.StepLR(opt_b, 10)
# single optimizer
model.configure_optimizers = lambda: opt_a
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 1 and len(lr_sched) == 0 and len(freq) == 0
# opt tuple
model.configure_optimizers = lambda: (opt_a, opt_b)
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 2 and optim[0] == opt_a and optim[1] == opt_b
assert len(lr_sched) == 0 and len(freq) == 0
# opt list
model.configure_optimizers = lambda: [opt_a, opt_b]
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 2 and optim[0] == opt_a and optim[1] == opt_b
assert len(lr_sched) == 0 and len(freq) == 0
# opt tuple of 2 lists
model.configure_optimizers = lambda: ([opt_a], [scheduler_a])
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 1 and len(lr_sched) == 1 and len(freq) == 0
assert optim[0] == opt_a
assert lr_sched[0] == dict(scheduler=scheduler_a, interval='epoch',
frequency=1, reduce_on_plateau=False, monitor='val_loss')
# opt single dictionary
model.configure_optimizers = lambda: {"optimizer": opt_a, "lr_scheduler": scheduler_a}
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 1 and len(lr_sched) == 1 and len(freq) == 0
assert optim[0] == opt_a
assert lr_sched[0] == dict(scheduler=scheduler_a, interval='epoch',
frequency=1, reduce_on_plateau=False, monitor='val_loss')
# opt multiple dictionaries with frequencies
model.configure_optimizers = lambda: (
{"optimizer": opt_a, "lr_scheduler": scheduler_a, "frequency": 1},
{"optimizer": opt_b, "lr_scheduler": scheduler_b, "frequency": 5},
)
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 2 and len(lr_sched) == 2 and len(freq) == 2
assert optim[0] == opt_a
assert lr_sched[0] == dict(scheduler=scheduler_a, interval='epoch',
frequency=1, reduce_on_plateau=False, monitor='val_loss')
assert freq == [1, 5]
def run_model_test(trainer_options,
model,
on_gpu: bool = True,
version=None,
with_hpc: bool = True):
reset_seed()
save_dir = trainer_options['default_root_dir']
# logger file to get meta
logger = get_default_logger(save_dir, version=version)
trainer_options.update(logger=logger)
if 'checkpoint_callback' not in trainer_options:
# logger file to get weights
checkpoint = init_checkpoint_callback(logger)
trainer_options.update(checkpoint_callback=checkpoint)
# fit model
trainer = Trainer(**trainer_options)
result = trainer.fit(model)
# correct result and ok accuracy
assert result == 1, 'amp + ddp model failed to complete'
# test model loading
pretrained_model = load_model_from_checkpoint(
logger, trainer.checkpoint_callback.dirpath)
# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
[
run_prediction(dataloader, pretrained_model)
for dataloader in test_loaders
]
if with_hpc:
if trainer.use_ddp or trainer.use_ddp2:
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers, trainer.optimizer_frequencies = \
trainer.init_optimizers(pretrained_model)
# test HPC loading / saving
trainer.hpc_save(save_dir, logger)
trainer.hpc_load(save_dir, on_gpu=on_gpu)
def run_model_test(trainer_options, model, on_gpu=True):
save_dir = trainer_options['default_save_path']
# logger file to get meta
logger = get_default_testtube_logger(save_dir, False)
# logger file to get weights
checkpoint = init_checkpoint_callback(logger)
# add these to the trainer options
trainer_options['checkpoint_callback'] = checkpoint
trainer_options['logger'] = logger
# fit model
trainer = Trainer(**trainer_options)
result = trainer.fit(model)
# correct result and ok accuracy
assert result == 1, 'amp + ddp model failed to complete'
# test model loading
pretrained_model = load_model(logger, trainer.checkpoint_callback.dirpath)
# test new model accuracy
test_loaders = model.test_dataloader()
if not isinstance(test_loaders, list):
test_loaders = [test_loaders]
[
run_prediction(dataloader, pretrained_model)
for dataloader in test_loaders
]
if trainer.use_ddp or trainer.use_ddp2:
# on hpc this would work fine... but need to hack it for the purpose of the test
trainer.model = pretrained_model
trainer.optimizers, trainer.lr_schedulers = trainer.init_optimizers(
pretrained_model.configure_optimizers())
# test HPC loading / saving
trainer.hpc_save(save_dir, logger)
trainer.hpc_load(save_dir, on_gpu=on_gpu)
def test_optimizer_return_options(tmpdir):
trainer = Trainer(default_root_dir=tmpdir)
model = BoringModel()
# single optimizer
opt_a = optim.Adam(model.parameters(), lr=0.002)
opt_b = optim.SGD(model.parameters(), lr=0.002)
scheduler_a = optim.lr_scheduler.StepLR(opt_a, 10)
scheduler_b = optim.lr_scheduler.StepLR(opt_b, 10)
# single optimizer
model.configure_optimizers = lambda: opt_a
opt, lr_sched, freq = trainer.init_optimizers(model)
assert len(opt) == 1 and len(lr_sched) == len(freq) == 0
# opt tuple
model.configure_optimizers = lambda: (opt_a, opt_b)
opt, lr_sched, freq = trainer.init_optimizers(model)
assert opt == [opt_a, opt_b]
assert len(lr_sched) == len(freq) == 0
# opt list
model.configure_optimizers = lambda: [opt_a, opt_b]
opt, lr_sched, freq = trainer.init_optimizers(model)
assert opt == [opt_a, opt_b]
assert len(lr_sched) == len(freq) == 0
ref_lr_sched = dict(
scheduler=scheduler_a,
interval="epoch",
frequency=1,
reduce_on_plateau=False,
monitor=None,
strict=True,
name=None,
opt_idx=None,
)
# opt tuple of 2 lists
model.configure_optimizers = lambda: ([opt_a], [scheduler_a])
opt, lr_sched, freq = trainer.init_optimizers(model)
assert len(opt) == len(lr_sched) == 1
assert len(freq) == 0
assert opt[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt tuple of 1 list
model.configure_optimizers = lambda: ([opt_a], scheduler_a)
opt, lr_sched, freq = trainer.init_optimizers(model)
assert len(opt) == len(lr_sched) == 1
assert len(freq) == 0
assert opt[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt single dictionary
model.configure_optimizers = lambda: {
"optimizer": opt_a,
"lr_scheduler": scheduler_a
}
opt, lr_sched, freq = trainer.init_optimizers(model)
assert len(opt) == len(lr_sched) == 1
assert len(freq) == 0
assert opt[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt multiple dictionaries with frequencies
model.configure_optimizers = lambda: (
{
"optimizer": opt_a,
"lr_scheduler": scheduler_a,
"frequency": 1
},
{
"optimizer": opt_b,
"lr_scheduler": scheduler_b,
"frequency": 5
},
)
opt, lr_sched, freq = trainer.init_optimizers(model)
assert len(opt) == len(lr_sched) == len(freq) == 2
assert opt[0] == opt_a
ref_lr_sched["opt_idx"] = 0
assert lr_sched[0] == ref_lr_sched
ref_lr_sched["scheduler"] = scheduler_b
ref_lr_sched["opt_idx"] = 1
assert lr_sched[1] == ref_lr_sched
assert freq == [1, 5]
def test_optimizer_return_options():
trainer = Trainer()
model = EvalModelTemplate()
# single optimizer
opt_a = torch.optim.Adam(model.parameters(), lr=0.002)
opt_b = torch.optim.SGD(model.parameters(), lr=0.002)
scheduler_a = torch.optim.lr_scheduler.StepLR(opt_a, 10)
scheduler_b = torch.optim.lr_scheduler.StepLR(opt_b, 10)
# single optimizer
model.configure_optimizers = lambda: opt_a
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == 1 and len(lr_sched) == len(freq) == 0
# opt tuple
model.configure_optimizers = lambda: (opt_a, opt_b)
optim, lr_sched, freq = trainer.init_optimizers(model)
assert optim == [opt_a, opt_b]
assert len(lr_sched) == len(freq) == 0
# opt list
model.configure_optimizers = lambda: [opt_a, opt_b]
optim, lr_sched, freq = trainer.init_optimizers(model)
assert optim == [opt_a, opt_b]
assert len(lr_sched) == len(freq) == 0
ref_lr_sched = dict(
scheduler=scheduler_a,
interval='epoch',
frequency=1,
reduce_on_plateau=False,
monitor=None,
strict=True,
name=None,
)
# opt tuple of 2 lists
model.configure_optimizers = lambda: ([opt_a], [scheduler_a])
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == len(lr_sched) == 1
assert len(freq) == 0
assert optim[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt tuple of 1 list
model.configure_optimizers = lambda: ([opt_a], scheduler_a)
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == len(lr_sched) == 1
assert len(freq) == 0
assert optim[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt single dictionary
model.configure_optimizers = lambda: {
"optimizer": opt_a,
"lr_scheduler": scheduler_a
}
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == len(lr_sched) == 1
assert len(freq) == 0
assert optim[0] == opt_a
assert lr_sched[0] == ref_lr_sched
# opt multiple dictionaries with frequencies
model.configure_optimizers = lambda: (
{
"optimizer": opt_a,
"lr_scheduler": scheduler_a,
"frequency": 1
},
{
"optimizer": opt_b,
"lr_scheduler": scheduler_b,
"frequency": 5
},
)
optim, lr_sched, freq = trainer.init_optimizers(model)
assert len(optim) == len(lr_sched) == len(freq) == 2
assert optim[0] == opt_a
assert lr_sched[0] == ref_lr_sched
assert freq == [1, 5]
