def test_sync_batchnorm_ddp(tmpdir):
seed_everything(234)
set_random_main_port()
# define datamodule and dataloader
dm = MNISTDataModule()
dm.prepare_data()
dm.setup(stage=None)
train_dataloader = dm.train_dataloader()
model = SyncBNModule()
bn_outputs = []
# shuffle is false by default
for batch_idx, batch in enumerate(train_dataloader):
x, _ = batch
_, out_bn = model.forward(x, batch_idx)
bn_outputs.append(out_bn)
# get 3 steps
if batch_idx == 2:
break
bn_outputs = [x.cuda() for x in bn_outputs]
# reset datamodule
# batch-size = 16 because 2 GPUs in DDP
dm = MNISTDataModule(batch_size=16, dist_sampler=True)
dm.prepare_data()
dm.setup(stage=None)
model = SyncBNModule(gpu_count=2, bn_targets=bn_outputs)
ddp = DDPSpawnStrategy(
parallel_devices=[torch.device("cuda", 0),
torch.device("cuda", 1)],
num_nodes=1,
sync_batchnorm=True,
cluster_environment=LightningEnvironment(),
find_unused_parameters=True,
)
trainer = Trainer(
default_root_dir=tmpdir,
gpus=2,
num_nodes=1,
strategy=ddp,
max_epochs=1,
max_steps=3,
sync_batchnorm=True,
num_sanity_val_steps=0,
replace_sampler_ddp=False,
)
trainer.fit(model, dm)
# the strategy is responsible for tearing down the batchnorm wrappers
assert not isinstance(model.bn_layer,
torch.nn.modules.batchnorm.SyncBatchNorm)
assert isinstance(model.bn_layer, torch.nn.modules.batchnorm._BatchNorm)
def test_sync_batchnorm_ddp(tmpdir):
seed_everything(234)
set_random_master_port()
# define datamodule and dataloader
dm = MNISTDataModule()
dm.prepare_data()
dm.setup(stage=None)
train_dataloader = dm.train_dataloader()
model = SyncBNModule()
bn_outputs = []
# shuffle is false by default
for batch_idx, batch in enumerate(train_dataloader):
x, _ = batch
_, out_bn = model.forward(x, batch_idx)
bn_outputs.append(out_bn)
# get 3 steps
if batch_idx == 2:
break
bn_outputs = [x.cuda() for x in bn_outputs]
# reset datamodule
# batch-size = 16 because 2 GPUs in DDP
dm = MNISTDataModule(batch_size=16, dist_sampler=True)
dm.prepare_data()
dm.setup(stage=None)
model = SyncBNModule(gpu_count=2, bn_targets=bn_outputs)
ddp = DDPSpawnPlugin(
parallel_devices=[torch.device("cuda", 0),
torch.device("cuda", 1)],
num_nodes=1,
sync_batchnorm=True,
cluster_environment=LightningEnvironment(),
find_unused_parameters=True,
)
trainer = Trainer(
default_root_dir=tmpdir,
gpus=2,
num_nodes=1,
accelerator="ddp_spawn",
max_epochs=1,
max_steps=3,
sync_batchnorm=True,
num_sanity_val_steps=0,
replace_sampler_ddp=False,
plugins=[ddp],
)
trainer.fit(model, dm)
assert trainer.state.finished, "Sync batchnorm failing with DDP"
def test_auto_scale_batch_size_set_model_attribute(tmpdir, use_hparams):
"""Test that new batch size gets written to the correct hyperparameter attribute."""
tutils.reset_seed()
hparams = EvalModelTemplate.get_default_hparams()
before_batch_size = hparams.get("batch_size")
class HparamsEvalModelTemplate(EvalModelTemplate):
def dataloader(self, *args, **kwargs):
# artificially set batch_size so we can get a dataloader
# remove it immediately after, because we want only self.hparams.batch_size
setattr(self, "batch_size", before_batch_size)
dataloader = super().dataloader(*args, **kwargs)
del self.batch_size
return dataloader
datamodule_fit = MNISTDataModule(data_dir=tmpdir,
batch_size=before_batch_size)
model_class = HparamsEvalModelTemplate if use_hparams else EvalModelTemplate
model = model_class(**hparams)
trainer = Trainer(default_root_dir=tmpdir,
max_epochs=1,
auto_scale_batch_size=True,
gpus=1)
trainer.tune(model, datamodule_fit)
after_batch_size = model.hparams.batch_size if use_hparams else model.batch_size
assert trainer.datamodule == datamodule_fit
assert before_batch_size != after_batch_size
assert after_batch_size <= len(trainer.train_dataloader.dataset)
assert datamodule_fit.batch_size == after_batch_size
def test_v1_7_0_datamodule_transform_properties(tmpdir):
dm = MNISTDataModule()
with pytest.deprecated_call(
match=
r"DataModule property `train_transforms` was deprecated in v1.5"):
dm.train_transforms = "a"
with pytest.deprecated_call(
match=r"DataModule property `val_transforms` was deprecated in v1.5"
):
dm.val_transforms = "b"
with pytest.deprecated_call(
match=
r"DataModule property `test_transforms` was deprecated in v1.5"):
dm.test_transforms = "c"
with pytest.deprecated_call(
match=
r"DataModule property `train_transforms` was deprecated in v1.5"):
_ = LightningDataModule(train_transforms="a")
with pytest.deprecated_call(
match=r"DataModule property `val_transforms` was deprecated in v1.5"
):
_ = LightningDataModule(val_transforms="b")
with pytest.deprecated_call(
match=
r"DataModule property `test_transforms` was deprecated in v1.5"):
_ = LightningDataModule(test_transforms="c")
with pytest.deprecated_call(
match=
r"DataModule property `test_transforms` was deprecated in v1.5"):
_ = LightningDataModule(test_transforms="c", dims=(1, 1, 1))
def test_sync_batchnorm_ddp(tmpdir):
seed_everything(234)
set_random_master_port()
# define datamodule and dataloader
dm = MNISTDataModule()
dm.prepare_data()
dm.setup(stage=None)
train_dataloader = dm.train_dataloader()
model = SyncBNModule()
bn_outputs = []
# shuffle is false by default
for batch_idx, batch in enumerate(train_dataloader):
x, _ = batch
_, out_bn = model.forward(x, batch_idx)
bn_outputs.append(out_bn)
# get 3 steps
if batch_idx == 2:
break
bn_outputs = [x.cuda() for x in bn_outputs]
# reset datamodule
# batch-size = 16 because 2 GPUs in DDP
dm = MNISTDataModule(batch_size=16, dist_sampler=True)
dm.prepare_data()
dm.setup(stage=None)
model = SyncBNModule(gpu_count=2, bn_targets=bn_outputs)
trainer = Trainer(gpus=2,
num_nodes=1,
accelerator='ddp_spawn',
max_epochs=1,
max_steps=3,
sync_batchnorm=True,
num_sanity_val_steps=0,
replace_sampler_ddp=False,
plugins=[DDPPlugin(find_unused_parameters=True)])
trainer.fit(model, dm)
assert trainer.state == TrainerState.FINISHED, "Sync batchnorm failing with DDP"
def test_v1_7_0_datamodule_dims_property(tmpdir):
dm = MNISTDataModule()
with pytest.deprecated_call(
match=r"DataModule property `dims` was deprecated in v1.5"):
_ = dm.dims
with pytest.deprecated_call(
match=r"DataModule property `dims` was deprecated in v1.5"):
_ = LightningDataModule(dims=(1, 1, 1))
def test_torchscript_properties(tmpdir, modelclass):
""" Test that scripted LightningModule has unnecessary methods removed. """
model = modelclass()
model.datamodule = MNISTDataModule(tmpdir)
script = model.to_torchscript()
assert not hasattr(script, "datamodule")
assert not hasattr(model, "batch_size") or hasattr(script, "batch_size")
assert not hasattr(model, "learning_rate") or hasattr(script, "learning_rate")
assert not callable(getattr(script, "training_step", None))
def test_v1_7_0_datamodule_size_property(tmpdir):
dm = MNISTDataModule()
with pytest.deprecated_call(match=r"DataModule property `size` was deprecated in v1.5"):
dm.size()