def run(
dataset,
pl_model: pl.LightningModule,
name: str,
path: Union[Path, str],
test_path: Union[Path, str],
seed: int,
args=args,
) -> None:
seed_everything(seed, workers=True)
datamodule: pl.LightningDataModule = DataModule(
dataset=dataset,
path=path,
test_path=test_path,
num_workers=8,
batch_size=args.batch_size,
seed=seed,
)
model: pl.LightningModule = pl_model()
callbacks: list[Callback] = build_callbacks()
csv_logger = CSVLogger(
save_dir="csv_logs",
name="seed_" + str(seed),
version=name,
)
if args.fast_dev_run:
trainer_kwargs = {"gpus": None, "auto_select_gpus": False}
else:
trainer_kwargs = {
"gpus": -1,
"auto_select_gpus": True,
"precision": 16
}
trainer: pl.Trainer = pl.Trainer.from_argparse_args(
args,
**trainer_kwargs,
deterministic=True, # ensure reproducible results
default_root_dir="ckpts",
logger=[csv_logger],
log_every_n_steps=10,
callbacks=callbacks,
max_epochs=35,
)
trainer.tune(model=model, datamodule=datamodule)
trainer.fit(model=model, datamodule=datamodule)
if not args.fast_dev_run:
test = trainer.test(model=model,
ckpt_path="best",
datamodule=datamodule)
pd.DataFrame(test).to_csv("csv_logs/seed_" + str(seed) + "_" + name +
"_test.csv")
csv_logger.save()
if args.save_to_hub:
model.model.push_to_hub(f"cjber/{args.save_to_hub}") # type: ignore
def test_file_logger_no_name(tmpdir, name):
"""Verify that None or empty name works."""
logger = CSVLogger(save_dir=tmpdir, name=name)
logger.save()
assert os.path.normpath(
logger.root_dir) == tmpdir # use os.path.normpath to handle trailing /
assert os.listdir(tmpdir / "version_0")
def test_file_logger_named_version(tmpdir):
"""Verify that manual versioning works for string versions, e.g. '2020-02-05-162402'"""
exp_name = "exp"
tmpdir.mkdir(exp_name)
expected_version = "2020-02-05-162402"
logger = CSVLogger(save_dir=tmpdir,
name=exp_name,
version=expected_version)
logger.log_hyperparams({"a": 1, "b": 2})
logger.save()
assert logger.version == expected_version
assert os.listdir(tmpdir / exp_name) == [expected_version]
assert os.listdir(tmpdir / exp_name / expected_version)
def test_file_logger_log_metrics(tmpdir, step_idx):
logger = CSVLogger(tmpdir)
metrics = {
"float": 0.3,
"int": 1,
"FloatTensor": torch.tensor(0.1),
"IntTensor": torch.tensor(1)
}
logger.log_metrics(metrics, step_idx)
logger.save()
path_csv = os.path.join(logger.log_dir, ExperimentWriter.NAME_METRICS_FILE)
with open(path_csv) as fp:
lines = fp.readlines()
assert len(lines) == 2
assert all(n in lines[0] for n in metrics)
def test_file_logger_log_hyperparams(tmpdir):
logger = CSVLogger(tmpdir)
hparams = {
"float": 0.3,
"int": 1,
"string": "abc",
"bool": True,
"dict": {
"a": {
"b": "c"
}
},
"list": [1, 2, 3],
"layer": torch.nn.BatchNorm1d,
}
logger.log_hyperparams(hparams)
logger.save()
path_yaml = os.path.join(logger.log_dir,
ExperimentWriter.NAME_HPARAMS_FILE)
params = load_hparams_from_yaml(path_yaml)
assert all(n in params for n in hparams)
def test_flush_n_steps(tmpdir):
logger = CSVLogger(tmpdir, flush_logs_every_n_steps=2)
metrics = {
"float": 0.3,
"int": 1,
"FloatTensor": torch.tensor(0.1),
"IntTensor": torch.tensor(1)
}
logger.save = MagicMock()
logger.log_metrics(metrics, step=0)
logger.save.assert_not_called()
logger.log_metrics(metrics, step=1)
logger.save.assert_called_once()
def test_file_logger_log_hyperparams(tmpdir):
logger = CSVLogger(tmpdir)
hparams = {
"float": 0.3,
"int": 1,
"string": "abc",
"bool": True,
"dict": {
'a': {
'b': 'c'
}
},
"list": [1, 2, 3],
"namespace": Namespace(foo=Namespace(bar='buzz')),
"layer": torch.nn.BatchNorm1d
}
logger.log_hyperparams(hparams)
logger.save()
path_yaml = os.path.join(logger.log_dir,
ExperimentWriter.NAME_HPARAMS_FILE)
params = load_hparams_from_yaml(path_yaml)
assert all([n in params for n in hparams])
def test_file_logger_no_name(tmpdir, name):
"""Verify that None or empty name works"""
logger = CSVLogger(save_dir=tmpdir, name=name)
logger.save()
assert logger.root_dir == tmpdir
assert os.listdir(tmpdir / "version_0")