def test_version(tmpdir):
"""Verify versions of loggers are concatenated properly."""
logger1 = CSVLogger(tmpdir, version=0)
logger2 = CSVLogger(tmpdir, version=2)
logger3 = CSVLogger(tmpdir, version=1)
logger4 = CSVLogger(tmpdir, version=0)
loggers = [logger1, logger2, logger3, logger4]
version = _version([])
assert version == ""
version = _version([logger3])
assert version == 1
version = _version(loggers)
assert version == "0_2_1"
version = _version(loggers, "-")
assert version == "0-2-1"
def __resolve_ckpt_dir(self, trainer: "pl.Trainer") -> None:
"""Determines model checkpoint save directory at runtime. Reference attributes from the trainer's logger to
determine where to save checkpoints. The path for saving weights is set in this priority:
1. The ``ModelCheckpoint``'s ``dirpath`` if passed in
2. The ``Trainer``'s ``weights_saved_path`` if passed in (deprecated)
3. The ``Logger``'s ``log_dir`` if the trainer has loggers
4. The ``Trainer``'s ``default_root_dir`` if the trainer has no loggers
The path gets extended with subdirectory "checkpoints".
"""
if self.dirpath is not None:
# short circuit if dirpath was passed to ModelCheckpoint
return
# TODO: Remove weights_save_path logic here in v1.8
if trainer._weights_save_path_internal != trainer.default_root_dir:
# the user has changed weights_save_path
ckpt_path = os.path.join(trainer._weights_save_path_internal,
"checkpoints")
elif trainer.loggers:
if len(trainer.loggers) == 1:
assert trainer.logger is not None
save_dir = trainer.logger.save_dir or trainer.default_root_dir
else:
save_dir = trainer.default_root_dir
name = _name(trainer.loggers)
version = _version(trainer.loggers)
version = version if isinstance(version,
str) else f"version_{version}"
ckpt_path = os.path.join(save_dir, str(name), version,
"checkpoints")
else:
# if no loggers, use default_root_dir
ckpt_path = os.path.join(trainer.default_root_dir, "checkpoints")
ckpt_path = trainer.strategy.broadcast(ckpt_path)
self.dirpath = ckpt_path
def __resolve_ckpt_dir(self, trainer: "pl.Trainer") -> None:
"""Determines model checkpoint save directory at runtime. References attributes from the trainer's logger
to determine where to save checkpoints. The base path for saving weights is set in this priority:
1. Checkpoint callback's path (if passed in)
2. The default_root_dir from trainer if trainer has no logger
3. The weights_save_path from trainer, if user provides it (deprecated)
4. User provided weights_saved_path
The base path gets extended with logger name and version (if these are available)
and subfolder "checkpoints".
"""
if self.dirpath is not None:
return # short circuit
# TODO: Remove weights_save_path logic here in v1.8
if trainer.loggers:
if trainer._weights_save_path_internal != trainer.default_root_dir:
# the user has changed weights_save_path, it overrides anything
save_dir = trainer._weights_save_path_internal
elif len(trainer.loggers) == 1:
save_dir = trainer.logger.save_dir or trainer.default_root_dir
else:
save_dir = trainer.default_root_dir
name = _name(trainer.loggers)
version = _version(trainer.loggers)
version = version if isinstance(version,
str) else f"version_{version}"
ckpt_path = os.path.join(save_dir, str(name), version,
"checkpoints")
else:
ckpt_path = os.path.join(trainer._weights_save_path_internal,
"checkpoints")
ckpt_path = trainer.strategy.broadcast(ckpt_path)
self.dirpath = ckpt_path
def get_standard_metrics(
trainer: "pl.Trainer",
pl_module: "pl.LightningModule") -> Dict[str, Union[int, str]]:
r"""
Returns several standard metrics displayed in the progress bar, including the average loss value,
split index of BPTT (if used) and the version of the experiment when using a logger.
.. code-block::
Epoch 1: 4%|▎ | 40/1095 [00:03<01:37, 10.84it/s, loss=4.501, v_num=10]
Return:
Dictionary with the standard metrics to be displayed in the progress bar.
"""
# call .item() only once but store elements without graphs
running_train_loss = trainer.fit_loop.running_loss.mean()
avg_training_loss = None
if running_train_loss is not None:
avg_training_loss = running_train_loss.cpu().item()
elif pl_module.automatic_optimization:
avg_training_loss = float("NaN")
items_dict: Dict[str, Union[int, str]] = {}
if avg_training_loss is not None:
items_dict["loss"] = f"{avg_training_loss:.3g}"
if pl_module.truncated_bptt_steps > 0:
items_dict["split_idx"] = trainer.fit_loop.split_idx
if trainer.loggers:
version = _version(trainer.loggers)
if version is not None:
if isinstance(version, str):
# show last 4 places of long version strings
version = version[-4:]
items_dict["v_num"] = version
return items_dict