def _do_save(self, trainer: 'pl.Trainer', filepath: str) -> None: # in debugging, track when we save checkpoints trainer.dev_debugger.track_checkpointing_history(filepath) # make paths if trainer.is_global_zero or tpu_training_and_local_rank_zero(trainer): self._fs.makedirs(os.path.dirname(filepath), exist_ok=True) # delegate the saving to the trainer trainer.save_checkpoint(filepath, self.save_weights_only)
def _save_none_monitor_checkpoint( self, trainer: 'pl.Trainer', monitor_candidates: Dict[str, _METRIC]) -> None: if self.monitor is not None or self.save_top_k == 0: return filepath = self._get_metric_interpolated_filepath_name( monitor_candidates, trainer) self._save_model(trainer, filepath) if (self.save_top_k is None and self.best_model_path and self.best_model_path != filepath and (trainer.is_global_zero or tpu_training_and_local_rank_zero(trainer))): self._del_model(self.best_model_path) self.best_model_path = filepath
def _save_last_checkpoint(self, trainer: 'pl.Trainer', monitor_candidates: Dict[str, _METRIC]) -> None: if not self.save_last: return filepath = self._format_checkpoint_name(self.CHECKPOINT_NAME_LAST, monitor_candidates) filepath = os.path.join(self.dirpath, f"{filepath}{self.FILE_EXTENSION}") self._save_model(trainer, filepath) if (self.last_model_path and self.last_model_path != filepath and (trainer.is_global_zero or tpu_training_and_local_rank_zero(trainer))): self._del_model(self.last_model_path) self.last_model_path = filepath
def __resolve_ckpt_dir(self, trainer: 'pl.Trainer') -> None: """ Determines model checkpoint save directory at runtime. References attributes from the trainer's logger to determine where to save checkpoints. The base path for saving weights is set in this priority: 1. Checkpoint callback's path (if passed in) 2. The default_root_dir from trainer if trainer has no logger 3. The weights_save_path from trainer, if user provides it 4. User provided weights_saved_path The base path gets extended with logger name and version (if these are available) and subfolder "checkpoints". """ # Todo: required argument `pl_module` is not used if self.dirpath is not None: return # short circuit if trainer.logger is not None: if trainer.weights_save_path != trainer.default_root_dir: # the user has changed weights_save_path, it overrides anything save_dir = trainer.weights_save_path else: save_dir = trainer.logger.save_dir or trainer.default_root_dir version = (trainer.logger.version if isinstance( trainer.logger.version, str) else f"version_{trainer.logger.version}") ckpt_path = os.path.join(save_dir, str(trainer.logger.name), version, "checkpoints") else: ckpt_path = os.path.join(trainer.weights_save_path, "checkpoints") ckpt_path = trainer.training_type_plugin.broadcast(ckpt_path) self.dirpath = ckpt_path if (not trainer.fast_dev_run and (trainer.is_global_zero or tpu_training_and_local_rank_zero(trainer))): self._fs.makedirs(self.dirpath, exist_ok=True)