def done(self) -> bool: """Evaluates when to leave the loop. Returns True if trainer.should_stop was set (e.g. by early stopping) or if the maximum number of steps or epochs is reached. """ # TODO(@awaelchli): Move track steps inside training loop and move part of these condition inside training loop stop_steps = _is_max_limit_reached(self.global_step, self.max_steps) stop_epochs = _is_max_limit_reached(self.current_epoch, self.max_epochs) should_stop = False if self.trainer.should_stop: # early stopping met_min_epochs = self.current_epoch >= self.min_epochs if self.min_epochs else True met_min_steps = self.global_step >= self.min_steps if self.min_steps else True if met_min_epochs and met_min_steps: should_stop = True else: log.info( "Trainer was signaled to stop but required minimum epochs" f" ({self.min_epochs}) or minimum steps ({self.min_steps}) has" " not been met. Training will continue..." ) self.trainer.should_stop = should_stop return stop_steps or should_stop or stop_epochs or self.trainer.num_training_batches == 0
def done(self) -> bool: """Evaluates when to leave the loop.""" # TODO(@awaelchli): Move track steps inside training loop and move part of these condition inside training loop stop_steps = _is_max_limit_reached(self.epoch_loop.global_step, self.max_steps) # `processed` is increased before `on_train_epoch_end`, the hook where checkpoints are typically saved. # we use it here because the checkpoint data won't have `completed` increased yet stop_epochs = _is_max_limit_reached( self.epoch_progress.current.processed, self.max_epochs) should_stop = False if self.trainer.should_stop: # early stopping met_min_epochs = self.epoch_progress.current.processed >= self.min_epochs if self.min_epochs else True met_min_steps = self.epoch_loop.global_step >= self.min_steps if self.min_steps else True if met_min_epochs and met_min_steps: should_stop = True else: log.info( "Trainer was signaled to stop but required minimum epochs" f" ({self.min_epochs}) or minimum steps ({self.min_steps}) has" " not been met. Training will continue...") self.trainer.should_stop = should_stop return stop_steps or should_stop or stop_epochs or self.trainer.num_training_batches == 0
def done(self) -> bool: """Evaluates when to leave the loop.""" if self.trainer.num_training_batches == 0: rank_zero_info("`Trainer.fit` stopped: No training batches.") return True # TODO(@awaelchli): Move track steps inside training loop and move part of these condition inside training loop stop_steps = _is_max_limit_reached(self.epoch_loop.global_step, self.max_steps) if stop_steps: rank_zero_info(f"`Trainer.fit` stopped: `max_steps={self.max_steps!r}` reached.") return True # `processed` is increased before `on_train_epoch_end`, the hook where checkpoints are typically saved. # we use it here because the checkpoint data won't have `completed` increased yet stop_epochs = _is_max_limit_reached(self.epoch_progress.current.processed, self.max_epochs) if stop_epochs: # in case they are not equal, override so `trainer.current_epoch` has the expected value self.epoch_progress.current.completed = self.epoch_progress.current.processed rank_zero_info(f"`Trainer.fit` stopped: `max_epochs={self.max_epochs!r}` reached.") return True if self.trainer.should_stop: # early stopping met_min_epochs = self.epoch_progress.current.processed >= self.min_epochs if self.min_epochs else True met_min_steps = self.epoch_loop.global_step >= self.min_steps if self.min_steps else True if met_min_epochs and met_min_steps: self.trainer.should_stop = True rank_zero_debug("`Trainer.fit` stopped: `trainer.should_stop` was set.") return True else: rank_zero_info( f"Trainer was signaled to stop but the required `min_epochs={self.min_epochs!r}` or" f" `min_steps={self.min_steps!r}` has not been met. Training will continue..." ) self.trainer.should_stop = False return False
def _is_training_done(self) -> bool: max_steps_reached = _is_max_limit_reached(self.global_step, self.max_steps) return max_steps_reached or self._num_ready_batches_reached()
def dump_checkpoint(self, weights_only: bool = False) -> dict: """Creating a model checkpoint dictionary object from various component states. Args: weights_only: saving model weights only Return: structured dictionary: { 'epoch': training epoch 'global_step': training global step 'pytorch-lightning_version': The version of PyTorch Lightning that produced this checkpoint 'callbacks': "callback specific state"[] # if not weights_only 'optimizer_states': "PT optim's state_dict"[] # if not weights_only 'lr_schedulers': "PT sched's state_dict"[] # if not weights_only 'native_amp_scaling_state': PT amp's state_dict # if not weights_only and use native amp 'amp_scaling_state': Apex's state_dict # if not weights_only and use apex amp 'state_dict': Model's state_dict (e.g. network weights) CHECKPOINT_HYPER_PARAMS_NAME: CHECKPOINT_HYPER_PARAMS_KEY: CHECKPOINT_HYPER_PARAMS_TYPE: something_cool_i_want_to_save: anything you define through model.on_save_checkpoint LightningDataModule.__class__.__name__: pl DataModule's state } """ # dump epoch/global_step/pytorch-lightning_version current_epoch = self.trainer.current_epoch global_step = self.trainer.global_step has_reached_max_steps = _is_max_limit_reached(global_step, self.trainer.max_steps) global_step += 1 if not has_reached_max_steps: current_epoch += 1 model = self.trainer.lightning_module checkpoint = { "epoch": current_epoch, "global_step": global_step, "pytorch-lightning_version": pl.__version__, "state_dict": self._get_lightning_module_state_dict(), "loops": self._get_loops_state_dict(), } if not weights_only: # dump callbacks checkpoint["callbacks"] = self.trainer._call_callbacks_on_save_checkpoint(checkpoint) optimizer_states = [] for i, optimizer in enumerate(self.trainer.optimizers): # Rely on accelerator to dump optimizer state optimizer_state = self.trainer.strategy.optimizer_state(optimizer) optimizer_states.append(optimizer_state) checkpoint["optimizer_states"] = optimizer_states # dump lr schedulers lr_schedulers = [] for config in self.trainer.lr_scheduler_configs: lr_schedulers.append(config.scheduler.state_dict()) checkpoint["lr_schedulers"] = lr_schedulers self.trainer.precision_plugin.on_save_checkpoint(checkpoint) # dump hyper-parameters if model.hparams: if hasattr(model, "_hparams_name"): checkpoint[pl.LightningModule.CHECKPOINT_HYPER_PARAMS_NAME] = model._hparams_name # dump arguments if _OMEGACONF_AVAILABLE and isinstance(model.hparams, Container): checkpoint[pl.LightningModule.CHECKPOINT_HYPER_PARAMS_KEY] = model.hparams checkpoint[pl.LightningModule.CHECKPOINT_HYPER_PARAMS_TYPE] = type(model.hparams) else: checkpoint[pl.LightningModule.CHECKPOINT_HYPER_PARAMS_KEY] = dict(model.hparams) # give the model a chance to dump a few things model.on_save_checkpoint(checkpoint) if self.trainer.datamodule is not None: self.trainer.datamodule.on_save_checkpoint(checkpoint) # TODO: remove this in v1.8. environment = self.trainer._accelerator_connector.cluster_environment if isinstance(environment, SLURMEnvironment) and environment.auto_requeue: model.on_hpc_save(checkpoint) return checkpoint