def __load_weights_on_main_process(self) -> None: model = self.lightning_module # load weights if not interrupted if on_colab_kaggle() and model.trainer.state == TrainerState.FITTING: self.load_spawn_weights(model) self._model = model
def __load_weights_on_main_process(self) -> None: model = self.lightning_module # load weights if not interrupted # TODO: check for trainer reference if on_colab_kaggle() and not model.trainer.testing: self.load_spawn_weights(model) self._model = model
def __load_weights_on_main_process(self) -> None: model = self.lightning_module # load weights if not interrupted if on_colab_kaggle() and model.trainer.state == TrainerState.FITTING: rank_zero_warn( "Calling load_spawn_weights from __load_weights_on_main_process" ) self.load_spawn_weights(model) rank_zero_warn("Finished load_weights_on_main_process") self._model = model
def __save_end_of_training_weights(self, model: LightningModule) -> None: # when training ends on these platforms dump weights to get out of the main process if on_colab_kaggle(): rank_zero_warn("cleaning up... please do not interrupt") self.save_spawn_weights(model)
def post_training(self) -> None: model = self.lightning_module if on_colab_kaggle(): rank_zero_warn("cleaning up... please do not interrupt") self.save_spawn_weights(model)