def __load_weights_on_main_process(self) -> None:
model = self.lightning_module
# load weights if not interrupted
if on_colab_kaggle() and model.trainer.state == TrainerState.FITTING:
self.load_spawn_weights(model)
self._model = model
def __load_weights_on_main_process(self) -> None:
model = self.lightning_module
# load weights if not interrupted
# TODO: check for trainer reference
if on_colab_kaggle() and not model.trainer.testing:
self.load_spawn_weights(model)
self._model = model
def __load_weights_on_main_process(self) -> None:
model = self.lightning_module
# load weights if not interrupted
if on_colab_kaggle() and model.trainer.state == TrainerState.FITTING:
rank_zero_warn(
"Calling load_spawn_weights from __load_weights_on_main_process"
)
self.load_spawn_weights(model)
rank_zero_warn("Finished load_weights_on_main_process")
self._model = model
def __save_end_of_training_weights(self, model: LightningModule) -> None:
# when training ends on these platforms dump weights to get out of the main process
if on_colab_kaggle():
rank_zero_warn("cleaning up... please do not interrupt")
self.save_spawn_weights(model)
def post_training(self) -> None:
model = self.lightning_module
if on_colab_kaggle():
rank_zero_warn("cleaning up... please do not interrupt")
self.save_spawn_weights(model)
