def _set_completed_steps_by_checkpoint(self, checkpoint_dir_for_init): if not checkpoint_dir_for_init: return if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init): raise ValueError("Invalid checkpoint directory {}".format( checkpoint_dir_for_init)) self._completed_steps = CheckpointSaver.get_version_from_checkpoint( checkpoint_dir_for_init)
def _set_completed_steps_by_checkpoint(self, checkpoint_dir_for_init): if not checkpoint_dir_for_init: return if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init): raise ValueError("Invalid checkpoint directory {}".format( checkpoint_dir_for_init)) model_verion = CheckpointSaver.get_version_from_checkpoint( checkpoint_dir_for_init) for callback in self.callbacks_list.callbacks: if isinstance(callback, MaxStepsStopping): callback.set_completed_steps(model_verion)
def _restore_params_from_checkpoint(self, checkpoint_dir_for_init): """Restore parameters from a checkpint directory for the PS instance """ if not checkpoint_dir_for_init: self.logger.info("checkpoint directory for init is None") return if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init): raise ValueError("Invalid checkpoint directory") self.parameters = CheckpointSaver.restore_params_from_checkpoint( checkpoint_dir_for_init, self.ps_id, self.num_ps_pods) self.parameters.init_status = True self.logger.info("The version of restored parameters is %d" % self.parameters.version)