def _set_completed_steps_by_checkpoint(self, checkpoint_dir_for_init):
if not checkpoint_dir_for_init:
return
if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init):
raise ValueError("Invalid checkpoint directory {}".format(
checkpoint_dir_for_init))
self._completed_steps = CheckpointSaver.get_version_from_checkpoint(
checkpoint_dir_for_init)
def _set_completed_steps_by_checkpoint(self, checkpoint_dir_for_init):
if not checkpoint_dir_for_init:
return
if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init):
raise ValueError("Invalid checkpoint directory {}".format(
checkpoint_dir_for_init))
model_verion = CheckpointSaver.get_version_from_checkpoint(
checkpoint_dir_for_init)
for callback in self.callbacks_list.callbacks:
if isinstance(callback, MaxStepsStopping):
callback.set_completed_steps(model_verion)
def _restore_params_from_checkpoint(self, checkpoint_dir_for_init):
"""Restore parameters from a checkpint directory for the PS instance
"""
if not checkpoint_dir_for_init:
self.logger.info("checkpoint directory for init is None")
return
if not CheckpointSaver.check_checkpoint_valid(checkpoint_dir_for_init):
raise ValueError("Invalid checkpoint directory")
self.parameters = CheckpointSaver.restore_params_from_checkpoint(
checkpoint_dir_for_init, self.ps_id, self.num_ps_pods)
self.parameters.init_status = True
self.logger.info("The version of restored parameters is %d" %
self.parameters.version)