def _load_global_step_from_checkpoint_dir(checkpoint_dir):
try:
checkpoint_reader = training.NewCheckpointReader(
training.latest_checkpoint(checkpoint_dir))
return checkpoint_reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)
except: # pylint: disable=bare-except
return 0
def _restore(self, path):
"""Restores this estimator from given path.
Note: will rebuild the graph and initialize all parameters,
and will ignore provided model.
Args:
path: Path to checkpoints and other information.
"""
# Currently Saver requires absolute path to work correctly.
path = os.path.abspath(path)
self._graph = ops.Graph()
with self._graph.as_default():
endpoints_filename = os.path.join(path, 'endpoints')
if not os.path.exists(endpoints_filename):
raise ValueError("Restore folder doesn't contain endpoints.")
with gfile.Open(endpoints_filename) as foutputs:
endpoints = foutputs.read().split('\n')
graph_filename = os.path.join(path, 'graph.pbtxt')
if not os.path.exists(graph_filename):
raise ValueError("Restore folder doesn't contain graph definition.")
with gfile.Open(graph_filename) as fgraph:
graph_def = graph_pb2.GraphDef()
text_format.Merge(fgraph.read(), graph_def)
(self._inp, self._out, self._model_predictions,
self._model_loss) = importer.import_graph_def(
graph_def, name='', return_elements=endpoints)
saver_filename = os.path.join(path, 'saver.pbtxt')
if not os.path.exists(saver_filename):
raise ValueError("Restore folder doesn't contain saver definition.")
with gfile.Open(saver_filename) as fsaver:
saver_def = train.SaverDef()
text_format.Merge(fsaver.read(), saver_def)
self._saver = train.Saver(saver_def=saver_def)
# Restore trainer
self._global_step = self._graph.get_tensor_by_name('global_step:0')
self._train = self._graph.get_operation_by_name('train')
# Restore summaries.
self._summaries = self._graph.get_operation_by_name(
'MergeSummary/MergeSummary')
# Restore session.
if not isinstance(self._config, RunConfig):
self._config = RunConfig(verbose=self.verbose)
self._session = session.Session(self._config.tf_master,
config=self._config.tf_config)
checkpoint_path = train.latest_checkpoint(path)
if checkpoint_path is None:
raise ValueError(
'Missing checkpoint files in the %s. Please '
'make sure you are you have checkpoint file that describes '
'latest checkpoints and appropriate checkpoints are there. '
'If you have moved the folder, you at this point need to '
'update manually update the paths in the checkpoint file.' % path)
self._saver.restore(self._session, checkpoint_path)
# Set to be initialized.
self._initialized = True
def _load_global_step_from_checkpoint_dir(checkpoint_dir):
try:
checkpoint_reader = training.NewCheckpointReader(
training.latest_checkpoint(checkpoint_dir))
return checkpoint_reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)
except: # pylint: disable=bare-except
return 0
def _load_global_step_from_checkpoint_dir(checkpoint_dir):
# noinspection PyBroadException
try:
checkpoint_reader = training.NewCheckpointReader(
training.latest_checkpoint(checkpoint_dir))
return checkpoint_reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)
except:
return 0
def _load_global_step(self):
try:
checkpoint_reader = tf_training.NewCheckpointReader(
tf_training.latest_checkpoint(self._checkpoint_dir))
step = checkpoint_reader.get_tensor(ops.GraphKeys.GLOBAL_STEP)
return step
except Exception as e:
print("Ignored: " + str(e.args))
return 0
def _restore(self, path):
"""Restores this estimator from given path.
Note: will rebuild the graph and initialize all parameters,
and will ignore provided model.
Args:
path: Path to checkpoints and other information.
"""
# Currently Saver requires absolute path to work correctly.
path = os.path.abspath(path)
self._graph = ops.Graph()
with self._graph.as_default():
endpoints_filename = os.path.join(path, 'endpoints')
if not os.path.exists(endpoints_filename):
raise ValueError("Restore folder doesn't contain endpoints.")
with gfile.Open(endpoints_filename) as foutputs:
endpoints = foutputs.read().split('\n')
graph_filename = os.path.join(path, 'graph.pbtxt')
if not os.path.exists(graph_filename):
raise ValueError(
"Restore folder doesn't contain graph definition.")
with gfile.Open(graph_filename) as fgraph:
graph_def = graph_pb2.GraphDef()
text_format.Merge(fgraph.read(), graph_def)
(self._inp, self._out, self._model_predictions,
self._model_loss) = importer.import_graph_def(
graph_def, name='', return_elements=endpoints)
saver_filename = os.path.join(path, 'saver.pbtxt')
if not os.path.exists(saver_filename):
raise ValueError(
"Restore folder doesn't contain saver defintion.")
with gfile.Open(saver_filename) as fsaver:
saver_def = train.SaverDef()
text_format.Merge(fsaver.read(), saver_def)
self._saver = train.Saver(saver_def=saver_def)
# Restore trainer
self._global_step = self._graph.get_tensor_by_name('global_step:0')
self._train = self._graph.get_operation_by_name('train')
# Restore summaries.
self._summaries = self._graph.get_operation_by_name(
'MergeSummary/MergeSummary')
# Restore session.
if not isinstance(self._config, RunConfig):
self._config = RunConfig(verbose=self.verbose)
self._session = session.Session(self._config.tf_master,
config=self._config.tf_config)
checkpoint_path = train.latest_checkpoint(path)
if checkpoint_path is None:
raise ValueError(
"Missing checkpoint files in the %s. Please "
"make sure you are you have checkpoint file that describes "
"latest checkpoints and appropriate checkpoints are there. "
"If you have moved the folder, you at this point need to "
"update manually update the paths in the checkpoint file."
% path)
self._saver.restore(self._session, checkpoint_path)
# Set to be initialized.
self._initialized = True
