def test_final_ops_is_scalar_tensor(self): with ops.Graph().as_default(): expected_value = 4 final_ops = constant_op.constant(expected_value) hook = basic_session_run_hooks.FinalOpsHook(final_ops) hook.begin() with session_lib.Session() as session: hook.end(session) self.assertEqual(expected_value, hook.final_ops_values)
def test_final_ops_is_tensor(self): with ops.Graph().as_default(): expected_values = [1, 6, 3, 5, 2, 4] final_ops = constant_op.constant(expected_values) hook = basic_session_run_hooks.FinalOpsHook(final_ops) hook.begin() with session_lib.Session() as session: hook.end(session) self.assertListEqual(expected_values, hook.final_ops_values.tolist())
def test_final_ops_with_dictionary(self): with ops.Graph().as_default(): expected_values = [4, -3] final_ops = array_ops.placeholder(dtype=dtypes.float32) final_ops_feed_dict = {final_ops: expected_values} hook = basic_session_run_hooks.FinalOpsHook( final_ops, final_ops_feed_dict) hook.begin() with session_lib.Session() as session: hook.end(session) self.assertListEqual(expected_values, hook.final_ops_values.tolist())
def evaluate_repeatedly(checkpoint_dir, master='', scaffold=None, eval_ops=None, feed_dict=None, final_ops=None, final_ops_feed_dict=None, eval_interval_secs=60, hooks=None, config=None, max_number_of_evaluations=None, timeout=None, timeout_fn=None): """Repeatedly searches for a checkpoint in `checkpoint_dir` and evaluates it. During a single evaluation, the `eval_ops` is run until the session is interrupted or requested to finish. This is typically requested via a `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running the requested number of times. Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is evaluated a single time after `eval_ops` has finished running and the fetched values of `final_ops` are returned. If `final_ops` is left as `None`, then `None` is returned. One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record summaries after the `eval_ops` have run. If `eval_ops` is `None`, the summaries run immedietly after the model checkpoint has been restored. Note that `evaluate_once` creates a local variable used to track the number of evaluations run via `tf.contrib.training.get_or_create_eval_step`. Consequently, if a custom local init op is provided via a `scaffold`, the caller should ensure that the local init op also initializes the eval step. Args: checkpoint_dir: The directory where checkpoints are stored. master: The BNS address of the TensorFlow master. scaffold: An tf.train.Scaffold instance for initializing variables and restoring variables. Note that `scaffold.init_fn` is used by the function to restore the checkpoint. If you supply a custom init_fn, then it must also take care of restoring the model from its checkpoint. eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to `Tensors`, which is run until the session is requested to stop, commonly done by a `tf.contrib.training.StopAfterNEvalsHook`. feed_dict: The feed dictionary to use when executing the `eval_ops`. final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to `Tensors`. final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`. eval_interval_secs: The minimum number of seconds between evaluations. hooks: List of `tf.train.SessionRunHook` callbacks which are run inside the evaluation loop. config: An instance of `tf.ConfigProto` that will be used to configure the `Session`. If left as `None`, the default will be used. max_number_of_evaluations: The maximum times to run the evaluation. If left as `None`, then evaluation runs indefinitely. timeout: The maximum amount of time to wait between checkpoints. If left as `None`, then the process will wait indefinitely. timeout_fn: Optional function to call after a timeout. If the function returns True, then it means that no new checkpoints will be generated and the iterator will exit. The function is called with no arguments. Returns: The fetched values of `final_ops` or `None` if `final_ops` is `None`. """ eval_step = get_or_create_eval_step() # Prepare the run hooks. hooks = hooks or [] if eval_ops is not None: update_eval_step = state_ops.assign_add(eval_step, 1) for h in hooks: if isinstance(h, StopAfterNEvalsHook): h._set_evals_completed_tensor(update_eval_step) # pylint: disable=protected-access if isinstance(eval_ops, dict): eval_ops['update_eval_step'] = update_eval_step elif isinstance(eval_ops, (tuple, list)): eval_ops = list(eval_ops) + [update_eval_step] else: eval_ops = [eval_ops, update_eval_step] final_ops_hook = basic_session_run_hooks.FinalOpsHook( final_ops, final_ops_feed_dict) hooks.append(final_ops_hook) num_evaluations = 0 for checkpoint_path in checkpoints_iterator( checkpoint_dir, min_interval_secs=eval_interval_secs, timeout=timeout, timeout_fn=timeout_fn): session_creator = monitored_session.ChiefSessionCreator( scaffold=scaffold, checkpoint_filename_with_path=checkpoint_path, master=master, config=config) with monitored_session.MonitoredSession( session_creator=session_creator, hooks=hooks) as session: logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) if eval_ops is not None: while not session.should_stop(): session.run(eval_ops, feed_dict) logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) num_evaluations += 1 if (max_number_of_evaluations is not None and num_evaluations >= max_number_of_evaluations): return final_ops_hook.final_ops_values return final_ops_hook.final_ops_values
def _evaluate_once(checkpoint_path, master='', scaffold=None, eval_ops=None, feed_dict=None, final_ops=None, final_ops_feed_dict=None, hooks=None, config=None): """Evaluates the model at the given checkpoint path. During a single evaluation, the `eval_ops` is run until the session is interrupted or requested to finish. This is typically requested via a `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running the requested number of times. Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is evaluated a single time after `eval_ops` has finished running and the fetched values of `final_ops` are returned. If `final_ops` is left as `None`, then `None` is returned. One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record summaries after the `eval_ops` have run. If `eval_ops` is `None`, the summaries run immediately after the model checkpoint has been restored. Note that `evaluate_once` creates a local variable used to track the number of evaluations run via `tf.contrib.training.get_or_create_eval_step`. Consequently, if a custom local init op is provided via a `scaffold`, the caller should ensure that the local init op also initializes the eval step. Args: checkpoint_path: The path to a checkpoint to use for evaluation. master: The BNS address of the TensorFlow master. scaffold: An tf.train.Scaffold instance for initializing variables and restoring variables. Note that `scaffold.init_fn` is used by the function to restore the checkpoint. If you supply a custom init_fn, then it must also take care of restoring the model from its checkpoint. eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to `Tensors`, which is run until the session is requested to stop, commonly done by a `tf.contrib.training.StopAfterNEvalsHook`. feed_dict: The feed dictionary to use when executing the `eval_ops`. final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to `Tensors`. final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`. hooks: List of `tf.train.SessionRunHook` callbacks which are run inside the evaluation loop. config: An instance of `tf.ConfigProto` that will be used to configure the `Session`. If left as `None`, the default will be used. Returns: The fetched values of `final_ops` or `None` if `final_ops` is `None`. """ eval_step = _get_or_create_eval_step() # Prepare the run hooks. hooks = list(hooks or []) if eval_ops is not None: if any(isinstance(h, _MultiStepStopAfterNEvalsHook) for h in hooks): steps_per_run_variable = \ basic_session_run_hooks.get_or_create_steps_per_run_variable() update_eval_step = state_ops.assign_add( eval_step, math_ops.cast(steps_per_run_variable, dtype=eval_step.dtype), use_locking=True) else: update_eval_step = state_ops.assign_add(eval_step, 1, use_locking=True) if isinstance(eval_ops, dict): eval_ops['update_eval_step'] = update_eval_step elif isinstance(eval_ops, (tuple, list)): eval_ops = list(eval_ops) + [update_eval_step] else: eval_ops = [eval_ops, update_eval_step] eval_step_value = _get_latest_eval_step_value(eval_ops) for h in hooks: if isinstance(h, (_StopAfterNEvalsHook, _MultiStepStopAfterNEvalsHook)): h._set_evals_completed_tensor(eval_step_value) # pylint: disable=protected-access logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())) # Prepare the session creator. session_creator = monitored_session.ChiefSessionCreator( scaffold=scaffold, checkpoint_filename_with_path=checkpoint_path, master=master, config=config) final_ops_hook = basic_session_run_hooks.FinalOpsHook( final_ops, final_ops_feed_dict) hooks.append(final_ops_hook) with monitored_session.MonitoredSession( session_creator=session_creator, hooks=hooks) as session: if eval_ops is not None: while not session.should_stop(): session.run(eval_ops, feed_dict) logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime())) return final_ops_hook.final_ops_values