def evaluate( summary_dir: Text, environment_name: Text, policy: py_tf_eager_policy.PyTFEagerPolicyBase, variable_container: reverb_variable_container.ReverbVariableContainer, suite_load_fn: Callable[ [Text], py_environment.PyEnvironment] = suite_mujoco.load, additional_metrics: Optional[Iterable[py_metric.PyStepMetric]] = None, is_running: Optional[Callable[[], bool]] = None) -> None: """Evaluates a policy iteratively fetching weights from variable container. Args: summary_dir: Directory which is used to store the summaries. environment_name: Name of the environment used to evaluate the policy. policy: The policy being evaluated. The weights of this policy are fetched from the variable container periodically. variable_container: Provides weights for the policy. suite_load_fn: Function that loads the environment (by calling it with the name of the environment) from a particular suite. additional_metrics: Optional collection of metrics that are computed as well during the evaluation. By default (`None`) it is empty. is_running: Optional callable which controls the running of the main evaluation loop (including fetching weights from the variable container and running the eval actor periodically). By default (`None`) this is a callable always returning `True` resulting in an infinite evaluation loop. """ additional_metrics = additional_metrics or [] is_running = is_running or (lambda: True) environment = suite_load_fn(environment_name) # Create the variable container. train_step = train_utils.create_train_step() variables = { reverb_variable_container.POLICY_KEY: policy.variables(), reverb_variable_container.TRAIN_STEP_KEY: train_step } variable_container.update(variables) prev_train_step_value = train_step.numpy() # Create the evaluator actor. eval_actor = actor.Actor(environment, policy, train_step, episodes_per_run=1, summary_dir=summary_dir, metrics=actor.collect_metrics(buffer_size=1) + additional_metrics, name='eval_actor') # Run the experience evaluation loop. while is_running(): eval_actor.run() logging.info('Evaluating using greedy policy at step: %d', train_step.numpy()) def is_train_step_the_same_or_behind(): # Checks if the `train_step` received from variable conainer is the same # (or behind) the latest evaluated train step (`prev_train_step_value`). variable_container.update(variables) return train_step.numpy() <= prev_train_step_value train_utils.wait_for_predicate( wait_predicate_fn=is_train_step_the_same_or_behind) prev_train_step_value = train_step.numpy()
def test_wait_for_predicate_second_false(self): """Tests predicate returning False on second call.""" predicate_mock = mock.MagicMock(side_effect=[True, False]) # 10 retry limit to avoid a near infinite loop on an error. train_utils.wait_for_predicate(predicate_mock, num_retries=10) self.assertEqual(predicate_mock.call_count, 2)
def test_wait_for_predicate_timeout(self): """Tests predicate returning True forever and then timing out.""" predicate_mock = mock.MagicMock(side_effect=[True, True, True]) with self.assertRaises(TimeoutError): train_utils.wait_for_predicate(predicate_mock, num_retries=3)
def evaluate( summary_dir: Text, environment_name: Text, policy: py_tf_eager_policy.PyTFEagerPolicyBase, variable_container: reverb_variable_container.ReverbVariableContainer, suite_load_fn: Callable[[Text], py_environment.PyEnvironment] = suite_mujoco.load, additional_metrics: Optional[Iterable[py_metric.PyStepMetric]] = None, is_running: Optional[Callable[[], bool]] = None, eval_interval: int = 1000, eval_episodes: int = 1, # TODO(b/178225158): Deprecate in favor of the reporting libray when ready. return_reporting_fn: Optional[Callable[[int, float], None]] = None ) -> None: """Evaluates a policy iteratively fetching weights from variable container. Args: summary_dir: Directory which is used to store the summaries. environment_name: Name of the environment used to evaluate the policy. policy: The policy being evaluated. The weights of this policy are fetched from the variable container periodically. variable_container: Provides weights for the policy. suite_load_fn: Function that loads the environment (by calling it with the name of the environment) from a particular suite. additional_metrics: Optional collection of metrics that are computed as well during the evaluation. By default (`None`) it is empty. is_running: Optional callable which controls the running of the main evaluation loop (including fetching weights from the variable container and running the eval actor periodically). By default (`None`) this is a callable always returning `True` resulting in an infinite evaluation loop. eval_interval: If set, eval is done at the given step interval or as close as possible based on polling. eval_episodes: Number of episodes to eval. return_reporting_fn: Optional callback function of the form `fn(train_step, average_return)` which reports the average return to a custom destination. """ additional_metrics = additional_metrics or [] is_running = is_running or (lambda: True) environment = suite_load_fn(environment_name) # Create the variable container. train_step = train_utils.create_train_step() variables = { reverb_variable_container.POLICY_KEY: policy.variables(), reverb_variable_container.TRAIN_STEP_KEY: train_step } variable_container.update(variables) prev_train_step_value = train_step.numpy() # Create the evaluator actor. metrics = actor.collect_metrics(buffer_size=eval_episodes) if return_reporting_fn: for m in metrics: if isinstance(m, py_metrics.AverageReturnMetric): average_return_metric = m break eval_actor = actor.Actor(environment, policy, train_step, episodes_per_run=eval_episodes, summary_dir=summary_dir, summary_interval=eval_interval, metrics=metrics + additional_metrics, name='eval_actor') # Run the experience evaluation loop. last_eval_step = 0 while is_running(): # Eval every step if no `eval_interval` is set, or if on the first step, or # if the step is equal or greater than `last_eval_step` + `eval_interval`. # It is very possible when logging a specific interval that the steps evaled # will not be exact, e.g. 1001 and then 2003 vs. 1000 and then 2000. if (train_step.numpy() == 0 or train_step.numpy() >= eval_interval + last_eval_step): logging.info('Evaluating using greedy policy at step: %d', train_step.numpy()) eval_actor.run() last_eval_step = train_step.numpy() def is_train_step_the_same_or_behind(): # Checks if the `train_step` received from variable conainer is the same # (or behind) the latest evaluated train step (`prev_train_step_value`). variable_container.update(variables) return train_step.numpy() <= prev_train_step_value train_utils.wait_for_predicate( wait_predicate_fn=is_train_step_the_same_or_behind) prev_train_step_value = train_step.numpy() # Optionally report the average return metric via a callback. if return_reporting_fn: return_reporting_fn(train_step.numpy(), average_return_metric.result())