def _create_embedded_rollout_worker(kwargs, send_fn): """Create a local rollout worker and a thread that samples from it. Args: kwargs (dict): args for the RolloutWorker constructor. send_fn (fn): function to send a JSON request to the server. """ # Since the server acts as an input datasource, we have to reset the # input config to the default, which runs env rollouts. kwargs = kwargs.copy() del kwargs["input_creator"] # Since the server also acts as an output writer, we might have to reset # the output config to the default, i.e. "output": None, otherwise a # local rollout worker might write to an unknown output directory del kwargs["output_creator"] # If server has no env (which is the expected case): # Generate a dummy ExternalEnv here using RandomEnv and the # given observation/action spaces. if kwargs["policy_config"].get("env") is None: from ray.rllib.examples.env.random_env import RandomEnv, RandomMultiAgentEnv config = { "action_space": kwargs["policy_config"]["action_space"], "observation_space": kwargs["policy_config"]["observation_space"], } _, is_ma = check_multi_agent(kwargs["policy_config"]) kwargs["env_creator"] = _auto_wrap_external( lambda _: (RandomMultiAgentEnv if is_ma else RandomEnv)(config) ) kwargs["policy_config"]["env"] = True # Otherwise, use the env specified by the server args. else: real_env_creator = kwargs["env_creator"] kwargs["env_creator"] = _auto_wrap_external(real_env_creator) logger.info("Creating rollout worker with kwargs={}".format(kwargs)) from ray.rllib.evaluation.rollout_worker import RolloutWorker rollout_worker = RolloutWorker(**kwargs) inference_thread = _LocalInferenceThread(rollout_worker, send_fn) inference_thread.start() return rollout_worker, inference_thread
def check_train_results(train_results): """Checks proper structure of a Trainer.train() returned dict. Args: train_results: The train results dict to check. Raises: AssertionError: If `train_results` doesn't have the proper structure or data in it. """ # Import these here to avoid circular dependencies. from ray.rllib.policy.sample_batch import DEFAULT_POLICY_ID from ray.rllib.utils.metrics.learner_info import LEARNER_INFO, \ LEARNER_STATS_KEY from ray.rllib.utils.pre_checks.multi_agent import check_multi_agent # Assert that some keys are where we would expect them. for key in [ "agent_timesteps_total", "config", "custom_metrics", "episode_len_mean", "episode_reward_max", "episode_reward_mean", "episode_reward_min", "episodes_total", "hist_stats", "info", "iterations_since_restore", "num_healthy_workers", "perf", "policy_reward_max", "policy_reward_mean", "policy_reward_min", "sampler_perf", "time_since_restore", "time_this_iter_s", "timesteps_since_restore", "timesteps_total", "timers", "time_total_s", "training_iteration", ]: assert key in train_results, \ f"'{key}' not found in `train_results` ({train_results})!" _, is_multi_agent = check_multi_agent(train_results["config"]) # Check in particular the "info" dict. info = train_results["info"] assert LEARNER_INFO in info, \ f"'learner' not in train_results['infos'] ({info})!" assert "num_steps_trained" in info or "num_env_steps_trained" in info, \ f"'num_(env_)?steps_trained' not in train_results['infos'] ({info})!" learner_info = info[LEARNER_INFO] # Make sure we have a default_policy key if we are not in a # multi-agent setup. if not is_multi_agent: # APEX algos sometimes have an empty learner info dict (no metrics # collected yet). assert len(learner_info) == 0 or DEFAULT_POLICY_ID in learner_info, \ f"'{DEFAULT_POLICY_ID}' not found in " \ f"train_results['infos']['learner'] ({learner_info})!" for pid, policy_stats in learner_info.items(): if pid == "batch_count": continue # Expect td-errors to be per batch-item. if "td_error" in policy_stats: configured_b = train_results["config"]["train_batch_size"] actual_b = policy_stats["td_error"].shape[0] # R2D2 case. if (configured_b - actual_b) / actual_b > 0.1: assert configured_b / ( train_results["config"]["model"]["max_seq_len"] + train_results["config"]["burn_in"]) == actual_b # Make sure each policy has the LEARNER_STATS_KEY under it. assert LEARNER_STATS_KEY in policy_stats learner_stats = policy_stats[LEARNER_STATS_KEY] for key, value in learner_stats.items(): # Min- and max-stats should be single values. if key.startswith("min_") or key.startswith("max_"): assert np.isscalar( value), f"'key' value not a scalar ({value})!" return train_results