def __init__(self, env_name, render, shared, maddpg, evaluate, env_num):
"""Instantiate the sampler object.
Parameters
----------
env_name : str
the name of the environment
render : bool
whether to render the environment
shared : bool
specifies whether agents in an environment are meant to share
policies. This is solely used by multi-agent Flow environments.
maddpg : bool
whether to use an environment variant that is compatible with the
MADDPG algorithm
evaluate : bool
specifies whether this is a training or evaluation environment
env_num : int
the environment number. Used to handle situations when multiple
parallel environments are being used.
"""
self.env, self._init_obs = create_env(
env=env_name,
render=render,
shared=shared,
maddpg=maddpg,
evaluate=evaluate,
)
self._env_num = env_num
self._render = render
def __init__(self, env_name, render, shared, maddpg, evaluate, env_num):
"""Instantiate the sampler object.
Parameters
----------
env_name : str
the name of the environment
render : bool
whether to render the environment
shared : bool
specifies whether agents in an environment are meant to share
policies. This is solely used by multi-agent Flow environments.
maddpg : bool
whether to use an environment variant that is compatible with the
MADDPG algorithm
evaluate : bool
specifies whether this is a training or evaluation environment
env_num : int
the environment number. Used to handle situations when multiple
parallel environments are being used.
"""
self.env, self._init_obs = create_env(
env=env_name,
render=render,
shared=shared,
maddpg=maddpg,
evaluate=evaluate,
)
# Collect the key for the info_dict variable.
if isinstance(self.env.action_space, dict):
initial_action = {
key: self.env.action_space[key].sample()
for key in self.env.action_space.keys()
}
elif env_name.startswith("multiagent") and shared:
init_obs = self._init_obs["obs"] if maddpg else self._init_obs
initial_action = {
key: self.env.action_space.sample()
for key in init_obs.keys()
}
else:
initial_action = self.env.action_space.sample()
_, _, _, info_dict = self.env.step(initial_action)
self._env_num = env_num
self._render = render
self._info_keys = list(info_dict.keys())
def __init__(self,
policy,
env,
eval_env=None,
nb_train_steps=1,
nb_rollout_steps=1,
nb_eval_episodes=50,
actor_update_freq=2,
meta_update_freq=10,
reward_scale=1.,
render=False,
render_eval=False,
eval_deterministic=True,
verbose=0,
policy_kwargs=None,
_init_setup_model=True):
"""Instantiate the algorithm object.
Parameters
----------
policy : type [ hbaselines.base_policies.ActorCriticPolicy ]
the policy model to use
env : gym.Env or str
the environment to learn from (if registered in Gym, can be str)
eval_env : gym.Env or str
the environment to evaluate from (if registered in Gym, can be str)
nb_train_steps : int
the number of training steps
nb_rollout_steps : int
the number of rollout steps
nb_eval_episodes : int
the number of evaluation episodes
actor_update_freq : int
number of training steps per actor policy update step. The critic
policy is updated every training step.
meta_update_freq : int
number of training steps per meta policy update step. The actor
policy of the meta-policy is further updated at the frequency
provided by the actor_update_freq variable. Note that this value is
only relevant when using the GoalConditionedPolicy policy.
reward_scale : float
the value the reward should be scaled by
render : bool
enable rendering of the training environment
render_eval : bool
enable rendering of the evaluation environment
eval_deterministic : bool
if set to True, the policy provides deterministic actions to the
evaluation environment. Otherwise, stochastic or noisy actions are
returned.
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
policy_kwargs : dict
policy-specific hyperparameters
_init_setup_model : bool
Whether or not to build the network at the creation of the instance
"""
shared = False if policy_kwargs is None else \
policy_kwargs.get("shared", False)
maddpg = False if policy_kwargs is None else \
policy_kwargs.get("maddpg", False)
self.policy = policy
self.env_name = deepcopy(env) if isinstance(env, str) \
else env.__str__()
self.env = create_env(env, render, shared, maddpg, evaluate=False)
self.eval_env = create_env(eval_env,
render_eval,
shared,
maddpg,
evaluate=True)
self.nb_train_steps = nb_train_steps
self.nb_rollout_steps = nb_rollout_steps
self.nb_eval_episodes = nb_eval_episodes
self.actor_update_freq = actor_update_freq
self.meta_update_freq = meta_update_freq
self.reward_scale = reward_scale
self.render = render
self.render_eval = render_eval
self.eval_deterministic = eval_deterministic
self.verbose = verbose
self.action_space = self.env.action_space
self.observation_space = self.env.observation_space
self.context_space = getattr(self.env, "context_space", None)
self.policy_kwargs = {'verbose': verbose}
# add the default policy kwargs to the policy_kwargs term
if is_feedforward_policy(policy):
self.policy_kwargs.update(FEEDFORWARD_PARAMS.copy())
elif is_goal_conditioned_policy(policy):
self.policy_kwargs.update(GOAL_CONDITIONED_PARAMS.copy())
self.policy_kwargs['env_name'] = self.env_name.__str__()
elif is_multiagent_policy(policy):
self.policy_kwargs.update(MULTI_FEEDFORWARD_PARAMS.copy())
self.policy_kwargs["all_ob_space"] = getattr(
self.env, "all_observation_space",
Box(-1, 1, (1, ), dtype=np.float32))
if is_td3_policy(policy):
self.policy_kwargs.update(TD3_PARAMS.copy())
elif is_sac_policy(policy):
self.policy_kwargs.update(SAC_PARAMS.copy())
self.policy_kwargs.update(policy_kwargs or {})
# Compute the time horizon, which is used to check if an environment
# terminated early and used to compute the done mask as per TD3
# implementation (see appendix A of their paper). If the horizon cannot
# be found, it is assumed to be 500 (default value for most gym
# environments).
if hasattr(self.env, "horizon"):
self.horizon = self.env.horizon
elif hasattr(self.env, "_max_episode_steps"):
self.horizon = self.env._max_episode_steps
elif hasattr(self.env, "env_params"):
# for Flow environments
self.horizon = self.env.env_params.horizon
else:
raise ValueError("Horizon attribute not found.")
# init
self.graph = None
self.policy_tf = None
self.sess = None
self.summary = None
self.obs = None
self.all_obs = None
self.episode_step = 0
self.episodes = 0
self.total_steps = 0
self.epoch_episode_steps = []
self.epoch_episode_rewards = []
self.epoch_episodes = 0
self.epoch = 0
self.episode_rew_history = deque(maxlen=100)
self.episode_reward = 0
self.rew_ph = None
self.rew_history_ph = None
self.eval_rew_ph = None
self.eval_success_ph = None
self.saver = None
# Append the fingerprint dimension to the observation dimension, if
# needed.
if self.policy_kwargs.get("use_fingerprints", False):
fingerprint_range = self.policy_kwargs["fingerprint_range"]
low = np.concatenate(
(self.observation_space.low, fingerprint_range[0]))
high = np.concatenate(
(self.observation_space.high, fingerprint_range[1]))
self.observation_space = Box(low=low, high=high, dtype=np.float32)
# Create the model variables and operations.
if _init_setup_model:
self.trainable_vars = self.setup_model()
def __init__(self,
policy,
env,
eval_env=None,
nb_train_steps=1,
nb_rollout_steps=1,
nb_eval_episodes=50,
actor_update_freq=2,
meta_update_freq=10,
reward_scale=1.,
render=False,
render_eval=False,
eval_deterministic=True,
save_replay_buffer=False,
num_envs=1,
verbose=0,
policy_kwargs=None,
_init_setup_model=True):
"""Instantiate the algorithm object.
Parameters
----------
policy : type [ hbaselines.base_policies.ActorCriticPolicy ]
the policy model to use
env : gym.Env or str
the environment to learn from (if registered in Gym, can be str)
eval_env : gym.Env or str
the environment to evaluate from (if registered in Gym, can be str)
nb_train_steps : int
the number of training steps
nb_rollout_steps : int
the number of rollout steps
nb_eval_episodes : int
the number of evaluation episodes
actor_update_freq : int
number of training steps per actor policy update step. The critic
policy is updated every training step.
meta_update_freq : int
number of training steps per meta policy update step. The actor
policy of the meta-policy is further updated at the frequency
provided by the actor_update_freq variable. Note that this value is
only relevant when using the GoalConditionedPolicy policy.
reward_scale : float
the value the reward should be scaled by
render : bool
enable rendering of the training environment
render_eval : bool
enable rendering of the evaluation environment
eval_deterministic : bool
if set to True, the policy provides deterministic actions to the
evaluation environment. Otherwise, stochastic or noisy actions are
returned.
save_replay_buffer : bool
whether to save the data from the replay buffer, at the frequency
that the model is saved. Only the most recent replay buffer is
stored.
num_envs : int
number of environments used to run simulations in parallel. Each
environment is run on a separate CPUS and uses the same policy as
the rest. Must be less than or equal to nb_rollout_steps.
verbose : int
the verbosity level: 0 none, 1 training information, 2 tensorflow
debug
policy_kwargs : dict
policy-specific hyperparameters
_init_setup_model : bool
Whether or not to build the network at the creation of the instance
Raises
------
AssertionError
if num_envs > nb_rollout_steps
"""
shared = False if policy_kwargs is None else \
policy_kwargs.get("shared", False)
maddpg = False if policy_kwargs is None else \
policy_kwargs.get("maddpg", False)
# Run assertions.
assert num_envs <= nb_rollout_steps, \
"num_envs must be less than or equal to nb_rollout_steps"
# Instantiate the ray instance.
if num_envs > 1:
ray.init(num_cpus=num_envs + 1, ignore_reinit_error=True)
self.policy = policy
self.env_name = deepcopy(env) if isinstance(env, str) \
else env.__str__()
self.eval_env, _ = create_env(eval_env,
render_eval,
shared,
maddpg,
evaluate=True)
self.nb_train_steps = nb_train_steps
self.nb_rollout_steps = nb_rollout_steps
self.nb_eval_episodes = nb_eval_episodes
self.actor_update_freq = actor_update_freq
self.meta_update_freq = meta_update_freq
self.reward_scale = reward_scale
self.render = render
self.render_eval = render_eval
self.eval_deterministic = eval_deterministic
self.save_replay_buffer = save_replay_buffer
self.num_envs = num_envs
self.verbose = verbose
self.policy_kwargs = {'verbose': verbose}
# Create the environment and collect the initial observations.
self.sampler, self.obs, self.all_obs = self.setup_sampler(
env, render, shared, maddpg)
# Collect the spaces of the environments.
self.ac_space, self.ob_space, self.co_space, all_ob_space = \
self.get_spaces()
# Add the default policy kwargs to the policy_kwargs term.
if is_feedforward_policy(policy):
self.policy_kwargs.update(FEEDFORWARD_PARAMS.copy())
if is_goal_conditioned_policy(policy):
self.policy_kwargs.update(GOAL_CONDITIONED_PARAMS.copy())
self.policy_kwargs['env_name'] = self.env_name.__str__()
self.policy_kwargs['num_envs'] = num_envs
if is_multiagent_policy(policy):
self.policy_kwargs.update(MULTIAGENT_PARAMS.copy())
self.policy_kwargs["all_ob_space"] = all_ob_space
if is_td3_policy(policy):
self.policy_kwargs.update(TD3_PARAMS.copy())
elif is_sac_policy(policy):
self.policy_kwargs.update(SAC_PARAMS.copy())
self.policy_kwargs = recursive_update(self.policy_kwargs, policy_kwargs
or {})
# Compute the time horizon, which is used to check if an environment
# terminated early and used to compute the done mask for TD3.
if self.num_envs > 1:
self.horizon = ray.get(self.sampler[0].horizon.remote())
else:
self.horizon = self.sampler[0].horizon()
# init
self.graph = None
self.policy_tf = None
self.sess = None
self.summary = None
self.episode_step = [0 for _ in range(num_envs)]
self.episodes = 0
self.total_steps = 0
self.epoch_episode_steps = []
self.epoch_episode_rewards = []
self.epoch_episodes = 0
self.epoch = 0
self.episode_rew_history = deque(maxlen=100)
self.episode_reward = [0 for _ in range(num_envs)]
self.rew_ph = None
self.rew_history_ph = None
self.eval_rew_ph = None
self.eval_success_ph = None
self.saver = None
# Create the model variables and operations.
if _init_setup_model:
self.trainable_vars = self.setup_model()
