def test_sync_vec_normalize():
env = DummyVecEnv([make_env])
assert unwrap_vec_normalize(env) is None
env = VecNormalize(env, norm_obs=True, norm_reward=True, clip_obs=10., clip_reward=10.)
assert isinstance(unwrap_vec_normalize(env), VecNormalize)
env = VecFrameStack(env, 1)
assert isinstance(unwrap_vec_normalize(env), VecNormalize)
eval_env = DummyVecEnv([make_env])
eval_env = VecNormalize(eval_env, training=False, norm_obs=True, norm_reward=True, clip_obs=10., clip_reward=10.)
eval_env = VecFrameStack(eval_env, 1)
env.reset()
# Initialize running mean
for _ in range(100):
env.step([env.action_space.sample()])
obs = env.reset()
original_obs = env.get_original_obs()
# Normalization must be different
assert not np.allclose(obs, eval_env.normalize_obs(original_obs))
sync_envs_normalization(env, eval_env)
# Now they must be synced
assert np.allclose(obs, eval_env.normalize_obs(original_obs))
def __init__(self,
policy,
env,
policy_base,
policy_kwargs=None,
verbose=0,
device='auto',
support_multi_env=False,
create_eval_env=False,
monitor_wrapper=True,
seed=None):
if isinstance(policy, str) and policy_base is not None:
self.policy_class = get_policy_from_name(policy_base, policy)
else:
self.policy_class = policy
self.env = env
# get VecNormalize object if needed
self._vec_normalize_env = unwrap_vec_normalize(env)
self.verbose = verbose
self.policy_kwargs = {} if policy_kwargs is None else policy_kwargs
self.observation_space = None
self.action_space = None
self.n_envs = None
self.num_timesteps = 0
self.eval_env = None
self.replay_buffer = None
self.seed = seed
self.action_noise = None
# Track the training progress (from 1 to 0)
# this is used to update the learning rate
self._current_progress = 1
# Create and wrap the env if needed
if env is not None:
if isinstance(env, str):
if create_eval_env:
eval_env = gym.make(env)
if monitor_wrapper:
eval_env = Monitor(eval_env, filename=None)
self.eval_env = DummyVecEnv([lambda: eval_env])
if self.verbose >= 1:
print(
"Creating environment from the given name, wrapped in a DummyVecEnv."
)
env = gym.make(env)
if monitor_wrapper:
env = Monitor(env, filename=None)
env = DummyVecEnv([lambda: env])
self.observation_space = env.observation_space
self.action_space = env.action_space
if not isinstance(env, VecEnv):
if self.verbose >= 1:
print("Wrapping the env in a DummyVecEnv.")
env = DummyVecEnv([lambda: env])
self.n_envs = env.num_envs
self.env = env
if not support_multi_env and self.n_envs > 1:
raise ValueError(
"Error: the model does not support multiple envs requires a single vectorized"
" environment.")
def custom_evaluate_policy(
model,
env,
n_eval_episodes=10,
deterministic=True,
render=False,
callback=None,
reward_threshold=None,
return_episode_rewards=False,
return_episode_info=False,
):
"""
Runs policy for `n_eval_episodes` episodes and returns average reward.
This is made to work only with one env.
:param model: (BaseRLModel) The RL agent you want to evaluate.
:param env: (gym.Env or VecEnv) The gym environment. In the case of a `VecEnv`
this must contain only one environment.
:param n_eval_episodes: (int) Number of episode to evaluate the agent
:param deterministic: (bool) Whether to use deterministic or stochastic actions
:param render: (bool) Whether to render the environment or not
:param callback: (callable) callback function to do additional checks,
called after each step.
:param reward_threshold: (float) Minimum expected reward per episode,
this will raise an error if the performance is not met
:param return_episode_rewards: (bool) If True, a list of reward per episode
will be returned instead of the mean.
:return: (float, float) Mean reward per episode, std of reward per episode
returns ([float], [int]) when `return_episode_rewards` is True
"""
if isinstance(env, VecEnv):
assert env.num_envs == 1, "You must pass only one environment when using this function"
# if isinstance(env, VecNormalize):
# env = env.venv
_vec_normalize_env = unwrap_vec_normalize(env)
episode_rewards, episode_lengths, episodes_info = [], [], []
if callback:
callback.on_eval_start()
for _ in range(n_eval_episodes):
obs = env.reset()
done, state = False, None
episode_reward = 0.0
episode_length = 0
episode_info = []
while not done:
action, state = model.predict(obs,
state=state,
deterministic=deterministic)
obs, reward, done, _info = env.step(action)
if _vec_normalize_env is not None:
reward_ = _vec_normalize_env.get_original_reward().squeeze()
episode_reward += reward_
else:
episode_reward += reward
episode_info.append(_info)
episode_length += 1
if render:
env.render("human")
episode_rewards.append(episode_reward)
episode_lengths.append(episode_length)
episodes_info.append(episode_info)
if callback:
if not callback.on_eval_episode_step():
break
mean_reward = np.mean(episode_rewards)
std_reward = np.std(episode_rewards)
if reward_threshold is not None:
assert mean_reward > reward_threshold, "Mean reward below threshold: " "{:.2f} < {:.2f}".format(
mean_reward, reward_threshold)
if return_episode_info and not return_episode_rewards:
return mean_reward, std_reward, episodes_info
elif return_episode_info and return_episode_rewards:
return episode_rewards, episode_lengths, episodes_info
if return_episode_rewards:
return episode_rewards, episode_lengths
return mean_reward, std_reward