def test_is_wrapped(): """Test that is_wrapped correctly detects wraps""" env = gym.make("Pendulum-v0") env = gym.Wrapper(env) assert not is_wrapped(env, Monitor) monitor_env = Monitor(env) assert is_wrapped(monitor_env, Monitor) env = gym.Wrapper(monitor_env) assert is_wrapped(env, Monitor) # Test that unwrap works as expected assert unwrap_wrapper(env, Monitor) == monitor_env
def _wrap_env(env: GymEnv, verbose: int = 0, monitor_wrapper: bool = True) -> VecEnv: """ " Wrap environment with the appropriate wrappers if needed. For instance, to have a vectorized environment or to re-order the image channels. :param env: :param verbose: :param monitor_wrapper: Whether to wrap the env in a ``Monitor`` when possible. :return: The wrapped environment. """ if not isinstance(env, VecEnv): if not is_wrapped(env, Monitor) and monitor_wrapper: if verbose >= 1: print("Wrapping the env with a `Monitor` wrapper") env = Monitor(env) if verbose >= 1: print("Wrapping the env in a DummyVecEnv.") env = DummyVecEnv([lambda: env]) if (is_image_space(env.observation_space) and not is_vecenv_wrapped(env, VecTransposeImage) and not is_image_space_channels_first(env.observation_space)): if verbose >= 1: print("Wrapping the env in a VecTransposeImage.") env = VecTransposeImage(env) # check if wrapper for dict support is needed when using HER if isinstance(env.observation_space, gym.spaces.dict.Dict): env = ObsDictWrapper(env) return env
def env_is_wrapped(self, wrapper_class: Type[gym.Wrapper], indices: VecEnvIndices = None) -> List[bool]: """Check if worker environments are wrapped with a given wrapper""" target_envs = self._get_target_envs(indices) # Import here to avoid a circular import from stable_baselines3.common import env_util return [env_util.is_wrapped(env_i, wrapper_class) for env_i in target_envs]
def env_is_wrapped(self, wrapper_class: Type[gym.Wrapper], indices: VecEnvIndices) -> List[bool]: target_envs = self._get_target_envs(indices) from stable_baselines3.common import env_util return [ env_util.is_wrapped(each_env, wrapper_class) for each_env in target_envs ]
def test_logger_wrapper(env_name, request): env = request.getfixturevalue(env_name) logger = env.logger env.reset() # Check CSV's have been created and linked in simulator correctly assert logger.log_progress_file == env.simulator._env_working_dir_parent + '/progress.csv' assert logger.log_file == env.simulator._eplus_working_dir + '/monitor.csv' tmp_log_file = logger.log_file # simulating short episode for _ in range(10): env.step(env.action_space.sample()) env.reset() assert os.path.isfile(logger.log_progress_file) assert os.path.isfile(tmp_log_file) # If env is wrapped with normalize obs... if is_wrapped(env, NormalizeObservation): print(logger.log_file[:-4] + '_normalized.csv') assert os.path.isfile(tmp_log_file[:-4] + '_normalized.csv') else: assert not os.path.isfile(tmp_log_file[:-4] + '_normalized.csv') # Check headers with open(tmp_log_file, mode='r', newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',') for row in reader: assert ','.join(row) == logger.monitor_header break with open(logger.log_progress_file, mode='r', newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',') for row in reader: assert ','.join(row) + '\n' == logger.progress_header break if is_wrapped(env, NormalizeObservation): with open(tmp_log_file[:-4] + '_normalized.csv', mode='r', newline='') as csvfile: reader = csv.reader(csvfile, delimiter=',') for row in reader: assert ','.join(row) == logger.monitor_header break env.close()
def _worker( remote: mp.connection.Connection, parent_remote: mp.connection.Connection, env_fn_wrapper: CloudpickleWrapper, render: bool, render_mode: str, ) -> None: # Import here to avoid a circular import from stable_baselines3.common.env_util import is_wrapped parent_remote.close() env = env_fn_wrapper.var() while True: try: cmd, data = remote.recv() if cmd == "step": observation, reward, done, info = env.step(data) if render: env.render(mode=render_mode) if done: # save final observation where user can get it, then reset info["terminal_observation"] = observation observation = env.reset() if render: env.render(mode=render_mode) remote.send((observation, reward, done, info)) elif cmd == "seed": remote.send(env.seed(data)) elif cmd == "reset": observation = env.reset() if render: env.render(mode=render_mode) remote.send(observation) elif cmd == "render": remote.send(env.render(data)) elif cmd == "close": env.close() remote.close() break elif cmd == "get_spaces": remote.send((env.observation_space, env.action_space)) elif cmd == "env_method": method = getattr(env, data[0]) remote.send(method(*data[1], **data[2])) elif cmd == "get_attr": remote.send(getattr(env, data)) elif cmd == "set_attr": remote.send(setattr(env, data[0], data[1])) elif cmd == "is_wrapped": remote.send(is_wrapped(env, data)) else: raise NotImplementedError( f"`{cmd}` is not implemented in the worker") except EOFError: break
def _wrap_env(env: GymEnv, verbose: int = 0, monitor_wrapper: bool = True) -> VecEnv: """ " Wrap environment with the appropriate wrappers if needed. For instance, to have a vectorized environment or to re-order the image channels. :param env: :param verbose: :param monitor_wrapper: Whether to wrap the env in a ``Monitor`` when possible. :return: The wrapped environment. """ if not isinstance(env, VecEnv): if not is_wrapped(env, Monitor) and monitor_wrapper: if verbose >= 1: print("Wrapping the env with a `Monitor` wrapper") env = Monitor(env) if verbose >= 1: print("Wrapping the env in a DummyVecEnv.") env = DummyVecEnv([lambda: env]) # Make sure that dict-spaces are not nested (not supported) check_for_nested_spaces(env.observation_space) if isinstance(env.observation_space, gym.spaces.Dict): for space in env.observation_space.spaces.values(): if isinstance(space, gym.spaces.Dict): raise ValueError( "Nested observation spaces are not supported (Dict spaces inside Dict space)." ) if not is_vecenv_wrapped(env, VecTransposeImage): wrap_with_vectranspose = False if isinstance(env.observation_space, gym.spaces.Dict): # If even one of the keys is a image-space in need of transpose, apply transpose # If the image spaces are not consistent (for instance one is channel first, # the other channel last), VecTransposeImage will throw an error for space in env.observation_space.spaces.values(): wrap_with_vectranspose = wrap_with_vectranspose or ( is_image_space(space) and not is_image_space_channels_first(space)) else: wrap_with_vectranspose = is_image_space( env.observation_space ) and not is_image_space_channels_first(env.observation_space) if wrap_with_vectranspose: if verbose >= 1: print("Wrapping the env in a VecTransposeImage.") env = VecTransposeImage(env) return env
def step( self, action: Union[int, np.ndarray] ) -> Tuple[np.ndarray, float, bool, Dict[str, Any]]: """Step the environment. Logging new information Args: action: Action executed in step Returns: (np.array(),float,bool,dict) tuple """ obs, reward, done, info = self.env.step(action) # We added some extra values (month,day,hour) manually in env, so we # need to delete them. if is_wrapped(self, NormalizeObservation): # Record action and new observation in simulator's csv self.logger.log_step_normalize( timestep=info['timestep'], date=[info['month'], info['day'], info['hour']], observation=obs[:-3], action=info['action_'], simulation_time=info['time_elapsed'], reward=reward, total_power_no_units=info['total_power_no_units'], comfort_penalty=info['comfort_penalty'], done=done) # Record original observation too self.logger.log_step( timestep=info['timestep'], date=[info['month'], info['day'], info['hour']], observation=self.env.get_unwrapped_obs()[:-3], action=info['action_'], simulation_time=info['time_elapsed'], reward=reward, total_power_no_units=info['total_power_no_units'], comfort_penalty=info['comfort_penalty'], power=info['total_power'], done=done) else: # Only record observation without normalization self.logger.log_step( timestep=info['timestep'], date=[info['month'], info['day'], info['hour']], observation=obs[:-3], action=info['action_'], simulation_time=info['time_elapsed'], reward=reward, total_power_no_units=info['total_power_no_units'], comfort_penalty=info['comfort_penalty'], power=info['total_power'], done=done) return obs, reward, done, info
def _on_training_start(self): # sinergym logger if is_wrapped(self.training_env, LoggerWrapper): if self.sinergym_logger: self.training_env.env_method('activate_logger') else: self.training_env.env_method('deactivate_logger') # record method depending on the type of algorithm if 'OnPolicyAlgorithm' in self.globals.keys(): self.record = self.logger.record elif 'OffPolicyAlgorithm' in self.globals.keys(): self.record = self.logger.record_mean else: raise KeyError
def evaluate_policy( model: "base_class.BaseAlgorithm", env: Union[gym.Env, VecEnv], n_eval_episodes: int = 10, deterministic: bool = True, render: bool = False, callback: Optional[Callable[[Dict[str, Any], Dict[str, Any]], None]] = None, reward_threshold: Optional[float] = None, return_episode_rewards: bool = False, warn: bool = True, ) -> Union[Tuple[float, float], Tuple[List[float], List[int]]]: """ Runs policy for ``n_eval_episodes`` episodes and returns average reward. This is made to work only with one env. .. note:: If environment has not been wrapped with ``Monitor`` wrapper, reward and episode lengths are counted as it appears with ``env.step`` calls. If the environment contains wrappers that modify rewards or episode lengths (e.g. reward scaling, early episode reset), these will affect the evaluation results as well. You can avoid this by wrapping environment with ``Monitor`` wrapper before anything else. :param model: The RL agent you want to evaluate. :param env: The gym environment. In the case of a ``VecEnv`` this must contain only one environment. :param n_eval_episodes: Number of episode to evaluate the agent :param deterministic: Whether to use deterministic or stochastic actions :param render: Whether to render the environment or not :param callback: callback function to do additional checks, called after each step. Gets locals() and globals() passed as parameters. :param reward_threshold: Minimum expected reward per episode, this will raise an error if the performance is not met :param return_episode_rewards: If True, a list of rewards and episde lengths per episode will be returned instead of the mean. :param warn: If True (default), warns user about lack of a Monitor wrapper in the evaluation environment. :return: Mean reward per episode, std of reward per episode. Returns ([float], [int]) when ``return_episode_rewards`` is True, first list containing per-episode rewards and second containing per-episode lengths (in number of steps). """ is_monitor_wrapped = False # Avoid circular import from stable_baselines3.common.env_util import is_wrapped from stable_baselines3.common.monitor import Monitor if isinstance(env, VecEnv): assert env.num_envs == 1, "You must pass only one environment when using this function" is_monitor_wrapped = env.env_is_wrapped(Monitor)[0] else: is_monitor_wrapped = is_wrapped(env, Monitor) if not is_monitor_wrapped and warn: warnings.warn( "Evaluation environment is not wrapped with a ``Monitor`` wrapper. " "This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. " "Consider wrapping environment first with ``Monitor`` wrapper.", UserWarning, ) episode_rewards, episode_lengths = [], [] not_reseted = True while len(episode_rewards) < n_eval_episodes: # Number of loops here might differ from true episodes # played, if underlying wrappers modify episode lengths. # Avoid double reset, as VecEnv are reset automatically. if not isinstance(env, VecEnv) or not_reseted: obs = env.reset() not_reseted = False done, state = False, None episode_reward = 0.0 episode_length = 0 while not done: action, state = model.predict(obs, state=state, deterministic=deterministic) obs, reward, done, info = env.step(action) episode_reward += reward if callback is not None: callback(locals(), globals()) episode_length += 1 if render: env.render() if is_monitor_wrapped: # Do not trust "done" with episode endings. # Remove vecenv stacking (if any) if isinstance(env, VecEnv): info = info[0] if "episode" in info.keys(): # Monitor wrapper includes "episode" key in info if environment # has been wrapped with it. Use those rewards instead. episode_rewards.append(info["episode"]["r"]) episode_lengths.append(info["episode"]["l"]) else: episode_rewards.append(episode_reward) episode_lengths.append(episode_length) episode_rewards = th.tensor(episode_rewards) mean_reward = th.mean(episode_rewards).item() std_reward = th.std(episode_rewards).item() if reward_threshold is not None: assert mean_reward > reward_threshold, "Mean reward below threshold: " f"{mean_reward:.2f} < {reward_threshold:.2f}" if return_episode_rewards: return episode_rewards, episode_lengths return mean_reward, std_reward
def on_training_end(self): if is_wrapped(self.training_env, LoggerWrapper): self.training_env.env_method('activate_logger')