def test_check_spaces_async_vector_env(shared_memory): # CartPole-v1 - observation_space: Box(4,), action_space: Discrete(2) env_fns = [make_env("CartPole-v1", i) for i in range(8)] # FrozenLake-v1 - Discrete(16), action_space: Discrete(4) env_fns[1] = make_env("FrozenLake-v1", 1) with pytest.raises(RuntimeError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close(terminate=True)
def test_create_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) finally: env.close() assert env.num_envs == 8
def test_check_observations_async_vector_env(shared_memory): # CubeCrash-v0 - observation_space: Box(40, 32, 3) env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] # MemorizeDigits-v0 - observation_space: Box(24, 32, 3) env_fns[1] = make_env("MemorizeDigits-v0", 1) with pytest.raises(RuntimeError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close(terminate=True)
def test_no_copy_async_vector_env(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory, copy=False) observations = env.reset() observations[0] = 0 finally: env.close()
def test_no_copy_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory, copy=False) observations = env.reset() observations[0] = 128 assert np.all(env.observations[0] == 128) finally: env.close()
def main(): env_id = "Ant-v3" num_envs = 5 vec_env = AsyncVectorEnv([make_env(env_id) for i in range(num_envs)]) state = vec_env.reset() for i in range(5000): action = vec_env.action_space.sample() state, reward, done, _ = vec_env.step(action) if any(done): done_idx = [i for i, e in enumerate(done) if e] print(f"{done_idx}")
def test_reset_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) observations = env.reset() finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape
def test_step_timeout_async_vector_env(shared_memory): env_fns = [make_slow_env(0.0, i) for i in range(4)] with pytest.raises(TimeoutError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.reset() env.step_async([0.1, 0.1, 0.3, 0.1]) observations, rewards, dones, _ = env.step_wait(timeout=0.1) finally: env.close(terminate=True)
def test_reset_timeout_async_vector_env(shared_memory): env_fns = [make_slow_env(0.3, i) for i in range(4)] with pytest.raises(TimeoutError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.reset_async() env.reset_wait(timeout=0.1) finally: env.close(terminate=True)
def test_set_attr_async_vector_env(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(4)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.set_attr("gravity", [9.81, 3.72, 8.87, 1.62]) gravity = env.get_attr("gravity") assert gravity == (9.81, 3.72, 8.87, 1.62) finally: env.close()
def make(id, num_envs=1, asynchronous=True, wrappers=None, **kwargs): """Create a vectorized environment from multiple copies of an environment, from its id Parameters ---------- id : str The environment ID. This must be a valid ID from the registry. num_envs : int Number of copies of the environment. asynchronous : bool (default: `True`) If `True`, wraps the environments in an `AsyncVectorEnv` (which uses `multiprocessing` to run the environments in parallel). If `False`, wraps the environments in a `SyncVectorEnv`. wrappers : Callable or Iterable of Callables (default: `None`) If not `None`, then apply the wrappers to each internal environment during creation. Returns ------- env : `gym.vector.VectorEnv` instance The vectorized environment. Example ------- >>> import gym >>> env = gym.vector.make('CartPole-v1', 3) >>> env.reset() array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827], [ 0.03073904, 0.00145001, -0.03088818, -0.03131252], [ 0.03468829, 0.01500225, 0.01230312, 0.01825218]], dtype=float32) """ from gym.envs import make as make_ def _make_env(): env = make_(id, **kwargs) if wrappers is not None: if callable(wrappers): env = wrappers(env) elif isinstance(wrappers, Iterable) and all( [callable(w) for w in wrappers] ): for wrapper in wrappers: env = wrapper(env) else: raise NotImplementedError return env env_fns = [_make_env for _ in range(num_envs)] return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def make( id: str, num_envs: int = 1, asynchronous: bool = True, wrappers: Optional[Union[callable, List[callable]]] = None, disable_env_checker: bool = False, **kwargs, ) -> VectorEnv: """Create a vectorized environment from multiple copies of an environment, from its id. Example:: >>> import gym >>> env = gym.vector.make('CartPole-v1', num_envs=3) >>> env.reset() array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827], [ 0.03073904, 0.00145001, -0.03088818, -0.03131252], [ 0.03468829, 0.01500225, 0.01230312, 0.01825218]], dtype=float32) Args: id: The environment ID. This must be a valid ID from the registry. num_envs: Number of copies of the environment. asynchronous: If `True`, wraps the environments in an :class:`AsyncVectorEnv` (which uses `multiprocessing`_ to run the environments in parallel). If ``False``, wraps the environments in a :class:`SyncVectorEnv`. wrappers: If not ``None``, then apply the wrappers to each internal environment during creation. disable_env_checker: If to disable the env checker, if True it will only run on the first environment created. **kwargs: Keywords arguments applied during gym.make Returns: The vectorized environment. """ def create_env(_disable_env_checker): """Creates an environment that can enable or disable the environment checker.""" def _make_env(): env = gym.envs.registration.make( id, disable_env_checker=_disable_env_checker, **kwargs) if wrappers is not None: if callable(wrappers): env = wrappers(env) elif isinstance(wrappers, Iterable) and all( [callable(w) for w in wrappers]): for wrapper in wrappers: env = wrapper(env) else: raise NotImplementedError return env return _make_env env_fns = [ create_env(disable_env_checker or env_num > 0) for env_num in range(num_envs) ] return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def test_step_async_vector_env(shared_memory, use_single_action_space): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) observations = env.reset() if use_single_action_space: actions = [env.single_action_space.sample() for _ in range(8)] else: actions = env.action_space.sample() observations, rewards, dones, _ = env.step(actions) finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape assert isinstance(rewards, np.ndarray) assert isinstance(rewards[0], (float, np.floating)) assert rewards.ndim == 1 assert rewards.size == 8 assert isinstance(dones, np.ndarray) assert dones.dtype == np.bool_ assert dones.ndim == 1 assert dones.size == 8
def test_custom_space_async_vector_env(): env_fns = [make_custom_space_env(i) for i in range(4)] try: env = AsyncVectorEnv(env_fns, shared_memory=False) reset_observations = env.reset() assert isinstance(env.single_action_space, CustomSpace) assert isinstance(env.action_space, Tuple) actions = ("action-2", "action-3", "action-5", "action-7") step_observations, rewards, dones, _ = env.step(actions) finally: env.close() assert isinstance(env.single_observation_space, CustomSpace) assert isinstance(env.observation_space, Tuple) assert isinstance(reset_observations, tuple) assert reset_observations == ("reset", "reset", "reset", "reset") assert isinstance(step_observations, tuple) assert step_observations == ( "step(action-2)", "step(action-3)", "step(action-5)", "step(action-7)", )
def main(env, n_envs, rollout_len, n_total_steps, log_interval, algorithm, n_epochs, num_mbatch, entcoef=0, gamma=0.99, lam=0.97, kl_threshold=0.075): env = AsyncVectorEnv([partial(make_monitored_env, env) for _ in range(n_envs)]) model = mlp_model(env) rollout_generator = RolloutGenerator(model, env, gamma=gamma, lam=lam) optimizer = optim.Adam(model.parameters()) n_batch = rollout_len * n_envs mbatch_size = int(n_batch / num_mbatch) epinfobuf = deque(maxlen=100) n_steps_per_second = deque(maxlen=log_interval) for update in range(1, int(n_total_steps / n_batch) + 1): update_start_time = time.time() obs, rews, dones, acs, old_ac_logps, vpreds, advs, vtargs, epinfos = ( rollout_generator.generate_rollout(rollout_len)) epinfobuf.extend(epinfos) if algorithm == 'a2c': train_info = train_a2c(model, optimizer, obs, acs, advs, vtargs) if algorithm == 'ppo_clip': train_info = train_ppo(model, optimizer, obs, acs, advs, vtargs, old_ac_logps, n_epochs=n_epochs, n_mbatch=num_mbatch, loss='clip', entcoef=entcoef, kl_threshold=kl_threshold) n_steps_per_second.append(n_batch / (time.time() - update_start_time)) if update % log_interval == 0: train_info = dict([(k, v.item()) for k, v in train_info.items()]) eprews = [epinfo['r'] for epinfo in epinfobuf] eplens = [epinfo['l'] for epinfo in epinfobuf] logger.logkv('n_steps_per_second', np.mean(n_steps_per_second)) logger.logkv('total_steps', update * n_batch) if len(epinfobuf) > 0: logger.logkv('eprew_mean', np.mean(eprews)) logger.logkv('eprew_std', np.std(eprews)) logger.logkv('eprew_min', np.min(eprews)) logger.logkv('eprew_max', np.max(eprews)) logger.logkv('eplen_mean', np.mean(eplens)) logger.logkv('eplen_std', np.std(eplens)) logger.logkv('eplen_min', np.min(eplens)) logger.logkv('eplen_max', np.max(eplens)) logger.logkv('vpred_mean', vpreds.mean().item()) logger.logkv('vpred_std', vpreds.std().item()) logger.logkv('vpred_min', vpreds.min().item()) logger.logkv('vpred_max', vpreds.max().item()) logger.logkvs(train_info) logger.dumpkvs()
def example_vector_env(env_id: str): def _make(): return gym.make(env_id) env = AsyncVectorEnv([_make for _ in range(3)]) print(env.observation_space) def actor(_): return env.action_space.sample() interactions = TransitionGenerator(env, actor, max_episode=2) for _ in interactions: pass
def vectorize_env(env_id: str, num_envs: int = 1, env_fn=make_env, seed=0) -> VectorEnv: env_fns = [partial(env_fn, env_id=env_id) for _ in range(num_envs)] if num_envs == 1: envs = SingleAsVectorEnv(env_fns[0]()) else: envs = AsyncVectorEnv(env_fns) dummy_env = env_fns[0]() if hasattr(dummy_env, "spec"): setattr(envs, "spec", dummy_env.spec) envs.seed(seed) return envs
def test_vector_env_equal(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(4)] num_steps = 100 try: async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) sync_env = SyncVectorEnv(env_fns) async_env.seed(0) sync_env.seed(0) assert async_env.num_envs == sync_env.num_envs assert async_env.observation_space == sync_env.observation_space assert async_env.single_observation_space == sync_env.single_observation_space assert async_env.action_space == sync_env.action_space assert async_env.single_action_space == sync_env.single_action_space async_observations = async_env.reset() sync_observations = sync_env.reset() assert np.all(async_observations == sync_observations) for _ in range(num_steps): actions = async_env.action_space.sample() assert actions in sync_env.action_space # fmt: off async_observations, async_rewards, async_dones, async_infos = async_env.step( actions) sync_observations, sync_rewards, sync_dones, sync_infos = sync_env.step( actions) # fmt: on for idx in range(len(sync_dones)): if sync_dones[idx]: assert "terminal_observation" in async_infos[idx] assert "terminal_observation" in sync_infos[idx] assert sync_dones[idx] assert np.all(async_observations == sync_observations) assert np.all(async_rewards == sync_rewards) assert np.all(async_dones == sync_dones) finally: async_env.close() sync_env.close()
def make_env( env_id: str, num_envs: int = 1, ): def _make(): _env = gym.make(env_id) return _env if num_envs == 1: env = SyncVectorEnv([_make]) if num_envs > 1: env = AsyncVectorEnv([_make for _ in range(num_envs)]) dummy_env = _make() setattr(env, "spec", dummy_env.spec) del dummy_env else: env = _make() return env
def make(id, num_envs=1, asynchronous=True, **kwargs): """Create a vectorized environment from multiple copies of an environment, from its id Parameters ---------- id : str The environment ID. This must be a valid ID from the registry. num_envs : int Number of copies of the environment. If `1`, then it returns an unwrapped (i.e. non-vectorized) environment. asynchronous : bool (default: `True`) If `True`, wraps the environments in an `AsyncVectorEnv` (which uses `multiprocessing` to run the environments in parallel). If `False`, wraps the environments in a `SyncVectorEnv`. Returns ------- env : `gym.vector.VectorEnv` instance The vectorized environment. Example ------- >>> import gym >>> env = gym.vector.make('CartPole-v1', 3) >>> env.reset() array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827], [ 0.03073904, 0.00145001, -0.03088818, -0.03131252], [ 0.03468829, 0.01500225, 0.01230312, 0.01825218]], dtype=float32) """ from gym.envs import make as make_ def _make_env(): return make_(id, **kwargs) if num_envs == 1: return _make_env() env_fns = [_make_env for _ in range(num_envs)] return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def test_call_async_vector_env(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(4)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) _ = env.reset() images = env.call("render", mode="rgb_array") gravity = env.call("gravity") finally: env.close() assert isinstance(images, tuple) assert len(images) == 4 for i in range(4): assert isinstance(images[i], np.ndarray) assert isinstance(gravity, tuple) assert len(gravity) == 4 for i in range(4): assert isinstance(gravity[i], float) assert gravity[i] == 9.8
def test_vector_env_equal(shared_memory): env_fns = [make_env('CubeCrash-v0', i) for i in range(4)] num_steps = 100 try: async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) sync_env = SyncVectorEnv(env_fns) async_env.seed(0) sync_env.seed(0) assert async_env.num_envs == sync_env.num_envs assert async_env.observation_space == sync_env.observation_space assert async_env.single_observation_space == sync_env.single_observation_space assert async_env.action_space == sync_env.action_space assert async_env.single_action_space == sync_env.single_action_space async_observations = async_env.reset() sync_observations = sync_env.reset() assert np.all(async_observations == sync_observations) for _ in range(num_steps): actions = async_env.action_space.sample() assert actions in sync_env.action_space async_observations, async_rewards, async_dones, _ = async_env.step( actions) sync_observations, sync_rewards, sync_dones, _ = sync_env.step( actions) assert np.all(async_observations == sync_observations) assert np.all(async_rewards == sync_rewards) assert np.all(async_dones == sync_dones) finally: async_env.close() sync_env.close()
def test_custom_space_async_vector_env(): env_fns = [make_custom_space_env(i) for i in range(4)] try: env = AsyncVectorEnv(env_fns, shared_memory=False) reset_observations = env.reset() actions = ('action-2', 'action-3', 'action-5', 'action-7') step_observations, rewards, dones, _ = env.step(actions) finally: env.close() assert isinstance(env.single_observation_space, CustomSpace) assert isinstance(env.observation_space, Tuple) assert isinstance(reset_observations, tuple) assert reset_observations == ('reset', 'reset', 'reset', 'reset') assert isinstance(step_observations, tuple) assert step_observations == ('step(action-2)', 'step(action-3)', 'step(action-5)', 'step(action-7)')
def test_vector_env_equal(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(4)] num_steps = 100 try: async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) sync_env = SyncVectorEnv(env_fns) assert async_env.num_envs == sync_env.num_envs assert async_env.observation_space == sync_env.observation_space assert async_env.single_observation_space == sync_env.single_observation_space assert async_env.action_space == sync_env.action_space assert async_env.single_action_space == sync_env.single_action_space async_observations = async_env.reset(seed=0) sync_observations = sync_env.reset(seed=0) assert np.all(async_observations == sync_observations) for _ in range(num_steps): actions = async_env.action_space.sample() assert actions in sync_env.action_space # fmt: off async_observations, async_rewards, async_dones, async_infos = async_env.step( actions) sync_observations, sync_rewards, sync_dones, sync_infos = sync_env.step( actions) # fmt: on if any(sync_dones): assert "terminal_observation" in async_infos assert "_terminal_observation" in async_infos assert "terminal_observation" in sync_infos assert "_terminal_observation" in sync_infos assert np.all(async_observations == sync_observations) assert np.all(async_rewards == sync_rewards) assert np.all(async_dones == sync_dones) finally: async_env.close() sync_env.close()
def test_reset_async_vector_env(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) observations = env.reset() finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) observations = env.reset(return_info=False) finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) observations, infos = env.reset(return_info=True) finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape assert isinstance(infos, list) assert all([isinstance(info, dict) for info in infos])
def test_custom_space_async_vector_env_shared_memory(): env_fns = [make_custom_space_env(i) for i in range(4)] with pytest.raises(ValueError): env = AsyncVectorEnv(env_fns, shared_memory=True) env.close(terminate=True)
def test_already_closed_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(4)] with pytest.raises(ClosedEnvironmentError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close() observations = env.reset()
def test_step_out_of_order_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(4)] with pytest.raises(NoAsyncCallError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) actions = env.action_space.sample() observations = env.reset() observations, rewards, dones, infos = env.step_wait() except AlreadyPendingCallError as exception: assert exception.name == "step" raise finally: env.close(terminate=True) with pytest.raises(AlreadyPendingCallError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) actions = env.action_space.sample() env.reset_async() env.step_async(actions) except AlreadyPendingCallError as exception: assert exception.name == "reset" raise finally: env.close(terminate=True)
import multiprocessing as mp import threading from gym.vector.tests.utils import make_env, make_slow_env from gym.vector.async_vector_env import AsyncVectorEnv import concurrent.futures from agent import Agent from agent_test import AgentTest print("Cores", mp.cpu_count()) if __name__ == '__main__': #Number of agents working in parallel num_agents = 100 env_fns = [make_env('CartPole-v0', num_agents) for _ in range(num_agents)] env = AsyncVectorEnv(env_fns) agent = Agent(env, state_size=4, action_size=2, num_agents=num_agents) env_test = gym.make('CartPole-v0') agent_test = AgentTest(env_test, state_size=4, action_size=2) one_set_of_weights = 0.1*np.random.randn(agent.get_weights_dim()) all_sets_of_weights = [] for i in range(num_agents): all_sets_of_weights.append(one_set_of_weights) start_time = time.time() for i in range(100): rewards = agent.evaluate(all_sets_of_weights, num_agents) print("Time needed for VecEnv approach: ", time.time() - start_time)
def test_already_closed_async_vector_env(shared_memory): env_fns = [make_env("CartPole-v1", i) for i in range(4)] with pytest.raises(ClosedEnvironmentError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close() env.reset()