def test_check_spaces_async_vector_env(shared_memory):
# CartPole-v1 - observation_space: Box(4,), action_space: Discrete(2)
env_fns = [make_env("CartPole-v1", i) for i in range(8)]
# FrozenLake-v1 - Discrete(16), action_space: Discrete(4)
env_fns[1] = make_env("FrozenLake-v1", 1)
with pytest.raises(RuntimeError):
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.close(terminate=True)
def test_create_async_vector_env(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
finally:
env.close()
assert env.num_envs == 8
def test_check_observations_async_vector_env(shared_memory):
# CubeCrash-v0 - observation_space: Box(40, 32, 3)
env_fns = [make_env("CubeCrash-v0", i) for i in range(8)]
# MemorizeDigits-v0 - observation_space: Box(24, 32, 3)
env_fns[1] = make_env("MemorizeDigits-v0", 1)
with pytest.raises(RuntimeError):
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.close(terminate=True)
def test_no_copy_async_vector_env(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory, copy=False)
observations = env.reset()
observations[0] = 0
finally:
env.close()
def test_no_copy_async_vector_env(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory, copy=False)
observations = env.reset()
observations[0] = 128
assert np.all(env.observations[0] == 128)
finally:
env.close()
def main():
env_id = "Ant-v3"
num_envs = 5
vec_env = AsyncVectorEnv([make_env(env_id) for i in range(num_envs)])
state = vec_env.reset()
for i in range(5000):
action = vec_env.action_space.sample()
state, reward, done, _ = vec_env.step(action)
if any(done):
done_idx = [i for i, e in enumerate(done) if e]
print(f"{done_idx}")
def test_reset_async_vector_env(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
observations = env.reset()
finally:
env.close()
assert isinstance(env.observation_space, Box)
assert isinstance(observations, np.ndarray)
assert observations.dtype == env.observation_space.dtype
assert observations.shape == (8, ) + env.single_observation_space.shape
assert observations.shape == env.observation_space.shape
def test_step_timeout_async_vector_env(shared_memory):
env_fns = [make_slow_env(0.0, i) for i in range(4)]
with pytest.raises(TimeoutError):
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.reset()
env.step_async([0.1, 0.1, 0.3, 0.1])
observations, rewards, dones, _ = env.step_wait(timeout=0.1)
finally:
env.close(terminate=True)
def test_reset_timeout_async_vector_env(shared_memory):
env_fns = [make_slow_env(0.3, i) for i in range(4)]
with pytest.raises(TimeoutError):
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.reset_async()
env.reset_wait(timeout=0.1)
finally:
env.close(terminate=True)
def test_set_attr_async_vector_env(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(4)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.set_attr("gravity", [9.81, 3.72, 8.87, 1.62])
gravity = env.get_attr("gravity")
assert gravity == (9.81, 3.72, 8.87, 1.62)
finally:
env.close()
def make(id, num_envs=1, asynchronous=True, wrappers=None, **kwargs):
"""Create a vectorized environment from multiple copies of an environment,
from its id
Parameters
----------
id : str
The environment ID. This must be a valid ID from the registry.
num_envs : int
Number of copies of the environment.
asynchronous : bool (default: `True`)
If `True`, wraps the environments in an `AsyncVectorEnv` (which uses
`multiprocessing` to run the environments in parallel). If `False`,
wraps the environments in a `SyncVectorEnv`.
wrappers : Callable or Iterable of Callables (default: `None`)
If not `None`, then apply the wrappers to each internal
environment during creation.
Returns
-------
env : `gym.vector.VectorEnv` instance
The vectorized environment.
Example
-------
>>> import gym
>>> env = gym.vector.make('CartPole-v1', 3)
>>> env.reset()
array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827],
[ 0.03073904, 0.00145001, -0.03088818, -0.03131252],
[ 0.03468829, 0.01500225, 0.01230312, 0.01825218]],
dtype=float32)
"""
from gym.envs import make as make_
def _make_env():
env = make_(id, **kwargs)
if wrappers is not None:
if callable(wrappers):
env = wrappers(env)
elif isinstance(wrappers, Iterable) and all(
[callable(w) for w in wrappers]
):
for wrapper in wrappers:
env = wrapper(env)
else:
raise NotImplementedError
return env
env_fns = [_make_env for _ in range(num_envs)]
return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def make(
id: str,
num_envs: int = 1,
asynchronous: bool = True,
wrappers: Optional[Union[callable, List[callable]]] = None,
disable_env_checker: bool = False,
**kwargs,
) -> VectorEnv:
"""Create a vectorized environment from multiple copies of an environment, from its id.
Example::
>>> import gym
>>> env = gym.vector.make('CartPole-v1', num_envs=3)
>>> env.reset()
array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827],
[ 0.03073904, 0.00145001, -0.03088818, -0.03131252],
[ 0.03468829, 0.01500225, 0.01230312, 0.01825218]],
dtype=float32)
Args:
id: The environment ID. This must be a valid ID from the registry.
num_envs: Number of copies of the environment.
asynchronous: If `True`, wraps the environments in an :class:`AsyncVectorEnv` (which uses `multiprocessing`_ to run the environments in parallel). If ``False``, wraps the environments in a :class:`SyncVectorEnv`.
wrappers: If not ``None``, then apply the wrappers to each internal environment during creation.
disable_env_checker: If to disable the env checker, if True it will only run on the first environment created.
**kwargs: Keywords arguments applied during gym.make
Returns:
The vectorized environment.
"""
def create_env(_disable_env_checker):
"""Creates an environment that can enable or disable the environment checker."""
def _make_env():
env = gym.envs.registration.make(
id, disable_env_checker=_disable_env_checker, **kwargs)
if wrappers is not None:
if callable(wrappers):
env = wrappers(env)
elif isinstance(wrappers, Iterable) and all(
[callable(w) for w in wrappers]):
for wrapper in wrappers:
env = wrapper(env)
else:
raise NotImplementedError
return env
return _make_env
env_fns = [
create_env(disable_env_checker or env_num > 0)
for env_num in range(num_envs)
]
return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def test_step_async_vector_env(shared_memory, use_single_action_space):
env_fns = [make_env("CubeCrash-v0", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
observations = env.reset()
if use_single_action_space:
actions = [env.single_action_space.sample() for _ in range(8)]
else:
actions = env.action_space.sample()
observations, rewards, dones, _ = env.step(actions)
finally:
env.close()
assert isinstance(env.observation_space, Box)
assert isinstance(observations, np.ndarray)
assert observations.dtype == env.observation_space.dtype
assert observations.shape == (8, ) + env.single_observation_space.shape
assert observations.shape == env.observation_space.shape
assert isinstance(rewards, np.ndarray)
assert isinstance(rewards[0], (float, np.floating))
assert rewards.ndim == 1
assert rewards.size == 8
assert isinstance(dones, np.ndarray)
assert dones.dtype == np.bool_
assert dones.ndim == 1
assert dones.size == 8
def test_custom_space_async_vector_env():
env_fns = [make_custom_space_env(i) for i in range(4)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=False)
reset_observations = env.reset()
assert isinstance(env.single_action_space, CustomSpace)
assert isinstance(env.action_space, Tuple)
actions = ("action-2", "action-3", "action-5", "action-7")
step_observations, rewards, dones, _ = env.step(actions)
finally:
env.close()
assert isinstance(env.single_observation_space, CustomSpace)
assert isinstance(env.observation_space, Tuple)
assert isinstance(reset_observations, tuple)
assert reset_observations == ("reset", "reset", "reset", "reset")
assert isinstance(step_observations, tuple)
assert step_observations == (
"step(action-2)",
"step(action-3)",
"step(action-5)",
"step(action-7)",
)
def main(env, n_envs, rollout_len, n_total_steps, log_interval, algorithm, n_epochs, num_mbatch,
entcoef=0, gamma=0.99, lam=0.97, kl_threshold=0.075):
env = AsyncVectorEnv([partial(make_monitored_env, env) for _ in range(n_envs)])
model = mlp_model(env)
rollout_generator = RolloutGenerator(model, env, gamma=gamma, lam=lam)
optimizer = optim.Adam(model.parameters())
n_batch = rollout_len * n_envs
mbatch_size = int(n_batch / num_mbatch)
epinfobuf = deque(maxlen=100)
n_steps_per_second = deque(maxlen=log_interval)
for update in range(1, int(n_total_steps / n_batch) + 1):
update_start_time = time.time()
obs, rews, dones, acs, old_ac_logps, vpreds, advs, vtargs, epinfos = (
rollout_generator.generate_rollout(rollout_len))
epinfobuf.extend(epinfos)
if algorithm == 'a2c':
train_info = train_a2c(model, optimizer, obs, acs, advs, vtargs)
if algorithm == 'ppo_clip':
train_info = train_ppo(model, optimizer, obs, acs, advs, vtargs, old_ac_logps,
n_epochs=n_epochs, n_mbatch=num_mbatch, loss='clip',
entcoef=entcoef, kl_threshold=kl_threshold)
n_steps_per_second.append(n_batch / (time.time() - update_start_time))
if update % log_interval == 0:
train_info = dict([(k, v.item()) for k, v in train_info.items()])
eprews = [epinfo['r'] for epinfo in epinfobuf]
eplens = [epinfo['l'] for epinfo in epinfobuf]
logger.logkv('n_steps_per_second', np.mean(n_steps_per_second))
logger.logkv('total_steps', update * n_batch)
if len(epinfobuf) > 0:
logger.logkv('eprew_mean', np.mean(eprews))
logger.logkv('eprew_std', np.std(eprews))
logger.logkv('eprew_min', np.min(eprews))
logger.logkv('eprew_max', np.max(eprews))
logger.logkv('eplen_mean', np.mean(eplens))
logger.logkv('eplen_std', np.std(eplens))
logger.logkv('eplen_min', np.min(eplens))
logger.logkv('eplen_max', np.max(eplens))
logger.logkv('vpred_mean', vpreds.mean().item())
logger.logkv('vpred_std', vpreds.std().item())
logger.logkv('vpred_min', vpreds.min().item())
logger.logkv('vpred_max', vpreds.max().item())
logger.logkvs(train_info)
logger.dumpkvs()
def example_vector_env(env_id: str):
def _make():
return gym.make(env_id)
env = AsyncVectorEnv([_make for _ in range(3)])
print(env.observation_space)
def actor(_):
return env.action_space.sample()
interactions = TransitionGenerator(env, actor, max_episode=2)
for _ in interactions:
pass
def vectorize_env(env_id: str, num_envs: int = 1, env_fn=make_env, seed=0) -> VectorEnv:
env_fns = [partial(env_fn, env_id=env_id) for _ in range(num_envs)]
if num_envs == 1:
envs = SingleAsVectorEnv(env_fns[0]())
else:
envs = AsyncVectorEnv(env_fns)
dummy_env = env_fns[0]()
if hasattr(dummy_env, "spec"):
setattr(envs, "spec", dummy_env.spec)
envs.seed(seed)
return envs
def test_vector_env_equal(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(4)]
num_steps = 100
try:
async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
sync_env = SyncVectorEnv(env_fns)
async_env.seed(0)
sync_env.seed(0)
assert async_env.num_envs == sync_env.num_envs
assert async_env.observation_space == sync_env.observation_space
assert async_env.single_observation_space == sync_env.single_observation_space
assert async_env.action_space == sync_env.action_space
assert async_env.single_action_space == sync_env.single_action_space
async_observations = async_env.reset()
sync_observations = sync_env.reset()
assert np.all(async_observations == sync_observations)
for _ in range(num_steps):
actions = async_env.action_space.sample()
assert actions in sync_env.action_space
# fmt: off
async_observations, async_rewards, async_dones, async_infos = async_env.step(
actions)
sync_observations, sync_rewards, sync_dones, sync_infos = sync_env.step(
actions)
# fmt: on
for idx in range(len(sync_dones)):
if sync_dones[idx]:
assert "terminal_observation" in async_infos[idx]
assert "terminal_observation" in sync_infos[idx]
assert sync_dones[idx]
assert np.all(async_observations == sync_observations)
assert np.all(async_rewards == sync_rewards)
assert np.all(async_dones == sync_dones)
finally:
async_env.close()
sync_env.close()
def make_env(
env_id: str,
num_envs: int = 1,
):
def _make():
_env = gym.make(env_id)
return _env
if num_envs == 1:
env = SyncVectorEnv([_make])
if num_envs > 1:
env = AsyncVectorEnv([_make for _ in range(num_envs)])
dummy_env = _make()
setattr(env, "spec", dummy_env.spec)
del dummy_env
else:
env = _make()
return env
def make(id, num_envs=1, asynchronous=True, **kwargs):
"""Create a vectorized environment from multiple copies of an environment,
from its id
Parameters
----------
id : str
The environment ID. This must be a valid ID from the registry.
num_envs : int
Number of copies of the environment. If `1`, then it returns an
unwrapped (i.e. non-vectorized) environment.
asynchronous : bool (default: `True`)
If `True`, wraps the environments in an `AsyncVectorEnv` (which uses
`multiprocessing` to run the environments in parallel). If `False`,
wraps the environments in a `SyncVectorEnv`.
Returns
-------
env : `gym.vector.VectorEnv` instance
The vectorized environment.
Example
-------
>>> import gym
>>> env = gym.vector.make('CartPole-v1', 3)
>>> env.reset()
array([[-0.04456399, 0.04653909, 0.01326909, -0.02099827],
[ 0.03073904, 0.00145001, -0.03088818, -0.03131252],
[ 0.03468829, 0.01500225, 0.01230312, 0.01825218]],
dtype=float32)
"""
from gym.envs import make as make_
def _make_env():
return make_(id, **kwargs)
if num_envs == 1:
return _make_env()
env_fns = [_make_env for _ in range(num_envs)]
return AsyncVectorEnv(env_fns) if asynchronous else SyncVectorEnv(env_fns)
def test_call_async_vector_env(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(4)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
_ = env.reset()
images = env.call("render", mode="rgb_array")
gravity = env.call("gravity")
finally:
env.close()
assert isinstance(images, tuple)
assert len(images) == 4
for i in range(4):
assert isinstance(images[i], np.ndarray)
assert isinstance(gravity, tuple)
assert len(gravity) == 4
for i in range(4):
assert isinstance(gravity[i], float)
assert gravity[i] == 9.8
def test_vector_env_equal(shared_memory):
env_fns = [make_env('CubeCrash-v0', i) for i in range(4)]
num_steps = 100
try:
async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
sync_env = SyncVectorEnv(env_fns)
async_env.seed(0)
sync_env.seed(0)
assert async_env.num_envs == sync_env.num_envs
assert async_env.observation_space == sync_env.observation_space
assert async_env.single_observation_space == sync_env.single_observation_space
assert async_env.action_space == sync_env.action_space
assert async_env.single_action_space == sync_env.single_action_space
async_observations = async_env.reset()
sync_observations = sync_env.reset()
assert np.all(async_observations == sync_observations)
for _ in range(num_steps):
actions = async_env.action_space.sample()
assert actions in sync_env.action_space
async_observations, async_rewards, async_dones, _ = async_env.step(
actions)
sync_observations, sync_rewards, sync_dones, _ = sync_env.step(
actions)
assert np.all(async_observations == sync_observations)
assert np.all(async_rewards == sync_rewards)
assert np.all(async_dones == sync_dones)
finally:
async_env.close()
sync_env.close()
def test_custom_space_async_vector_env():
env_fns = [make_custom_space_env(i) for i in range(4)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=False)
reset_observations = env.reset()
actions = ('action-2', 'action-3', 'action-5', 'action-7')
step_observations, rewards, dones, _ = env.step(actions)
finally:
env.close()
assert isinstance(env.single_observation_space, CustomSpace)
assert isinstance(env.observation_space, Tuple)
assert isinstance(reset_observations, tuple)
assert reset_observations == ('reset', 'reset', 'reset', 'reset')
assert isinstance(step_observations, tuple)
assert step_observations == ('step(action-2)', 'step(action-3)',
'step(action-5)', 'step(action-7)')
def test_vector_env_equal(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(4)]
num_steps = 100
try:
async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
sync_env = SyncVectorEnv(env_fns)
assert async_env.num_envs == sync_env.num_envs
assert async_env.observation_space == sync_env.observation_space
assert async_env.single_observation_space == sync_env.single_observation_space
assert async_env.action_space == sync_env.action_space
assert async_env.single_action_space == sync_env.single_action_space
async_observations = async_env.reset(seed=0)
sync_observations = sync_env.reset(seed=0)
assert np.all(async_observations == sync_observations)
for _ in range(num_steps):
actions = async_env.action_space.sample()
assert actions in sync_env.action_space
# fmt: off
async_observations, async_rewards, async_dones, async_infos = async_env.step(
actions)
sync_observations, sync_rewards, sync_dones, sync_infos = sync_env.step(
actions)
# fmt: on
if any(sync_dones):
assert "terminal_observation" in async_infos
assert "_terminal_observation" in async_infos
assert "terminal_observation" in sync_infos
assert "_terminal_observation" in sync_infos
assert np.all(async_observations == sync_observations)
assert np.all(async_rewards == sync_rewards)
assert np.all(async_dones == sync_dones)
finally:
async_env.close()
sync_env.close()
def test_reset_async_vector_env(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(8)]
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
observations = env.reset()
finally:
env.close()
assert isinstance(env.observation_space, Box)
assert isinstance(observations, np.ndarray)
assert observations.dtype == env.observation_space.dtype
assert observations.shape == (8, ) + env.single_observation_space.shape
assert observations.shape == env.observation_space.shape
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
observations = env.reset(return_info=False)
finally:
env.close()
assert isinstance(env.observation_space, Box)
assert isinstance(observations, np.ndarray)
assert observations.dtype == env.observation_space.dtype
assert observations.shape == (8, ) + env.single_observation_space.shape
assert observations.shape == env.observation_space.shape
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
observations, infos = env.reset(return_info=True)
finally:
env.close()
assert isinstance(env.observation_space, Box)
assert isinstance(observations, np.ndarray)
assert observations.dtype == env.observation_space.dtype
assert observations.shape == (8, ) + env.single_observation_space.shape
assert observations.shape == env.observation_space.shape
assert isinstance(infos, list)
assert all([isinstance(info, dict) for info in infos])
def test_custom_space_async_vector_env_shared_memory():
env_fns = [make_custom_space_env(i) for i in range(4)]
with pytest.raises(ValueError):
env = AsyncVectorEnv(env_fns, shared_memory=True)
env.close(terminate=True)
def test_already_closed_async_vector_env(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(4)]
with pytest.raises(ClosedEnvironmentError):
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.close()
observations = env.reset()
def test_step_out_of_order_async_vector_env(shared_memory):
env_fns = [make_env("CubeCrash-v0", i) for i in range(4)]
with pytest.raises(NoAsyncCallError):
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
actions = env.action_space.sample()
observations = env.reset()
observations, rewards, dones, infos = env.step_wait()
except AlreadyPendingCallError as exception:
assert exception.name == "step"
raise
finally:
env.close(terminate=True)
with pytest.raises(AlreadyPendingCallError):
try:
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
actions = env.action_space.sample()
env.reset_async()
env.step_async(actions)
except AlreadyPendingCallError as exception:
assert exception.name == "reset"
raise
finally:
env.close(terminate=True)
import multiprocessing as mp
import threading
from gym.vector.tests.utils import make_env, make_slow_env
from gym.vector.async_vector_env import AsyncVectorEnv
import concurrent.futures
from agent import Agent
from agent_test import AgentTest
print("Cores", mp.cpu_count())
if __name__ == '__main__':
#Number of agents working in parallel
num_agents = 100
env_fns = [make_env('CartPole-v0', num_agents) for _ in range(num_agents)]
env = AsyncVectorEnv(env_fns)
agent = Agent(env, state_size=4, action_size=2, num_agents=num_agents)
env_test = gym.make('CartPole-v0')
agent_test = AgentTest(env_test, state_size=4, action_size=2)
one_set_of_weights = 0.1*np.random.randn(agent.get_weights_dim())
all_sets_of_weights = []
for i in range(num_agents):
all_sets_of_weights.append(one_set_of_weights)
start_time = time.time()
for i in range(100):
rewards = agent.evaluate(all_sets_of_weights, num_agents)
print("Time needed for VecEnv approach: ", time.time() - start_time)
def test_already_closed_async_vector_env(shared_memory):
env_fns = [make_env("CartPole-v1", i) for i in range(4)]
with pytest.raises(ClosedEnvironmentError):
env = AsyncVectorEnv(env_fns, shared_memory=shared_memory)
env.close()
env.reset()
