def test_check_observations_async_vector_env(shared_memory): # CubeCrash-v0 - observation_space: Box(40, 32, 3) env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] # MemorizeDigits-v0 - observation_space: Box(24, 32, 3) env_fns[1] = make_env("MemorizeDigits-v0", 1) with pytest.raises(RuntimeError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close(terminate=True)
def test_check_observations_sync_vector_env(): # CubeCrash-v0 - observation_space: Box(40, 32, 3) env_fns = [make_env('CubeCrash-v0', i) for i in range(8)] # MemorizeDigits-v0 - observation_space: Box(24, 32, 3) env_fns[1] = make_env('MemorizeDigits-v0', 1) with pytest.raises(RuntimeError): env = SyncVectorEnv(env_fns) env.close()
def test_step_sync_vector_env(use_single_action_space): env_fns = [make_env('CubeCrash-v0', i) for i in range(8)] try: env = SyncVectorEnv(env_fns) observations = env.reset() if use_single_action_space: actions = [env.single_action_space.sample() for _ in range(8)] else: actions = env.action_space.sample() observations, rewards, dones, _ = env.step(actions) finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape assert isinstance(rewards, np.ndarray) assert isinstance(rewards[0], (float, np.floating)) assert rewards.ndim == 1 assert rewards.size == 8 assert isinstance(dones, np.ndarray) assert dones.dtype == np.bool_ assert dones.ndim == 1 assert dones.size == 8
def test_step_out_of_order_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(4)] with pytest.raises(NoAsyncCallError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) actions = env.action_space.sample() observations = env.reset() observations, rewards, dones, infos = env.step_wait() except AlreadyPendingCallError as exception: assert exception.name == "step" raise finally: env.close(terminate=True) with pytest.raises(AlreadyPendingCallError): try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) actions = env.action_space.sample() env.reset_async() env.step_async(actions) except AlreadyPendingCallError as exception: assert exception.name == "reset" raise finally: env.close(terminate=True)
def test_create_sync_vector_env(): env_fns = [make_env('CubeCrash-v0', i) for i in range(8)] try: env = SyncVectorEnv(env_fns) finally: env.close() assert env.num_envs == 8
def test_create_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) finally: env.close() assert env.num_envs == 8
def test_no_copy_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(8)] try: env = AsyncVectorEnv(env_fns, shared_memory=shared_memory, copy=False) observations = env.reset() observations[0] = 128 assert np.all(env.observations[0] == 128) finally: env.close()
def test_reset_sync_vector_env(): env_fns = [make_env('CubeCrash-v0', i) for i in range(8)] try: env = SyncVectorEnv(env_fns) observations = env.reset() finally: env.close() assert isinstance(env.observation_space, Box) assert isinstance(observations, np.ndarray) assert observations.dtype == env.observation_space.dtype assert observations.shape == (8, ) + env.single_observation_space.shape assert observations.shape == env.observation_space.shape
def main(): np.set_printoptions(suppress=True, formatter={'float_kind': '{:0.2f}'.format}) env_fns = [make_env('MountainCar-v0', i) for i in range(4)] try: env = SyncVectorEnv(env_fns) finally: env.close() state_size = env.observation_space.shape[1] action_size = env.action_space[0].n NUM_EPISODES = 1000 STEPS_PER_EPISODE = 200 batch_size = 32 eps_mean_reward = [0.0] * NUM_EPISODES agent = DQNAgent(state_size, action_size) start_time = datetime.now() for ep_count in range(NUM_EPISODES): episode_rew = 0 state = env.reset() if (ep_count == 0): print("ep={} state.shape={}".format(ep_count, state.shape)) #state = np.reshape(state, [-1, state_size]) ep_start_time = datetime.now() for time in range(STEPS_PER_EPISODE): # env.render() action = agent.act(state) next_state, reward, done, _ = env.step(action) episode_rew += np.sum(reward) #next_state = np.reshape(next_state, [-1, state_size]) if (time == 0): print("ep={} time={} action.len={} next_state.shape={} elaps_time={}".format( \ ep_count, time, len(action), next_state.shape, (datetime.now() - ep_start_time)) ) #add to DQN buffer for idx in range(0, env.num_envs): agent.memorize(state[idx], action[idx], reward[idx], next_state[idx], done[idx]) state = next_state if time >= STEPS_PER_EPISODE - 1: eps_mean_reward[ep_count] = np.mean(episode_rew) / time print("ep: {}/{}, mean_avg_reward: {}, exec_time= {}".format( \ ep_count , NUM_EPISODES, eps_mean_reward[ep_count], (datetime.now() - ep_start_time))) #update DQN model if there are enough samples if len(agent.memory) > batch_size and time % 8 == 0: agent.replay(batch_size) #if ep_count % 2 == 0: # agent.save(str(os.path.join(save_path,'ma-foraging-dqn.h5'))) print("Finish train DQN Agent with {} episodes in {}".format( NUM_EPISODES, (datetime.now() - start_time)))
def test_vector_env_equal(shared_memory): env_fns = [make_env('CubeCrash-v0', i) for i in range(4)] num_steps = 100 try: async_env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) sync_env = SyncVectorEnv(env_fns) async_env.seed(0) sync_env.seed(0) assert async_env.num_envs == sync_env.num_envs assert async_env.observation_space == sync_env.observation_space assert async_env.single_observation_space == sync_env.single_observation_space assert async_env.action_space == sync_env.action_space assert async_env.single_action_space == sync_env.single_action_space async_observations = async_env.reset() sync_observations = sync_env.reset() assert np.all(async_observations == sync_observations) for _ in range(num_steps): actions = async_env.action_space.sample() assert actions in sync_env.action_space async_observations, async_rewards, async_dones, _ = async_env.step( actions) sync_observations, sync_rewards, sync_dones, _ = sync_env.step( actions) assert np.all(async_observations == sync_observations) assert np.all(async_rewards == sync_rewards) assert np.all(async_dones == sync_dones) finally: async_env.close() sync_env.close()
import time import multiprocessing as mp import threading from gym.vector.tests.utils import make_env, make_slow_env from gym.vector.async_vector_env import AsyncVectorEnv import concurrent.futures from agent import Agent from agent_test import AgentTest print("Cores", mp.cpu_count()) if __name__ == '__main__': #Number of agents working in parallel num_agents = 100 env_fns = [make_env('CartPole-v0', num_agents) for _ in range(num_agents)] env = AsyncVectorEnv(env_fns) agent = Agent(env, state_size=4, action_size=2, num_agents=num_agents) env_test = gym.make('CartPole-v0') agent_test = AgentTest(env_test, state_size=4, action_size=2) one_set_of_weights = 0.1*np.random.randn(agent.get_weights_dim()) all_sets_of_weights = [] for i in range(num_agents): all_sets_of_weights.append(one_set_of_weights) start_time = time.time() for i in range(100): rewards = agent.evaluate(all_sets_of_weights, num_agents) print("Time needed for VecEnv approach: ", time.time() - start_time)
def test_already_closed_async_vector_env(shared_memory): env_fns = [make_env("CubeCrash-v0", i) for i in range(4)] with pytest.raises(ClosedEnvironmentError): env = AsyncVectorEnv(env_fns, shared_memory=shared_memory) env.close() observations = env.reset()