def test_run_environment(env_name): """ Run the gym test using the specified environment :param env_name: Name of the Unity environment binary to launch """ u_env = UnityEnvironment(env_name, worker_id=1, no_graphics=True) env = UnityToGymWrapper(u_env) try: # Examine environment parameters print(str(env)) # Reset the environment initial_observations = env.reset() if len(env.observation_space.shape) == 1: # Examine the initial vector observation print("Agent observations look like: \n{}".format( initial_observations)) for _episode in range(10): env.reset() done = False episode_rewards = 0 while not done: actions = env.action_space.sample() obs, reward, done, _ = env.step(actions) episode_rewards += reward print("Total reward this episode: {}".format(episode_rewards)) finally: env.close()
class UnityEnvWrapper(gym.Env): def __init__(self, env_config): self.worker_index = 0 if 'SM_CHANNEL_TRAIN' in os.environ: env_name = os.environ['SM_CHANNEL_TRAIN'] +'/'+ env_config['env_name'] os.chmod(env_name, 0o755) print("Changed environment binary into executable mode.") # Try connecting to the Unity3D game instance. while True: try: channel = EnvironmentParametersChannel() unity_env = UnityEnvironment( env_name, no_graphics=True, worker_id=self.worker_index, side_channels=[channel], additional_args=['-logFile', 'unity.log']) channel.set_float_parameter("simulation_mode", 1.0) except UnityWorkerInUseException: self.worker_index += 1 else: break else: env_name = env_config['env_name'] while True: try: unity_env = default_registry[env_name].make( no_graphics=True, worker_id=self.worker_index, additional_args=['-logFile', 'unity.log']) except UnityWorkerInUseException: self.worker_index += 1 else: break self.env = UnityToGymWrapper(unity_env) self.action_space = self.env.action_space self.observation_space = self.env.observation_space def reset(self): return self.env.reset() def step(self, action): return self.env.step(action) def close(self): try: self.env.close() except Exception: pass
def test_closing(env_name): """ Run the gym test and closes the environment multiple times :param env_name: Name of the Unity environment binary to launch """ try: env1 = UnityToGymWrapper( UnityEnvironment(env_name, worker_id=1, no_graphics=True)) env1.close() env1 = UnityToGymWrapper( UnityEnvironment(env_name, worker_id=1, no_graphics=True)) env2 = UnityToGymWrapper( UnityEnvironment(env_name, worker_id=2, no_graphics=True)) env2.reset() finally: env1.close() env2.close()
class FooCarEnv(gym.Env): _channel = EnvironmentParametersChannel() PathSpace = { 'xyz': 0, 'xy': 2, 'yz': 2, 'xz': 2 } def __init__(self, no_graphics:bool=False, seed:int=1, **config): self._config = config worker_id = 0 if 'worker_id' in config: worker_id = config['worker_id'] self._unity_env = UnityEnvironment( file_name=UNITY_ENV_EXE_FILE, # file_name=None, # Unity Editor Mode (debug) no_graphics=no_graphics, seed=seed, side_channels=[self._channel], worker_id=worker_id ) for key, value in config.items(): self._channel.set_float_parameter(key, float(value)) self._gym_env = UnityToGymWrapper(self._unity_env) def step(self, action): obs, reward, done, info = self._gym_env.step(action) size = self.observation_size return obs[:size], reward, done, info def reset(self): obs = self._gym_env.reset() size = self.observation_size return obs[:size] def render(self, mode="rgb_array"): return self._gym_env.render(mode=mode) def seed(self, seed=None): self._gym_env.seed(seed=seed) # it will throw a warning def close(self): self._gym_env.close() @property def metadata(self): return self._gym_env.metadata @property def reward_range(self) -> Tuple[float, float]: return self._gym_env.reward_range @property def action_space(self): return self._gym_env.action_space @property def observation_space(self): config = self._config space = self.PathSpace path_space = config['path_space'] if 'path_space' in config else space['xz'] r = config['radius_anchor_circle'] if 'radius_anchor_circle' in config else 8.0 r_e = config['radius_epsilon_ratio'] if 'radius_epsilon_ratio' in config else 0.7 h = config['max_anchor_height'] if 'max_anchor_height' in config else 1.0 xyz_mode = (path_space == space['xyz']) bound = max(r * (1 + r_e), h if xyz_mode else 0) shape = (self.observation_size,) return gym.spaces.Box(-bound, +bound, dtype=np.float32, shape=shape) @property def observation_size(self): # Reference: readonly variable (Unity)FooCar/CarAgent.ObservationSize config = self._config space = self.PathSpace path_space = config['path_space'] if 'path_space' in config else space['xz'] ticker_end = config['ticker_end'] if 'ticker_end' in config else 5 ticker_start = config['ticker_start'] if 'ticker_start' in config else -3 xyz_mode = (path_space == space['xyz']) basic_num = 6 point_dim = 3 if xyz_mode else 2 return basic_num + 2 * point_dim * (ticker_end - ticker_start + 1)
state = torch.load('D:/RL_project/FInal Project/RLCar/Path_folder/46305_0.172707200050354.pth') def get_action(state): if len(state) == 34: state = get_il_state(state) with torch.no_grad(): state = torch.Tensor(state).view(1,-1).to(device) print("state.shape=",state.shape) action = model_req(state) return action.cpu().numpy() def il_eval(): state = env.reset() score = 0 max_t = 10000 for t in range(max_t): action = get_action(state) next_state, reward, done, _ = env.step(action) next_state = get_il_state(next_state) state = next_state score += reward if done: break #env = UnityToGymWrapper(UnityEnvironment(base_port=5004), 0) env = UnityToGymWrapper(UnityEnvironment('D:/RL_project/FInal Project/RLCar/Build/RLCar.exe'), 0) il_eval() env.close()
def objective(trial): # Domain setup # windows_path = "../crawler_single/UnityEnvironment" # build_path = windows_path linux_path = "../crawler_single/linux/dynamic_server/crawler_dynamic.x86_64" build_path = linux_path unity_env = UnityEnvironment(file_name=build_path, seed=1, side_channels=[], no_graphics=False) env = UnityToGymWrapper(unity_env=unity_env) training_episodes = 10000 params = {} params["nr_output_features"] = env.action_space.shape[0] params["nr_input_features"] = env.observation_space.shape[0] params["env"] = env params["lr"] = 3e-4 params["clip"] = 0.2 params["hidden_units"] = 512 params["update_episodes"] = 10 params["minibatch_size"] = 32 params["tau"] = 0.95 params["std"] = 0.35 params["update_episodes"] = trial.suggest_int(name='update_episodes', low=5, high=30, step=5) params["ppo_epochs"] = trial.suggest_int(name='ppo_epochs', low=2, high=10, step=2) params["gamma"] = trial.suggest_float(name='gamma', low=0.98, high=0.99, log=True) params["beta"] = trial.suggest_float(name='beta', low=0.08, high=0.12, log=True) print(params) time_str = time.strftime("%y%m%d_%H") t = "{}_{}".format(worker_id, time_str) print(t) writer = SummaryWriter(log_dir='runs/alex/{}'.format(time_str), filename_suffix=t) agent = a.PPOLearner(params, writer) returns = [ episode(env, agent, params, writer, i) for i in range(training_episodes) ] torch.save( agent.ppo_net, "../Net_Crawler/Alex/PPONet_crawler{}_{}.pt".format( worker_id, time_str)) mean_reward, std_reward = evaluate_model(agent.ppo_net, env, n_eval_episodes=10) print("{}, {}".format(mean_reward, std_reward)) writer.close() env.close() return mean_reward
class ActorUnity(Actor, RoadworkActorInterface): def __init__(self, ctx, actor_id): super(ActorUnity, self).__init__(ctx, actor_id) self.env = None # Placeholder self.actor_id = actor_id async def sim_call_method(self, data) -> object: method = data['method'] args = data['args'] # Array of arguments - [] kwargs = data['kwargs'] # Dict return getattr(self.env, method)(*args, **kwargs) async def sim_get_state(self, data) -> object: key = data['key'] has_value, val = await self._state_manager.try_get_state(key) return val async def sim_set_state(self, data) -> None: key = data['key'] value = data['value'] print(f'Setting Sim State for key {key}', flush=True) await self._state_manager.set_state(key, value) await self._state_manager.save_state() async def _on_activate(self) -> None: """An callback which will be called whenever actor is activated.""" print(f'Activate {self.__class__.__name__} actor!', flush=True) async def _on_deactivate(self) -> None: """An callback which will be called whenever actor is deactivated.""" print(f'Deactivate {self.__class__.__name__} actor!', flush=True) # see behavior_spec: https://github.com/Unity-Technologies/ml-agents/blob/release_4_docs/docs/Python-API.md#interacting-with-a-unity-environment # behavior_spec.action_type and behavior_spec.action_shape is what we need here async def sim_action_space(self) -> object: behavior_names = list(self.env.behavior_specs.keys()) # the behavior_names which map to a Behavior Spec with observation_shapes, action_type, action_shape behavior_idx = 0 # we currently support only 1 behavior spec! even though Unity can support multiple (@TODO) behavior_spec = self.env.behavior_specs[behavior_names[behavior_idx]] print(f"Action Type: {behavior_spec.action_type}", flush=True) print(f"Action Shape: {behavior_spec.action_shape}", flush=True) # We can use /src/Lib/python/roadwork/roadwork/json/unserializer.py as an example # Currently only ActionType.DISCRETE implemented, all ActionTypes can be found here: https://github.com/Unity-Technologies/ml-agents/blob/3901bad5b0b4e094e119af2f9d0d1304ad3f97ae/ml-agents-envs/mlagents_envs/base_env.py#L247 # Note: Unity supports DISCRETE or CONTINUOUS action spaces @TODO: implement continuous in a specific env (which one??) if behavior_spec.is_action_discrete() == True: self.env.action_space = spaces.Discrete(behavior_spec.action_shape[0]) print(f"Converted Action Space: {self.env.action_space}", flush=True) res = Serializer.serializeMeta(self.env.action_space) return res # see behavior_spec: https://github.com/Unity-Technologies/ml-agents/blob/release_4_docs/docs/Python-API.md#interacting-with-a-unity-environment # behavior_spec.observation_shapes is what we need, this is an array of tuples [ (), (), (), ... ] which represents variables? (@TODO: Confirm) (e.g. https://github.com/Unity-Technologies/ml-agents/blob/master/docs/Learning-Environment-Examples.md#basic) # @TODO: This sounds as a MultiDiscrete environment (https://github.com/openai/gym/blob/master/gym/spaces/multi_discrete.py) so we map to this currently async def sim_observation_space(self) -> object: behavior_names = list(self.env.behavior_specs.keys()) # the behavior_names which map to a Behavior Spec with observation_shapes, action_type, action_shape behavior_idx = 0 # we currently support only 1 behavior spec! even though Unity can support multiple (@TODO) behavior_spec = self.env.behavior_specs[behavior_names[behavior_idx]] print(f"Observation Shapes: {behavior_spec.observation_shapes}", flush=True) observation_space_n_vec = [] for i in range(0, len(behavior_spec.observation_shapes)): observation_space_n_vec.append(behavior_spec.observation_shapes[i][0]) # Get el 0 from the tuple, containing the size print(f"Converted Observation Space: {observation_space_n_vec}", flush=True) self.env.observation_space = spaces.MultiDiscrete(observation_space_n_vec) res = Serializer.serializeMeta(self.env.observation_space) return res async def sim_create(self, data) -> None: """An actor method to create a sim environment.""" env_id = data['env_id'] # seed = data['seed'] print(f'Creating sim with value {env_id}', flush=True) print(f"Current dir: {os.getcwd()}", flush=True) try: print("[Server] Creating Unity Environment", flush=True) self.env = UnityEnvironment(f"{os.getcwd()}/src/Server/Unity/envs/{env_id}/{env_id}") print("[Server] Resetting environment already", flush=True) self.env.reset() # we need to reset first in Unity # self.unity_env = UnityEnvironment("./environments/GridWorld") # self.env = gym.make(env_id) # if seed: # self.env.seed(seed) except gym.error.Error as e: print(e) raise Exception("Attempted to look up malformed environment ID '{}'".format(env_id)) except Exception as e: print(e) raise Exception(e) except: print(sys.exc_info()) traceback.print_tb(sys.exc_info()[2]) raise async def sim_reset(self) -> object: observation = self.env.reset() # observation is a ndarray, we need to serialize this # therefore, change it to list type which is serializable if isinstance(observation, np.ndarray): observation = observation.tolist() return observation async def sim_render(self) -> None: self.env.render() async def sim_monitor_start(self, data) -> None: episodeInterval = 10 # Create a recording every X episodes if data['episode_interval']: episodeInterval = int(data['episode_interval']) v_c = lambda count: count % episodeInterval == 0 # Create every X episodes #self.env = gym.wrappers.Monitor(self.env, f'./output/{self.actor_id}', resume=False, force=True, video_callable=v_c) #self.env = UnityToGymWrapper(self.unity_environment) #defaults to BaseEnv self.env = UnityToGymWrapper() async def sim_monitor_stop(self) -> None: self.env.close() async def sim_action_sample(self) -> object: action = self.env.action_space.sample() return action async def sim_step(self, data) -> object: action = data['action'] # Unity requires us to set the action with env.set_actions(behavior_name, action) where action is an array behavior_names = list(self.env.behavior_specs.keys()) # the behavior_names which map to a Behavior Spec with observation_shapes, action_type, action_shape behavior_idx = 0 # we currently support only 1 behavior spec! even though Unity can support multiple (@TODO) behavior_name = behavior_names[behavior_idx] self.env.set_actions(behavior_name, np.array([ [ action ] ])) # first dimension = number of agents, second dimension = action? self.env.step() # step does not return in Unity # Get the DecisionSteps and TerminalSteps # -> they both contain: # DecisionSteps: Which agents need an action this step? (Note: contains action masks!) # E.g.: DecisionStep(obs=[array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.], dtype=float32)], reward=-0.01, agent_id=0, action_mask=[array([False, False, False])]) # TerminalSteps: Which agents their episode ended? decision_steps, terminal_steps = self.env.get_steps(behavior_names[behavior_idx]) # print(decision_steps, flush=True) # print(terminal_steps, flush=True) # print(decision_steps[0], flush=True) # print(terminal_steps[0], flush=True) # We support 1 decision step currently, get its observation # TODO decision_step_idx = 0 decision_step = decision_steps[decision_step_idx] obs, reward, agent_id, action_mask = decision_step observation = obs[decision_step_idx] reward = float(reward) isDone = False info = {} # @TODO: terminal_steps should be implemented, it requires a reset # observation is a ndarray, we need to serialize this # therefore, change it to list type which is serializable if isinstance(observation, np.ndarray): observation = observation.tolist() return observation, reward, isDone, info
def train(path): # env = gym.make("LunarLander-v2") # env = wrappers.Monitor(env, "tmp/lunar-lander", video_callable=lambda episode_id: True, force=True) unityenv = UnityEnvironment(path) env = UnityToGymWrapper(unity_env=unityenv, flatten_branched=True) ddqnAgent = DDQNAgent(alpha=0.0001, gamma=0.99, nActions=7, epsilon=1.0, batchSize=512, inputShape=210) nEpisodes = 1000 ddqnScores = [] ddqnAverageScores = [] epsilonHistory = [] stepsPerEpisode = [] for episode in range(nEpisodes): StartTime = time.time() done = False score = 0 steps = 0 observation = env.reset() while not done: action = ddqnAgent.chooseAction(observation) observationNew, reward, done, info = env.step(action) score += reward ddqnAgent.remember(state=observation, stateNew=observationNew, action=action, reward=reward, done=done) observation = observationNew ddqnAgent.learn() steps += 1 epsilonHistory.append(ddqnAgent.epsilon) ddqnScores.append(score) averageScore = np.mean(ddqnScores) ddqnAverageScores.append(averageScore) stepsPerEpisode.append(steps) ElapsedTime = time.time() - StartTime ElapsedTime = ElapsedTime / 60 print("Episode:", episode, "Score: %.2f" % score, "Average Score: %.2f" % averageScore, "Run Time:", ElapsedTime, "Minutes", "Epsilon:", ddqnAgent.epsilon, "Steps:", steps) if episode > 1 and episode % 9 == 0: ddqnAgent.saveModel() env.close() x = [i for i in range(nEpisodes)] fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2, 2, figsize=(10, 10)) fig.suptitle("DDQN Hallway") ax1.plot(x, ddqnScores, "C1") ax1.set_title('Episodes vs Scores') ax1.set(xlabel='Episodes', ylabel='Scores') ax2.plot(x, ddqnAverageScores, "C2") ax2.set_title('Episodes vs Average Scores') ax2.set(xlabel='Episodes', ylabel='Average Scores') ax3.plot(x, epsilonHistory, "C3") ax3.set_title('Episodes vs Epsilon Decay') ax3.set(xlabel='Episodes', ylabel='Epsilon Decay') ax4.plot(x, stepsPerEpisode, "C4") ax4.set_title('Episodes vs Steps Per Epsisode') ax4.set(xlabel='Episodes', ylabel='Steps') plt.savefig('Hallway.png')