def __init__(self, level: LevelSelection, seed: int, frame_skip: int, human_control: bool, custom_reward_threshold: Union[int, float], visualization_parameters: VisualizationParameters, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters, **kwargs) self.state_space = StateSpace({ "observation": VectorObservationSpace(shape=4) # "observation": PlanarMapsObservationSpace(shape=np.array([84, 84, 2]), low=0, high=1) # "observation": TensorObservationSpace(shape=np.array([2, 2]), low=0, high=1) # "player": VectorObservationSpace(shape=2), # "map": PlanarMapsObservationSpace(shape=np.array([84, 84, 1]), low=0, high=2), }) self.goal_space = VectorObservationSpace(shape=4) # self.state_space = PlanarMapsObservationSpace(shape=np.array([84, 84, 2]), low=0, high=1) self.action_space = DiscreteActionSpace(num_actions=4, descriptions={ "0": "up", "1": "down", "2": "left", "3": "right" }) self.env = FacEnv() self.reward_limit = -np.power( np.sum( np.arange(1 + np.abs(self.env.target['x']) + np.abs(self.env.target['y']))) / 2, 2)
def test_filter(): filter = AttentionDiscretization(2) # passing an output space that is wrong with pytest.raises(ValueError): filter.validate_output_action_space(DiscreteActionSpace(10)) with pytest.raises(ValueError): filter.validate_output_action_space(BoxActionSpace(10)) # 1 dimensional box output_space = AttentionActionSpace(2, 0, 83) input_space = filter.get_unfiltered_action_space(output_space) assert np.all(filter.target_actions == np.array( [[[0., 0.], [41.5, 41.5]], [[0., 41.5], [41.5, 83.]], [[41.5, 0], [83., 41.5]], [[41.5, 41.5], [83., 83.]]])) assert input_space.actions == list(range(4)) action = 2 result = filter.filter(action) assert np.all(result == np.array([[41.5, 0], [83., 41.5]])) assert output_space.contains(result) # force int bins filter = AttentionDiscretization(2, force_int_bins=True) input_space = filter.get_unfiltered_action_space(output_space) assert np.all(filter.target_actions == np.array( [[[0., 0.], [41, 41]], [[0., 41], [41, 83.]], [[41, 0], [83., 41]], [[41, 41], [83., 83.]]]))
def test_init(): # discrete control action_space = DiscreteActionSpace(3) # OU process doesn't work for discrete controls with pytest.raises(ValueError): policy = OUProcess(action_space, mu=0, theta=0.1, sigma=0.2, dt=0.01)
def get_unfiltered_action_space( self, output_action_space: ActionSpace) -> DiscreteActionSpace: self.output_action_space = output_action_space self.input_action_space = DiscreteActionSpace( len(self.target_actions), self.descriptions, filtered_action_space=output_action_space) return self.input_action_space
def test_init(): # discrete control action_space = DiscreteActionSpace(3) noise_schedule = LinearSchedule(1.0, 1.0, 1000) # additive noise requires a bounded range for the actions action_space = BoxActionSpace(np.array([10])) with pytest.raises(ValueError): policy = AdditiveNoise(action_space, noise_schedule, 0)
def test_get_action(): # discrete control action_space = DiscreteActionSpace(3) policy = Greedy(action_space) best_action, _ = policy.get_action(np.array([10, 20, 30])) assert best_action == 2 # continuous control action_space = BoxActionSpace(np.array([10])) policy = Greedy(action_space) best_action = policy.get_action(np.array([1, 1, 1])) assert np.all(best_action == np.array([1, 1, 1]))
def test_filter(): filter = BoxMasking(10, 20) # passing an output space that is wrong with pytest.raises(ValueError): filter.validate_output_action_space(DiscreteActionSpace(10)) # 1 dimensional box output_space = BoxActionSpace(1, 5, 30) input_space = filter.get_unfiltered_action_space(output_space) action = np.array([2]) result = filter.filter(action) assert result == np.array([12]) assert output_space.contains(result)
def test_ppo_v_head(): agent_parameters = ClippedPPOAgentParameters() action_space = DiscreteActionSpace(num_actions=5) spaces = SpacesDefinition(state=None, goal=None, action=action_space, reward=None) value_net = VHead(agent_parameters=agent_parameters, spaces=spaces, network_name="test_v_head") value_net.initialize() batch_size = 15 middleware_data = mx.nd.random.uniform(shape=(batch_size, 100)) values = value_net(middleware_data) assert values.ndim == 1 # (batch_size) assert values.shape[0] == batch_size
def test_filter(): filter = LinearBoxToBoxMap(10, 20) # passing an output space that is wrong with pytest.raises(ValueError): filter.validate_output_action_space(DiscreteActionSpace(10)) # 1 dimensional box output_space = BoxActionSpace(1, 5, 35) input_space = filter.get_unfiltered_action_space(output_space) action = np.array([2]) action = np.array([12]) result = filter.filter(action) assert result == np.array([11]) assert output_space.val_matches_space_definition(result)
def test_change_phase(): # discrete control action_space = DiscreteActionSpace(3) epsilon_schedule = LinearSchedule(1.0, 0.1, 1000) policy = EGreedy(action_space, epsilon_schedule, evaluation_epsilon=0.01) # verify schedule not applying if not in training phase assert policy.get_control_param() == 1.0 policy.change_phase(RunPhase.TEST) best_action = policy.get_action(np.array([10, 20, 30])) assert policy.epsilon_schedule.current_value == 1.0 policy.change_phase(RunPhase.HEATUP) best_action = policy.get_action(np.array([10, 20, 30])) assert policy.epsilon_schedule.current_value == 1.0 policy.change_phase(RunPhase.UNDEFINED) best_action = policy.get_action(np.array([10, 20, 30])) assert policy.epsilon_schedule.current_value == 1.0
def test_get_action(): # discrete control action_space = DiscreteActionSpace(3) epsilon_schedule = LinearSchedule(1.0, 1.0, 1000) policy = EGreedy(action_space, epsilon_schedule, evaluation_epsilon=0) # verify that test phase gives greedy actions (evaluation_epsilon = 0) policy.change_phase(RunPhase.TEST) for i in range(100): best_action, _ = policy.get_action(np.array([10, 20, 30])) assert best_action == 2 # verify that train phase gives uniform actions (exploration = 1) policy.change_phase(RunPhase.TRAIN) counters = np.array([0, 0, 0]) for i in range(30000): best_action, _ = policy.get_action(np.array([10, 20, 30])) counters[best_action] += 1 assert np.all(counters > 9500) # this is noisy so we allow 5% error
def test_ppo_head(): agent_parameters = ClippedPPOAgentParameters() num_actions = 5 action_space = DiscreteActionSpace(num_actions=num_actions) spaces = SpacesDefinition(state=None, goal=None, action=action_space, reward=None) head = PPOHead(agent_parameters=agent_parameters, spaces=spaces, network_name="test_ppo_head") head.initialize() batch_size = 15 middleware_data = mx.nd.random.uniform(shape=(batch_size, 100)) probs = head(middleware_data) assert probs.ndim == 2 # (batch_size, num_actions) assert probs.shape[0] == batch_size assert probs.shape[1] == num_actions
def __init__(self, level: LevelSelection, seed: int, frame_skip: int, human_control: bool, custom_reward_threshold: Union[int, float], visualization_parameters: VisualizationParameters, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters, **kwargs) self.state_space = StateSpace({ "observation": PlanarMapsObservationSpace(np.array( [self.map_height, self.map_width, len(Channels)]), low=0, high=20) }) self.init_env() self.action_space = DiscreteActionSpace(num_actions=21, descriptions={ "0": "up", "1": "down", "2": "left", "3": "right", "4": "up-left", "5": "up-right", "6": "down-left", "7": "down-right", "8": "mine-up", "9": "mine-down", "10": "mine-left", "11": "mine-right", "12": "belt-up", "13": "belt-down", "14": "belt-left", "15": "belt-right", "16": "inserter-up", "17": "inserter-down", "18": "inserter-left", "19": "inserter-right", "20": "chest" })
def load_action_space(path_to_json): '''Loads the action space from a given json file, loads default action space is file upload fails. path_to_json - Absolute path to the json file containing the action space ''' json_actions = None try: # Try loading the custom model metadata (may or may not be present) with open(path_to_json, 'r') as file: model_metadata = json.load(file) json_actions = model_metadata['action_space'] LOGGER.info("Loaded action space from file: %s", json_actions) except Exception as ex: # Failed to load, fall back on the default action space from markov.defaults import model_metadata json_actions = model_metadata['action_space'] LOGGER.info("Exception %s on loading custom action space, using default: %s", \ ex, json_actions) action_space = DiscreteActionSpace(num_actions=len(json_actions), default_action=next((i for i, v in enumerate(json_actions) \ if v['steering_angle'] == 0), None)) return action_space, json_actions
def train_on_csv_file(csv_file, n_epochs, dataset_size, obs_dim, act_dim): tf.reset_default_graph( ) # just to clean things up; only needed for the tutorial schedule_params = set_schedule_params(n_epochs, dataset_size) ######### # Agent # ######### # note that we have moved to BCQ, which will help the training to converge better and faster agent_params = set_agent_params(DDQNBCQAgentParameters) # additional setting for DDQNBCQAgentParameters agent parameters # can use either a kNN or a NN based model for predicting which actions not to max over in the bellman equation # agent_params.algorithm.action_drop_method_parameters = KNNParameters() agent_params.algorithm.action_drop_method_parameters = NNImitationModelParameters( ) DATATSET_PATH = csv_file agent_params.memory.load_memory_from_file_path = CsvDataset( DATATSET_PATH, is_episodic=True) spaces = SpacesDefinition(state=StateSpace( {'observation': VectorObservationSpace(shape=obs_dim)}), goal=None, action=DiscreteActionSpace(act_dim), reward=RewardSpace(1)) graph_manager = BatchRLGraphManager( agent_params=agent_params, env_params=None, spaces_definition=spaces, schedule_params=schedule_params, vis_params=VisualizationParameters( dump_signals_to_csv_every_x_episodes=1), reward_model_num_epochs=30, train_to_eval_ratio=0.4) graph_manager.create_graph(task_parameters) graph_manager.improve() return
def test_get_control_param(): # discrete control action_space = DiscreteActionSpace(3) epsilon_schedule = LinearSchedule(1.0, 0.1, 1000) policy = EGreedy(action_space, epsilon_schedule, evaluation_epsilon=0.01) # verify schedule applies to TRAIN phase policy.change_phase(RunPhase.TRAIN) for i in range(999): best_action = policy.get_action(np.array([10, 20, 30])) assert 1.0 > policy.get_control_param() > 0.1 best_action = policy.get_action(np.array([10, 20, 30])) assert policy.get_control_param() == 0.1 # test phases policy.change_phase(RunPhase.TEST) assert policy.get_control_param() == 0.01 policy.change_phase(RunPhase.TRAIN) assert policy.get_control_param() == 0.1 policy.change_phase(RunPhase.HEATUP) assert policy.get_control_param() == 0.1
def test_filter(): filter = BoxDiscretization(9) # passing an output space that is wrong with pytest.raises(ValueError): filter.validate_output_action_space(DiscreteActionSpace(10)) # 1 dimensional box output_space = BoxActionSpace(1, 5, 15) input_space = filter.get_unfiltered_action_space(output_space) assert filter.target_actions == [[5.], [6.25], [7.5], [8.75], [10.], [11.25], [12.5], [13.75], [15.]] assert input_space.actions == list(range(9)) action = 2 result = filter.filter(action) assert result == [7.5] assert output_space.contains(result) # 2 dimensional box filter = BoxDiscretization(3) output_space = BoxActionSpace(2, 5, 15) input_space = filter.get_unfiltered_action_space(output_space) assert filter.target_actions == [[5., 5.], [5., 10.], [5., 15.], [10., 5.], [10., 10.], [10., 15.], [15., 5.], [15., 10.], [15., 15.]] assert input_space.actions == list(range(9)) action = 2 result = filter.filter(action) assert result == [5., 15.] assert output_space.contains(result)
def __init__(self, level: LevelSelection, frame_skip: int, visualization_parameters: VisualizationParameters, target_success_rate: float = 1.0, additional_simulator_parameters: Dict[str, Any] = {}, seed: Union[None, int] = None, human_control: bool = False, custom_reward_threshold: Union[int, float] = None, random_initialization_steps: int = 1, max_over_num_frames: int = 1, observation_space_type: ObservationSpaceType = None, **kwargs): """ :param level: (str) A string representing the gym level to run. This can also be a LevelSelection object. For example, BreakoutDeterministic-v0 :param frame_skip: (int) The number of frames to skip between any two actions given by the agent. The action will be repeated for all the skipped frames. :param visualization_parameters: (VisualizationParameters) The parameters used for visualizing the environment, such as the render flag, storing videos etc. :param additional_simulator_parameters: (Dict[str, Any]) Any additional parameters that the user can pass to the Gym environment. These parameters should be accepted by the __init__ function of the implemented Gym environment. :param seed: (int) A seed to use for the random number generator when running the environment. :param human_control: (bool) A flag that allows controlling the environment using the keyboard keys. :param custom_reward_threshold: (float) Allows defining a custom reward that will be used to decide when the agent succeeded in passing the environment. If not set, this value will be taken from the Gym environment definition. :param random_initialization_steps: (int) The number of random steps that will be taken in the environment after each reset. This is a feature presented in the DQN paper, which improves the variability of the episodes the agent sees. :param max_over_num_frames: (int) This value will be used for merging multiple frames into a single frame by taking the maximum value for each of the pixels in the frame. This is particularly used in Atari games, where the frames flicker, and objects can be seen in one frame but disappear in the next. :param observation_space_type: This value will be used for generating observation space. Allows a custom space. Should be one of ObservationSpaceType. If not specified, observation space is inferred from the number of dimensions of the observation: 1D: Vector space, 3D: Image space if 1 or 3 channels, PlanarMaps space otherwise. """ super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters, target_success_rate) self.random_initialization_steps = random_initialization_steps self.max_over_num_frames = max_over_num_frames self.additional_simulator_parameters = additional_simulator_parameters # hide warnings gym.logger.set_level(40) """ load and initialize environment environment ids can be defined in 3 ways: 1. Native gym environments like BreakoutDeterministic-v0 for example 2. Custom gym environments written and installed as python packages. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space For example: my_environment_package:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class 3. Custom gym environments written as an independent module which is not installed. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space. For example: path_to_my_environment.sub_directory.my_module:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class which is located in the module in the relative path path_to_my_environment.sub_directory.my_module """ if ':' in self.env_id: # custom environments if '/' in self.env_id or '.' in self.env_id: # environment in a an absolute path module written as a unix path or in a relative path module # written as a python import path env_class = short_dynamic_import(self.env_id) else: # environment in a python package env_class = gym.envs.registration.load(self.env_id) # instantiate the environment try: self.env = env_class(**self.additional_simulator_parameters) except: screen.error( "Failed to instantiate Gym environment class %s with arguments %s" % (env_class, self.additional_simulator_parameters), crash=False) raise else: self.env = gym.make(self.env_id) # for classic control we want to use the native renderer because otherwise we will get 2 renderer windows environment_to_always_use_with_native_rendering = [ 'classic_control', 'mujoco', 'robotics' ] self.native_rendering = self.native_rendering or \ any([env in str(self.env.unwrapped.__class__) for env in environment_to_always_use_with_native_rendering]) if self.native_rendering: if hasattr(self, 'renderer'): self.renderer.close() # seed if self.seed is not None: self.env.seed(self.seed) np.random.seed(self.seed) random.seed(self.seed) # frame skip and max between consecutive frames self.is_mujoco_env = 'mujoco' in str(self.env.unwrapped.__class__) self.is_roboschool_env = 'roboschool' in str( self.env.unwrapped.__class__) self.is_atari_env = 'Atari' in str(self.env.unwrapped.__class__) if self.is_atari_env: self.env.unwrapped.frameskip = 1 # this accesses the atari env that is wrapped with a timelimit wrapper env if self.env_id == "SpaceInvadersDeterministic-v4" and self.frame_skip == 4: screen.warning( "Warning: The frame-skip for Space Invaders was automatically updated from 4 to 3. " "This is following the DQN paper where it was noticed that a frame-skip of 3 makes the " "laser rays disappear. To force frame-skip of 4, please use SpaceInvadersNoFrameskip-v4." ) self.frame_skip = 3 self.env = MaxOverFramesAndFrameskipEnvWrapper( self.env, frameskip=self.frame_skip, max_over_num_frames=self.max_over_num_frames) else: self.env.unwrapped.frameskip = self.frame_skip self.state_space = StateSpace({}) # observations if not isinstance(self.env.observation_space, gym.spaces.dict.Dict): state_space = {'observation': self.env.observation_space} else: state_space = self.env.observation_space.spaces for observation_space_name, observation_space in state_space.items(): if observation_space_type == ObservationSpaceType.Tensor: # we consider arbitrary input tensor which does not necessarily represent images self.state_space[ observation_space_name] = TensorObservationSpace( shape=np.array(observation_space.shape), low=observation_space.low, high=observation_space.high) elif observation_space_type == ObservationSpaceType.Image or len( observation_space.shape) == 3: # we assume gym has image observations (with arbitrary number of channels) where their values are # within 0-255, and where the channel dimension is the last dimension if observation_space.shape[-1] in [1, 3]: self.state_space[ observation_space_name] = ImageObservationSpace( shape=np.array(observation_space.shape), high=255, channels_axis=-1) else: # For any number of channels other than 1 or 3, use the generic PlanarMaps space self.state_space[ observation_space_name] = PlanarMapsObservationSpace( shape=np.array(observation_space.shape), low=0, high=255, channels_axis=-1) elif observation_space_type == ObservationSpaceType.Vector or len( observation_space.shape) == 1: self.state_space[ observation_space_name] = VectorObservationSpace( shape=observation_space.shape[0], low=observation_space.low, high=observation_space.high) else: raise screen.error( "Failed to instantiate Gym environment class %s with observation space type %s" % (env_class, observation_space_type), crash=True) if 'desired_goal' in state_space.keys(): self.goal_space = self.state_space['desired_goal'] # actions if type(self.env.action_space) == gym.spaces.box.Box: self.action_space = BoxActionSpace( shape=self.env.action_space.shape, low=self.env.action_space.low, high=self.env.action_space.high) elif type(self.env.action_space) == gym.spaces.discrete.Discrete: actions_description = [] if hasattr(self.env.unwrapped, 'get_action_meanings'): actions_description = self.env.unwrapped.get_action_meanings() self.action_space = DiscreteActionSpace( num_actions=self.env.action_space.n, descriptions=actions_description) else: raise screen.error(( "Failed to instantiate gym environment class {} due to unsupported " "action space {}. Expected BoxActionSpace or DiscreteActionSpace." ).format(env_class, self.env.action_space), crash=True) if self.human_control: # TODO: add this to the action space # map keyboard keys to actions self.key_to_action = {} if hasattr(self.env.unwrapped, 'get_keys_to_action'): self.key_to_action = self.env.unwrapped.get_keys_to_action() else: screen.error( "Error: Environment {} does not support human control.". format(self.env), crash=True) # initialize the state by getting a new state from the environment self.reset_internal_state(True) # render if self.is_rendered: image = self.get_rendered_image() scale = 1 if self.human_control: scale = 2 if not self.native_rendering: self.renderer.create_screen(image.shape[1] * scale, image.shape[0] * scale) # the info is only updated after the first step self.state = self.step(self.action_space.default_action).next_state self.state_space['measurements'] = VectorObservationSpace( shape=len(self.info.keys())) if self.env.spec and custom_reward_threshold is None: self.reward_success_threshold = self.env.spec.reward_threshold self.reward_space = RewardSpace( 1, reward_success_threshold=self.reward_success_threshold) self.target_success_rate = target_success_rate
agent_params.exploration.epsilon_schedule = LinearSchedule(initial_value=0, final_value=0, decay_steps=1) agent_params.exploration.evaluation_epsilon = 0 # can use either a kNN or a NN based model for predicting which actions not to max over in the bellman equation agent_params.algorithm.action_drop_method_parameters = KNNParameters() ########### # Dataset # ########### DATATSET_PATH = 'cartpole_dataset.csv' agent_params.memory = EpisodicExperienceReplayParameters() agent_params.memory.load_memory_from_file_path = CsvDataset(DATATSET_PATH, is_episodic = True) spaces = SpacesDefinition(state=StateSpace({'observation': VectorObservationSpace(shape=4)}), goal=None, action=DiscreteActionSpace(2), reward=RewardSpace(1)) ################# # Visualization # ################# vis_params = VisualizationParameters() vis_params.dump_gifs = True ######## # Test # ######## preset_validation_params = PresetValidationParameters() preset_validation_params.test = True preset_validation_params.min_reward_threshold = 150
def test_get_control_param(): action_space = DiscreteActionSpace(3) policy = Greedy(action_space) assert policy.get_control_param() == 0
def test_init(): # discrete control action_space = DiscreteActionSpace(3)
def __init__( self, level: LevelSelection, seed: int, frame_skip: int, human_control: bool, custom_reward_threshold: Union[int, float], visualization_parameters: VisualizationParameters, host: str, port: int, timeout: float, number_of_vehicles: int, number_of_walkers: int, weather_id: int, #rendering_mode: bool, ego_vehicle_filter: str, display_size: int, sensors: List[SensorTypes], camera_height: int, camera_width: int, lidar_bin: float, obs_range: float, display_route: bool, render_pygame: bool, d_behind: float, out_lane_thres: float, desired_speed: float, max_past_step: int, dt: float, discrete: bool, discrete_acc: List[float], discrete_steer: List[float], continuous_accel_range: List[float], continuous_steer_range: List[float], max_ego_spawn_times: int, max_time_episode: int, max_waypt: int, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters) self.level = level # self.frame_skip = frame_skip # self.seed = seed # self.human_control = human_control # self.custom_reward_threshold = custom_reward_threshold # self.visualization_paramters = visualization_parameters self.host = host self.port = port self.timeout = timeout self.number_of_vehicles = number_of_vehicles self.number_of_walkers = number_of_walkers self.weather_id = weather_id self.ego_vehicle_filter = ego_vehicle_filter self.display_size = display_size self.sensors = sensors self.camera_height = camera_height self.camera_width = camera_width self.obs_range = obs_range self.lidar_bin = lidar_bin self.obs_size = int(self.obs_range / self.lidar_bin) self.display_route = display_route self.render_pygame = render_pygame self.d_behind = d_behind self.out_lane_thres = out_lane_thres self.desired_speed = desired_speed self.max_past_step = max_past_step self.dt = dt self.discrete = discrete self.discrete_acc = discrete_acc self.discrete_steer = discrete_steer self.continuous_accel_range = continuous_accel_range self.continuous_steer_range = continuous_steer_range self.max_ego_spawn_times = max_ego_spawn_times self.max_time_episode = max_time_episode self.max_waypt = max_waypt # Connect to carla server and get world object print('connecting to Carla server...') self.client = carla.Client(self.host, self.port) self.client.set_timeout(self.timeout) self.traffic_manager = self.client.get_trafficmanager() self.world = self.client.load_world(level) print('Carla server connected!') # Set weather self.world.set_weather(CARLA_WEATHER_PRESETS[self.weather_id]) # Get spawn points self._get_spawn_points() # Create the ego vehicle blueprint self.ego_bp = self._create_vehicle_bluepprint(self.ego_vehicle_filter, color='49,8,8') # Collision sensor self.collision_hist = [] # The collision history self.collision_hist_l = 1 # collision history length self.collision_bp = self.world.get_blueprint_library().find( 'sensor.other.collision') # Lidar sensor self.lidar_data = None self.lidar_height = 2.1 self.lidar_trans = carla.Transform( carla.Location(x=0.0, z=self.lidar_height)) self.lidar_bp = self.world.get_blueprint_library().find( 'sensor.lidar.ray_cast') self.lidar_bp.set_attribute('channels', '32') self.lidar_bp.set_attribute('range', '5000') # Camera sensor self.camera_img = np.zeros((self.camera_height, self.camera_width, 3), dtype=np.uint8) self.camera_trans = carla.Transform(carla.Location(x=0.8, z=1.7)) self.camera_bp = self.world.get_blueprint_library().find( 'sensor.camera.rgb') # Modify the attributes of the blueprint to set image resolution and field of view. self.camera_bp.set_attribute('image_size_x', str(self.camera_width)) self.camera_bp.set_attribute('image_size_y', str(self.camera_height)) self.camera_bp.set_attribute('fov', '110') # Set the time in seconds between sensor captures self.camera_bp.set_attribute('sensor_tick', '0.02') # Set fixed simulation step for synchronous mode self.settings = self.world.get_settings() self.settings.fixed_delta_seconds = self.dt #self.settings.no_rendering_mode = rendering_mode self._set_synchronous_mode(True) # Record the time of total steps and resetting steps self.reset_step = 0 self.total_step = 0 # Action space if self.discrete: self.discrete_act = [discrete_acc, discrete_steer] self.n_acc = len(self.discrete_act[0]) self.n_steer = len(self.discrete_act[1]) self.action_space = DiscreteActionSpace( num_actions=self.n_acc * self.n_steer, descriptions=["acceleration", "steering"]) else: self.action_space = BoxActionSpace( shape=2, low=np.array( [continuous_accel_range[0], continuous_steer_range[0]]), high=np.array( [continuous_accel_range[1], continuous_steer_range[1]]), descriptions=["acceleration", "steering"]) # Observation space self.state_space = StateSpace({ "measurements": VectorObservationSpace(shape=4, low=np.array([-2, -1, -5, 0]), high=np.array([2, 1, 30, 1]), measurements_names=[ "lat_dist", "heading_error", "ego_speed", "safety_margin" ]) }) if SensorTypes.FRONT_CAMERA in self.sensors: self.state_space[ SensorTypes.FRONT_CAMERA.value] = ImageObservationSpace( shape=np.array([self.camera_height, self.camera_width, 3]), high=255) if SensorTypes.LIDAR in self.sensors: self.state_space[ SensorTypes.LIDAR.value] = PlanarMapsObservationSpace( shape=np.array([self.obs_size, self.obs_size, 3]), low=0, high=255) if SensorTypes.BIRDEYE in self.sensors: self.state_space[ SensorTypes.BIRDEYE.value] = ImageObservationSpace( shape=np.array([self.obs_size, self.obs_size, 3]), high=255) # Initialize the renderer self._init_renderer() self.reset_internal_state(True)
def __init__(self, level: LevelSelection, frame_skip: int, visualization_parameters: VisualizationParameters, target_success_rate: float = 1.0, seed: Union[None, int] = None, human_control: bool = False, custom_reward_threshold: Union[int, float] = None, screen_size: int = 84, minimap_size: int = 64, feature_minimap_maps_to_use: List = range(7), feature_screen_maps_to_use: List = range(17), observation_type: StarcraftObservationType = StarcraftObservationType.Features, disable_fog: bool = False, auto_select_all_army: bool = True, use_full_action_space: bool = False, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters, target_success_rate) self.screen_size = screen_size self.minimap_size = minimap_size self.feature_minimap_maps_to_use = feature_minimap_maps_to_use self.feature_screen_maps_to_use = feature_screen_maps_to_use self.observation_type = observation_type self.features_screen_size = None self.feature_minimap_size = None self.rgb_screen_size = None self.rgb_minimap_size = None if self.observation_type == StarcraftObservationType.Features: self.features_screen_size = screen_size self.feature_minimap_size = minimap_size elif self.observation_type == StarcraftObservationType.RGB: self.rgb_screen_size = screen_size self.rgb_minimap_size = minimap_size self.disable_fog = disable_fog self.auto_select_all_army = auto_select_all_army self.use_full_action_space = use_full_action_space # step_mul is the equivalent to frame skipping. Not sure if it repeats actions in between or not though. self.env = sc2_env.SC2Env( map_name=self.env_id, step_mul=frame_skip, visualize=self.is_rendered, agent_interface_format=sc2_env.AgentInterfaceFormat( feature_dimensions=sc2_env.Dimensions( screen=self.features_screen_size, minimap=self.feature_minimap_size) # rgb_dimensions=sc2_env.Dimensions( # screen=self.rgb_screen_size, # minimap=self.rgb_screen_size # ) ), # feature_screen_size=self.features_screen_size, # feature_minimap_size=self.feature_minimap_size, # rgb_screen_size=self.rgb_screen_size, # rgb_minimap_size=self.rgb_screen_size, disable_fog=disable_fog, random_seed=self.seed) # print all the available actions # self.env = available_actions_printer.AvailableActionsPrinter(self.env) self.reset_internal_state(True) """ feature_screen: [height_map, visibility_map, creep, power, player_id, player_relative, unit_type, selected, unit_hit_points, unit_hit_points_ratio, unit_energy, unit_energy_ratio, unit_shields, unit_shields_ratio, unit_density, unit_density_aa, effects] feature_minimap: [height_map, visibility_map, creep, camera, player_id, player_relative, selecte d] player: [player_id, minerals, vespene, food_cap, food_army, food_workers, idle_worker_dount, army_count, warp_gate_count, larva_count] """ self.screen_shape = np.array( self.env.observation_spec()[0]['feature_screen']) self.screen_shape[0] = len(self.feature_screen_maps_to_use) self.minimap_shape = np.array( self.env.observation_spec()[0]['feature_minimap']) self.minimap_shape[0] = len(self.feature_minimap_maps_to_use) self.state_space = StateSpace({ "screen": PlanarMapsObservationSpace(shape=self.screen_shape, low=0, high=255, channels_axis=0), "minimap": PlanarMapsObservationSpace(shape=self.minimap_shape, low=0, high=255, channels_axis=0), "measurements": VectorObservationSpace(self.env.observation_spec()[0]["player"][0]) }) if self.use_full_action_space: action_identifiers = list(self.env.action_spec()[0].functions) num_action_identifiers = len(action_identifiers) action_arguments = [(arg.name, arg.sizes) for arg in self.env.action_spec()[0].types] sub_action_spaces = [DiscreteActionSpace(num_action_identifiers)] for argument in action_arguments: for dimension in argument[1]: sub_action_spaces.append(DiscreteActionSpace(dimension)) self.action_space = CompoundActionSpace(sub_action_spaces) else: self.action_space = BoxActionSpace(2, 0, self.screen_size - 1, ["X-Axis, Y-Axis"], default_action=np.array([ self.screen_size / 2, self.screen_size / 2 ])) self.target_success_rate = target_success_rate
def get_action_space(self, json_actions): """Return the action space for the training algorithm Args: json_actions (dict): The json object containing the value of action_space key in model_metadata.json Returns: ActionSpace: Action space object for the particular training algorithm """ try: if self.action_space_type == ActionSpaceTypes.CONTINUOUS.value: # Setting the low and high values to -1 to +1 and scaling in the environment low = np.array([-1.0, -1.0]) high = np.array([1.0, 1.0]) # action_space for CLIPPED_PPO_CONTINUOUS training algorithm is assumed to be a dict with the format: # action_space:{steering_angle:{low:-30.0 , high:30.0}, speed:{low:0.1, high:4.0}} if (json_actions[ModelMetadataKeys.STEERING_ANGLE.value][ ModelMetadataKeys.LOW.value] >= json_actions[ModelMetadataKeys.STEERING_ANGLE.value][ ModelMetadataKeys.HIGH.value] or json_actions[ModelMetadataKeys.SPEED.value][ ModelMetadataKeys.LOW.value] >= json_actions[ModelMetadataKeys.SPEED.value][ ModelMetadataKeys.HIGH.value]): log_and_exit( "Action space bounds are incorrect while defining ClippedPPOActionSpaceConfig. \ json_actions: {}".format(json_actions), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, ) original_low = np.array([ json_actions[ModelMetadataKeys.STEERING_ANGLE.value][ ModelMetadataKeys.LOW.value], json_actions[ModelMetadataKeys.SPEED.value][ ModelMetadataKeys.LOW.value], ]) original_high = np.array([ json_actions[ModelMetadataKeys.STEERING_ANGLE.value][ ModelMetadataKeys.HIGH.value], json_actions[ModelMetadataKeys.SPEED.value][ ModelMetadataKeys.HIGH.value], ]) action_space = ScalableBoxActionSpace( 2, low, high, default_action=0.5 * (high + low), scale_action_space=True, scaled_up_action_space_bounds={ ModelMetadataKeys.LOW.value: original_low, ModelMetadataKeys.HIGH.value: original_high, }, ) elif self.action_space_type == ActionSpaceTypes.DISCRETE.value: action_space = DiscreteActionSpace( num_actions=len(json_actions), default_action=next( (i for i, v in enumerate(json_actions) if v[ModelMetadataKeys.STEERING_ANGLE.value] == 0), None, ), ) return action_space except Exception as ex: log_and_exit( "Error while getting action space in ClippedPPOActionSpaceConfig: {}" .format(ex), SIMAPP_SIMULATION_WORKER_EXCEPTION, SIMAPP_EVENT_ERROR_CODE_500, )
# ER - we'll be needing an episodic replay buffer for off-policy evaluation agent_params.memory = EpisodicExperienceReplayParameters() # E-Greedy schedule - there is no exploration in Batch RL. Disabling E-Greedy. agent_params.exploration.epsilon_schedule = LinearSchedule(initial_value=0, final_value=0, decay_steps=1) agent_params.exploration.evaluation_epsilon = 0 # can use either a kNN or a NN based model for predicting which actions not to max over in the bellman equation #agent_params.algorithm.action_drop_method_parameters = KNNParameters() DATATSET_PATH = 'acrobot_dataset.csv' agent_params.memory = EpisodicExperienceReplayParameters() agent_params.memory.load_memory_from_file_path = CsvDataset(DATATSET_PATH, is_episodic = True) spaces = SpacesDefinition(state=StateSpace({'observation': VectorObservationSpace(shape=6)}), goal=None, action=DiscreteActionSpace(3), reward=RewardSpace(1)) graph_manager = BatchRLGraphManager(agent_params=agent_params, env_params=None, spaces_definition=spaces, schedule_params=schedule_params, vis_params=VisualizationParameters(dump_signals_to_csv_every_x_episodes=1), reward_model_num_epochs=30, train_to_eval_ratio=0.4) graph_manager.create_graph(task_parameters) graph_manager.improve()
def test_discrete(): action_space = DiscreteActionSpace(3, ["zero", "one", "two"]) assert action_space.shape == 1 for i in range(100): assert 3 > action_space.sample() >= 0 action_info = action_space.sample_with_info() assert action_info.action_probability == 1. / 3 assert action_space.high == 2 assert action_space.low == 0 # list descriptions assert action_space.get_description(1) == "one" # dict descriptions action_space = DiscreteActionSpace(3, {1: "one", 2: "two", 0: "zero"}) assert action_space.get_description(0) == "zero" # no descriptions action_space = DiscreteActionSpace(3) assert action_space.get_description(0) == "0" # descriptions for invalid action with pytest.raises(ValueError): assert action_space.get_description(3) == "0"
def __init__(self, level: LevelSelection, frame_skip: int, visualization_parameters: VisualizationParameters, additional_simulator_parameters: Dict[str, Any] = None, seed: Union[None, int] = None, human_control: bool = False, custom_reward_threshold: Union[int, float] = None, random_initialization_steps: int = 1, max_over_num_frames: int = 1, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters) self.random_initialization_steps = random_initialization_steps self.max_over_num_frames = max_over_num_frames self.additional_simulator_parameters = additional_simulator_parameters # hide warnings gym.logger.set_level(40) """ load and initialize environment environment ids can be defined in 3 ways: 1. Native gym environments like BreakoutDeterministic-v0 for example 2. Custom gym environments written and installed as python packages. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space For example: my_environment_package:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class 3. Custom gym environments written as an independent module which is not installed. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space. For example: path_to_my_environment.sub_directory.my_module:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class which is located in the module in the relative path path_to_my_environment.sub_directory.my_module """ if ':' in self.env_id: # custom environments if '/' in self.env_id or '.' in self.env_id: # environment in a an absolute path module written as a unix path or in a relative path module # written as a python import path env_class = short_dynamic_import(self.env_id) else: # environment in a python package env_class = gym.envs.registration.load(self.env_id) # instantiate the environment if self.additional_simulator_parameters: self.env = env_class(**self.additional_simulator_parameters) else: self.env = env_class() else: self.env = gym.make(self.env_id) # for classic control we want to use the native renderer because otherwise we will get 2 renderer windows environment_to_always_use_with_native_rendering = [ 'classic_control', 'mujoco', 'robotics' ] self.native_rendering = self.native_rendering or \ any([env in str(self.env.unwrapped.__class__) for env in environment_to_always_use_with_native_rendering]) if self.native_rendering: if hasattr(self, 'renderer'): self.renderer.close() # seed if self.seed is not None: self.env.seed(self.seed) np.random.seed(self.seed) random.seed(self.seed) # frame skip and max between consecutive frames self.is_robotics_env = 'robotics' in str(self.env.unwrapped.__class__) self.is_mujoco_env = 'mujoco' in str(self.env.unwrapped.__class__) self.is_atari_env = 'Atari' in str(self.env.unwrapped.__class__) self.timelimit_env_wrapper = self.env if self.is_atari_env: self.env.unwrapped.frameskip = 1 # this accesses the atari env that is wrapped with a timelimit wrapper env if self.env_id == "SpaceInvadersDeterministic-v4" and self.frame_skip == 4: screen.warning( "Warning: The frame-skip for Space Invaders was automatically updated from 4 to 3. " "This is following the DQN paper where it was noticed that a frame-skip of 3 makes the " "laser rays disappear. To force frame-skip of 4, please use SpaceInvadersNoFrameskip-v4." ) self.frame_skip = 3 self.env = MaxOverFramesAndFrameskipEnvWrapper( self.env, frameskip=self.frame_skip, max_over_num_frames=self.max_over_num_frames) else: self.env.unwrapped.frameskip = self.frame_skip self.state_space = StateSpace({}) # observations if not isinstance(self.env.observation_space, gym.spaces.dict_space.Dict): state_space = {'observation': self.env.observation_space} else: state_space = self.env.observation_space.spaces for observation_space_name, observation_space in state_space.items(): if len(observation_space.shape ) == 3 and observation_space.shape[-1] == 3: # we assume gym has image observations which are RGB and where their values are within 0-255 self.state_space[ observation_space_name] = ImageObservationSpace( shape=np.array(observation_space.shape), high=255, channels_axis=-1) else: self.state_space[ observation_space_name] = VectorObservationSpace( shape=observation_space.shape[0], low=observation_space.low, high=observation_space.high) if 'desired_goal' in state_space.keys(): self.goal_space = self.state_space['desired_goal'] # actions if type(self.env.action_space) == gym.spaces.box.Box: self.action_space = BoxActionSpace( shape=self.env.action_space.shape, low=self.env.action_space.low, high=self.env.action_space.high) elif type(self.env.action_space) == gym.spaces.discrete.Discrete: actions_description = [] if hasattr(self.env.unwrapped, 'get_action_meanings'): actions_description = self.env.unwrapped.get_action_meanings() self.action_space = DiscreteActionSpace( num_actions=self.env.action_space.n, descriptions=actions_description) if self.human_control: # TODO: add this to the action space # map keyboard keys to actions self.key_to_action = {} if hasattr(self.env.unwrapped, 'get_keys_to_action'): self.key_to_action = self.env.unwrapped.get_keys_to_action() # initialize the state by getting a new state from the environment self.reset_internal_state(True) # render if self.is_rendered: image = self.get_rendered_image() scale = 1 if self.human_control: scale = 2 if not self.native_rendering: self.renderer.create_screen(image.shape[1] * scale, image.shape[0] * scale) # measurements if self.env.spec is not None: self.timestep_limit = self.env.spec.timestep_limit else: self.timestep_limit = None # the info is only updated after the first step self.state = self.step(self.action_space.default_action).next_state self.state_space['measurements'] = VectorObservationSpace( shape=len(self.info.keys())) if self.env.spec and custom_reward_threshold is None: self.reward_success_threshold = self.env.spec.reward_threshold self.reward_space = RewardSpace( 1, reward_success_threshold=self.reward_success_threshold)
class GymEnvironment(Environment): def __init__(self, level: LevelSelection, frame_skip: int, visualization_parameters: VisualizationParameters, additional_simulator_parameters: Dict[str, Any] = None, seed: Union[None, int] = None, human_control: bool = False, custom_reward_threshold: Union[int, float] = None, random_initialization_steps: int = 1, max_over_num_frames: int = 1, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters) self.random_initialization_steps = random_initialization_steps self.max_over_num_frames = max_over_num_frames self.additional_simulator_parameters = additional_simulator_parameters # hide warnings gym.logger.set_level(40) """ load and initialize environment environment ids can be defined in 3 ways: 1. Native gym environments like BreakoutDeterministic-v0 for example 2. Custom gym environments written and installed as python packages. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space For example: my_environment_package:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class 3. Custom gym environments written as an independent module which is not installed. This environments should have a python module with a class inheriting gym.Env, implementing the relevant functions (_reset, _step, _render) and defining the observation and action space. For example: path_to_my_environment.sub_directory.my_module:MyEnvironmentClass will run an environment defined in the MyEnvironmentClass class which is located in the module in the relative path path_to_my_environment.sub_directory.my_module """ if ':' in self.env_id: # custom environments if '/' in self.env_id or '.' in self.env_id: # environment in a an absolute path module written as a unix path or in a relative path module # written as a python import path env_class = short_dynamic_import(self.env_id) else: # environment in a python package env_class = gym.envs.registration.load(self.env_id) # instantiate the environment if self.additional_simulator_parameters: self.env = env_class(**self.additional_simulator_parameters) else: self.env = env_class() else: self.env = gym.make(self.env_id) # for classic control we want to use the native renderer because otherwise we will get 2 renderer windows environment_to_always_use_with_native_rendering = [ 'classic_control', 'mujoco', 'robotics' ] self.native_rendering = self.native_rendering or \ any([env in str(self.env.unwrapped.__class__) for env in environment_to_always_use_with_native_rendering]) if self.native_rendering: if hasattr(self, 'renderer'): self.renderer.close() # seed if self.seed is not None: self.env.seed(self.seed) np.random.seed(self.seed) random.seed(self.seed) # frame skip and max between consecutive frames self.is_robotics_env = 'robotics' in str(self.env.unwrapped.__class__) self.is_mujoco_env = 'mujoco' in str(self.env.unwrapped.__class__) self.is_atari_env = 'Atari' in str(self.env.unwrapped.__class__) self.timelimit_env_wrapper = self.env if self.is_atari_env: self.env.unwrapped.frameskip = 1 # this accesses the atari env that is wrapped with a timelimit wrapper env if self.env_id == "SpaceInvadersDeterministic-v4" and self.frame_skip == 4: screen.warning( "Warning: The frame-skip for Space Invaders was automatically updated from 4 to 3. " "This is following the DQN paper where it was noticed that a frame-skip of 3 makes the " "laser rays disappear. To force frame-skip of 4, please use SpaceInvadersNoFrameskip-v4." ) self.frame_skip = 3 self.env = MaxOverFramesAndFrameskipEnvWrapper( self.env, frameskip=self.frame_skip, max_over_num_frames=self.max_over_num_frames) else: self.env.unwrapped.frameskip = self.frame_skip self.state_space = StateSpace({}) # observations if not isinstance(self.env.observation_space, gym.spaces.dict_space.Dict): state_space = {'observation': self.env.observation_space} else: state_space = self.env.observation_space.spaces for observation_space_name, observation_space in state_space.items(): if len(observation_space.shape ) == 3 and observation_space.shape[-1] == 3: # we assume gym has image observations which are RGB and where their values are within 0-255 self.state_space[ observation_space_name] = ImageObservationSpace( shape=np.array(observation_space.shape), high=255, channels_axis=-1) else: self.state_space[ observation_space_name] = VectorObservationSpace( shape=observation_space.shape[0], low=observation_space.low, high=observation_space.high) if 'desired_goal' in state_space.keys(): self.goal_space = self.state_space['desired_goal'] # actions if type(self.env.action_space) == gym.spaces.box.Box: self.action_space = BoxActionSpace( shape=self.env.action_space.shape, low=self.env.action_space.low, high=self.env.action_space.high) elif type(self.env.action_space) == gym.spaces.discrete.Discrete: actions_description = [] if hasattr(self.env.unwrapped, 'get_action_meanings'): actions_description = self.env.unwrapped.get_action_meanings() self.action_space = DiscreteActionSpace( num_actions=self.env.action_space.n, descriptions=actions_description) if self.human_control: # TODO: add this to the action space # map keyboard keys to actions self.key_to_action = {} if hasattr(self.env.unwrapped, 'get_keys_to_action'): self.key_to_action = self.env.unwrapped.get_keys_to_action() # initialize the state by getting a new state from the environment self.reset_internal_state(True) # render if self.is_rendered: image = self.get_rendered_image() scale = 1 if self.human_control: scale = 2 if not self.native_rendering: self.renderer.create_screen(image.shape[1] * scale, image.shape[0] * scale) # measurements if self.env.spec is not None: self.timestep_limit = self.env.spec.timestep_limit else: self.timestep_limit = None # the info is only updated after the first step self.state = self.step(self.action_space.default_action).next_state self.state_space['measurements'] = VectorObservationSpace( shape=len(self.info.keys())) if self.env.spec and custom_reward_threshold is None: self.reward_success_threshold = self.env.spec.reward_threshold self.reward_space = RewardSpace( 1, reward_success_threshold=self.reward_success_threshold) def _wrap_state(self, state): if not isinstance(self.env.observation_space, gym.spaces.Dict): return {'observation': state} return state def _update_state(self): if self.is_atari_env and hasattr(self, 'current_ale_lives') \ and self.current_ale_lives != self.env.unwrapped.ale.lives(): if self.phase == RunPhase.TRAIN or self.phase == RunPhase.HEATUP: # signal termination for life loss self.done = True elif self.phase == RunPhase.TEST and not self.done: # the episode is not terminated in evaluation, but we need to press fire again self._press_fire() self._update_ale_lives() # TODO: update the measurements if self.state and "desired_goal" in self.state.keys(): self.goal = self.state['desired_goal'] def _take_action(self, action): if type(self.action_space) == BoxActionSpace: action = self.action_space.clip_action_to_space(action) self.state, self.reward, self.done, self.info = self.env.step(action) self.state = self._wrap_state(self.state) def _random_noop(self): # simulate a random initial environment state by stepping for a random number of times between 0 and 30 step_count = 0 random_initialization_steps = random.randint( 0, self.random_initialization_steps) while self.action_space is not None and ( self.state is None or step_count < random_initialization_steps): step_count += 1 self.step(self.action_space.default_action) def _press_fire(self): fire_action = 1 if self.is_atari_env and self.env.unwrapped.get_action_meanings( )[fire_action] == 'FIRE': self.current_ale_lives = self.env.unwrapped.ale.lives() self.step(fire_action) if self.done: self.reset_internal_state() def _update_ale_lives(self): if self.is_atari_env: self.current_ale_lives = self.env.unwrapped.ale.lives() def _restart_environment_episode(self, force_environment_reset=False): # prevent reset of environment if there are ale lives left if (self.is_atari_env and self.env.unwrapped.ale.lives() > 0) \ and not force_environment_reset and not self.timelimit_env_wrapper._past_limit(): self.step(self.action_space.default_action) else: self.state = self.env.reset() self.state = self._wrap_state(self.state) self._update_ale_lives() if self.is_atari_env: self._random_noop() self._press_fire() # initialize the number of lives self._update_ale_lives() def _set_mujoco_camera(self, camera_idx: int): """ This function can be used to set the camera for rendering the mujoco simulator :param camera_idx: The index of the camera to use. Should be defined in the model :return: None """ if self.env.unwrapped.viewer is not None and self.env.unwrapped.viewer.cam.fixedcamid != camera_idx and\ self.env.unwrapped.viewer._ncam > camera_idx: from mujoco_py.generated import const self.env.unwrapped.viewer.cam.type = const.CAMERA_FIXED self.env.unwrapped.viewer.cam.fixedcamid = camera_idx def _get_robotics_image(self): self.env.render() image = self.env.unwrapped._get_viewer().read_pixels( 1600, 900, depth=False)[::-1, :, :] image = scipy.misc.imresize(image, (270, 480, 3)) return image def _render(self): self.env.render(mode='human') # required for setting up a fixed camera for mujoco if self.is_mujoco_env: self._set_mujoco_camera(0) def get_rendered_image(self): if self.is_robotics_env: # necessary for fetch since the rendered image is cropped to an irrelevant part of the simulator image = self._get_robotics_image() else: image = self.env.render(mode='rgb_array') # required for setting up a fixed camera for mujoco if self.is_mujoco_env: self._set_mujoco_camera(0) return image
def __init__(self, level: LevelSelection, frame_skip: int, visualization_parameters: VisualizationParameters, seed: Union[None, int] = None, human_control: bool = False, custom_reward_threshold: Union[int, float] = None, width: int = 84, height: int = 84, num_rotations: int = 24, episode_length: int = 50000, **kwargs): super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters) # seed if self.seed is not None: np.random.seed(self.seed) random.seed(self.seed) self.object = Object2D(width, height, num_rotations) self.last_result = self.object.reset() self.state_space = StateSpace({}) # image observations self.state_space['observation'] = PlanarMapsObservationSpace( shape=np.array([width, height, 3]), low=0, high=255) # measurements observations measurements_space_size = 4 measurements_names = [ 'x-position', 'y-position', 'rotation', 'num_covered_states' ] self.state_space['measurements'] = VectorObservationSpace( shape=measurements_space_size, measurements_names=measurements_names) # actions self.num_actions = 6 self.action_space = DiscreteActionSpace(self.num_actions) self.steps = 0 self.episode_length = episode_length self.width = width self.height = height self.num_rotations = num_rotations self.bin_size = 1 self.covered_states = np.zeros( (int(self.width / self.bin_size), int(self.height / self.bin_size), self.num_rotations)) self.num_covered_states = 0 # render if self.is_rendered: image = np.squeeze(self.object.render()) self.renderer.create_screen(image.shape[1], image.shape[0]) # initialize the state by getting a new state from the environment self.reset_internal_state(True)
# NN configuration agent_params.network_wrappers['main'].learning_rate = 0.0001 agent_params.network_wrappers['main'].replace_mse_with_huber_loss = False agent_params.network_wrappers['main'].softmax_temperature = 0.2 # ER size agent_params.memory = EpisodicExperienceReplayParameters() DATATSET_PATH = os.path.join(os.path.expanduser('~'), 'share/Data/MLA/L2P/L2_data_rnd_RL.csv') agent_params.memory.load_memory_from_file_path = CsvDataset( DATATSET_PATH, True) # E-Greedy schedule agent_params.exploration.epsilon_schedule = LinearSchedule(0, 0, 10000) agent_params.exploration.evaluation_epsilon = 0 spaces = SpacesDefinition(state=StateSpace( {'observation': VectorObservationSpace(shape=7)}), goal=None, action=DiscreteActionSpace(4), reward=RewardSpace(1)) graph_manager = BatchRLGraphManager( agent_params=agent_params, env_params=None, spaces_definition=spaces, schedule_params=schedule_params, vis_params=VisualizationParameters(dump_signals_to_csv_every_x_episodes=1), reward_model_num_epochs=30, train_to_eval_ratio=0.4)