def __init__(self, goal_reward=10, actuation_cost_coeff=30, distance_cost_coeff=1, init_sigma=0.1): self.dynamics = PointDynamics(dim=2, sigma=0) self.init_mu = np.zeros(2, dtype=np.float32) self.init_sigma = init_sigma self.goal_positions = np.array([[5, 0], [-5, 0], [0, 5], [0, -5]], dtype=np.float32) self.goal_threshold = 1. self.goal_reward = goal_reward self.action_cost_coeff = actuation_cost_coeff self.distance_cost_coeff = distance_cost_coeff self.xlim = (-7, 7) self.ylim = (-7, 7) self.vel_bound = 1. self.reset() self.observation = None self.reward_range = (-float('inf'), float('inf')) self.metadata = {'render.modes': []} self.spec = None self._ax = None self._env_lines = [] self.fixed_plots = None self.dynamic_plots = [] super().__init__() Serializable.quick_init(self, locals())
def __init__(self, env, policy, backup_policy, mix_policy, pos_eps_policy, neg_eps_policy, baseline, minibatch_size=500, n_sub_itr=10, optimizer=None, optimizer_args=None, delta=0.01, **kwargs): Serializable.quick_init(self, locals()) self.optimizer = optimizer if optimizer is None: if optimizer_args is None: optimizer_args = dict() self.optimizer = CGOptimizer(**optimizer_args) self.opt_info = None self.backup_policy = backup_policy self.mix_policy = mix_policy self.pos_eps_policy = pos_eps_policy self.neg_eps_policy = neg_eps_policy self.minibatch_size = minibatch_size self.n_sub_itr = n_sub_itr self.delta = delta super(CATRPO, self).__init__(env=env, policy=policy, baseline=baseline, **kwargs)
def __init__(self, env_spec, obs_pl, action, scope_name=None): Serializable.quick_init(self, locals()) self._obs_pl = obs_pl self._action = action self._scope_name = (tf.get_variable_scope().name if not scope_name else scope_name) super(NNPolicy, self).__init__(env_spec)
def __init__(self, inputs, name, hidden_layer_sizes): Parameterized.__init__(self) Serializable.quick_init(self, locals()) self._name = name self._inputs = inputs self._layer_sizes = list(hidden_layer_sizes) + [1] self._output = self._output_for(self._inputs)
def __init__(self, inputs, name, hidden_layer_sizes): Parameterized.__init__(self) Serializable.quick_init(self, locals()) self._name = name self._inputs = inputs self._layer_sizes = list(hidden_layer_sizes) + [1] self._output = self._output_for(self._inputs)
def __init__(self, radar_range=2, radar_resolution=1, discretized=True, use_maps='all', states_cache=None): """ :param radar_range: how many measurements does 'radar' make to each of 4 sides (and combinations) :param radar_resolution: distance between two measurements of agent`s 'radar' :param discretized: discretized actions from {<-1,-0.33> , <-0.33,0.33> , <0.33,1>} to [-1, 0, 1] :param use_maps: which maps to use, list of indexes or 'all' :param states_cache: pre-populated cache to use (observation -> set of states) """ Serializable.quick_init(self, locals()) self.radar_range = radar_range self.radar_resolution = radar_resolution self.discretized = discretized if states_cache is None: self.states_cache = dict() else: self.states_cache = states_cache self.agent_width = 2.4 / np.pi self.max_action_distance = 0.2 self.do_render_init = True self.render_prev_pos = np.zeros(2) self.do_caching = True self.current_map_idx = None self.agent_pos = None self.agent_ori = None # Maps initialization if use_maps == 'all': raw_maps = self.all_maps else: # noinspection PyTypeChecker raw_maps = [self.all_maps[i] for i in use_maps] self.maps = [] self.bit_maps = [] for i in range(len(raw_maps)): # Normalize char map m = np.array([list(row.upper()) for row in raw_maps[i]]) m[np.logical_or(m == '.', m == ' ')] = 'F' m[np.logical_or(m == 'X', m == '#')] = 'W' m[m == 'O'] = 'H' self.maps.append(m) # Make bit map bm = np.zeros(m.shape) bm[np.logical_or(m == 'W', m == 'H')] = 1 self.bit_maps.append(bm)
def __init__(self, env_name, record_video=False, video_schedule=None, log_dir=None, record_log=False, force_reset=True): if log_dir is None: if logger.get_snapshot_dir() is None: logger.log( "Warning: skipping Gym environment monitoring since snapshot_dir not configured." ) else: log_dir = os.path.join(logger.get_snapshot_dir(), "gym_log") Serializable.quick_init(self, locals()) env = gym.envs.make(env_name) # HACK: Gets rid of the TimeLimit wrapper that sets 'done = True' when # the time limit specified for each environment has been passed and # therefore the environment is not Markovian (terminal condition depends # on time rather than state). env = env.env self.env = env self.env_id = env.spec.id assert not (not record_log and record_video) if log_dir is None or record_log is False: self.monitoring = False else: if not record_video: video_schedule = NoVideoSchedule() else: if video_schedule is None: video_schedule = CappedCubicVideoSchedule() self.env = gym.wrappers.Monitor(self.env, log_dir, video_callable=video_schedule, force=True) self.monitoring = True self._observation_space = convert_gym_space(env.observation_space) logger.log("observation space: {}".format(self._observation_space)) self._action_space = convert_gym_space(env.action_space) logger.log("action space: {}".format(self._action_space)) self._horizon = env.spec.tags[ 'wrapper_config.TimeLimit.max_episode_steps'] self._log_dir = log_dir self._force_reset = force_reset
def __init__(self, env_spec, hidden_layer_sizes=(100, 100), name='value_function'): Serializable.quick_init(self, locals()) self._Do = flat_dim(env_spec.observation_space) self._observations_ph = tf.placeholder( tf.float32, shape=[None, self._Do], name='observations') super(NNVFunction, self).__init__( inputs=(self._observations_ph,), name=name, hidden_layer_sizes=hidden_layer_sizes)
def __init__(self, env_spec, q_functions): Serializable.quick_init(self, locals()) self.q_functions = q_functions self._Da = flat_dim(env_spec.action_space) self._Do = flat_dim(env_spec.observation_space) self._observations_ph = tf.placeholder( tf.float32, shape=[None, self._Do], name='observations') self._actions_ph = tf.placeholder( tf.float32, shape=[None, self._Da], name='actions') self._output = self.output_for( self._observations_ph, self._actions_ph, reuse=True)
def __init__(self, env, num_orig_skills, subpath_infos=None): """ Creates a top-level environment for a HRL agent. Original env`s actions are replaced by N discrete actions, N being the number of skills. :param env: AsaEnv environment to wrap :param num_orig_skills: number of pre-trained skill that will prepared be in HRL policy :param subpath_infos: 'all' or list of subpath information to keep, defaults to ['env_infos'] """ Serializable.quick_init(self, locals()) super().__init__(env) self._num_orig_skills = num_orig_skills self.action_space = Discrete(self._num_orig_skills) self.hrl_policy = None if subpath_infos is None: subpath_infos = ['env_infos'] self.subpath_infos = subpath_infos
def __init__(self, env, start_obss, end_obss): """ Creates an environment tailored to train a single (missing) skill. Trajectories are initialized in start_obss state and terminated (and reward is generated) upon reaching end_obs state. :param env: AsaEnv environment to wrap. Environment is cloned to sustain integrity of original env. :param start_obss: Tensor of experienced starting observations (where skill should initiate) :param end_obss: Tensor of experienced ending observations (where skill should terminate) """ Serializable.quick_init(self, locals()) Wrapper.__init__(self, AsaEnv.clone_wrapped( env)) # this clones base env along with all wrappers if start_obss.shape != end_obss.shape: raise ValueError( 'start_obss ({}) and end_obss ({}) must be of same shape'. format(start_obss.shape, end_obss.shape)) self._end_obss = end_obss.reshape((end_obss.shape[0], -1)) self._start_obss = start_obss.reshape((start_obss.shape[0], -1)) self.current_obs_idx = None
def __init__(self, goal=(0, -1), arm_distance_coeff=0): """ goal (`list`): List of two elements denoting the x and y coordinates of the goal location. Either of the coordinate can also be a string 'any' to make the reward not to depend on the corresponding coordinate. arm_distance_coeff ('float'): Coefficient for the arm-to-object distance cost. """ super(PusherEnv, self).__init__(file_path=self.FILE_PATH) Serializable.quick_init(self, locals()) self._goal_mask = [coordinate != 'any' for coordinate in goal] self._goal = np.array(goal)[self._goal_mask].astype(np.float32) self._arm_distance_coeff = arm_distance_coeff self._action_cost_coeff = 0.1 # Make the the complete robot visible when visualizing. self.model.stat.extent = 10
def __init__(self, goal=(0, -1), arm_distance_coeff=0): """ goal (`list`): List of two elements denoting the x and y coordinates of the goal location. Either of the coordinate can also be a string 'any' to make the reward not to depend on the corresponding coordinate. arm_distance_coeff ('float'): Coefficient for the arm-to-object distance cost. """ super(PusherEnv, self).__init__(file_path=self.FILE_PATH) Serializable.quick_init(self, locals()) self._goal_mask = [coordinate != 'any' for coordinate in goal] self._goal = np.array(goal)[self._goal_mask].astype(np.float32) self._arm_distance_coeff = arm_distance_coeff self._action_cost_coeff = 0.1 # Make the the complete robot visible when visualizing. self.model.stat.extent = 10
def __init__(self, env_name, record_video=False, video_schedule=None, log_dir=None, record_log=False, force_reset=True): if log_dir is None: if logger.get_snapshot_dir() is None: logger.log("Warning: skipping Gym environment monitoring since snapshot_dir not configured.") else: log_dir = os.path.join(logger.get_snapshot_dir(), "gym_log") Serializable.quick_init(self, locals()) env = gym.envs.make(env_name) # HACK: Gets rid of the TimeLimit wrapper that sets 'done = True' when # the time limit specified for each environment has been passed and # therefore the environment is not Markovian (terminal condition depends # on time rather than state). env = env.env self.env = env self.env_id = env.spec.id assert not (not record_log and record_video) if log_dir is None or record_log is False: self.monitoring = False else: if not record_video: video_schedule = NoVideoSchedule() else: if video_schedule is None: video_schedule = CappedCubicVideoSchedule() self.env = gym.wrappers.Monitor(self.env, log_dir, video_callable=video_schedule, force=True) self.monitoring = True self._observation_space = convert_gym_space(env.observation_space) logger.log("observation space: {}".format(self._observation_space)) self._action_space = convert_gym_space(env.action_space) logger.log("action space: {}".format(self._action_space)) self._horizon = env.spec.tags['wrapper_config.TimeLimit.max_episode_steps'] self._log_dir = log_dir self._force_reset = force_reset
def __init__(self, env_spec, hidden_layer_sizes, squash=True, name='policy'): Serializable.quick_init(self, locals()) self._action_dim = flat_dim(env_spec.action_space) self._observation_dim = flat_dim(env_spec.observation_space) self._layer_sizes = list(hidden_layer_sizes) + [self._action_dim] self._squash = squash self._name = name self._observation_ph = tf.placeholder( tf.float32, shape=[None, self._observation_dim], name='observation') self._actions = self.actions_for(self._observation_ph) super(StochasticNNPolicy, self).__init__( env_spec, self._observation_ph, self._actions, self._name)
def clone_wrapped(env): """ Clone Serializable AsaEnv wrapped into multiple other environment wrappers. This performs Serializable.clone on inner env and all its wrappers, if possible. Supported wrappers: - TfEnv: no cloning needed, only wrap in new instance - NormalizedEnv: clone and wrap - other Serializable: clone and wrap, display warning - other non-Serializable: wrap in new instance, display warning :param env: AsaEnv wrapped in multiple other environment wrappers """ # Unwrap stack = [] while not isinstance(env, AsaEnv): stack.append((type(env), env)) env = env.env # Clone inner env new_env = Serializable.clone(env) # Re-wrap, cloning wrappers on the way while stack: wrapper_cls, wrapper_env = stack.pop() if wrapper_cls is TfEnv: new_env = TfEnv(new_env) elif wrapper_cls is NormalizedEnv: # WARNING: obs_mean and obs_var are not copied to original env after skill training! new_env = Serializable.clone(wrapper_env, env=new_env) elif isinstance(wrapper_env, Serializable): new_env = Serializable.clone(wrapper_env, env=new_env) warn_once( 'AsaEnv: clone_wrapped performed on unknown Serializable wrapper "{}". ' 'Wrapper was cloned and applied.'.format(wrapper_cls)) else: new_env = wrapper_cls(env=new_env) warn_once( 'AsaEnv: clone_wrapped performed on unknown non-Serializable wrapper "{}". ' 'Wrapper was initiated with default parameters and applied.' .format(wrapper_cls)) return new_env
def __init__(self, env_spec, max_replay_buffer_size): super(SimpleReplayBuffer, self).__init__() Serializable.quick_init(self, locals()) max_replay_buffer_size = int(max_replay_buffer_size) self._env_spec = env_spec self._observation_dim = flat_dim(env_spec.observation_space) self._action_dim = flat_dim(env_spec.action_space) self._max_buffer_size = max_replay_buffer_size self._observations = np.zeros((max_replay_buffer_size, self._observation_dim)) # It's a bit memory inefficient to save the observations twice, # but it makes the code *much* easier since you no longer have to # worry about termination conditions. self._next_obs = np.zeros((max_replay_buffer_size, self._observation_dim)) self._actions = np.zeros((max_replay_buffer_size, self._action_dim)) self._rewards = np.zeros(max_replay_buffer_size) # self._terminals[i] = a terminal was received at time i self._terminals = np.zeros(max_replay_buffer_size, dtype='uint8') self._top = 0 self._size = 0
def __init__(self, env_spec, max_replay_buffer_size): super(SimpleReplayBuffer, self).__init__() Serializable.quick_init(self, locals()) max_replay_buffer_size = int(max_replay_buffer_size) self._env_spec = env_spec self._observation_dim = flat_dim(env_spec.observation_space) self._action_dim = flat_dim(env_spec.action_space) self._max_buffer_size = max_replay_buffer_size self._observations = np.zeros( (max_replay_buffer_size, self._observation_dim)) # It's a bit memory inefficient to save the observations twice, # but it makes the code *much* easier since you no longer have to # worry about termination conditions. self._next_obs = np.zeros( (max_replay_buffer_size, self._observation_dim)) self._actions = np.zeros((max_replay_buffer_size, self._action_dim)) self._rewards = np.zeros(max_replay_buffer_size) # self._terminals[i] = a terminal was received at time i self._terminals = np.zeros(max_replay_buffer_size, dtype='uint8') self._top = 0 self._size = 0
def __init__(self, env_spec, hidden_layer_sizes, squash=True, name='policy'): Serializable.quick_init(self, locals()) self._action_dim = flat_dim(env_spec.action_space) self._observation_dim = flat_dim(env_spec.observation_space) self._layer_sizes = list(hidden_layer_sizes) + [self._action_dim] self._squash = squash self._name = name self._observation_ph = tf.placeholder( tf.float32, shape=[None, self._observation_dim], name='observation') self._actions = self.actions_for(self._observation_ph) super(StochasticNNPolicy, self).__init__(env_spec, self._observation_ph, self._actions, self._name)
def __init__(self, goal_reward=10, actuation_cost_coeff=30, distance_cost_coeff=1, init_sigma=0.1): self.dynamics = PointDynamics(dim=2, sigma=0) self.init_mu = np.zeros(2, dtype=np.float32) self.init_sigma = init_sigma self.goal_positions = np.array( [ [5, 0], [-5, 0], [0, 5], [0, -5] ], dtype=np.float32 ) self.goal_threshold = 1. self.goal_reward = goal_reward self.action_cost_coeff = actuation_cost_coeff self.distance_cost_coeff = distance_cost_coeff self.xlim = (-7, 7) self.ylim = (-7, 7) self.vel_bound = 1. self.reset() self.observation = None self.reward_range = (-float('inf'), float('inf')) self.metadata = {'render.modes': []} self.spec = None self._ax = None self._env_lines = [] self.fixed_plots = None self.dynamic_plots = [] super().__init__() Serializable.quick_init(self, locals())
def __init__(self, top_policy, skill_policy_prototype, skill_policies, skill_stop_functions=None, skill_max_timesteps=100): """ :param top_policy: policy for top-level agent, to be trained :param skill_policies: list of trained skill policies :param skill_policy_prototype: an empty policy serving as a prototype for newly created skill policies. New policies are generated by calling Serializable.clone() upon this prototype, producing a new instance of the policy initialized with same parameters as the prototype. :param skill_stop_functions: list of stopping functions (path_dict -> bool) for trained skills :param skill_max_timesteps: maximum length of skill execution """ Serializable.quick_init(self, locals()) self.top_policy = top_policy self.skill_policy_prototype = skill_policy_prototype self.skill_policies = skill_policies self.skill_max_timesteps = skill_max_timesteps num_orig_skills = len(skill_policies) # pad _skills_end_obss to align indexes with skill_policies self._skills_end_obss = [None for _ in range(num_orig_skills)] # if _skill_stop_functions is not provided, default stopping function (return False) is assigned to all self._skill_stop_functions = skill_stop_functions if skill_stop_functions is not None \ else [lambda path: False for _ in range(num_orig_skills)] assert (len(self._skill_stop_functions) == num_orig_skills) # Check top-level policy if not isinstance(top_policy.action_space, Discrete) \ or top_policy.action_space.n != self.num_skills: raise TypeError( 'Top level policy must have Discrete(num_skills) action space.' )
def create_new_skill(self, end_obss): """ Create new untrained skill and add it to skills list, along with its stopping function. :return: new skill policy and skill ID (index of the skill) :rtype: tuple(garage.policies.base.Policy, int) """ new_skill_id = len(self.skill_policies) new_skill_pol = Serializable.clone( obj=self.skill_policy_prototype, name='{}Skill{}'.format( type(self.skill_policy_prototype).__name__, new_skill_id)) self.skill_policies.append(new_skill_pol) self._skills_end_obss.append(np.copy(end_obss)) unique_end_obss = np.unique(self._skills_end_obss[new_skill_id], axis=0) self._skill_stop_functions.append( # lambda path: path['observations'][-1] in self._skills_end_obss[new_skill_id] lambda path: (path['observations'][-1] == unique_end_obss).all(axis=1).any()) return new_skill_pol, new_skill_id
def __setstate__(self, d): Serializable.__setstate__(self, d) global load_params if load_params: self.set_param_values(d['params'])
def __getstate__(self): d = Serializable.__getstate__(self) d['params'] = self.get_param_values() return d
def __init__(self, env, delay=0.01): Serializable.quick_init(self, locals()) gym.Wrapper.__init__(self, env) self._delay = delay
def __init__(self, *args, **kwargs): Serializable.quick_init(self, locals()) self.reward_range = None self.metadata = None super().__init__(SequenceReacherEnv(*args, **kwargs))