def __init__(self,
env_spec,
name='discrete_mlp_q_function',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
obs_dim = env_spec.observation_space.shape
action_dim = env_spec.action_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
obs_ph = tf.placeholder(tf.float32, (None, ) + obs_dim, name='obs')
with tf.variable_scope(name) as vs:
self._variable_scope = vs
self.model.build(obs_ph)
def __init__(self,
env_spec,
name='ContinuousMLPPolicy',
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.tanh,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name='MLPModel',
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def __init__(self,
env_spec,
name='CategoricalMLPPolicy',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
assert isinstance(
env_spec.action_space,
Discrete), ('CategoricalMLPPolicy only works with akro.tf.Discrete'
'action space.')
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.n
self.model = MLPModel(output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel')
self._initialize()
def test_is_pickleable(self, output_dim, hidden_sizes):
model = MLPModel(
output_dim=output_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=None,
hidden_w_init=tf.ones_initializer(),
output_w_init=tf.ones_initializer())
outputs = model.build(self.input_var)
# assign bias to all one
with tf.compat.v1.variable_scope('MLPModel/mlp', reuse=True):
bias = tf.compat.v1.get_variable('hidden_0/bias')
bias.load(tf.ones_like(bias).eval())
output1 = self.sess.run(outputs, feed_dict={self.input_var: self.obs})
h = pickle.dumps(model)
with tf.compat.v1.Session(graph=tf.Graph()) as sess:
input_var = tf.compat.v1.placeholder(tf.float32, shape=(None, 5))
model_pickled = pickle.loads(h)
outputs = model_pickled.build(input_var)
output2 = sess.run(outputs, feed_dict={input_var: self.obs})
assert np.array_equal(output1, output2)
def __init__(self,
env_spec,
name='DeterministicMLPPolicy',
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.tanh,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
input_include_goal=False,
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
if input_include_goal:
self.obs_dim = env_spec.observation_space.flat_dim_with_keys(
['observation', 'desired_goal'])
else:
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name='MLPModel',
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def test_output_values(self, output_dim, hidden_sizes):
model = MLPModel(output_dim=output_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=None,
hidden_w_init=tf.ones_initializer(),
output_w_init=tf.ones_initializer())
outputs = model.build(self.input_var)
output = self.sess.run(outputs, feed_dict={self.input_var: self.obs})
expected_output = np.full([1, output_dim], 5 * np.prod(hidden_sizes))
assert np.array_equal(output, expected_output)
def __init__(self,
env_spec,
name='MLPTerminalFunction',
hidden_sizes=(20, 20),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
input_include_goal=False,
layer_normalization=False):
super().__init__(name)
self._env_spec = env_spec
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._input_include_goal = input_include_goal
self._layer_normalization = layer_normalization
if self._input_include_goal:
self._obs_dim = env_spec.observation_space.flat_dim_with_keys(
['observation', 'desired_goal'])
else:
self._obs_dim = env_spec.observation_space.flat_dim
self._action_dim = env_spec.action_space.flat_dim
self.model = MLPModel(output_dim=2,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def __init__(self,
env_spec,
name=None,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
dueling=False,
layer_normalization=False):
super().__init__(name)
self._env_spec = env_spec
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._dueling = dueling
self._layer_normalization = layer_normalization
self.obs_dim = env_spec.observation_space.shape
action_dim = env_spec.action_space.flat_dim
if not dueling:
self.model = MLPModel(output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
else:
self.model = MLPDuelingModel(
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._network = None
self._initialize()
def __init__(self,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pad,
name='CategoricalCNNPolicy',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
if not isinstance(env_spec.action_space, akro.Discrete):
raise ValueError(
'CategoricalCNNPolicy only works with akro.Discrete action '
'space.')
if not isinstance(env_spec.observation_space, akro.Box) or \
not len(env_spec.observation_space.shape) in (2, 3):
raise ValueError(
'{} can only process 2D, 3D akro.Image or'
' akro.Box observations, but received an env_spec with '
'observation_space of type {} and shape {}'.format(
type(self).__name__,
type(env_spec.observation_space).__name__,
env_spec.observation_space.shape))
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.shape
self.action_dim = env_spec.action_space.n
self.model = Sequential(
CNNModel(filter_dims=conv_filter_sizes,
num_filters=conv_filters,
strides=conv_strides,
padding=conv_pad,
hidden_nonlinearity=hidden_nonlinearity,
name='CNNModel'),
MLPModel(output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel'))
self._initialize()
def __init__(self,
env_spec,
name="CategoricalMLPPolicy",
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.nn.softmax,
layer_normalization=False):
assert isinstance(
env_spec.action_space,
Discrete), ("CategoricalMLPPolicy only works with akro.tf.Discrete"
"action space.")
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.n
self.model = MLPModel(
output_dim=self.action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
layer_normalization=layer_normalization)
self._initialize()
def __init__(self,
env_spec,
name="DeterministicMLPPolicy",
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh,
input_include_goal=False,
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
if input_include_goal:
self.obs_dim = env_spec.observation_space.flat_dim_with_keys(
["observation", "desired_goal"])
else:
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
layer_normalization=layer_normalization)
self._initialize()
def test_is_pickleable(self, output_dim, hidden_sizes):
model = MLPModel(
output_dim=output_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=None,
hidden_w_init=tf.ones_initializer(),
output_w_init=tf.ones_initializer())
model_pickled = pickle.loads(pickle.dumps(model))
with tf.Session(graph=tf.Graph()) as sess:
input_var = tf.placeholder(tf.float32, shape=(None, 5))
outputs = model_pickled.build(input_var)
output = sess.run(outputs, feed_dict={input_var: self.obs})
expected_output = np.full([1, output_dim],
5 * np.prod(hidden_sizes))
assert np.array_equal(output, expected_output)
def test_comp_mech_policy(self):
y1 = 0.1
v1 = 0.1
env = TfEnv(MassSpringEnv_OptK_HwAsPolicy(params))
comp_policy_model = MLPModel(output_dim=1,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
hidden_b_init=tf.zeros_initializer(),
hidden_w_init=tf.zeros_initializer(),
output_b_init=tf.zeros_initializer(),
output_w_init=tf.zeros_initializer())
mech_policy_model = MechPolicyModel_OptK_HwAsPolicy(params)
with tf.compat.v1.Session() as sess:
comp_mech_policy = CompMechPolicy_OptK_HwAsPolicy(
name='test_comp_mech_policy',
env_spec=env.spec,
comp_policy_model=comp_policy_model,
mech_policy_model=mech_policy_model)
actions = comp_mech_policy.get_actions([[y1, v1]])
print('actions: ', actions)
# self.assertTrue(np.allclose(actions[1]['mean'], np.array([[params.half_force_range*0-params.k_init*y1, 0.0]])))
self.assertTrue(
np.allclose(
actions[1]['log_std'],
np.array([[
params.f_log_std_init_action,
params.f_log_std_init_auxiliary
]])))
action = comp_mech_policy.get_action([y1, v1])
print('single action: ', action)
# self.assertTrue(np.allclose(action[1]['mean'], np.array([params.half_force_range*0-params.k_init*y1, 0.0])))
self.assertTrue(
np.allclose(
action[1]['log_std'],
np.array([
params.f_log_std_init_action,
params.f_log_std_init_auxiliary
])))
print(comp_mech_policy.distribution)
def test_comp_mech_policy(self):
print('\n Testing CompMechPolicy_OptK_HwAsAction ...')
y1 = 0.1
v1 = 0.1
env = TfEnv(MassSpringEnv_OptK_HwAsAction(params))
comp_policy_model = MLPModel(output_dim=1,
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
hidden_b_init=tf.zeros_initializer(),
hidden_w_init=tf.zeros_initializer(),
output_b_init=tf.zeros_initializer(),
output_w_init=tf.zeros_initializer())
mech_policy_model = MechPolicyModel_OptK_HwAsAction(params=params)
with tf.compat.v1.Session() as sess:
comp_mech_policy = CompMechPolicy_OptK_HwAsAction(
name='test_comp_mech_policy',
env_spec=env.spec,
comp_policy_model=comp_policy_model,
mech_policy_model=mech_policy_model)
actions = comp_mech_policy.get_actions([[y1, v1]])
print('actions: ', actions)
# self.assertTrue(np.allclose(actions[1]['mean'], np.array([[0.0, params.k_init]])))
self.assertTrue(
np.allclose(actions[1]['log_std'],
np.array([[params.f_log_std_init_action] + [
params.k_log_std_init_action,
] * params.n_springs]),
atol=1e-3))
action = comp_mech_policy.get_action([y1, v1])
print('single action: ', action)
# self.assertTrue(np.allclose(actions[1]['mean'], np.array([0.0, params.k_init])))
self.assertTrue(
np.allclose(actions[1]['log_std'],
np.array([[params.f_log_std_init_action] + [
params.k_log_std_init_action,
] * params.n_springs]),
atol=1e-3))
print(comp_mech_policy.distribution)
def __init__(self,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pad,
name='CategoricalConvPolicy',
hidden_sizes=[],
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
assert isinstance(env_spec.action_space, akro.Discrete), (
'CategoricalConvPolicy only works with akro.Discrete action '
'space.')
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.shape
self.action_dim = env_spec.action_space.n
self.model = Sequential(
CNNModel(
filter_dims=conv_filter_sizes,
num_filters=conv_filters,
strides=conv_strides,
padding=conv_pad,
hidden_nonlinearity=hidden_nonlinearity,
name='CNNModel'),
MLPModel(
output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel'))
self._initialize()
class CategoricalMLPPolicy(StochasticPolicy):
"""Estimate action distribution with Categorical parameterized by a MLP.
A policy that contains a MLP to make prediction based on
a categorical distribution.
It only works with akro.Discrete action space.
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Policy name, also the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this policy consists of two
hidden layers, each with 32 hidden units.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
layer_normalization (bool): Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='CategoricalMLPPolicy',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
assert isinstance(env_spec.action_space, akro.Discrete), (
'CategoricalMLPPolicy only works with akro.Discrete action '
'space.')
super().__init__(name, env_spec)
self._obs_dim = env_spec.observation_space.flat_dim
self._action_dim = env_spec.action_space.n
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self.model = MLPModel(output_dim=self._action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel')
self._initialize()
def _initialize(self):
state_input = tf.compat.v1.placeholder(tf.float32,
shape=(None, self._obs_dim))
with tf.compat.v1.variable_scope(self.name) as vs:
self._variable_scope = vs
self.model.build(state_input)
self._f_prob = tf.compat.v1.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
@property
def vectorized(self):
"""Vectorized or not.
Returns:
bool: True if primitive supports vectorized operations.
"""
return True
def dist_info_sym(self, obs_var, state_info_vars=None, name=None):
"""Build a symbolic graph of the distribution parameters.
Args:
obs_var (tf.Tensor): Tensor input for symbolic graph.
state_info_vars (dict[tf.Tensor]): Extra state information, e.g.
previous action.
name (str): Name for symbolic graph.
Returns:
dict[tf.Tensor]: Outputs of the symbolic graph of distribution
parameters.
"""
with tf.compat.v1.variable_scope(self._variable_scope):
prob = self.model.build(obs_var, name=name)
return dict(prob=prob)
def dist_info(self, obs, state_infos=None):
"""Build a symbolic graph of the distribution parameters.
Args:
obs (np.ndarray): Input for symbolic graph.
state_infos (dict[np.ndarray]): Extra state information, e.g.
previous action.
Returns:
dict[np.ndarray]: Outputs of the symbolic graph of distribution
parameters.
"""
prob = self._f_prob(obs)
return dict(prob=prob)
def get_action(self, observation):
"""Get single action from this policy for the input observation.
Args:
observation (numpy.ndarray): Observation from environment.
Returns:
numpy.ndarray: Predicted action.
dict[str: np.ndarray]: Action distribution.
"""
flat_obs = self.observation_space.flatten(observation)
prob = self._f_prob([flat_obs])[0]
action = self.action_space.weighted_sample(prob)
return action, dict(prob=prob)
def get_actions(self, observations):
"""Get multiple actions from this policy for the input observations.
Args:
observations (numpy.ndarray): Observations from environment.
Returns:
numpy.ndarray: Predicted actions.
dict[str: np.ndarray]: Action distributions.
"""
flat_obs = self.observation_space.flatten_n(observations)
probs = self._f_prob(flat_obs)
actions = list(map(self.action_space.weighted_sample, probs))
return actions, dict(prob=probs)
def get_regularizable_vars(self):
"""Get regularizable weight variables under the Policy scope.
Returns:
list(tf.Variable): List of regularizable variables.
"""
trainable = self.get_trainable_vars()
return [
var for var in trainable
if 'hidden' in var.name and 'kernel' in var.name
]
@property
def distribution(self):
"""Policy distribution.
Returns:
garage.tf.distributions.Categorical: Policy distribution.
"""
return Categorical(self._action_dim)
def clone(self, name):
"""Return a clone of the policy.
It only copies the configuration of the Q-function,
not the parameters.
Args:
name (str): Name of the newly created policy.
Returns:
garage.tf.policies.CategoricalMLPPolicy: Clone of this object
"""
return self.__class__(name=name,
env_spec=self._env_spec,
hidden_sizes=self._hidden_sizes,
hidden_nonlinearity=self._hidden_nonlinearity,
hidden_w_init=self._hidden_w_init,
hidden_b_init=self._hidden_b_init,
output_nonlinearity=self._output_nonlinearity,
output_w_init=self._output_w_init,
output_b_init=self._output_b_init,
layer_normalization=self._layer_normalization)
def __getstate__(self):
"""Object.__getstate__.
Returns:
dict: the state to be pickled for the instance.
"""
new_dict = super().__getstate__()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__.
Args:
state (dict): Unpickled state.
"""
super().__setstate__(state)
self._initialize()
class DeterministicMLPPolicyWithModel(Policy2):
"""
DeterministicMLPPolicy with model.
The policy network selects action based on the state of the environment.
It uses neural nets to fit the function of pi(s).
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Policy name, also the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this policy consists of two
hidden layers, each with 32 hidden units.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
input_include_goal (bool): Include goal in the observation or not.
layer_normalization (bool): Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='DeterministicMLPPolicy',
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.tanh,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
input_include_goal=False,
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
if input_include_goal:
self.obs_dim = env_spec.observation_space.flat_dim_with_keys(
['observation', 'desired_goal'])
else:
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name='MLPModel',
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
state_input = tf.placeholder(tf.float32, shape=(None, self.obs_dim))
with tf.variable_scope(self._variable_scope):
self.model.build(state_input)
self._f_prob = tf.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
def get_action_sym(self, obs_var, name=None, **kwargs):
"""Return action sym according to obs_var."""
with tf.variable_scope(self._variable_scope):
action = self.model.build(obs_var, name=name)
action = tf.reshape(action, self.action_space.shape)
return action
@overrides
def get_action(self, observation):
"""Return a single action."""
flat_obs = self.observation_space.flatten(observation)
action = self._f_prob([flat_obs])
action = self.action_space.unflatten(action)
return action, dict()
@overrides
def get_actions(self, observations):
"""Return multiple actions."""
flat_obs = self.observation_space.flatten_n(observations)
actions = self._f_prob(flat_obs)
actions = self.action_space.unflatten_n(actions)
return actions, dict()
@property
def vectorized(self):
"""Vectorized or not."""
return True
def __getstate__(self):
"""Object.__getstate__."""
new_dict = self.__dict__.copy()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
self.__dict__.update(state)
self._initialize()
class CategoricalMLPPolicyWithModel(StochasticPolicy2):
"""
CategoricalMLPPolicy with model.
A policy that contains a MLP to make prediction based on
a categorical distribution.
It only works with akro.tf.Discrete action space.
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Policy name, also the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this policy consists of two
hidden layers, each with 32 hidden units.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
layer_normalization (bool): Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='CategoricalMLPPolicy',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.softmax,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
assert isinstance(
env_spec.action_space,
Discrete), ('CategoricalMLPPolicy only works with akro.tf.Discrete'
'action space.')
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.n
self.model = MLPModel(output_dim=self.action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization,
name='MLPModel')
self._initialize()
def _initialize(self):
state_input = tf.placeholder(tf.float32, shape=(None, self.obs_dim))
with tf.variable_scope(self._variable_scope):
self.model.build(state_input)
self._f_prob = tf.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
@property
def vectorized(self):
"""Vectorized or not."""
return True
@overrides
def dist_info_sym(self, obs_var, state_info_vars=None, name=None):
"""Symbolic graph of the distribution."""
with tf.variable_scope(self._variable_scope):
prob = self.model.build(obs_var, name=name)
return dict(prob=prob)
@overrides
def dist_info(self, obs, state_infos=None):
"""Distribution info."""
prob = self._f_prob(obs)
return dict(prob=prob)
@overrides
def get_action(self, observation):
"""Return a single action."""
flat_obs = self.observation_space.flatten(observation)
prob = self._f_prob([flat_obs])[0]
action = self.action_space.weighted_sample(prob)
return action, dict(prob=prob)
def get_actions(self, observations):
"""Return multiple actions."""
flat_obs = self.observation_space.flatten_n(observations)
probs = self._f_prob(flat_obs)
actions = list(map(self.action_space.weighted_sample, probs))
return actions, dict(prob=probs)
@property
def distribution(self):
"""Policy distribution."""
return Categorical(self.action_dim)
def __getstate__(self):
"""Object.__getstate__."""
new_dict = self.__dict__.copy()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
self.__dict__.update(state)
self._initialize()
class MLPTerminalFunction(EnvFunction2):
"""
Continuous MLP QFunction.
This class implements a q value network to predict q based on the input
state and action. It uses an MLP to fit the function of Q(s, a).
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Name of the q-function, also serves as the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this q-function consists of
two hidden layers, each with 32 hidden units.
action_merge_layer (int): The index of layers at which to concatenate
action inputs with the network. The indexing works like standard
python list indexing. Index of 0 refers to the input layer
(observation input) while an index of -1 points to the last
hidden layer. Default parameter points to second layer from the
end.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
input_include_goal (bool): Whether input includes goal.
layer_normalization (bool): Bool for using layer normalization.
"""
def __init__(self,
env_spec,
name='MLPTerminalFunction',
hidden_sizes=(20, 20),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
input_include_goal=False,
layer_normalization=False):
super().__init__(name)
self._env_spec = env_spec
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._input_include_goal = input_include_goal
self._layer_normalization = layer_normalization
if self._input_include_goal:
self._obs_dim = env_spec.observation_space.flat_dim_with_keys(
['observation', 'desired_goal'])
else:
self._obs_dim = env_spec.observation_space.flat_dim
self._action_dim = env_spec.action_space.flat_dim
self.model = MLPModel(output_dim=2,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
obs_ph = tf.compat.v1.placeholder(tf.float32, (None, self._obs_dim),
name='obs')
with tf.compat.v1.variable_scope(self.name) as vs:
self._variable_scope = vs
self.model.build(obs_ph)
self._f_qval = tf.compat.v1.get_default_session().make_callable(
self.model.networks['default'].outputs, feed_list=[obs_ph])
def get_obs_val(self, observation):
"""Q Value of the network."""
return self._f_qval(observation)
@property
def inputs(self):
"""Return the input tensor."""
return self.model.networks['default'].inputs
@overrides
def get_fval_sym(self, state_input, name):
"""
Symbolic graph for q-network.
Args:
state_input (tf.Tensor): The state input tf.Tensor to the network.
name (str): Network variable scope.
Return:
The tf.Tensor output of Discrete MLP QFunction.
"""
with tf.compat.v1.variable_scope(self._variable_scope):
return self.model.build(state_input, name=name)
def clone(self, name):
"""
Return a clone of the Q-function.
It only copies the configuration of the Q-function,
not the parameters.
Args:
name (str): Name of the newly created q-function.
"""
return self.__class__(name=name,
env_spec=self._env_spec,
hidden_sizes=self._hidden_sizes,
hidden_nonlinearity=self._hidden_nonlinearity,
hidden_w_init=self._hidden_w_init,
hidden_b_init=self._hidden_b_init,
output_nonlinearity=self._output_nonlinearity,
output_w_init=self._output_w_init,
output_b_init=self._output_b_init,
layer_normalization=self._layer_normalization)
def __getstate__(self):
"""Object.__getstate__."""
new_dict = self.__dict__.copy()
del new_dict['_f_qval']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
self.__dict__.update(state)
self._initialize()
class DiscreteMLPQFunction:
"""
Discrete MLP Q Function.
This class implements a Q-value network. It predicts Q-value based on the
input state and action. It uses an MLP to fit the function Q(s, a).
Args:
env_spec: Environment specification.
name: Name of the q-function, also serves as the variable scope.
hidden_sizes: Output dimension of dense layer(s).
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
layer_normalization: Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='discrete_mlp_q_function',
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
obs_dim = env_spec.observation_space.shape
action_dim = env_spec.action_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
obs_ph = tf.placeholder(tf.float32, (None, ) + obs_dim, name='obs')
with tf.variable_scope(name) as vs:
self._variable_scope = vs
self.model.build(obs_ph)
@property
def q_vals(self):
return self.model.networks['default'].outputs
@property
def input(self):
return self.model.networks['default'].input
@overrides
def get_qval_sym(self, state_input, name):
"""
Symbolic graph for q-network.
Args:
state_input: The state input tf.Tensor to the network.
name: Network variable scope.
"""
with tf.variable_scope(self._variable_scope):
return self.model.build(state_input, name=name)
class DeterministicMLPPolicyWithModel(Policy2):
"""
DeterministicMLPPolicy with model.
The policy network selects action based on the state of the environment.
It uses neural nets to fit the function of pi(s).
Args:
env_spec: Environment specification.
name: Variable scope of the mlp.
hidden_sizes: Output dimension of dense layer(s).
hidden_nonlinearity: Activation function for
intermediate dense layer(s).
output_nonlinearity: Activation function for
output dense layer.
input_include_goal: Include goal or not.
layer_normalization: Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name="DeterministicMLPPolicy",
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.tanh,
input_include_goal=False,
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
if input_include_goal:
self.obs_dim = env_spec.observation_space.flat_dim_with_keys(
["observation", "desired_goal"])
else:
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
state_input = tf.placeholder(tf.float32, shape=(None, self.obs_dim))
with tf.variable_scope(self._variable_scope):
self.model.build(state_input)
self._f_prob = tf.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
def get_action_sym(self, obs_var, name=None, **kwargs):
"""Return action sym according to obs_var."""
with tf.variable_scope(self._variable_scope):
action = self.model.build(obs_var, name=name)
action = tf.reshape(action, self.action_space.shape)
return action
@overrides
def get_action(self, observation):
"""Return a single action."""
flat_obs = self.observation_space.flatten(observation)
action = self._f_prob([flat_obs])
action = self.action_space.unflatten(action)
return action, dict()
@overrides
def get_actions(self, observations):
"""Return multiple actions."""
flat_obs = self.observation_space.flatten_n(observations)
actions = self._f_prob(flat_obs)
actions = self.action_space.unflatten_n(actions)
return actions, dict()
@property
def vectorized(self):
"""Vectorized or not."""
return True
def __getstate__(self):
"""Object.__getstate__."""
new_dict = self.__dict__.copy()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
self.__dict__.update(state)
self._initialize()
def __init__(self,
env_spec,
filter_dims,
num_filters,
strides,
hidden_sizes=[256],
name=None,
padding='SAME',
max_pooling=False,
pool_strides=(2, 2),
pool_shapes=(2, 2),
cnn_hidden_nonlinearity=tf.nn.relu,
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
dueling=False,
layer_normalization=False):
super().__init__(name)
self._env_spec = env_spec
self._action_dim = env_spec.action_space.n
self._filter_dims = filter_dims
self._num_filters = num_filters
self._strides = strides
self._hidden_sizes = hidden_sizes
self._padding = padding
self._max_pooling = max_pooling
self._pool_strides = pool_strides
self._pool_shapes = pool_shapes
self._cnn_hidden_nonlinearity = cnn_hidden_nonlinearity
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self._dueling = dueling
self.obs_dim = self._env_spec.observation_space.shape
action_dim = self._env_spec.action_space.flat_dim
if not max_pooling:
cnn_model = CNNModel(filter_dims=filter_dims,
num_filters=num_filters,
strides=strides,
padding=padding,
hidden_nonlinearity=cnn_hidden_nonlinearity)
else:
cnn_model = CNNModelWithMaxPooling(
filter_dims=filter_dims,
num_filters=num_filters,
strides=strides,
padding=padding,
pool_strides=pool_strides,
pool_shapes=pool_shapes,
hidden_nonlinearity=cnn_hidden_nonlinearity)
if not dueling:
output_model = MLPModel(output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
else:
output_model = MLPDuelingModel(
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self.model = Sequential(cnn_model, output_model)
self._initialize()
class CategoricalMLPPolicyWithModel(StochasticPolicy2):
"""
CategoricalMLPPolicy with model.
It only works with akro.tf.Discrete action space.
Args:
env_spec: Environment specification.
name: variable scope of the mlp.
hidden_sizes: Output dimension of dense layer(s).
hidden_nonlinearity: Activation function for
intermediate dense layer(s).
output_nonlinearity: Activation function for
output dense layer.
layer_normalization: Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name="CategoricalMLPPolicy",
hidden_sizes=(32, 32),
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=tf.nn.softmax,
layer_normalization=False):
assert isinstance(
env_spec.action_space,
Discrete), ("CategoricalMLPPolicy only works with akro.tf.Discrete"
"action space.")
super().__init__(name, env_spec)
self.obs_dim = env_spec.observation_space.flat_dim
self.action_dim = env_spec.action_space.n
self.model = MLPModel(
output_dim=self.action_dim,
name=name,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
state_input = tf.placeholder(tf.float32, shape=(None, self.obs_dim))
with tf.variable_scope(self._variable_scope):
self.model.build(state_input)
self._f_prob = tf.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
@property
def vectorized(self):
"""Vectorized or not."""
return True
@overrides
def dist_info_sym(self, obs_var, state_info_vars=None, name=None):
"""Symbolic graph of the distribution."""
with tf.variable_scope(self._variable_scope):
prob = self.model.build(obs_var, name=name)
return dict(prob=prob)
@overrides
def dist_info(self, obs, state_infos=None):
"""Distribution info."""
prob = self._f_prob(obs)
return dict(prob=prob)
@overrides
def get_action(self, observation):
"""Return a single action."""
flat_obs = self.observation_space.flatten(observation)
prob = self._f_prob([flat_obs])[0]
action = self.action_space.weighted_sample(prob)
return action, dict(prob=prob)
def get_actions(self, observations):
"""Return multiple actions."""
flat_obs = self.observation_space.flatten_n(observations)
probs = self._f_prob(flat_obs)
actions = list(map(self.action_space.weighted_sample, probs))
return actions, dict(prob=probs)
@property
def distribution(self):
"""Policy distribution."""
return Categorical(self.action_dim)
def __getstate__(self):
"""Object.__getstate__."""
new_dict = self.__dict__.copy()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
self.__dict__.update(state)
self._initialize()
def __init__(self,
env_spec,
filters,
strides,
hidden_sizes=(256, ),
name=None,
padding='SAME',
max_pooling=False,
pool_strides=(2, 2),
pool_shapes=(2, 2),
cnn_hidden_nonlinearity=tf.nn.relu,
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=None,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
dueling=False,
layer_normalization=False):
if not isinstance(env_spec.observation_space, akro.Box) or \
not len(env_spec.observation_space.shape) in (2, 3):
raise ValueError(
'{} can only process 2D, 3D akro.Image or'
' akro.Box observations, but received an env_spec with '
'observation_space of type {} and shape {}'.format(
type(self).__name__,
type(env_spec.observation_space).__name__,
env_spec.observation_space.shape))
super().__init__(name)
self._env_spec = env_spec
self._action_dim = env_spec.action_space.n
self._filters = filters
self._strides = strides
self._hidden_sizes = hidden_sizes
self._padding = padding
self._max_pooling = max_pooling
self._pool_strides = pool_strides
self._pool_shapes = pool_shapes
self._cnn_hidden_nonlinearity = cnn_hidden_nonlinearity
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self._dueling = dueling
self.obs_dim = self._env_spec.observation_space.shape
action_dim = self._env_spec.action_space.flat_dim
if not max_pooling:
cnn_model = CNNModel(filters=filters,
strides=strides,
padding=padding,
hidden_nonlinearity=cnn_hidden_nonlinearity)
else:
cnn_model = CNNModelWithMaxPooling(
filters=filters,
strides=strides,
padding=padding,
pool_strides=pool_strides,
pool_shapes=pool_shapes,
hidden_nonlinearity=cnn_hidden_nonlinearity)
if not dueling:
output_model = MLPModel(output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
else:
output_model = MLPDuelingModel(
output_dim=action_dim,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self.model = Sequential(cnn_model, output_model)
self._initialize()
class ContinuousMLPPolicy(Policy):
"""ContinuousMLPPolicy
The policy network selects action based on the state of the environment.
It uses neural nets to fit the function of pi(s).
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Policy name, also the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this policy consists of two
hidden layers, each with 32 hidden units.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
input_include_goal (bool): Include goal in the observation or not.
layer_normalization (bool): Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='ContinuousMLPPolicy',
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.glorot_uniform_initializer(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.tanh,
output_w_init=tf.glorot_uniform_initializer(),
output_b_init=tf.zeros_initializer(),
input_include_goal=False,
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._input_include_goal = input_include_goal
self._layer_normalization = layer_normalization
if self._input_include_goal:
self.obs_dim = env_spec.observation_space.flat_dim_with_keys(
['observation', 'desired_goal'])
else:
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name='MLPModel',
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
state_input = tf.compat.v1.placeholder(tf.float32,
shape=(None, self.obs_dim))
with tf.compat.v1.variable_scope(self.name) as vs:
self._variable_scope = vs
self.model.build(state_input)
self._f_prob = tf.compat.v1.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
def get_action_sym(self, obs_var, name=None):
"""Symbolic graph of the action.
Args:
obs_var (tf.Tensor): Tensor input for symbolic graph.
name (str): Name for symbolic graph.
"""
with tf.compat.v1.variable_scope(self._variable_scope):
return self.model.build(obs_var, name=name)
@overrides
def get_action(self, observation):
"""Get single action from this policy for the input observation.
Args:
observation (numpy.ndarray): Observation from environment.
Returns:
action (numpy.ndarray): Predicted action.
agent_info (dict): Empty dict since this policy does
not model a distribution.
"""
flat_obs = self.observation_space.flatten(observation)
action = self._f_prob([flat_obs])
action = self.action_space.unflatten(action)
return action, dict()
@overrides
def get_actions(self, observations):
"""Get multiple actions from this policy for the input observations.
Args:
observations (numpy.ndarray): Observations from environment.
Returns:
actions (numpy.ndarray): Predicted actions.
agent_infos (dict): Empty dict since this policy does
not model a distribution.
"""
flat_obs = self.observation_space.flatten_n(observations)
actions = self._f_prob(flat_obs)
actions = self.action_space.unflatten_n(actions)
return actions, dict()
@property
def vectorized(self):
"""Vectorized or not."""
return True
def clone(self, name):
"""Return a clone of the policy.
It only copies the configuration of the Q-function,
not the parameters.
Args:
name (str): Name of the newly created policy.
"""
return self.__class__(name=name,
env_spec=self._env_spec,
hidden_sizes=self._hidden_sizes,
hidden_nonlinearity=self._hidden_nonlinearity,
hidden_w_init=self._hidden_w_init,
hidden_b_init=self._hidden_b_init,
output_nonlinearity=self._output_nonlinearity,
output_w_init=self._output_w_init,
output_b_init=self._output_b_init,
input_include_goal=self._input_include_goal,
layer_normalization=self._layer_normalization)
def __getstate__(self):
"""Object.__getstate__."""
new_dict = super().__getstate__()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__."""
super().__setstate__(state)
self._initialize()
class ContinuousMLPPolicy(Policy):
"""Continuous MLP Policy Network.
The policy network selects action based on the state of the environment.
It uses neural nets to fit the function of pi(s).
Args:
env_spec (garage.envs.env_spec.EnvSpec): Environment specification.
name (str): Policy name, also the variable scope.
hidden_sizes (list[int]): Output dimension of dense layer(s).
For example, (32, 32) means the MLP of this policy consists of two
hidden layers, each with 32 hidden units.
hidden_nonlinearity (callable): Activation function for intermediate
dense layer(s). It should return a tf.Tensor. Set it to
None to maintain a linear activation.
hidden_w_init (callable): Initializer function for the weight
of intermediate dense layer(s). The function should return a
tf.Tensor.
hidden_b_init (callable): Initializer function for the bias
of intermediate dense layer(s). The function should return a
tf.Tensor.
output_nonlinearity (callable): Activation function for output dense
layer. It should return a tf.Tensor. Set it to None to
maintain a linear activation.
output_w_init (callable): Initializer function for the weight
of output dense layer(s). The function should return a
tf.Tensor.
output_b_init (callable): Initializer function for the bias
of output dense layer(s). The function should return a
tf.Tensor.
layer_normalization (bool): Bool for using layer normalization or not.
"""
def __init__(self,
env_spec,
name='ContinuousMLPPolicy',
hidden_sizes=(64, 64),
hidden_nonlinearity=tf.nn.relu,
hidden_w_init=tf.initializers.glorot_uniform(),
hidden_b_init=tf.zeros_initializer(),
output_nonlinearity=tf.nn.tanh,
output_w_init=tf.initializers.glorot_uniform(),
output_b_init=tf.zeros_initializer(),
layer_normalization=False):
super().__init__(name, env_spec)
action_dim = env_spec.action_space.flat_dim
self._hidden_sizes = hidden_sizes
self._hidden_nonlinearity = hidden_nonlinearity
self._hidden_w_init = hidden_w_init
self._hidden_b_init = hidden_b_init
self._output_nonlinearity = output_nonlinearity
self._output_w_init = output_w_init
self._output_b_init = output_b_init
self._layer_normalization = layer_normalization
self.obs_dim = env_spec.observation_space.flat_dim
self.model = MLPModel(output_dim=action_dim,
name='MLPModel',
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
hidden_w_init=hidden_w_init,
hidden_b_init=hidden_b_init,
output_nonlinearity=output_nonlinearity,
output_w_init=output_w_init,
output_b_init=output_b_init,
layer_normalization=layer_normalization)
self._initialize()
def _initialize(self):
state_input = tf.compat.v1.placeholder(tf.float32,
shape=(None, self.obs_dim))
with tf.compat.v1.variable_scope(self.name) as vs:
self._variable_scope = vs
self.model.build(state_input)
self._f_prob = tf.compat.v1.get_default_session().make_callable(
self.model.networks['default'].outputs,
feed_list=[self.model.networks['default'].input])
def get_action_sym(self, obs_var, name=None):
"""Symbolic graph of the action.
Args:
obs_var (tf.Tensor): Tensor input for symbolic graph.
name (str): Name for symbolic graph.
Returns:
tf.Tensor: symbolic graph of the action.
"""
with tf.compat.v1.variable_scope(self._variable_scope):
return self.model.build(obs_var, name=name)
def get_action(self, observation):
"""Get single action from this policy for the input observation.
Args:
observation (numpy.ndarray): Observation from environment.
Returns:
numpy.ndarray: Predicted action.
dict: Empty dict since this policy does not model a distribution.
"""
action = self._f_prob([observation])
action = self.action_space.unflatten(action)
return action, dict()
def get_actions(self, observations):
"""Get multiple actions from this policy for the input observations.
Args:
observations (numpy.ndarray): Observations from environment.
Returns:
numpy.ndarray: Predicted actions.
dict: Empty dict since this policy does not model a distribution.
"""
actions = self._f_prob(observations)
actions = self.action_space.unflatten_n(actions)
return actions, dict()
def get_regularizable_vars(self):
"""Get regularizable weight variables under the Policy scope.
Returns:
list(tf.Variable): List of regularizable variables.
"""
trainable = self.get_trainable_vars()
return [
var for var in trainable
if 'hidden' in var.name and 'kernel' in var.name
]
@property
def vectorized(self):
"""Vectorized or not.
Returns:
bool: vectorized or not.
"""
return True
def clone(self, name):
"""Return a clone of the policy.
It only copies the configuration of the Q-function,
not the parameters.
Args:
name (str): Name of the newly created policy.
Returns:
garage.tf.policies.ContinuousMLPPolicy: Clone of this object
"""
return self.__class__(name=name,
env_spec=self._env_spec,
hidden_sizes=self._hidden_sizes,
hidden_nonlinearity=self._hidden_nonlinearity,
hidden_w_init=self._hidden_w_init,
hidden_b_init=self._hidden_b_init,
output_nonlinearity=self._output_nonlinearity,
output_w_init=self._output_w_init,
output_b_init=self._output_b_init,
layer_normalization=self._layer_normalization)
def __getstate__(self):
"""Object.__getstate__.
Returns:
dict: the state to be pickled as the contents for the instance.
"""
new_dict = super().__getstate__()
del new_dict['_f_prob']
return new_dict
def __setstate__(self, state):
"""Object.__setstate__.
Args:
state (dict): unpickled state.
"""
super().__setstate__(state)
self._initialize()
