def __init__(
self,
env_spec,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
hidden_sizes=[],
hidden_nonlinearity=tf.nn.relu,
output_nonlinearity=tf.nn.softmax,
prob_network=None,
name="CategoricalConvPolicy",
):
"""
:param env_spec: A spec for the mdp.
:param hidden_sizes: list of sizes for the fully connected
hidden layers
:param hidden_nonlinearity: nonlinearity used for each hidden layer
:param prob_network: manually specified network for this policy, other
network params
are ignored
:return:
"""
assert isinstance(env_spec.action_space, Discrete)
Serializable.quick_init(self, locals())
self._name = name
self._env_spec = env_spec
self._prob_network_name = "prob_network"
with tf.variable_scope(name, "CategoricalConvPolicy"):
if prob_network is None:
prob_network = ConvNetwork(
input_shape=env_spec.observation_space.shape,
output_dim=env_spec.action_space.n,
conv_filters=conv_filters,
conv_filter_sizes=conv_filter_sizes,
conv_strides=conv_strides,
conv_pads=conv_pads,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
name="conv_prob_network",
)
with tf.name_scope(self._prob_network_name):
out_prob = L.get_output(prob_network.output_layer)
self._l_prob = prob_network.output_layer
self._l_obs = prob_network.input_layer
self._f_prob = tensor_utils.compile_function(
[prob_network.input_layer.input_var], [out_prob])
self._dist = Categorical(env_spec.action_space.n)
super(CategoricalConvPolicy, self).__init__(env_spec)
LayersPowered.__init__(self, [prob_network.output_layer])
def __init__(self,
input_shape,
output_dim,
conv_filters,
conv_filter_sizes,
conv_strides,
conv_pads,
hidden_sizes,
hidden_nonlinearity=tf.nn.tanh,
output_nonlinearity=None,
name='GaussianConvRegressor',
mean_network=None,
learn_std=True,
init_std=1.0,
adaptive_std=False,
std_share_network=False,
std_conv_filters=[],
std_conv_filter_sizes=[],
std_conv_strides=[],
std_conv_pads=[],
std_hidden_sizes=[],
std_hidden_nonlinearity=None,
std_output_nonlinearity=None,
normalize_inputs=True,
normalize_outputs=True,
subsample_factor=1.,
optimizer=None,
optimizer_args=dict(),
use_trust_region=True,
max_kl_step=0.01):
Parameterized.__init__(self)
Serializable.quick_init(self, locals())
self._mean_network_name = 'mean_network'
self._std_network_name = 'std_network'
with tf.compat.v1.variable_scope(name):
if optimizer is None:
if use_trust_region:
optimizer = PenaltyLbfgsOptimizer(**optimizer_args)
else:
optimizer = LbfgsOptimizer(**optimizer_args)
else:
optimizer = optimizer(**optimizer_args)
self._optimizer = optimizer
self._subsample_factor = subsample_factor
if mean_network is None:
if std_share_network:
b = np.concatenate(
[
np.zeros(output_dim),
np.full(output_dim, np.log(init_std))
],
axis=0) # yapf: disable
b = tf.constant_initializer(b)
mean_network = ConvNetwork(
name=self._mean_network_name,
input_shape=input_shape,
output_dim=2 * output_dim,
conv_filters=conv_filters,
conv_filter_sizes=conv_filter_sizes,
conv_strides=conv_strides,
conv_pads=conv_pads,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity,
output_b_init=b)
l_mean = layers.SliceLayer(
mean_network.output_layer,
slice(output_dim),
name='mean_slice',
)
else:
mean_network = ConvNetwork(
name=self._mean_network_name,
input_shape=input_shape,
output_dim=output_dim,
conv_filters=conv_filters,
conv_filter_sizes=conv_filter_sizes,
conv_strides=conv_strides,
conv_pads=conv_pads,
hidden_sizes=hidden_sizes,
hidden_nonlinearity=hidden_nonlinearity,
output_nonlinearity=output_nonlinearity)
l_mean = mean_network.output_layer
if adaptive_std:
l_log_std = ConvNetwork(
name=self._std_network_name,
input_shape=input_shape,
output_dim=output_dim,
conv_filters=std_conv_filters,
conv_filter_sizes=std_conv_filter_sizes,
conv_strides=std_conv_strides,
conv_pads=std_conv_pads,
hidden_sizes=std_hidden_sizes,
hidden_nonlinearity=std_hidden_nonlinearity,
output_nonlinearity=std_output_nonlinearity,
output_b_init=tf.constant_initializer(np.log(init_std)),
).output_layer
elif std_share_network:
l_log_std = layers.SliceLayer(
mean_network.output_layer,
slice(output_dim, 2 * output_dim),
name='log_std_slice',
)
else:
l_log_std = layers.ParamLayer(
mean_network.input_layer,
num_units=output_dim,
param=tf.constant_initializer(np.log(init_std)),
trainable=learn_std,
name=self._std_network_name,
)
LayersPowered.__init__(self, [l_mean, l_log_std])
xs_var = mean_network.input_layer.input_var
ys_var = tf.compat.v1.placeholder(
dtype=tf.float32, name='ys', shape=(None, output_dim))
old_means_var = tf.compat.v1.placeholder(
dtype=tf.float32, name='ys', shape=(None, output_dim))
old_log_stds_var = tf.compat.v1.placeholder(
dtype=tf.float32,
name='old_log_stds',
shape=(None, output_dim))
x_mean_var = tf.Variable(
np.zeros((1, np.prod(input_shape)), dtype=np.float32),
name='x_mean',
)
x_std_var = tf.Variable(
np.ones((1, np.prod(input_shape)), dtype=np.float32),
name='x_std',
)
y_mean_var = tf.Variable(
np.zeros((1, output_dim), dtype=np.float32),
name='y_mean',
)
y_std_var = tf.Variable(
np.ones((1, output_dim), dtype=np.float32),
name='y_std',
)
normalized_xs_var = (xs_var - x_mean_var) / x_std_var
normalized_ys_var = (ys_var - y_mean_var) / y_std_var
with tf.name_scope(
self._mean_network_name, values=[normalized_xs_var]):
normalized_means_var = layers.get_output(
l_mean, {mean_network.input_layer: normalized_xs_var})
with tf.name_scope(
self._std_network_name, values=[normalized_xs_var]):
normalized_log_stds_var = layers.get_output(
l_log_std, {mean_network.input_layer: normalized_xs_var})
means_var = normalized_means_var * y_std_var + y_mean_var
log_stds_var = normalized_log_stds_var + tf.math.log(y_std_var)
normalized_old_means_var = (old_means_var - y_mean_var) / y_std_var
normalized_old_log_stds_var = (
old_log_stds_var - tf.math.log(y_std_var))
dist = self._dist = DiagonalGaussian(output_dim)
normalized_dist_info_vars = dict(
mean=normalized_means_var, log_std=normalized_log_stds_var)
mean_kl = tf.reduce_mean(
dist.kl_sym(
dict(
mean=normalized_old_means_var,
log_std=normalized_old_log_stds_var),
normalized_dist_info_vars,
))
loss = -tf.reduce_mean(
dist.log_likelihood_sym(normalized_ys_var,
normalized_dist_info_vars))
self._f_predict = tensor_utils.compile_function([xs_var],
means_var)
self._f_pdists = tensor_utils.compile_function(
[xs_var], [means_var, log_stds_var])
self._l_mean = l_mean
self._l_log_std = l_log_std
optimizer_args = dict(
loss=loss,
target=self,
network_outputs=[
normalized_means_var, normalized_log_stds_var
],
)
if use_trust_region:
optimizer_args['leq_constraint'] = (mean_kl, max_kl_step)
optimizer_args['inputs'] = [
xs_var, ys_var, old_means_var, old_log_stds_var
]
else:
optimizer_args['inputs'] = [xs_var, ys_var]
self._optimizer.update_opt(**optimizer_args)
self._use_trust_region = use_trust_region
self._name = name
self._normalize_inputs = normalize_inputs
self._normalize_outputs = normalize_outputs
self._mean_network = mean_network
self._x_mean_var = x_mean_var
self._x_std_var = x_std_var
self._y_mean_var = y_mean_var
self._y_std_var = y_std_var