def __init__(self,
input_tensor_spec,
output_tensor_spec,
preprocessing_layers=None,
preprocessing_combiner=None,
conv_layer_params=None,
input_fc_layer_params=(200, 100),
input_dropout_layer_params=None,
lstm_size=(40, ),
output_fc_layer_params=(200, 100),
activation_fn=tf.keras.activations.relu,
dtype=tf.float32,
discrete_projection_net=_categorical_projection_net,
continuous_projection_net=_normal_projection_net,
name='ActorDistributionRnnNetwork'):
"""Creates an instance of `ActorDistributionRnnNetwork`.
Args:
input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the
input.
output_tensor_spec: A nest of `tensor_spec.BoundedTensorSpec` representing
the output.
preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer`
representing preprocessing for the different observations.
All of these layers must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
preprocessing_combiner: (Optional.) A keras layer that takes a flat list
of tensors and combines them. Good options include
`tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`.
This layer must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
conv_layer_params: Optional list of convolution layers parameters, where
each item is a length-three tuple indicating (filters, kernel_size,
stride).
input_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied before
the LSTM cell.
input_dropout_layer_params: Optional list of dropout layer parameters,
each item is the fraction of input units to drop or a dictionary of
parameters according to the keras.Dropout documentation. The additional
parameter `permanent', if set to True, allows to apply dropout at
inference for approximated Bayesian inference. The dropout layers are
interleaved with the fully connected layers; there is a dropout layer
after each fully connected layer, except if the entry in the list is
None. This list must have the same length of input_fc_layer_params, or
be None.
lstm_size: An iterable of ints specifying the LSTM cell sizes to use.
output_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied after the
LSTM cell.
activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ...
dtype: The dtype to use by the convolution and fully connected layers.
discrete_projection_net: Callable that generates a discrete projection
network to be called with some hidden state and the outer_rank of the
state.
continuous_projection_net: Callable that generates a continuous projection
network to be called with some hidden state and the outer_rank of the
state.
name: A string representing name of the network.
Raises:
ValueError: If 'input_dropout_layer_params' is not None.
"""
if input_dropout_layer_params:
raise ValueError('Dropout layer is not supported.')
lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork(
input_tensor_spec=input_tensor_spec,
preprocessing_layers=preprocessing_layers,
preprocessing_combiner=preprocessing_combiner,
conv_layer_params=conv_layer_params,
input_fc_layer_params=input_fc_layer_params,
lstm_size=lstm_size,
output_fc_layer_params=output_fc_layer_params,
activation_fn=activation_fn,
dtype=dtype,
name=name)
def map_proj(spec):
if tensor_spec.is_discrete(spec):
return discrete_projection_net(spec)
else:
return continuous_projection_net(spec)
projection_networks = tf.nest.map_structure(map_proj,
output_tensor_spec)
output_spec = tf.nest.map_structure(
lambda proj_net: proj_net.output_spec, projection_networks)
super(ActorDistributionRnnNetwork,
self).__init__(input_tensor_spec=input_tensor_spec,
state_spec=lstm_encoder.state_spec,
output_spec=output_spec,
name=name)
self._lstm_encoder = lstm_encoder
self._projection_networks = projection_networks
self._output_tensor_spec = output_tensor_spec
def __init__(self,
input_tensor_spec,
output_tensor_spec,
preprocessing_layers=None,
preprocessing_combiner=None,
conv_layer_params=None,
input_fc_layer_params=(200, 100),
lstm_size=(40, ),
output_fc_layer_params=(200, 100),
activation_fn=tf.keras.activations.relu,
name='ActorRnnNetwork'):
"""Creates an instance of `ActorRnnNetwork`.
This ActorRnnNetwork supports handling complex observations with preprocessing_layer
and preprocessing_combiner.
Args:
input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the
input observations.
output_tensor_spec: A nest of `tensor_spec.BoundedTensorSpec` representing
the actions.
preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer`
representing preprocessing for the different observations.
All of these layers must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
preprocessing_combiner: (Optional.) A keras layer that takes a flat list
of tensors and combines them. Good options include
`tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`.
This layer must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
conv_layer_params: Optional list of convolution layers parameters, where
each item is a length-three tuple indicating (filters, kernel_size,
stride).
input_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied before
the LSTM cell.
lstm_size: An iterable of ints specifying the LSTM cell sizes to use.
output_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied after the
LSTM cell.
activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ...
name: A string representing name of the network.
"""
lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork(
input_tensor_spec=input_tensor_spec,
preprocessing_layers=preprocessing_layers,
preprocessing_combiner=preprocessing_combiner,
conv_layer_params=conv_layer_params,
input_fc_layer_params=input_fc_layer_params,
lstm_size=lstm_size,
output_fc_layer_params=output_fc_layer_params,
activation_fn=activation_fn,
name=name)
flat_action_spec = tf.nest.flatten(output_tensor_spec)
action_layers = [
tf.keras.layers.Dense(
single_action_spec.shape.num_elements(),
activation=tf.keras.activations.tanh,
kernel_initializer=tf.keras.initializers.RandomUniform(
minval=-0.003, maxval=0.003),
name='action') for single_action_spec in flat_action_spec
]
super(ActorRnnNetwork,
self).__init__(input_tensor_spec=input_tensor_spec,
state_spec=lstm_encoder.state_spec,
name=name)
self._output_tensor_spec = output_tensor_spec
self._flat_action_spec = flat_action_spec
self._lstm_encoder = lstm_encoder
self._action_layers = action_layers
def __init__(self,
input_tensor_spec,
preprocessing_layers=None,
preprocessing_combiner=None,
conv_layer_params=None,
input_fc_layer_params=(75, 40),
input_dropout_layer_params=None,
lstm_size=(40,),
output_fc_layer_params=(75, 40),
activation_fn=tf.keras.activations.relu,
dtype=tf.float32,
name='ValueRnnNetwork'):
"""Creates an instance of `ValueRnnNetwork`.
Network supports calls with shape outer_rank + input_tensor_shape.shape.
Note outer_rank must be at least 1.
Args:
input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the
input observations.
preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer`
representing preprocessing for the different observations.
All of these layers must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
preprocessing_combiner: (Optional.) A keras layer that takes a flat list
of tensors and combines them. Good options include
`tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`.
This layer must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
conv_layer_params: Optional list of convolution layers parameters, where
each item is a length-three tuple indicating (filters, kernel_size,
stride).
input_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied before
the LSTM cell.
input_dropout_layer_params: Optional list of dropout layer parameters,
where each item is the fraction of input units to drop. The dropout
layers are interleaved with the fully connected layers; there is a
dropout layer after each fully connected layer, except if the entry in
the list is None. This list must have the same length of
input_fc_layer_params, or be None.
lstm_size: An iterable of ints specifying the LSTM cell sizes to use.
output_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied after the
LSTM cell.
activation_fn: Activation function, e.g. tf.keras.activations.relu,.
dtype: The dtype to use by the convolution, LSTM, and fully connected
layers.
name: A string representing name of the network.
"""
del input_dropout_layer_params
lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork(
input_tensor_spec=input_tensor_spec,
preprocessing_layers=preprocessing_layers,
preprocessing_combiner=preprocessing_combiner,
conv_layer_params=conv_layer_params,
input_fc_layer_params=input_fc_layer_params,
lstm_size=lstm_size,
output_fc_layer_params=output_fc_layer_params,
activation_fn=activation_fn,
dtype=dtype,
name=name)
postprocessing_layers = tf.keras.layers.Dense(
1,
activation=None,
kernel_initializer=tf.compat.v1.initializers.random_uniform(
minval=-0.03, maxval=0.03))
super(ValueRnnNetwork, self).__init__(
input_tensor_spec=input_tensor_spec,
state_spec=lstm_encoder.state_spec,
name=name)
self._lstm_encoder = lstm_encoder
self._postprocessing_layers = postprocessing_layers
def __init__(self,
input_tensor_spec,
observation_preprocessing_layers=None,
observation_preprocessing_combiner=None,
observation_conv_layer_params=None,
observation_fc_layer_params=(200, ),
action_fc_layer_params=(200, ),
joint_fc_layer_params=(100),
lstm_size=(40, ),
output_fc_layer_params=(200, 100),
activation_fn=tf.keras.activations.relu,
dtype=tf.float32,
name='CriticRnnNetwork'):
"""Creates an instance of `CriticRnnNetwork`.
This CriticRnnNetwork supports handling complex observations with preprocessing_layer
and preprocessing_combiner.
Args:
input_tensor_spec: A tuple of (observation, action) each of type
`tensor_spec.TensorSpec` representing the inputs.
observation_preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer`
representing preprocessing for the different observations.
All of these layers must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
observation_preprocessing_combiner: (Optional.) A keras layer that takes a flat list
of tensors and combines them. Good options include
`tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`.
This layer must not be already built. For more details see
the documentation of `networks.EncodingNetwork`.
observation_conv_layer_params: Optional list of convolution layers
parameters to apply to the observations, where each item is a
length-three tuple indicating (filters, kernel_size, stride).
observation_fc_layer_params: Optional list of fully_connected parameters,
where each item is the number of units in the layer. This is applied
after the observation convultional layer.
action_fc_layer_params: Optional list of parameters for a fully_connected
layer to apply to the actions, where each item is the number of units
in the layer.
joint_fc_layer_params: Optional list of parameters for a fully_connected
layer to apply after merging observations and actions, where each item
is the number of units in the layer.
lstm_size: An iterable of ints specifying the LSTM cell sizes to use.
output_fc_layer_params: Optional list of fully_connected parameters, where
each item is the number of units in the layer. This is applied after the
LSTM cell.
activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ...
name: A string representing name of the network.
Raises:
ValueError: If `action_spec` contains more than one item.
"""
observation_spec, action_spec = input_tensor_spec
if len(tf.nest.flatten(action_spec)) > 1:
raise ValueError(
'Only a single action is supported by this network.')
kernel_initializer = tf.compat.v1.variance_scaling_initializer(
scale=2.0, mode='fan_in', distribution='truncated_normal')
obs_encoder = encoding_network.EncodingNetwork(
observation_spec,
preprocessing_layers=observation_preprocessing_layers,
preprocessing_combiner=observation_preprocessing_combiner,
conv_layer_params=observation_conv_layer_params,
fc_layer_params=observation_fc_layer_params,
activation_fn=activation_fn,
kernel_initializer=kernel_initializer,
dtype=dtype,
name='obs_encoding')
action_layers = sequential_layer.SequentialLayer(
utils.mlp_layers(fc_layer_params=action_fc_layer_params,
activation_fn=activation_fn,
kernel_initializer=tf.compat.v1.keras.
initializers.VarianceScaling(
scale=1. / 3.,
mode='fan_in',
distribution='uniform'),
name='action_encoding'))
obs_encoding_spec = tf.TensorSpec(
shape=(observation_fc_layer_params[-1], ), dtype=tf.float32)
lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork(
input_tensor_spec=(obs_encoding_spec, action_spec),
preprocessing_layers=(tf.keras.layers.Flatten(), action_layers),
preprocessing_combiner=tf.keras.layers.Concatenate(axis=-1),
input_fc_layer_params=joint_fc_layer_params,
lstm_size=lstm_size,
output_fc_layer_params=output_fc_layer_params,
activation_fn=activation_fn,
dtype=dtype,
name='lstm')
output_layers = [
tf.keras.layers.Dense(
1,
activation=None,
kernel_initializer=tf.keras.initializers.RandomUniform(
minval=-0.003, maxval=0.003),
name='value')
]
super(CriticRnnNetwork,
self).__init__(input_tensor_spec=input_tensor_spec,
state_spec=lstm_encoder.state_spec,
name=name)
self._obs_encoder = obs_encoder
self._lstm_encoder = lstm_encoder
self._output_layers = output_layers
