def __init__(self, input_tensor_spec, output_tensor_spec, preprocessing_layers=None, preprocessing_combiner=None, conv_layer_params=None, input_fc_layer_params=(200, 100), input_dropout_layer_params=None, lstm_size=(40, ), output_fc_layer_params=(200, 100), activation_fn=tf.keras.activations.relu, dtype=tf.float32, discrete_projection_net=_categorical_projection_net, continuous_projection_net=_normal_projection_net, name='ActorDistributionRnnNetwork'): """Creates an instance of `ActorDistributionRnnNetwork`. Args: input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the input. output_tensor_spec: A nest of `tensor_spec.BoundedTensorSpec` representing the output. preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer` representing preprocessing for the different observations. All of these layers must not be already built. For more details see the documentation of `networks.EncodingNetwork`. preprocessing_combiner: (Optional.) A keras layer that takes a flat list of tensors and combines them. Good options include `tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`. This layer must not be already built. For more details see the documentation of `networks.EncodingNetwork`. conv_layer_params: Optional list of convolution layers parameters, where each item is a length-three tuple indicating (filters, kernel_size, stride). input_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied before the LSTM cell. input_dropout_layer_params: Optional list of dropout layer parameters, each item is the fraction of input units to drop or a dictionary of parameters according to the keras.Dropout documentation. The additional parameter `permanent', if set to True, allows to apply dropout at inference for approximated Bayesian inference. The dropout layers are interleaved with the fully connected layers; there is a dropout layer after each fully connected layer, except if the entry in the list is None. This list must have the same length of input_fc_layer_params, or be None. lstm_size: An iterable of ints specifying the LSTM cell sizes to use. output_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied after the LSTM cell. activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ... dtype: The dtype to use by the convolution and fully connected layers. discrete_projection_net: Callable that generates a discrete projection network to be called with some hidden state and the outer_rank of the state. continuous_projection_net: Callable that generates a continuous projection network to be called with some hidden state and the outer_rank of the state. name: A string representing name of the network. Raises: ValueError: If 'input_dropout_layer_params' is not None. """ if input_dropout_layer_params: raise ValueError('Dropout layer is not supported.') lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork( input_tensor_spec=input_tensor_spec, preprocessing_layers=preprocessing_layers, preprocessing_combiner=preprocessing_combiner, conv_layer_params=conv_layer_params, input_fc_layer_params=input_fc_layer_params, lstm_size=lstm_size, output_fc_layer_params=output_fc_layer_params, activation_fn=activation_fn, dtype=dtype, name=name) def map_proj(spec): if tensor_spec.is_discrete(spec): return discrete_projection_net(spec) else: return continuous_projection_net(spec) projection_networks = tf.nest.map_structure(map_proj, output_tensor_spec) output_spec = tf.nest.map_structure( lambda proj_net: proj_net.output_spec, projection_networks) super(ActorDistributionRnnNetwork, self).__init__(input_tensor_spec=input_tensor_spec, state_spec=lstm_encoder.state_spec, output_spec=output_spec, name=name) self._lstm_encoder = lstm_encoder self._projection_networks = projection_networks self._output_tensor_spec = output_tensor_spec
def __init__(self, input_tensor_spec, output_tensor_spec, preprocessing_layers=None, preprocessing_combiner=None, conv_layer_params=None, input_fc_layer_params=(200, 100), lstm_size=(40, ), output_fc_layer_params=(200, 100), activation_fn=tf.keras.activations.relu, name='ActorRnnNetwork'): """Creates an instance of `ActorRnnNetwork`. This ActorRnnNetwork supports handling complex observations with preprocessing_layer and preprocessing_combiner. Args: input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the input observations. output_tensor_spec: A nest of `tensor_spec.BoundedTensorSpec` representing the actions. preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer` representing preprocessing for the different observations. All of these layers must not be already built. For more details see the documentation of `networks.EncodingNetwork`. preprocessing_combiner: (Optional.) A keras layer that takes a flat list of tensors and combines them. Good options include `tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`. This layer must not be already built. For more details see the documentation of `networks.EncodingNetwork`. conv_layer_params: Optional list of convolution layers parameters, where each item is a length-three tuple indicating (filters, kernel_size, stride). input_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied before the LSTM cell. lstm_size: An iterable of ints specifying the LSTM cell sizes to use. output_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied after the LSTM cell. activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ... name: A string representing name of the network. """ lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork( input_tensor_spec=input_tensor_spec, preprocessing_layers=preprocessing_layers, preprocessing_combiner=preprocessing_combiner, conv_layer_params=conv_layer_params, input_fc_layer_params=input_fc_layer_params, lstm_size=lstm_size, output_fc_layer_params=output_fc_layer_params, activation_fn=activation_fn, name=name) flat_action_spec = tf.nest.flatten(output_tensor_spec) action_layers = [ tf.keras.layers.Dense( single_action_spec.shape.num_elements(), activation=tf.keras.activations.tanh, kernel_initializer=tf.keras.initializers.RandomUniform( minval=-0.003, maxval=0.003), name='action') for single_action_spec in flat_action_spec ] super(ActorRnnNetwork, self).__init__(input_tensor_spec=input_tensor_spec, state_spec=lstm_encoder.state_spec, name=name) self._output_tensor_spec = output_tensor_spec self._flat_action_spec = flat_action_spec self._lstm_encoder = lstm_encoder self._action_layers = action_layers
def __init__(self, input_tensor_spec, preprocessing_layers=None, preprocessing_combiner=None, conv_layer_params=None, input_fc_layer_params=(75, 40), input_dropout_layer_params=None, lstm_size=(40,), output_fc_layer_params=(75, 40), activation_fn=tf.keras.activations.relu, dtype=tf.float32, name='ValueRnnNetwork'): """Creates an instance of `ValueRnnNetwork`. Network supports calls with shape outer_rank + input_tensor_shape.shape. Note outer_rank must be at least 1. Args: input_tensor_spec: A nest of `tensor_spec.TensorSpec` representing the input observations. preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer` representing preprocessing for the different observations. All of these layers must not be already built. For more details see the documentation of `networks.EncodingNetwork`. preprocessing_combiner: (Optional.) A keras layer that takes a flat list of tensors and combines them. Good options include `tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`. This layer must not be already built. For more details see the documentation of `networks.EncodingNetwork`. conv_layer_params: Optional list of convolution layers parameters, where each item is a length-three tuple indicating (filters, kernel_size, stride). input_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied before the LSTM cell. input_dropout_layer_params: Optional list of dropout layer parameters, where each item is the fraction of input units to drop. The dropout layers are interleaved with the fully connected layers; there is a dropout layer after each fully connected layer, except if the entry in the list is None. This list must have the same length of input_fc_layer_params, or be None. lstm_size: An iterable of ints specifying the LSTM cell sizes to use. output_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied after the LSTM cell. activation_fn: Activation function, e.g. tf.keras.activations.relu,. dtype: The dtype to use by the convolution, LSTM, and fully connected layers. name: A string representing name of the network. """ del input_dropout_layer_params lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork( input_tensor_spec=input_tensor_spec, preprocessing_layers=preprocessing_layers, preprocessing_combiner=preprocessing_combiner, conv_layer_params=conv_layer_params, input_fc_layer_params=input_fc_layer_params, lstm_size=lstm_size, output_fc_layer_params=output_fc_layer_params, activation_fn=activation_fn, dtype=dtype, name=name) postprocessing_layers = tf.keras.layers.Dense( 1, activation=None, kernel_initializer=tf.compat.v1.initializers.random_uniform( minval=-0.03, maxval=0.03)) super(ValueRnnNetwork, self).__init__( input_tensor_spec=input_tensor_spec, state_spec=lstm_encoder.state_spec, name=name) self._lstm_encoder = lstm_encoder self._postprocessing_layers = postprocessing_layers
def __init__(self, input_tensor_spec, observation_preprocessing_layers=None, observation_preprocessing_combiner=None, observation_conv_layer_params=None, observation_fc_layer_params=(200, ), action_fc_layer_params=(200, ), joint_fc_layer_params=(100), lstm_size=(40, ), output_fc_layer_params=(200, 100), activation_fn=tf.keras.activations.relu, dtype=tf.float32, name='CriticRnnNetwork'): """Creates an instance of `CriticRnnNetwork`. This CriticRnnNetwork supports handling complex observations with preprocessing_layer and preprocessing_combiner. Args: input_tensor_spec: A tuple of (observation, action) each of type `tensor_spec.TensorSpec` representing the inputs. observation_preprocessing_layers: (Optional.) A nest of `tf.keras.layers.Layer` representing preprocessing for the different observations. All of these layers must not be already built. For more details see the documentation of `networks.EncodingNetwork`. observation_preprocessing_combiner: (Optional.) A keras layer that takes a flat list of tensors and combines them. Good options include `tf.keras.layers.Add` and `tf.keras.layers.Concatenate(axis=-1)`. This layer must not be already built. For more details see the documentation of `networks.EncodingNetwork`. observation_conv_layer_params: Optional list of convolution layers parameters to apply to the observations, where each item is a length-three tuple indicating (filters, kernel_size, stride). observation_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied after the observation convultional layer. action_fc_layer_params: Optional list of parameters for a fully_connected layer to apply to the actions, where each item is the number of units in the layer. joint_fc_layer_params: Optional list of parameters for a fully_connected layer to apply after merging observations and actions, where each item is the number of units in the layer. lstm_size: An iterable of ints specifying the LSTM cell sizes to use. output_fc_layer_params: Optional list of fully_connected parameters, where each item is the number of units in the layer. This is applied after the LSTM cell. activation_fn: Activation function, e.g. tf.nn.relu, slim.leaky_relu, ... name: A string representing name of the network. Raises: ValueError: If `action_spec` contains more than one item. """ observation_spec, action_spec = input_tensor_spec if len(tf.nest.flatten(action_spec)) > 1: raise ValueError( 'Only a single action is supported by this network.') kernel_initializer = tf.compat.v1.variance_scaling_initializer( scale=2.0, mode='fan_in', distribution='truncated_normal') obs_encoder = encoding_network.EncodingNetwork( observation_spec, preprocessing_layers=observation_preprocessing_layers, preprocessing_combiner=observation_preprocessing_combiner, conv_layer_params=observation_conv_layer_params, fc_layer_params=observation_fc_layer_params, activation_fn=activation_fn, kernel_initializer=kernel_initializer, dtype=dtype, name='obs_encoding') action_layers = sequential_layer.SequentialLayer( utils.mlp_layers(fc_layer_params=action_fc_layer_params, activation_fn=activation_fn, kernel_initializer=tf.compat.v1.keras. initializers.VarianceScaling( scale=1. / 3., mode='fan_in', distribution='uniform'), name='action_encoding')) obs_encoding_spec = tf.TensorSpec( shape=(observation_fc_layer_params[-1], ), dtype=tf.float32) lstm_encoder = lstm_encoding_network.LSTMEncodingNetwork( input_tensor_spec=(obs_encoding_spec, action_spec), preprocessing_layers=(tf.keras.layers.Flatten(), action_layers), preprocessing_combiner=tf.keras.layers.Concatenate(axis=-1), input_fc_layer_params=joint_fc_layer_params, lstm_size=lstm_size, output_fc_layer_params=output_fc_layer_params, activation_fn=activation_fn, dtype=dtype, name='lstm') output_layers = [ tf.keras.layers.Dense( 1, activation=None, kernel_initializer=tf.keras.initializers.RandomUniform( minval=-0.003, maxval=0.003), name='value') ] super(CriticRnnNetwork, self).__init__(input_tensor_spec=input_tensor_spec, state_spec=lstm_encoder.state_spec, name=name) self._obs_encoder = obs_encoder self._lstm_encoder = lstm_encoder self._output_layers = output_layers