def test_switch_dropout(self):
"""Tests dropout mode.
"""
emb_dim = 4
num_units = 64
hparams = {
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
},
}
mode = tf.placeholder(tf.string)
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_, mode)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output_train, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN})
self.assertEqual(output_train.shape[0], batch_size)
output_test, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL})
self.assertEqual(output_test.shape[0], batch_size)
def test_get_rnn_cell(self):
"""Tests :func:`texar.tf.core.layers.get_rnn_cell`.
"""
emb_dim = 4
num_units = 64
# Given instance
hparams = {"type": rnn.LSTMCell(num_units)}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given class
hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given string, and complex hyperparameters
keep_prob_x = tf.placeholder(name='keep_prob',
shape=[],
dtype=tf.float32)
hparams = {
"type": "tensorflow.contrib.rnn.GRUCell",
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
"state_keep_prob": keep_prob_x,
"variational_recurrent": True,
"input_size": [emb_dim, num_units]
},
"residual": True,
"highway": True
}
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {
keep_prob_x: 1.0,
context.global_mode(): tf.estimator.ModeKeys.TRAIN
}
output_, state_ = sess.run([output, state], feed_dict=feed_dict)
self.assertEqual(output_.shape[0], batch_size)
if isinstance(state_, (list, tuple)):
self.assertEqual(state_[0].shape[0], batch_size)
self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units)
else:
self.assertEqual(state_.shape[0], batch_size)
self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
The hyperparameters are the same as in
:meth:`~texar.tf.modules.BasicRNNDecoder.default_hparams` of
:class:`~texar.tf.modules.BasicRNNDecoder`, except that the default
"name" here is "rnn_decoder".
"""
return {
"rnn_cell": layers.default_rnn_cell_hparams(),
"helper_train": rnn_decoder_helpers.default_helper_train_hparams(),
"helper_infer": rnn_decoder_helpers.default_helper_infer_hparams(),
"max_decoding_length_train": None,
"max_decoding_length_infer": None,
"name": "rnn_decoder",
"output_layer_bias": True,
}
def setUp(self):
tf.test.TestCase.setUp(self)
self._batch_size = 100
self._decoder_cell = layers.get_rnn_cell(
layers.default_rnn_cell_hparams())
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell_fw": default_rnn_cell_hparams(),
"rnn_cell_bw": default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"output_layer_bw": {
# Same hyperparams and default values as "output_layer_fw"
# ...
},
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
}
Here:
"rnn_cell_fw": dict
Hyperparameters of the forward RNN cell.
Ignored if :attr:`cell_fw` is given to the encoder constructor.
The default value is defined in
:func:`~texar.tf.core.default_rnn_cell_hparams`.
"rnn_cell_bw": dict
Hyperparameters of the backward RNN cell.
Ignored if :attr:`cell_bw` is given to the encoder constructor
, or if :attr:`"rnn_cell_share_config"` is `True`.
The default value is defined in
:meth:`~texar.tf.core.default_rnn_cell_hparams`.
"rnn_cell_share_config": bool
Whether share hyperparameters of the backward cell with the
forward cell. Note that the cell parameters (variables) are not
shared.
"output_layer_fw": dict
Hyperparameters of the forward output layer. Ignored if
:attr:`output_layer_fw` is given to the constructor.
See the "output_layer" field of
:meth:`~texar.tf.modules.UnidirectionalRNNEncoder.default_hparams` for
details.
"output_layer_bw": dict
Hyperparameters of the backward output layer. Ignored if
:attr:`output_layer_bw` is given to the constructor. Have the
same structure and defaults with :attr:`"output_layer_fw"`.
Ignored if :attr:`"output_layer_share_config"` is True.
"output_layer_share_config": bool
Whether share hyperparameters of the backward output layer
with the forward output layer. Note that the layer parameters
(variables) are not shared.
"name": str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell_fw": layers.default_rnn_cell_hparams(),
"rnn_cell_bw": layers.default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": _default_output_layer_hparams(),
"output_layer_bw": _default_output_layer_hparams(),
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
})
return hparams
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell": default_rnn_cell_hparams(),
"output_layer": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"name": "unidirectional_rnn_encoder"
}
Here:
"rnn_cell": dict
A dictionary of RNN cell hyperparameters. Ignored if
:attr:`cell` is given to the encoder constructor.
The default value is defined in
:func:`~texar.tf.core.default_rnn_cell_hparams`.
"output_layer": dict
Output layer hyperparameters. Ignored if :attr:`output_layer`
is given to the encoder constructor. Includes:
"num_layers": int
The number of output (dense) layers. Set to 0 to avoid any
output layers applied to the cell outputs..
"layer_size": int or list
The size of each of the output (dense) layers.
If an `int`, each output layer will have the same size. If
a list, the length must equal to :attr:`num_layers`.
"activation": str or callable or None
Activation function for each of the output (dense)
layer except for the final layer. This can be
a function, or its string name or module path.
If function name is given, the function must be from
module :tf_main:`tf.nn <nn>` or :tf_main:`tf < >`.
For example
.. code-block:: python
"activation": "relu" # function name
"activation": "my_module.my_activation_fn" # module path
"activation": my_module.my_activation_fn # function
Default is `None` which maintains a linear activation.
"final_layer_activation": str or callable or None
The activation function for the final output layer.
"other_dense_kwargs": dict or None
Other keyword arguments to construct each of the output
dense layers, e.g., `use_bias`. See
:tf_main:`Dense <layers/Dense>` for the keyword arguments.
"dropout_layer_ids": int or list
The indexes of layers (starting from `0`) whose inputs
are applied with dropout. The index = :attr:`num_layers`
means dropout applies to the final layer output. E.g.,
.. code-block:: python
{
"num_layers": 2,
"dropout_layer_ids": [0, 2]
}
will leads to a series of layers as
`-dropout-layer0-layer1-dropout-`.
The dropout mode (training or not) is controlled
by the :attr:`mode` argument of :meth:`_build`.
"dropout_rate": float
The dropout rate, between 0 and 1. E.g.,
`"dropout_rate": 0.1` would drop out 10% of elements.
"variational_dropout": bool
Whether the dropout mask is the same across all time steps.
"name": str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell": layers.default_rnn_cell_hparams(),
"output_layer": _default_output_layer_hparams(),
"name": "unidirectional_rnn_encoder"
})
return hparams