def test_switch_dropout(self):
"""Tests dropout mode.
"""
emb_dim = 4
num_units = 64
hparams = {
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
},
}
mode = tf.placeholder(tf.string)
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_, mode)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
output_train, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN})
self.assertEqual(output_train.shape[0], batch_size)
output_test, _ = sess.run(
[output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL})
self.assertEqual(output_test.shape[0], batch_size)
def __init__(self, *args, **kwargs):
super(TestConnectors, self).__init__(*args, **kwargs)
self._batch_size = 100
self._decoder_cell = layers.get_rnn_cell(
256, layers.default_rnn_cell_hparams())
def test_get_rnn_cell(self):
"""Tests :func:`texar.core.layers.get_rnn_cell`.
"""
emb_dim = 4
num_units = 64
# Given instance
hparams = {"type": rnn.LSTMCell(num_units)}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given class
hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}}
cell = layers.get_rnn_cell(hparams)
self.assertTrue(isinstance(cell, rnn.LSTMCell))
# Given string, and complex hyperparameters
keep_prob_x = tf.placeholder(name='keep_prob',
shape=[],
dtype=tf.float32)
hparams = {
"type": "tensorflow.contrib.rnn.GRUCell",
"kwargs": {
"num_units": num_units
},
"num_layers": 2,
"dropout": {
"input_keep_prob": 0.8,
"state_keep_prob": keep_prob_x,
"variational_recurrent": True,
"input_size": [emb_dim, num_units]
},
"residual": True,
"highway": True
}
hparams_ = HParams(hparams, layers.default_rnn_cell_hparams())
cell = layers.get_rnn_cell(hparams_)
batch_size = 16
inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32)
output, state = cell(inputs,
cell.zero_state(batch_size, dtype=tf.float32))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
feed_dict = {
keep_prob_x: 1.0,
context.global_mode(): tf.estimator.ModeKeys.TRAIN
}
output_, state_ = sess.run([output, state], feed_dict=feed_dict)
self.assertEqual(output_.shape[0], batch_size)
if isinstance(state_, (list, tuple)):
self.assertEqual(state_[0].shape[0], batch_size)
self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units)
else:
self.assertEqual(state_.shape[0], batch_size)
self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
The hyperparameters are the same as in
:meth:`~texar.modules.BasicRNNDecoder.default_hparams` of
:class:`~texar.modules.BasicRNNDecoder`, except that the default
"name" here is "rnn_decoder".
"""
return {
"rnn_cell": layers.default_rnn_cell_hparams(),
"helper_train": rnn_decoder_helpers.default_helper_train_hparams(),
"helper_infer": rnn_decoder_helpers.default_helper_infer_hparams(),
"max_decoding_length_train": None,
"max_decoding_length_infer": None,
"name": "rnn_decoder"
}
def default_hparams():
r"""Returns a dictionary of hyperparameters with default values.
The hyperparameters are the same as in
:meth:`~texar.modules.BasicRNNDecoder.default_hparams` of
:class:`~texar.modules.BasicRNNDecoder`, except that the default
``"name"`` here is ``"rnn_decoder"``.
"""
return {
'rnn_cell': layers.default_rnn_cell_hparams(),
'helper_train': decoder_helpers.default_helper_train_hparams(),
'helper_infer': decoder_helpers.default_helper_infer_hparams(),
'max_decoding_length_train': None,
'max_decoding_length_infer': None,
'name': 'rnn_decoder',
"output_layer_bias": True,
}
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
Returns:
.. code-block:: python
{
"rnn_cell_fw": default_rnn_cell_hparams(),
"rnn_cell_bw": default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"output_layer_bw": {
# Same as "output_layer_fw"
# ...
},
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
}
Here:
"rnn_cell_fw" : dict
Hyperparameters of the forward RNN cell.
Ignored if :attr:`cell_fw` is given when constructing
the encoder.
The default value is defined in
:meth:`~texar.core.layers.default_rnn_cell_hparams`.
"rnn_cell_bw" : dict
Hyperparameters of the backward RNN cell.
Ignored if :attr:`cell_bw` is given when constructing
the encoder, or if :attr:`"rnn_cell_share_config"` is `True`.
The default value is defined in
:meth:`~texar.core.layers.default_rnn_cell_hparams`.
"rnn_cell_share_config" : bool
Whether share hyperparameters of the backward cell with the
forward cell. Note that the cell parameters are not shared.
If `True` (default), :attr:`"rnn_cell_bw"` is ignored.
"output_layer_fw" : dict
Hyperparameters of the forward output layer. Ignored if
:attr:`output_layer_fw` is given in the constructor. Includes:
"num_layers" : int
The number of output (dense) layers. Set to 0 to avoid any
output layers applied to the cell outputs..
"layer_size" : int or list
The size of each of the output (dense) layers.
If an `int`, each output layer will have the same size. If
a list, the length must equal to :attr:`num_layers`.
"activation" : str or callable or None
The activation function for each of the output (dense)
layer except for the final layer. This can be
the function itself, or its string name or full path.
E.g., `"activation": tensorflow.nn.relu`
or `"activation": "relu"`
or `"activation": "tensorflow.nn.relu"`
Default is `None` which maintains a linear activation.
"final_layer_activation" : str or callable or None
The activation function for the final output layer.
"other_dense_kwargs" : dict or None
Other keyword arguments to construct each of the output
dense layers, e.g., :attr:`use_bias`. See
:tf_main:`Dense <layers/Dense>` for the arguments.
E.g., `"other_dense_kwargs": { "use_bias": False }`.
"dropout_layer_ids" : int or list
The indexes of layers (starting from `0`) whose inputs
are applied with dropout. The index = :attr:`num_layers`
means dropout applies to the final layer output. E.g.,
.. code-block:: python
{
"num_layers": 2,
"dropout_layer_ids": [0, 2]
}
will leads to a series of layers as
`-dropout-layer0-layer1-dropout-`.
The dropout mode (training or not) is controlled
by the :attr:`mode` argument when calling the encoder.
"dropout_rate" : float
The dropout rate, between 0 and 1. E.g.,
`"dropout_rate": 0.1` would drop out 10% of elements.
"variational_dropout": bool
Whether the dropout mask is the same across all time steps.
"output_layer_bw" : dict
Hyperparameters of the backward output layer. Ignored if
:attr:`output_layer_bw` is given in the constructor. Have the
same structure and defaults with :attr:`"output_layer_fw"`.
"output_layer_share_config" : bool
Whether share hyperparameters of the backward output layer
with the forward output layer. Note that the layer parameters
are not shared.
If `True` (default), :attr:`"output_layer_bw"` is ignored.
"name" : str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell_fw": layers.default_rnn_cell_hparams(),
"rnn_cell_bw": layers.default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": _default_output_layer_hparams(),
"output_layer_bw": _default_output_layer_hparams(),
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
})
return hparams
def default_hparams() -> Dict[str, Any]:
r"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell_fw": default_rnn_cell_hparams(),
"rnn_cell_bw": default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"output_layer_bw": {
# Same hyperparams and default values as "output_layer_fw"
# ...
},
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
}
Here:
`"rnn_cell_fw"`: dict
Hyperparameters of the forward RNN cell.
Ignored if :attr:`cell_fw` is given to the encoder constructor.
The default value is defined in
:func:`~texar.core.default_rnn_cell_hparams`.
`"rnn_cell_bw"`: dict
Hyperparameters of the backward RNN cell.
Ignored if :attr:`cell_bw` is given to the encoder constructor,
or if `"rnn_cell_share_config"` is `True`.
The default value is defined in
:meth:`~texar.core.default_rnn_cell_hparams`.
`"rnn_cell_share_config"`: bool
Whether share hyperparameters of the backward cell with the
forward cell. Note that the cell parameters (variables) are not
shared.
`"output_layer_fw"`: dict
Hyperparameters of the forward output layer. Ignored if
``output_layer_fw`` is given to the constructor.
See the ``"output_layer"`` field of
:meth:`~texar.modules.UnidirectionalRNNEncoder.default_hparams` for
details.
`"output_layer_bw"`: dict
Hyperparameters of the backward output layer. Ignored if
:attr:`output_layer_bw` is given to the constructor. Have the
same structure and defaults with :attr:`"output_layer_fw"`.
Ignored if ``output_layer_share_config`` is `True`.
`"output_layer_share_config"`: bool
Whether share hyperparameters of the backward output layer
with the forward output layer. Note that the layer parameters
(variables) are not shared.
`"name"`: str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell_fw": layers.default_rnn_cell_hparams(),
"rnn_cell_bw": layers.default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": _default_output_layer_hparams(),
"output_layer_bw": _default_output_layer_hparams(),
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
})
return hparams
def default_hparams() -> Dict[str, Any]:
r"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell": default_rnn_cell_hparams(),
"output_layer": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"name": "unidirectional_rnn_encoder"
}
Here:
`"rnn_cell"`: dict
A dictionary of RNN cell hyperparameters. Ignored if
:attr:`cell` is given to the encoder constructor.
The default value is defined in
:func:`~texar.core.default_rnn_cell_hparams`.
`"output_layer"`: dict
Output layer hyperparameters. Ignored if :attr:`output_layer`
is given to the encoder constructor. Includes:
`"num_layers"`: int
The number of output (dense) layers. Set to 0 to avoid any
output layers applied to the cell outputs.
`"layer_size"`: int or list
The size of each of the output (dense) layers.
If an `int`, each output layer will have the same size. If
a list, the length must equal to :attr:`num_layers`.
`"activation"`: str or callable or None
Activation function for each of the output (dense)
layer except for the final layer. This can be
a function, or its string name or module path.
If function name is given, the function must be from
:mod:`torch.nn`.
For example:
.. code-block:: python
"activation": "relu" # function name
"activation": "my_module.my_activation_fn" # module path
"activation": my_module.my_activation_fn # function
Default is `None` which results in an identity activation.
`"final_layer_activation"`: str or callable or None
The activation function for the final output layer.
`"other_dense_kwargs"`: dict or None
Other keyword arguments to construct each of the output
dense layers, e.g., ``bias``. See
:torch_nn:`Linear` for the keyword arguments.
`"dropout_layer_ids"`: int or list
The indexes of layers (starting from 0) whose inputs
are applied with dropout. The index = :attr:`num_layers`
means dropout applies to the final layer output. E.g.,
.. code-block:: python
{
"num_layers": 2,
"dropout_layer_ids": [0, 2]
}
will leads to a series of layers as
`-dropout-layer0-layer1-dropout-`.
The dropout mode (training or not) is controlled
by :attr:`self.training`.
`"dropout_rate"`: float
The dropout rate, between 0 and 1. E.g.,
``"dropout_rate": 0.1`` would drop out 10% of elements.
`"variational_dropout"`: bool
Whether the dropout mask is the same across all time steps.
`"name"`: str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell": layers.default_rnn_cell_hparams(),
"output_layer": _default_output_layer_hparams(),
"name": "unidirectional_rnn_encoder"
})
return hparams
def setUp(self):
tf.test.TestCase.setUp(self)
self._batch_size = 100
self._decoder_cell = layers.get_rnn_cell(
layers.default_rnn_cell_hparams())
def test_get_rnn_cell(self):
r"""Tests the HParams class.
"""
input_size = 10
hparams = {
'type': 'LSTMCell',
'kwargs': {
'num_units': 20,
'forget_bias': 1.0,
},
'num_layers': 3,
'dropout': {
'input_keep_prob': 0.5,
'output_keep_prob': 0.5,
'state_keep_prob': 0.5,
'variational_recurrent': True
},
'residual': True,
'highway': True,
}
hparams = HParams(hparams, default_rnn_cell_hparams())
rnn_cell = get_rnn_cell(input_size, hparams)
self.assertIsInstance(rnn_cell, wrappers.MultiRNNCell)
self.assertEqual(len(rnn_cell._cell), hparams.num_layers)
self.assertEqual(rnn_cell.input_size, input_size)
self.assertEqual(rnn_cell.hidden_size, hparams.kwargs.num_units)
for idx, cell in enumerate(rnn_cell._cell):
layer_input_size = (input_size
if idx == 0 else hparams.kwargs.num_units)
self.assertEqual(cell.input_size, layer_input_size)
self.assertEqual(cell.hidden_size, hparams.kwargs.num_units)
if idx > 0:
highway = cell
residual = highway._cell
dropout = residual._cell
self.assertIsInstance(highway, wrappers.HighwayWrapper)
self.assertIsInstance(residual, wrappers.ResidualWrapper)
else:
dropout = cell
lstm = dropout._cell
builtin_lstm = lstm._cell
self.assertIsInstance(dropout, wrappers.DropoutWrapper)
self.assertIsInstance(lstm, wrappers.LSTMCell)
self.assertIsInstance(builtin_lstm, nn.LSTMCell)
h = hparams.kwargs.num_units
forget_bias = builtin_lstm.bias_ih[h:(2 * h)]
self.assertTrue((forget_bias == hparams.kwargs.forget_bias).all())
for key in ['input', 'output', 'state']:
self.assertEqual(getattr(dropout, f'_{key}_keep_prob'),
hparams.dropout[f'{key}_keep_prob'])
self.assertTrue(dropout._variational_recurrent)
batch_size = 8
seq_len = 6
state = None
for step in range(seq_len):
input = torch.zeros(batch_size, input_size)
output, state = rnn_cell(input, state)
self.assertEqual(output.shape,
(batch_size, hparams.kwargs.num_units))
self.assertEqual(len(state), hparams.num_layers)
utils.map_structure(
lambda s: self.assertEqual(s.shape, (batch_size, hparams.kwargs
.num_units)), state)
