def __init__(self, *args, **kwargs):
super(TestConnectors, self).__init__(*args, **kwargs)
self._batch_size = 100
self._decoder_cell = layers.get_rnn_cell(
256, layers.default_rnn_cell_hparams())
def default_hparams():
r"""Returns a dictionary of hyperparameters with default values.
The hyperparameters are the same as in
:meth:`~texar.torch.modules.BasicRNNDecoder.default_hparams` of
:class:`~texar.torch.modules.BasicRNNDecoder`, except that the default
``"name"`` here is ``"rnn_decoder"``.
"""
return {
'rnn_cell': layers.default_rnn_cell_hparams(),
'helper_train': decoder_helpers.default_helper_train_hparams(),
'helper_infer': decoder_helpers.default_helper_infer_hparams(),
'max_decoding_length_train': None,
'max_decoding_length_infer': None,
'name': 'rnn_decoder',
"output_layer_bias": True,
}
def setUp(self) -> None:
self._batch_size = 100
self._decoder_cell = layers.get_rnn_cell(
256, layers.default_rnn_cell_hparams())
def default_hparams() -> Dict[str, Any]:
r"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell_fw": default_rnn_cell_hparams(),
"rnn_cell_bw": default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"output_layer_bw": {
# Same hyperparams and default values as "output_layer_fw"
# ...
},
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
}
Here:
`"rnn_cell_fw"`: dict
Hyperparameters of the forward RNN cell.
Ignored if :attr:`cell_fw` is given to the encoder constructor.
The default value is defined in
:func:`~texar.torch.core.default_rnn_cell_hparams`.
`"rnn_cell_bw"`: dict
Hyperparameters of the backward RNN cell.
Ignored if :attr:`cell_bw` is given to the encoder constructor,
or if `"rnn_cell_share_config"` is `True`.
The default value is defined in
:meth:`~texar.torch.core.default_rnn_cell_hparams`.
`"rnn_cell_share_config"`: bool
Whether share hyperparameters of the backward cell with the
forward cell. Note that the cell parameters (variables) are not
shared.
`"output_layer_fw"`: dict
Hyperparameters of the forward output layer. Ignored if
``output_layer_fw`` is given to the constructor.
See the ``"output_layer"`` field of
:meth:`~texar.torch.modules.UnidirectionalRNNEncoder` for details.
`"output_layer_bw"`: dict
Hyperparameters of the backward output layer. Ignored if
:attr:`output_layer_bw` is given to the constructor. Have the
same structure and defaults with :attr:`"output_layer_fw"`.
Ignored if ``output_layer_share_config`` is `True`.
`"output_layer_share_config"`: bool
Whether share hyperparameters of the backward output layer
with the forward output layer. Note that the layer parameters
(variables) are not shared.
`"name"`: str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell_fw": layers.default_rnn_cell_hparams(),
"rnn_cell_bw": layers.default_rnn_cell_hparams(),
"rnn_cell_share_config": True,
"output_layer_fw": _default_output_layer_hparams(),
"output_layer_bw": _default_output_layer_hparams(),
"output_layer_share_config": True,
"name": "bidirectional_rnn_encoder"
})
return hparams
def default_hparams() -> Dict[str, Any]:
r"""Returns a dictionary of hyperparameters with default values.
.. code-block:: python
{
"rnn_cell": default_rnn_cell_hparams(),
"output_layer": {
"num_layers": 0,
"layer_size": 128,
"activation": "identity",
"final_layer_activation": None,
"other_dense_kwargs": None,
"dropout_layer_ids": [],
"dropout_rate": 0.5,
"variational_dropout": False
},
"name": "unidirectional_rnn_encoder"
}
Here:
`"rnn_cell"`: dict
A dictionary of RNN cell hyperparameters. Ignored if
:attr:`cell` is given to the encoder constructor.
The default value is defined in
:func:`~texar.torch.core.default_rnn_cell_hparams`.
`"output_layer"`: dict
Output layer hyperparameters. Ignored if :attr:`output_layer`
is given to the encoder constructor. Includes:
`"num_layers"`: int
The number of output (dense) layers. Set to 0 to avoid any
output layers applied to the cell outputs.
`"layer_size"`: int or list
The size of each of the output (dense) layers.
If an `int`, each output layer will have the same size. If
a list, the length must equal to :attr:`num_layers`.
`"activation"`: str or callable or None
Activation function for each of the output (dense)
layer except for the final layer. This can be
a function, or its string name or module path.
If function name is given, the function must be from
:mod:`torch.nn`.
For example:
.. code-block:: python
"activation": "relu" # function name
"activation": "my_module.my_activation_fn" # module path
"activation": my_module.my_activation_fn # function
Default is `None` which results in an identity activation.
`"final_layer_activation"`: str or callable or None
The activation function for the final output layer.
`"other_dense_kwargs"`: dict or None
Other keyword arguments to construct each of the output
dense layers, e.g., ``bias``. See
:torch_nn:`Linear` for the keyword arguments.
`"dropout_layer_ids"`: int or list
The indexes of layers (starting from 0) whose inputs
are applied with dropout. The index = :attr:`num_layers`
means dropout applies to the final layer output. For example,
.. code-block:: python
{
"num_layers": 2,
"dropout_layer_ids": [0, 2]
}
will leads to a series of layers as
`-dropout-layer0-layer1-dropout-`.
The dropout mode (training or not) is controlled
by :attr:`self.training`.
`"dropout_rate"`: float
The dropout rate, between 0 and 1. For example,
``"dropout_rate": 0.1`` would zero out 10% of elements.
`"variational_dropout"`: bool
Whether the dropout mask is the same across all time steps.
`"name"`: str
Name of the encoder
"""
hparams = RNNEncoderBase.default_hparams()
hparams.update({
"rnn_cell": layers.default_rnn_cell_hparams(),
"output_layer": _default_output_layer_hparams(),
"name": "unidirectional_rnn_encoder"
})
return hparams
def test_get_rnn_cell(self):
r"""Tests the HParams class.
"""
input_size = 10
hparams = {
'type': 'LSTMCell',
'kwargs': {
'num_units': 20,
'forget_bias': 1.0,
},
'num_layers': 3,
'dropout': {
'input_keep_prob': 0.5,
'output_keep_prob': 0.5,
'state_keep_prob': 0.5,
'variational_recurrent': True
},
'residual': True,
'highway': True,
}
hparams = HParams(hparams, default_rnn_cell_hparams())
rnn_cell = get_rnn_cell(input_size, hparams)
self.assertIsInstance(rnn_cell, wrappers.MultiRNNCell)
self.assertEqual(len(rnn_cell._cell), hparams.num_layers)
self.assertEqual(rnn_cell.input_size, input_size)
self.assertEqual(rnn_cell.hidden_size, hparams.kwargs.num_units)
for idx, cell in enumerate(rnn_cell._cell):
layer_input_size = (input_size
if idx == 0 else hparams.kwargs.num_units)
self.assertEqual(cell.input_size, layer_input_size)
self.assertEqual(cell.hidden_size, hparams.kwargs.num_units)
if idx > 0:
highway = cell
residual = highway._cell
dropout = residual._cell
self.assertIsInstance(highway, wrappers.HighwayWrapper)
self.assertIsInstance(residual, wrappers.ResidualWrapper)
else:
dropout = cell
lstm = dropout._cell
builtin_lstm = lstm._cell
self.assertIsInstance(dropout, wrappers.DropoutWrapper)
self.assertIsInstance(lstm, wrappers.LSTMCell)
self.assertIsInstance(builtin_lstm, nn.LSTMCell)
h = hparams.kwargs.num_units
forget_bias = builtin_lstm.bias_ih[h:(2 * h)]
self.assertTrue((forget_bias == hparams.kwargs.forget_bias).all())
for key in ['input', 'output', 'state']:
self.assertEqual(getattr(dropout, f'_{key}_keep_prob'),
hparams.dropout[f'{key}_keep_prob'])
self.assertTrue(dropout._variational_recurrent)
batch_size = 8
seq_len = 6
state = None
for step in range(seq_len):
input = torch.zeros(batch_size, input_size)
output, state = rnn_cell(input, state)
self.assertEqual(output.shape,
(batch_size, hparams.kwargs.num_units))
self.assertEqual(len(state), hparams.num_layers)
utils.map_structure(
lambda s: self.assertEqual(s.shape, (batch_size, hparams.kwargs
.num_units)), state)
