def __init__(self, *args, **kwargs): super(TestConnectors, self).__init__(*args, **kwargs) self._batch_size = 100 self._decoder_cell = layers.get_rnn_cell( 256, layers.default_rnn_cell_hparams())
def default_hparams(): r"""Returns a dictionary of hyperparameters with default values. The hyperparameters are the same as in :meth:`~texar.torch.modules.BasicRNNDecoder.default_hparams` of :class:`~texar.torch.modules.BasicRNNDecoder`, except that the default ``"name"`` here is ``"rnn_decoder"``. """ return { 'rnn_cell': layers.default_rnn_cell_hparams(), 'helper_train': decoder_helpers.default_helper_train_hparams(), 'helper_infer': decoder_helpers.default_helper_infer_hparams(), 'max_decoding_length_train': None, 'max_decoding_length_infer': None, 'name': 'rnn_decoder', "output_layer_bias": True, }
def setUp(self) -> None: self._batch_size = 100 self._decoder_cell = layers.get_rnn_cell( 256, layers.default_rnn_cell_hparams())
def default_hparams() -> Dict[str, Any]: r"""Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell_fw": default_rnn_cell_hparams(), "rnn_cell_bw": default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "output_layer_bw": { # Same hyperparams and default values as "output_layer_fw" # ... }, "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" } Here: `"rnn_cell_fw"`: dict Hyperparameters of the forward RNN cell. Ignored if :attr:`cell_fw` is given to the encoder constructor. The default value is defined in :func:`~texar.torch.core.default_rnn_cell_hparams`. `"rnn_cell_bw"`: dict Hyperparameters of the backward RNN cell. Ignored if :attr:`cell_bw` is given to the encoder constructor, or if `"rnn_cell_share_config"` is `True`. The default value is defined in :meth:`~texar.torch.core.default_rnn_cell_hparams`. `"rnn_cell_share_config"`: bool Whether share hyperparameters of the backward cell with the forward cell. Note that the cell parameters (variables) are not shared. `"output_layer_fw"`: dict Hyperparameters of the forward output layer. Ignored if ``output_layer_fw`` is given to the constructor. See the ``"output_layer"`` field of :meth:`~texar.torch.modules.UnidirectionalRNNEncoder` for details. `"output_layer_bw"`: dict Hyperparameters of the backward output layer. Ignored if :attr:`output_layer_bw` is given to the constructor. Have the same structure and defaults with :attr:`"output_layer_fw"`. Ignored if ``output_layer_share_config`` is `True`. `"output_layer_share_config"`: bool Whether share hyperparameters of the backward output layer with the forward output layer. Note that the layer parameters (variables) are not shared. `"name"`: str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell_fw": layers.default_rnn_cell_hparams(), "rnn_cell_bw": layers.default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": _default_output_layer_hparams(), "output_layer_bw": _default_output_layer_hparams(), "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" }) return hparams
def default_hparams() -> Dict[str, Any]: r"""Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "output_layer": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "name": "unidirectional_rnn_encoder" } Here: `"rnn_cell"`: dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given to the encoder constructor. The default value is defined in :func:`~texar.torch.core.default_rnn_cell_hparams`. `"output_layer"`: dict Output layer hyperparameters. Ignored if :attr:`output_layer` is given to the encoder constructor. Includes: `"num_layers"`: int The number of output (dense) layers. Set to 0 to avoid any output layers applied to the cell outputs. `"layer_size"`: int or list The size of each of the output (dense) layers. If an `int`, each output layer will have the same size. If a list, the length must equal to :attr:`num_layers`. `"activation"`: str or callable or None Activation function for each of the output (dense) layer except for the final layer. This can be a function, or its string name or module path. If function name is given, the function must be from :mod:`torch.nn`. For example: .. code-block:: python "activation": "relu" # function name "activation": "my_module.my_activation_fn" # module path "activation": my_module.my_activation_fn # function Default is `None` which results in an identity activation. `"final_layer_activation"`: str or callable or None The activation function for the final output layer. `"other_dense_kwargs"`: dict or None Other keyword arguments to construct each of the output dense layers, e.g., ``bias``. See :torch_nn:`Linear` for the keyword arguments. `"dropout_layer_ids"`: int or list The indexes of layers (starting from 0) whose inputs are applied with dropout. The index = :attr:`num_layers` means dropout applies to the final layer output. For example, .. code-block:: python { "num_layers": 2, "dropout_layer_ids": [0, 2] } will leads to a series of layers as `-dropout-layer0-layer1-dropout-`. The dropout mode (training or not) is controlled by :attr:`self.training`. `"dropout_rate"`: float The dropout rate, between 0 and 1. For example, ``"dropout_rate": 0.1`` would zero out 10% of elements. `"variational_dropout"`: bool Whether the dropout mask is the same across all time steps. `"name"`: str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell": layers.default_rnn_cell_hparams(), "output_layer": _default_output_layer_hparams(), "name": "unidirectional_rnn_encoder" }) return hparams
def test_get_rnn_cell(self): r"""Tests the HParams class. """ input_size = 10 hparams = { 'type': 'LSTMCell', 'kwargs': { 'num_units': 20, 'forget_bias': 1.0, }, 'num_layers': 3, 'dropout': { 'input_keep_prob': 0.5, 'output_keep_prob': 0.5, 'state_keep_prob': 0.5, 'variational_recurrent': True }, 'residual': True, 'highway': True, } hparams = HParams(hparams, default_rnn_cell_hparams()) rnn_cell = get_rnn_cell(input_size, hparams) self.assertIsInstance(rnn_cell, wrappers.MultiRNNCell) self.assertEqual(len(rnn_cell._cell), hparams.num_layers) self.assertEqual(rnn_cell.input_size, input_size) self.assertEqual(rnn_cell.hidden_size, hparams.kwargs.num_units) for idx, cell in enumerate(rnn_cell._cell): layer_input_size = (input_size if idx == 0 else hparams.kwargs.num_units) self.assertEqual(cell.input_size, layer_input_size) self.assertEqual(cell.hidden_size, hparams.kwargs.num_units) if idx > 0: highway = cell residual = highway._cell dropout = residual._cell self.assertIsInstance(highway, wrappers.HighwayWrapper) self.assertIsInstance(residual, wrappers.ResidualWrapper) else: dropout = cell lstm = dropout._cell builtin_lstm = lstm._cell self.assertIsInstance(dropout, wrappers.DropoutWrapper) self.assertIsInstance(lstm, wrappers.LSTMCell) self.assertIsInstance(builtin_lstm, nn.LSTMCell) h = hparams.kwargs.num_units forget_bias = builtin_lstm.bias_ih[h:(2 * h)] self.assertTrue((forget_bias == hparams.kwargs.forget_bias).all()) for key in ['input', 'output', 'state']: self.assertEqual(getattr(dropout, f'_{key}_keep_prob'), hparams.dropout[f'{key}_keep_prob']) self.assertTrue(dropout._variational_recurrent) batch_size = 8 seq_len = 6 state = None for step in range(seq_len): input = torch.zeros(batch_size, input_size) output, state = rnn_cell(input, state) self.assertEqual(output.shape, (batch_size, hparams.kwargs.num_units)) self.assertEqual(len(state), hparams.num_layers) utils.map_structure( lambda s: self.assertEqual(s.shape, (batch_size, hparams.kwargs .num_units)), state)