def test_switch_dropout(self): """Tests dropout mode. """ emb_dim = 4 num_units = 64 hparams = { "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, }, } mode = tf.placeholder(tf.string) hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_, mode) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) output_train, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN}) self.assertEqual(output_train.shape[0], batch_size) output_test, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL}) self.assertEqual(output_test.shape[0], batch_size)
def __init__(self, *args, **kwargs): super(TestConnectors, self).__init__(*args, **kwargs) self._batch_size = 100 self._decoder_cell = layers.get_rnn_cell( 256, layers.default_rnn_cell_hparams())
def test_get_rnn_cell(self): """Tests :func:`texar.core.layers.get_rnn_cell`. """ emb_dim = 4 num_units = 64 # Given instance hparams = {"type": rnn.LSTMCell(num_units)} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given class hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given string, and complex hyperparameters keep_prob_x = tf.placeholder(name='keep_prob', shape=[], dtype=tf.float32) hparams = { "type": "tensorflow.contrib.rnn.GRUCell", "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, "state_keep_prob": keep_prob_x, "variational_recurrent": True, "input_size": [emb_dim, num_units] }, "residual": True, "highway": True } hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) feed_dict = { keep_prob_x: 1.0, context.global_mode(): tf.estimator.ModeKeys.TRAIN } output_, state_ = sess.run([output, state], feed_dict=feed_dict) self.assertEqual(output_.shape[0], batch_size) if isinstance(state_, (list, tuple)): self.assertEqual(state_[0].shape[0], batch_size) self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units) else: self.assertEqual(state_.shape[0], batch_size) self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def default_hparams(): """Returns a dictionary of hyperparameters with default values. The hyperparameters are the same as in :meth:`~texar.modules.BasicRNNDecoder.default_hparams` of :class:`~texar.modules.BasicRNNDecoder`, except that the default "name" here is "rnn_decoder". """ return { "rnn_cell": layers.default_rnn_cell_hparams(), "helper_train": rnn_decoder_helpers.default_helper_train_hparams(), "helper_infer": rnn_decoder_helpers.default_helper_infer_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "name": "rnn_decoder" }
def default_hparams(): r"""Returns a dictionary of hyperparameters with default values. The hyperparameters are the same as in :meth:`~texar.modules.BasicRNNDecoder.default_hparams` of :class:`~texar.modules.BasicRNNDecoder`, except that the default ``"name"`` here is ``"rnn_decoder"``. """ return { 'rnn_cell': layers.default_rnn_cell_hparams(), 'helper_train': decoder_helpers.default_helper_train_hparams(), 'helper_infer': decoder_helpers.default_helper_infer_hparams(), 'max_decoding_length_train': None, 'max_decoding_length_infer': None, 'name': 'rnn_decoder', "output_layer_bias": True, }
def default_hparams(): """Returns a dictionary of hyperparameters with default values. Returns: .. code-block:: python { "rnn_cell_fw": default_rnn_cell_hparams(), "rnn_cell_bw": default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "output_layer_bw": { # Same as "output_layer_fw" # ... }, "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" } Here: "rnn_cell_fw" : dict Hyperparameters of the forward RNN cell. Ignored if :attr:`cell_fw` is given when constructing the encoder. The default value is defined in :meth:`~texar.core.layers.default_rnn_cell_hparams`. "rnn_cell_bw" : dict Hyperparameters of the backward RNN cell. Ignored if :attr:`cell_bw` is given when constructing the encoder, or if :attr:`"rnn_cell_share_config"` is `True`. The default value is defined in :meth:`~texar.core.layers.default_rnn_cell_hparams`. "rnn_cell_share_config" : bool Whether share hyperparameters of the backward cell with the forward cell. Note that the cell parameters are not shared. If `True` (default), :attr:`"rnn_cell_bw"` is ignored. "output_layer_fw" : dict Hyperparameters of the forward output layer. Ignored if :attr:`output_layer_fw` is given in the constructor. Includes: "num_layers" : int The number of output (dense) layers. Set to 0 to avoid any output layers applied to the cell outputs.. "layer_size" : int or list The size of each of the output (dense) layers. If an `int`, each output layer will have the same size. If a list, the length must equal to :attr:`num_layers`. "activation" : str or callable or None The activation function for each of the output (dense) layer except for the final layer. This can be the function itself, or its string name or full path. E.g., `"activation": tensorflow.nn.relu` or `"activation": "relu"` or `"activation": "tensorflow.nn.relu"` Default is `None` which maintains a linear activation. "final_layer_activation" : str or callable or None The activation function for the final output layer. "other_dense_kwargs" : dict or None Other keyword arguments to construct each of the output dense layers, e.g., :attr:`use_bias`. See :tf_main:`Dense <layers/Dense>` for the arguments. E.g., `"other_dense_kwargs": { "use_bias": False }`. "dropout_layer_ids" : int or list The indexes of layers (starting from `0`) whose inputs are applied with dropout. The index = :attr:`num_layers` means dropout applies to the final layer output. E.g., .. code-block:: python { "num_layers": 2, "dropout_layer_ids": [0, 2] } will leads to a series of layers as `-dropout-layer0-layer1-dropout-`. The dropout mode (training or not) is controlled by the :attr:`mode` argument when calling the encoder. "dropout_rate" : float The dropout rate, between 0 and 1. E.g., `"dropout_rate": 0.1` would drop out 10% of elements. "variational_dropout": bool Whether the dropout mask is the same across all time steps. "output_layer_bw" : dict Hyperparameters of the backward output layer. Ignored if :attr:`output_layer_bw` is given in the constructor. Have the same structure and defaults with :attr:`"output_layer_fw"`. "output_layer_share_config" : bool Whether share hyperparameters of the backward output layer with the forward output layer. Note that the layer parameters are not shared. If `True` (default), :attr:`"output_layer_bw"` is ignored. "name" : str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell_fw": layers.default_rnn_cell_hparams(), "rnn_cell_bw": layers.default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": _default_output_layer_hparams(), "output_layer_bw": _default_output_layer_hparams(), "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" }) return hparams
def default_hparams() -> Dict[str, Any]: r"""Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell_fw": default_rnn_cell_hparams(), "rnn_cell_bw": default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "output_layer_bw": { # Same hyperparams and default values as "output_layer_fw" # ... }, "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" } Here: `"rnn_cell_fw"`: dict Hyperparameters of the forward RNN cell. Ignored if :attr:`cell_fw` is given to the encoder constructor. The default value is defined in :func:`~texar.core.default_rnn_cell_hparams`. `"rnn_cell_bw"`: dict Hyperparameters of the backward RNN cell. Ignored if :attr:`cell_bw` is given to the encoder constructor, or if `"rnn_cell_share_config"` is `True`. The default value is defined in :meth:`~texar.core.default_rnn_cell_hparams`. `"rnn_cell_share_config"`: bool Whether share hyperparameters of the backward cell with the forward cell. Note that the cell parameters (variables) are not shared. `"output_layer_fw"`: dict Hyperparameters of the forward output layer. Ignored if ``output_layer_fw`` is given to the constructor. See the ``"output_layer"`` field of :meth:`~texar.modules.UnidirectionalRNNEncoder.default_hparams` for details. `"output_layer_bw"`: dict Hyperparameters of the backward output layer. Ignored if :attr:`output_layer_bw` is given to the constructor. Have the same structure and defaults with :attr:`"output_layer_fw"`. Ignored if ``output_layer_share_config`` is `True`. `"output_layer_share_config"`: bool Whether share hyperparameters of the backward output layer with the forward output layer. Note that the layer parameters (variables) are not shared. `"name"`: str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell_fw": layers.default_rnn_cell_hparams(), "rnn_cell_bw": layers.default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": _default_output_layer_hparams(), "output_layer_bw": _default_output_layer_hparams(), "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" }) return hparams
def default_hparams() -> Dict[str, Any]: r"""Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "output_layer": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "name": "unidirectional_rnn_encoder" } Here: `"rnn_cell"`: dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given to the encoder constructor. The default value is defined in :func:`~texar.core.default_rnn_cell_hparams`. `"output_layer"`: dict Output layer hyperparameters. Ignored if :attr:`output_layer` is given to the encoder constructor. Includes: `"num_layers"`: int The number of output (dense) layers. Set to 0 to avoid any output layers applied to the cell outputs. `"layer_size"`: int or list The size of each of the output (dense) layers. If an `int`, each output layer will have the same size. If a list, the length must equal to :attr:`num_layers`. `"activation"`: str or callable or None Activation function for each of the output (dense) layer except for the final layer. This can be a function, or its string name or module path. If function name is given, the function must be from :mod:`torch.nn`. For example: .. code-block:: python "activation": "relu" # function name "activation": "my_module.my_activation_fn" # module path "activation": my_module.my_activation_fn # function Default is `None` which results in an identity activation. `"final_layer_activation"`: str or callable or None The activation function for the final output layer. `"other_dense_kwargs"`: dict or None Other keyword arguments to construct each of the output dense layers, e.g., ``bias``. See :torch_nn:`Linear` for the keyword arguments. `"dropout_layer_ids"`: int or list The indexes of layers (starting from 0) whose inputs are applied with dropout. The index = :attr:`num_layers` means dropout applies to the final layer output. E.g., .. code-block:: python { "num_layers": 2, "dropout_layer_ids": [0, 2] } will leads to a series of layers as `-dropout-layer0-layer1-dropout-`. The dropout mode (training or not) is controlled by :attr:`self.training`. `"dropout_rate"`: float The dropout rate, between 0 and 1. E.g., ``"dropout_rate": 0.1`` would drop out 10% of elements. `"variational_dropout"`: bool Whether the dropout mask is the same across all time steps. `"name"`: str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell": layers.default_rnn_cell_hparams(), "output_layer": _default_output_layer_hparams(), "name": "unidirectional_rnn_encoder" }) return hparams
def setUp(self): tf.test.TestCase.setUp(self) self._batch_size = 100 self._decoder_cell = layers.get_rnn_cell( layers.default_rnn_cell_hparams())
def test_get_rnn_cell(self): r"""Tests the HParams class. """ input_size = 10 hparams = { 'type': 'LSTMCell', 'kwargs': { 'num_units': 20, 'forget_bias': 1.0, }, 'num_layers': 3, 'dropout': { 'input_keep_prob': 0.5, 'output_keep_prob': 0.5, 'state_keep_prob': 0.5, 'variational_recurrent': True }, 'residual': True, 'highway': True, } hparams = HParams(hparams, default_rnn_cell_hparams()) rnn_cell = get_rnn_cell(input_size, hparams) self.assertIsInstance(rnn_cell, wrappers.MultiRNNCell) self.assertEqual(len(rnn_cell._cell), hparams.num_layers) self.assertEqual(rnn_cell.input_size, input_size) self.assertEqual(rnn_cell.hidden_size, hparams.kwargs.num_units) for idx, cell in enumerate(rnn_cell._cell): layer_input_size = (input_size if idx == 0 else hparams.kwargs.num_units) self.assertEqual(cell.input_size, layer_input_size) self.assertEqual(cell.hidden_size, hparams.kwargs.num_units) if idx > 0: highway = cell residual = highway._cell dropout = residual._cell self.assertIsInstance(highway, wrappers.HighwayWrapper) self.assertIsInstance(residual, wrappers.ResidualWrapper) else: dropout = cell lstm = dropout._cell builtin_lstm = lstm._cell self.assertIsInstance(dropout, wrappers.DropoutWrapper) self.assertIsInstance(lstm, wrappers.LSTMCell) self.assertIsInstance(builtin_lstm, nn.LSTMCell) h = hparams.kwargs.num_units forget_bias = builtin_lstm.bias_ih[h:(2 * h)] self.assertTrue((forget_bias == hparams.kwargs.forget_bias).all()) for key in ['input', 'output', 'state']: self.assertEqual(getattr(dropout, f'_{key}_keep_prob'), hparams.dropout[f'{key}_keep_prob']) self.assertTrue(dropout._variational_recurrent) batch_size = 8 seq_len = 6 state = None for step in range(seq_len): input = torch.zeros(batch_size, input_size) output, state = rnn_cell(input, state) self.assertEqual(output.shape, (batch_size, hparams.kwargs.num_units)) self.assertEqual(len(state), hparams.num_layers) utils.map_structure( lambda s: self.assertEqual(s.shape, (batch_size, hparams.kwargs .num_units)), state)