def test_switch_dropout(self): """Tests dropout mode. """ emb_dim = 4 num_units = 64 hparams = { "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, }, } mode = tf.placeholder(tf.string) hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_, mode) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) output_train, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.TRAIN}) self.assertEqual(output_train.shape[0], batch_size) output_test, _ = sess.run( [output, state], feed_dict={mode: tf.estimator.ModeKeys.EVAL}) self.assertEqual(output_test.shape[0], batch_size)
def test_get_rnn_cell(self): """Tests :func:`texar.tf.core.layers.get_rnn_cell`. """ emb_dim = 4 num_units = 64 # Given instance hparams = {"type": rnn.LSTMCell(num_units)} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given class hparams = {"type": rnn.LSTMCell, "kwargs": {"num_units": 10}} cell = layers.get_rnn_cell(hparams) self.assertTrue(isinstance(cell, rnn.LSTMCell)) # Given string, and complex hyperparameters keep_prob_x = tf.placeholder(name='keep_prob', shape=[], dtype=tf.float32) hparams = { "type": "tensorflow.contrib.rnn.GRUCell", "kwargs": { "num_units": num_units }, "num_layers": 2, "dropout": { "input_keep_prob": 0.8, "state_keep_prob": keep_prob_x, "variational_recurrent": True, "input_size": [emb_dim, num_units] }, "residual": True, "highway": True } hparams_ = HParams(hparams, layers.default_rnn_cell_hparams()) cell = layers.get_rnn_cell(hparams_) batch_size = 16 inputs = tf.zeros([batch_size, emb_dim], dtype=tf.float32) output, state = cell(inputs, cell.zero_state(batch_size, dtype=tf.float32)) with self.test_session() as sess: sess.run(tf.global_variables_initializer()) feed_dict = { keep_prob_x: 1.0, context.global_mode(): tf.estimator.ModeKeys.TRAIN } output_, state_ = sess.run([output, state], feed_dict=feed_dict) self.assertEqual(output_.shape[0], batch_size) if isinstance(state_, (list, tuple)): self.assertEqual(state_[0].shape[0], batch_size) self.assertEqual(state_[0].shape[1], hparams_.kwargs.num_units) else: self.assertEqual(state_.shape[0], batch_size) self.assertEqual(state_.shape[1], hparams_.kwargs.num_units)
def default_hparams(): """Returns a dictionary of hyperparameters with default values. The hyperparameters are the same as in :meth:`~texar.tf.modules.BasicRNNDecoder.default_hparams` of :class:`~texar.tf.modules.BasicRNNDecoder`, except that the default "name" here is "rnn_decoder". """ return { "rnn_cell": layers.default_rnn_cell_hparams(), "helper_train": rnn_decoder_helpers.default_helper_train_hparams(), "helper_infer": rnn_decoder_helpers.default_helper_infer_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "name": "rnn_decoder", "output_layer_bias": True, }
def setUp(self): tf.test.TestCase.setUp(self) self._batch_size = 100 self._decoder_cell = layers.get_rnn_cell( layers.default_rnn_cell_hparams())
def default_hparams(): """Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell_fw": default_rnn_cell_hparams(), "rnn_cell_bw": default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "output_layer_bw": { # Same hyperparams and default values as "output_layer_fw" # ... }, "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" } Here: "rnn_cell_fw": dict Hyperparameters of the forward RNN cell. Ignored if :attr:`cell_fw` is given to the encoder constructor. The default value is defined in :func:`~texar.tf.core.default_rnn_cell_hparams`. "rnn_cell_bw": dict Hyperparameters of the backward RNN cell. Ignored if :attr:`cell_bw` is given to the encoder constructor , or if :attr:`"rnn_cell_share_config"` is `True`. The default value is defined in :meth:`~texar.tf.core.default_rnn_cell_hparams`. "rnn_cell_share_config": bool Whether share hyperparameters of the backward cell with the forward cell. Note that the cell parameters (variables) are not shared. "output_layer_fw": dict Hyperparameters of the forward output layer. Ignored if :attr:`output_layer_fw` is given to the constructor. See the "output_layer" field of :meth:`~texar.tf.modules.UnidirectionalRNNEncoder.default_hparams` for details. "output_layer_bw": dict Hyperparameters of the backward output layer. Ignored if :attr:`output_layer_bw` is given to the constructor. Have the same structure and defaults with :attr:`"output_layer_fw"`. Ignored if :attr:`"output_layer_share_config"` is True. "output_layer_share_config": bool Whether share hyperparameters of the backward output layer with the forward output layer. Note that the layer parameters (variables) are not shared. "name": str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell_fw": layers.default_rnn_cell_hparams(), "rnn_cell_bw": layers.default_rnn_cell_hparams(), "rnn_cell_share_config": True, "output_layer_fw": _default_output_layer_hparams(), "output_layer_bw": _default_output_layer_hparams(), "output_layer_share_config": True, "name": "bidirectional_rnn_encoder" }) return hparams
def default_hparams(): """Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "output_layer": { "num_layers": 0, "layer_size": 128, "activation": "identity", "final_layer_activation": None, "other_dense_kwargs": None, "dropout_layer_ids": [], "dropout_rate": 0.5, "variational_dropout": False }, "name": "unidirectional_rnn_encoder" } Here: "rnn_cell": dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given to the encoder constructor. The default value is defined in :func:`~texar.tf.core.default_rnn_cell_hparams`. "output_layer": dict Output layer hyperparameters. Ignored if :attr:`output_layer` is given to the encoder constructor. Includes: "num_layers": int The number of output (dense) layers. Set to 0 to avoid any output layers applied to the cell outputs.. "layer_size": int or list The size of each of the output (dense) layers. If an `int`, each output layer will have the same size. If a list, the length must equal to :attr:`num_layers`. "activation": str or callable or None Activation function for each of the output (dense) layer except for the final layer. This can be a function, or its string name or module path. If function name is given, the function must be from module :tf_main:`tf.nn <nn>` or :tf_main:`tf < >`. For example .. code-block:: python "activation": "relu" # function name "activation": "my_module.my_activation_fn" # module path "activation": my_module.my_activation_fn # function Default is `None` which maintains a linear activation. "final_layer_activation": str or callable or None The activation function for the final output layer. "other_dense_kwargs": dict or None Other keyword arguments to construct each of the output dense layers, e.g., `use_bias`. See :tf_main:`Dense <layers/Dense>` for the keyword arguments. "dropout_layer_ids": int or list The indexes of layers (starting from `0`) whose inputs are applied with dropout. The index = :attr:`num_layers` means dropout applies to the final layer output. E.g., .. code-block:: python { "num_layers": 2, "dropout_layer_ids": [0, 2] } will leads to a series of layers as `-dropout-layer0-layer1-dropout-`. The dropout mode (training or not) is controlled by the :attr:`mode` argument of :meth:`_build`. "dropout_rate": float The dropout rate, between 0 and 1. E.g., `"dropout_rate": 0.1` would drop out 10% of elements. "variational_dropout": bool Whether the dropout mask is the same across all time steps. "name": str Name of the encoder """ hparams = RNNEncoderBase.default_hparams() hparams.update({ "rnn_cell": layers.default_rnn_cell_hparams(), "output_layer": _default_output_layer_hparams(), "name": "unidirectional_rnn_encoder" }) return hparams