def __init__( self, memory, memory_sequence_length=None, cell=None, cell_dropout_mode=None, vocab_size=None, output_layer=None, #attention_layer=None, # TODO(zhiting): only valid for tf>=1.0 cell_input_fn=None, hparams=None): RNNDecoderBase.__init__(self, cell, vocab_size, output_layer, cell_dropout_mode, hparams) attn_hparams = self._hparams['attention'] attn_kwargs = attn_hparams['kwargs'].todict() # Parse the 'probability_fn' argument if 'probability_fn' in attn_kwargs: prob_fn = attn_kwargs['probability_fn'] if prob_fn is not None and not callable(prob_fn): prob_fn = utils.get_function(prob_fn, [ 'tensorflow.nn', 'tensorflow.contrib.sparsemax', 'tensorflow.contrib.seq2seq' ]) attn_kwargs['probability_fn'] = prob_fn attn_kwargs.update({ "memory_sequence_length": memory_sequence_length, "memory": memory }) self._attn_kwargs = attn_kwargs attn_modules = ['tensorflow.contrib.seq2seq', 'texar.custom'] # Use variable_scope to ensure all trainable variables created in # the attention mechanism are collected with tf.variable_scope(self.variable_scope): attention_mechanism = utils.check_or_get_instance( attn_hparams["type"], attn_kwargs, attn_modules, classtype=tf.contrib.seq2seq.AttentionMechanism) self._attn_cell_kwargs = { "attention_layer_size": attn_hparams["attention_layer_size"], "alignment_history": attn_hparams["alignment_history"], "output_attention": attn_hparams["output_attention"], } self._cell_input_fn = cell_input_fn # Use variable_scope to ensure all trainable variables created in # AttentionWrapper are collected with tf.variable_scope(self.variable_scope): #if attention_layer is not None: # self._attn_cell_kwargs["attention_layer_size"] = None attn_cell = AttentionWrapper( self._cell, attention_mechanism, cell_input_fn=self._cell_input_fn, #attention_layer=attention_layer, **self._attn_cell_kwargs) self._cell = attn_cell
def __init__(self, cell=None, cell_dropout_mode=None, vocab_size=None, output_layer=None, hparams=None): RNNDecoderBase.__init__(self, cell, vocab_size, output_layer, cell_dropout_mode, hparams)
def __init__(self, cell=None, cell_dropout_mode=None, vocab_size=None, output_layer=None, position_embedder=None, hparams=None): RNNDecoderBase.__init__(self, cell, vocab_size, output_layer, cell_dropout_mode, hparams) self.position_embedder = position_embedder self.current_segment_id = -1
def default_hparams(): """Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "helper_train": { "type": "TrainingHelper", "kwargs": {} } "helper_infer": { "type": "SampleEmbeddingHelper", "kwargs": {} } "name": "basic_rnn_decoder" } Here: "rnn_cell" : dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given to the decoder constructor. The default value is defined in :func:`~texar.core.default_rnn_cell_hparams`. "max_decoding_length_train": int or None Maximum allowed number of decoding steps in training mode. If `None` (default), decoding is performed until fully done, e.g., encountering the <EOS> token. Ignored if `max_decoding_length` is given when calling the decoder. "max_decoding_length_infer" : int or None Same as "max_decoding_length_train" but for inference mode. "helper_train" : dict The hyperparameters of the helper used in training. "type" can be a helper class, its name or module path, or a helper instance. If a class name is given, the class must be from module :tf_main:`tf.contrib.seq2seq <contrib/seq2seq>`, :mod:`texar.modules`, or :mod:`texar.custom`. This is used only when both `decoding_strategy` and `helper` augments are `None` when calling the decoder. See :meth:`~texar.modules.RNNDecoderBase._build` for more details. "helper_infer": dict Same as "helper_train" but during inference mode. "name" : str Name of the decoder. The default value is "basic_rnn_decoder". """ hparams = RNNDecoderBase.default_hparams() hparams["name"] = "basic_rnn_decoder" return hparams
def default_hparams(): """Returns a dictionary of hyperparameters with default values. Returns: .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "helper_train": default_helper_train_hparams(), "helper_infer": default_helper_infer_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "name": "basic_rnn_decoder" } Here: "rnn_cell" : dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given when constructing the decoder. The default value is defined in :meth:`~texar.core.layers.default_rnn_cell_hparams`. "helper_train" : dict A dictionary of :class:`Helper` hyperparameters. The helper is used in training phase. The default value is defined in :meth:`~texar.modules.default_helper_train_hparams` "helper_infer": dict A dictionary of :class:`Helper` hyperparameters. The helper is used in inference phase. The default value is defined in :meth:`~texar.modules.default_helper_infer_hparams` "max_decoding_length_train": int or None Maximum allowed number of decoding steps in training mode.. The default is `None`, which means decoding is performed until fully done, e.g., encountering the <EOS> token. "max_decoding_length_infer" : int or None Maximum allowed number of decoding steps in inference mode. The default is `None`, which means decoding is performed until fully done, e.g., encountering the <EOS> token. "name" : str Name of the decoder. The default value is "basic_rnn_decoder". """ hparams = RNNDecoderBase.default_hparams() hparams["name"] = "basic_rnn_decoder" return hparams
def default_hparams(): r"""Returns a dictionary of hyperparameters with default values. .. code-block:: python { "rnn_cell": default_rnn_cell_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "helper_train": { "type": "TrainingHelper", "kwargs": {} } "helper_infer": { "type": "SampleEmbeddingHelper", "kwargs": {} } "name": "basic_rnn_decoder" } Here: `"rnn_cell"`: dict A dictionary of RNN cell hyperparameters. Ignored if :attr:`cell` is given to the decoder constructor. The default value is defined in :func:`~texar.core.default_rnn_cell_hparams`. `"max_decoding_length_train"`: int or None Maximum allowed number of decoding steps in training mode. If `None` (default), decoding is performed until fully done, e.g., encountering the ``<EOS>`` token. Ignored if ``"max_decoding_length"`` is not `None` given when calling the decoder. `"max_decoding_length_infer"`: int or None Same as ``"max_decoding_length_train"`` but for inference mode. `"helper_train"`: dict The hyperparameters of the helper used in training. ``"type"`` can be a helper class, its name or module path, or a helper instance. If a class name is given, the class must be from module :mod:`texar.modules`, or :mod:`texar.custom`. This is used only when both ``"decoding_strategy"`` and ``"helper"`` arguments are `None` when calling the decoder. See :meth:`~texar.modules.RNNDecoderBase.forward` for more details. `"helper_infer"`: dict Same as ``"helper_train"`` but during inference mode. `"name"`: str Name of the decoder. The default value is ``"basic_rnn_decoder"``. """ hparams = RNNDecoderBase.default_hparams() hparams['name'] = 'basic_rnn_decoder' return hparams
def default_hparams(): r"""Returns a dictionary of hyperparameters with default values. Common hyperparameters are the same as in :class:`~texar.modules.BasicRNNDecoder`. :meth:`~texar.modules.BasicRNNDecoder.default_hparams`. Additional hyperparameters are for attention mechanism configuration. .. code-block:: python { "attention": { "type": "LuongAttention", "kwargs": { "num_units": 256, }, "attention_layer_size": None, "alignment_history": False, "output_attention": True, }, # The following hyperparameters are the same as with # `BasicRNNDecoder` "rnn_cell": default_rnn_cell_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "helper_train": { "type": "TrainingHelper", "kwargs": {} } "helper_infer": { "type": "SampleEmbeddingHelper", "kwargs": {} } "name": "attention_rnn_decoder" } Here: `"attention"`: dict Attention hyperparameters, including: `"type"`: str or class or instance The attention type. Can be an attention class, its name or module path, or a class instance. The class must be a subclass of :class:`~texar.core.AttentionMechanism`. If class name is given, the class must be from modules :mod:`texar.core` or :mod:`texar.custom`. Example: .. code-block:: python # class name "type": "LuongAttention" "type": "BahdanauAttention" # module path "type": "texar.core.BahdanauMonotonicAttention" "type": "my_module.MyAttentionMechanismClass" # class "type": texar.core.LuongMonotonicAttention # instance "type": LuongAttention(...) `"kwargs"`: dict keyword arguments for the attention class constructor. Arguments :attr:`memory` and :attr:`memory_sequence_length` should **not** be specified here because they are given to the decoder constructor. Ignored if "type" is an attention class instance. For example: .. code-block:: python "type": "LuongAttention", "kwargs": { "num_units": 256, "probability_fn": torch.nn.functional.softmax, } Here `"probability_fn"` can also be set to the string name or module path to a probability function. `"attention_layer_size"`: int or None The depth of the attention (output) layer. The context and cell output are fed into the attention layer to generate attention at each time step. If `None` (default), use the context as attention at each time step. `"alignment_history"`: bool whether to store alignment history from all time steps in the final output state. (Stored as a time major `TensorArray` on which you must call `stack()`.) `"output_attention"`: bool If `True` (default), the output at each time step is the attention value. This is the behavior of Luong-style attention mechanisms. If `False`, the output at each time step is the output of `cell`. This is the behavior of Bahdanau-style attention mechanisms. In both cases, the `attention` tensor is propagated to the next time step via the state and is used there. This flag only controls whether the attention mechanism is propagated up to the next cell in an RNN stack or to the top RNN output. """ hparams = RNNDecoderBase.default_hparams() hparams["name"] = "attention_rnn_decoder" hparams["attention"] = { "type": "LuongAttention", "kwargs": { "num_units": 256, }, "attention_layer_size": None, "alignment_history": False, "output_attention": True, } return hparams
def default_hparams(): """Returns a dictionary of hyperparameters with default values: Common hyperparameters are the same as in :class:`~texar.modules.BasicRNNDecoder` (see :meth:`texar.modules.BasicRNNDecoder.default_hparams`). Additional hyperparameters are included for attention mechanism configuration. Returns: .. code-block:: python { "attention": { "type": "LuongAttention", "kwargs": { "num_units": 256, }, "attention_layer_size": None, "alignment_history": False, "output_attention": True, }, # The following hyperparameters are the same with # `BasicRNNDecoder` "rnn_cell": default_rnn_cell_hparams(), "helper_train": default_helper_train_hparams(), "helper_infer": default_helper_infer_hparams(), "max_decoding_length_train": None, "max_decoding_length_infer": None, "name": "attention_rnn_decoder" } Here: "attention" : dict Attention hyperparameters, which includes: "type" : str Name or full path to the attention class which can be - Built-in attentions defined in \ `tensorflow.contrib.seq2seq`, including \ :class:`~tensorflow.contrib.seq2seq.LuongAttention`,\ :class:`~tensorflow.contrib.seq2seq.BahdanauAttention`,\ :class:`~tensorflow.contrib.seq2seq.BahdanauMonotonicAttention`\ and \ :class:`~tensorflow.contrib.seq2seq.LuongMonotonicAttention`. - User-defined attention classes in :mod:`texar.custom`. - External attention classes. Must provide the full path, \ e.g., "my_module.MyAttentionClass". The default value is "LuongAttention". "kwargs" : dict A dictionary of arguments for constructor of the attention class. Any arguments besides the ones in the default value are allowed, except :attr:`memory` and :attr:`memory_sequence_length` (if exist) which are provided by :attr:`attention_keys` and :attr:`attention_values_length` in the decoder's constructor, respectively. The default value is: .. code-block:: python { "num_units": 256, } - :attr:`"num_units"` is the depth of the attention \ mechanism. E.g., We can specify :attr:`probability_fn` for :tf_main:`LuongAttention <contrib/seq2seq/LuongAttention>` or :tf_main:`BahdanauAttention <contrib/seq2seq/BahdanauAttention>` like: .. code-block:: python { "probability_fn": pf_value, ... } where :attr:`pf_value` is a callable or its name or full path to that converts the attention score to probabilities. The callable can be :tf_main:`tf.nn.softmax <nn/softmax>` (default), :tf_main:`tf.contrib.seq2seq.hardmax <contrib/seq2seq/hardmax>`, or :tf_main:`tf.contrib.sparsemax.sparsemax <contrib/sparsemax/sparsemax>`. Its signature should be: `probabilities = probability_fn(score)` "attention_layer_size" : int or None The depth of the attention (output) layer. If `None` (default), use the context as attention at each time step. Otherwise, feed the context and cell output into the attention layer to generate attention at each time step. The default value is `None`. TODO(zhiting): what does this mean? "alignment_history": bool whether to store alignment history from all time steps in the final output state. The default value is `False`. TODO(zhiting): what does this mean? "output_attention": bool If `True` (default), the output at each time step is the attention value. This is the behavior of Luong-style attention mechanisms. If `False`, the output at each time step is the output of `cell`. This is the beahvior of Bhadanau-style attention mechanisms. In both cases, the `attention` tensor is propagated to the next time step via the state and is used there. This flag only controls whether the attention mechanism is propagated up to the next cell in an RNN stack or to the top RNN output. The default value is `True`. """ hparams = RNNDecoderBase.default_hparams() hparams["name"] = "attention_rnn_decoder" hparams["attention"] = { "type": "LuongAttention", "kwargs": { "num_units": 256, }, "attention_layer_size": None, "alignment_history": False, "output_attention": True, } return hparams