Exemple #1
0
    def _build(self, memory=None, query=None, soft_memory=None, soft_query=None,
               mode=None, **kwargs):
        """Pass the :attr:`memory` and :attr:`query` through the memory network
        and return the :attr:`logits` after the final matrix.

        Only one of :attr:`memory` and :attr:`soft_memory` can be specified.
        They should not be specified at the same time.

        Args:
            memory (optional): Memory used in A/C operations. By default, it
                should be an integer tensor of shape
                `[batch_size, memory_size]`,
                containing the ids to embed if provided.
            query (optional): Query vectors as the intial input of the memory
                network.
                If you'd like to apply some transformation (e.g., embedding)
                on it before it's fed into the network, please set `use_B` to
                True and add `query_embed_fn` when constructing this instance.
                If `query_embed_fn` is set to
                :meth:`~texar.tf.modules.MemNetBase.get_default_embed_fn`,
                it should be of shape `[batch_size]`.
                If `use_B` is not set, it should be of shape
                `[batch_size, memory_dim]`.
            soft_memory (optional): Soft memory used in A/C operations. By
                default, it should be a tensor of shape
                `[batch_size, memory_size, raw_memory_dim]`,
                containing the weights used to mix the embedding vectors.
                If you'd like to apply a matrix multiplication on the memory,
                this option can also be used.
            soft_query (optional): Query vectors as the intial input of the
                memory network.
                If you'd like to apply some transformation (e.g., embedding)
                on it before it's fed into the network, please set `use_B` to
                True and add `query_embed_fn` when constructing this instance.
                Similar to :attr:`soft_memory`, if `query_embed_fn` is set to
                :meth:`~texar.tf.modules.MemNetBase.get_default_embed_fn`,
                then it must be of shape `[batch_size, raw_memory_dim]`.
                Ignored if `use_B` is not set.
            mode (optional): A tensor taking value in
                :tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including
                `TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout is
                controlled by :func:`texar.tf.global_mode`.
        """
        if self._B is not None:
            def _unsqueeze(x):
                return x if x is None else tf.expand_dims(x, 1)
            query = tf.squeeze(
                self._B(_unsqueeze(query), _unsqueeze(soft_query), mode=mode),
                1)
        self._u = [query]
        self._m = self._A(memory, soft_memory, mode=mode)
        self._c = self._C(memory, soft_memory, mode=mode)

        keep_prob = switch_dropout(1 - self.hparams.dropout_rate, mode=mode)
        if self.hparams.variational:
            with tf.variable_scope("variational_dropout"):
                noise = tf.random_uniform(tf.shape(self._u[-1]))
                random_tensor = keep_prob + noise
                binary_tensor = tf.floor(random_tensor)

            def _variational_dropout(val):
                return tf.math.div(val, keep_prob) * binary_tensor

        for _ in range(self._n_hops):
            u_ = self._AC(self._u[-1], self._m, self._c)
            if self._relu_dim == 0:
                pass
            elif self._relu_dim == self._memory_dim:
                u_ = tf.nn.relu(u_)
            elif 0 < self._relu_dim < self._memory_dim:
                linear_part = u_[:, : self._memory_dim - self._relu_dim]
                relu_part = u_[:, self._memory_dim - self._relu_dim:]
                relued_part = tf.nn.relu(relu_part)
                u_ = tf.concat(axis=1, values=[linear_part, relued_part])
            else:
                raise ValueError(
                    "relu_dim = {} is illegal".format(self._relu_dim))
            if self.hparams.variational:
                u_ = _variational_dropout(u_)
            else:
                u_ = tf.nn.dropout(u_, keep_prob)
            self._u.append(u_)

        logits = self._W(self._u[-1])

        if not self._built:
            self._add_internal_trainable_variables()
            self._built = True

        return logits
Exemple #2
0
def get_rnn_cell(hparams=None, mode=None):
    """Creates an RNN cell.

    See :func:`~texar.tf.core.default_rnn_cell_hparams` for all
    hyperparameters and default values.

    Args:
        hparams (dict or HParams, optional): Cell hyperparameters. Missing
            hyperparameters are set to default values.
        mode (optional): A Tensor taking value in
            :tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including
            `TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout will be
            controlled by :func:`texar.tf.global_mode`.

    Returns:
        A cell instance.

    Raises:
        ValueError: If hparams["num_layers"]>1 and hparams["type"] is a class
            instance.
        ValueError: The cell is not an
            :tf_main:`RNNCell <contrib/rnn/RNNCell>` instance.
    """
    if hparams is None or isinstance(hparams, dict):
        hparams = HParams(hparams, default_rnn_cell_hparams())

    d_hp = hparams["dropout"]
    if d_hp["variational_recurrent"] and \
            len(d_hp["input_size"]) != hparams["num_layers"]:
        raise ValueError(
            "If variational_recurrent=True, input_size must be a list of "
            "num_layers(%d) integers. Got len(input_size)=%d." %
            (hparams["num_layers"], len(d_hp["input_size"])))

    cells = []
    cell_kwargs = hparams["kwargs"].todict()
    num_layers = hparams["num_layers"]
    for layer_i in range(num_layers):
        # Create the basic cell
        cell_type = hparams["type"]
        if not is_str(cell_type) and not isinstance(cell_type, type):
            if num_layers > 1:
                raise ValueError(
                    "If 'num_layers'>1, then 'type' must be a cell class or "
                    "its name/module path, rather than a cell instance.")
        cell_modules = [
            'tensorflow.nn.rnn_cell', 'tensorflow.contrib.rnn',
            'texar.tf.custom'
        ]
        cell = utils.check_or_get_instance(cell_type, cell_kwargs,
                                           cell_modules, rnn.RNNCell)

        # Optionally add dropout
        if d_hp["input_keep_prob"] < 1.0 or \
                d_hp["output_keep_prob"] < 1.0 or \
                d_hp["state_keep_prob"] < 1.0:
            vr_kwargs = {}
            if d_hp["variational_recurrent"]:
                vr_kwargs = {
                    "variational_recurrent": True,
                    "input_size": d_hp["input_size"][layer_i],
                    "dtype": tf.float32
                }
            input_keep_prob = switch_dropout(d_hp["input_keep_prob"], mode)
            output_keep_prob = switch_dropout(d_hp["output_keep_prob"], mode)
            state_keep_prob = switch_dropout(d_hp["state_keep_prob"], mode)
            cell = rnn.DropoutWrapper(cell=cell,
                                      input_keep_prob=input_keep_prob,
                                      output_keep_prob=output_keep_prob,
                                      state_keep_prob=state_keep_prob,
                                      **vr_kwargs)

        # Optionally add residual and highway connections
        if layer_i > 0:
            if hparams["residual"]:
                cell = rnn.ResidualWrapper(cell)
            if hparams["highway"]:
                cell = rnn.HighwayWrapper(cell)

        cells.append(cell)

    if hparams["num_layers"] > 1:
        cell = rnn.MultiRNNCell(cells)
    else:
        cell = cells[0]

    return cell