def _build(self, memory=None, query=None, soft_memory=None, soft_query=None,
mode=None, **kwargs):
"""Pass the :attr:`memory` and :attr:`query` through the memory network
and return the :attr:`logits` after the final matrix.
Only one of :attr:`memory` and :attr:`soft_memory` can be specified.
They should not be specified at the same time.
Args:
memory (optional): Memory used in A/C operations. By default, it
should be an integer tensor of shape
`[batch_size, memory_size]`,
containing the ids to embed if provided.
query (optional): Query vectors as the intial input of the memory
network.
If you'd like to apply some transformation (e.g., embedding)
on it before it's fed into the network, please set `use_B` to
True and add `query_embed_fn` when constructing this instance.
If `query_embed_fn` is set to
:meth:`~texar.tf.modules.MemNetBase.get_default_embed_fn`,
it should be of shape `[batch_size]`.
If `use_B` is not set, it should be of shape
`[batch_size, memory_dim]`.
soft_memory (optional): Soft memory used in A/C operations. By
default, it should be a tensor of shape
`[batch_size, memory_size, raw_memory_dim]`,
containing the weights used to mix the embedding vectors.
If you'd like to apply a matrix multiplication on the memory,
this option can also be used.
soft_query (optional): Query vectors as the intial input of the
memory network.
If you'd like to apply some transformation (e.g., embedding)
on it before it's fed into the network, please set `use_B` to
True and add `query_embed_fn` when constructing this instance.
Similar to :attr:`soft_memory`, if `query_embed_fn` is set to
:meth:`~texar.tf.modules.MemNetBase.get_default_embed_fn`,
then it must be of shape `[batch_size, raw_memory_dim]`.
Ignored if `use_B` is not set.
mode (optional): A tensor taking value in
:tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including
`TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout is
controlled by :func:`texar.tf.global_mode`.
"""
if self._B is not None:
def _unsqueeze(x):
return x if x is None else tf.expand_dims(x, 1)
query = tf.squeeze(
self._B(_unsqueeze(query), _unsqueeze(soft_query), mode=mode),
1)
self._u = [query]
self._m = self._A(memory, soft_memory, mode=mode)
self._c = self._C(memory, soft_memory, mode=mode)
keep_prob = switch_dropout(1 - self.hparams.dropout_rate, mode=mode)
if self.hparams.variational:
with tf.variable_scope("variational_dropout"):
noise = tf.random_uniform(tf.shape(self._u[-1]))
random_tensor = keep_prob + noise
binary_tensor = tf.floor(random_tensor)
def _variational_dropout(val):
return tf.math.div(val, keep_prob) * binary_tensor
for _ in range(self._n_hops):
u_ = self._AC(self._u[-1], self._m, self._c)
if self._relu_dim == 0:
pass
elif self._relu_dim == self._memory_dim:
u_ = tf.nn.relu(u_)
elif 0 < self._relu_dim < self._memory_dim:
linear_part = u_[:, : self._memory_dim - self._relu_dim]
relu_part = u_[:, self._memory_dim - self._relu_dim:]
relued_part = tf.nn.relu(relu_part)
u_ = tf.concat(axis=1, values=[linear_part, relued_part])
else:
raise ValueError(
"relu_dim = {} is illegal".format(self._relu_dim))
if self.hparams.variational:
u_ = _variational_dropout(u_)
else:
u_ = tf.nn.dropout(u_, keep_prob)
self._u.append(u_)
logits = self._W(self._u[-1])
if not self._built:
self._add_internal_trainable_variables()
self._built = True
return logits
def get_rnn_cell(hparams=None, mode=None):
"""Creates an RNN cell.
See :func:`~texar.tf.core.default_rnn_cell_hparams` for all
hyperparameters and default values.
Args:
hparams (dict or HParams, optional): Cell hyperparameters. Missing
hyperparameters are set to default values.
mode (optional): A Tensor taking value in
:tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including
`TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout will be
controlled by :func:`texar.tf.global_mode`.
Returns:
A cell instance.
Raises:
ValueError: If hparams["num_layers"]>1 and hparams["type"] is a class
instance.
ValueError: The cell is not an
:tf_main:`RNNCell <contrib/rnn/RNNCell>` instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_rnn_cell_hparams())
d_hp = hparams["dropout"]
if d_hp["variational_recurrent"] and \
len(d_hp["input_size"]) != hparams["num_layers"]:
raise ValueError(
"If variational_recurrent=True, input_size must be a list of "
"num_layers(%d) integers. Got len(input_size)=%d." %
(hparams["num_layers"], len(d_hp["input_size"])))
cells = []
cell_kwargs = hparams["kwargs"].todict()
num_layers = hparams["num_layers"]
for layer_i in range(num_layers):
# Create the basic cell
cell_type = hparams["type"]
if not is_str(cell_type) and not isinstance(cell_type, type):
if num_layers > 1:
raise ValueError(
"If 'num_layers'>1, then 'type' must be a cell class or "
"its name/module path, rather than a cell instance.")
cell_modules = [
'tensorflow.nn.rnn_cell', 'tensorflow.contrib.rnn',
'texar.tf.custom'
]
cell = utils.check_or_get_instance(cell_type, cell_kwargs,
cell_modules, rnn.RNNCell)
# Optionally add dropout
if d_hp["input_keep_prob"] < 1.0 or \
d_hp["output_keep_prob"] < 1.0 or \
d_hp["state_keep_prob"] < 1.0:
vr_kwargs = {}
if d_hp["variational_recurrent"]:
vr_kwargs = {
"variational_recurrent": True,
"input_size": d_hp["input_size"][layer_i],
"dtype": tf.float32
}
input_keep_prob = switch_dropout(d_hp["input_keep_prob"], mode)
output_keep_prob = switch_dropout(d_hp["output_keep_prob"], mode)
state_keep_prob = switch_dropout(d_hp["state_keep_prob"], mode)
cell = rnn.DropoutWrapper(cell=cell,
input_keep_prob=input_keep_prob,
output_keep_prob=output_keep_prob,
state_keep_prob=state_keep_prob,
**vr_kwargs)
# Optionally add residual and highway connections
if layer_i > 0:
if hparams["residual"]:
cell = rnn.ResidualWrapper(cell)
if hparams["highway"]:
cell = rnn.HighwayWrapper(cell)
cells.append(cell)
if hparams["num_layers"] > 1:
cell = rnn.MultiRNNCell(cells)
else:
cell = cells[0]
return cell