def __init__(self,
num_mixtures,
memory,
memory_sequence_length=None,
check_inner_dims_defined=True,
score_mask_value=None,
name='GmmAttention'):
self.dtype = memory.dtype
self.num_mixtures = num_mixtures
self.query_layer = tf.layers.Dense(3 * num_mixtures,
name='gmm_query_layer',
use_bias=True,
dtype=self.dtype)
with tf.name_scope(name, 'GmmAttentionMechanismInit'):
if score_mask_value is None:
score_mask_value = 0.
self._maybe_mask_score = functools.partial(
_maybe_mask_score,
memory_sequence_length=memory_sequence_length,
score_mask_value=score_mask_value)
self._value = _prepare_memory(memory, memory_sequence_length,
check_inner_dims_defined)
self._batch_size = (self._value.shape[0].value
or tf.shape(self._value)[0])
self._alignments_size = (self._value.shape[1].value
or tf.shape(self._value)[1])
def __init__(self,
cell,
memory,
memory_sequence_length,
output_layer,
max_oovs,
batch_size,
memory_full_vocab,
first_lv_sim_func,
second_lv_sim_func,
attention_layer_size=None,
alignment_history=False,
cell_input_fn=None,
output_attention=False,
output_generation_distribution=False,
output_copy_distribution=False,
output_combined_distribution=True,
initial_cell_state=None,
unk_id=None,
name=None):
super(HierarchicalAttnPointerWrapper, self).__init__(name=name)
if not rnn_cell_impl._like_rnncell(cell): # pylint: disable=protected-access
raise TypeError("cell must be an RNNCell, saw type: %s" %
type(cell).__name__)
self._is_multi = False
if cell_input_fn is None:
cell_input_fn = (lambda inputs, attention: array_ops.concat(
[inputs, attention], -1))
else:
if not callable(cell_input_fn):
raise TypeError(
"cell_input_fn must be callable, saw type: %s" %
type(cell_input_fn).__name__)
if attention_layer_size is not None:
attention_layer_sizes = tuple(attention_layer_size if isinstance(
attention_layer_size, (list,
tuple)) else (attention_layer_size, ))
if len(attention_layer_sizes) != 1:
raise ValueError(
"If provided, attention_layer_size must contain exactly one "
"integer per attention_mechanism, saw: %d vs 1" %
(len(attention_layer_sizes)))
self._attention_layers = tuple(
layers_core.Dense(attention_layer_size,
name="attention_layer",
use_bias=False)
for attention_layer_size in attention_layer_sizes)
self._attention_layer_size = sum(attention_layer_sizes)
else:
self._attention_layers = None
self._attention_layer_size = memory.get_shape()[-1].value
self._cell = cell
self._cell_input_fn = cell_input_fn
self._output_attention = output_attention
self._output_generation_distribution = output_generation_distribution
self._output_copy_distribution = output_copy_distribution
self._output_combined_distribution = output_combined_distribution
self._unk_id = unk_id
self._alignment_history = alignment_history
self._output_layer = output_layer
self._max_oovs = max_oovs
self._batch_size = batch_size
[self._b, self._k, _, h] = memory.get_shape().as_list()
#self._k = tf.shape(memory)[1].value
#self._b = tf.shape(memory)[0].value
#h = tf.shape(memory)[-1].value
b = self._b
k = self._k
mem_reshaped = tf.reshape(memory, [b * k, -1, h])
print(mem_reshaped.get_shape().as_list())
mem_mask_reshaped = tf.reshape(memory_sequence_length, [-1])
self._memory = tf.reshape(
_prepare_memory(mem_reshaped, mem_mask_reshaped, False),
[b, k, -1, h])
self._memory_full_vocab = memory_full_vocab
self._attention_mechanisms = [None] # placeholder
with tf.variable_scope("first_lv_attn"):
self._first_lv_sim_func = first_lv_sim_func
with tf.variable_scope("second_lv_attn"):
self._second_lv_sim_func = second_lv_sim_func
if self._output_combined_distribution or \
self._output_generation_distribution or \
self._output_copy_distribution or \
self._output_attention:
assert self._output_combined_distribution ^\
self._output_generation_distribution ^\
self._output_copy_distribution ^\
self._output_attention, "Can only output one type!"
if self._output_combined_distribution or self._output_copy_distribution:
assert self._unk_id is not None
with ops.name_scope(name, "AttnPointerWrapperInit"):
if initial_cell_state is None:
self._initial_cell_state = None
else:
final_state_tensor = nest.flatten(initial_cell_state)[-1]
state_batch_size = (final_state_tensor.shape[0].value
or array_ops.shape(final_state_tensor)[0])
error_message = (
"When constructing AttnPointerWrapper %s: " %
self._base_name +
"Non-matching batch sizes between the memory "
"(encoder output) and initial_cell_state. Are you using "
"the BeamSearchDecoder? You may need to tile your initial state "
"via the tf.contrib.seq2seq.tile_batch function with argument "
"multiple=beam_width.")
with ops.control_dependencies(
self._batch_size_checks(state_batch_size,
error_message)):
self._initial_cell_state = nest.map_structure(
lambda s: array_ops.identity(
s, name="check_initial_cell_state"),
initial_cell_state)
def __init__(self,
attention_v,
dec_query_layer,
enc_query_layer,
memory_layer,
memory,
memory_sequence_length=None,
probability_fn=None,
score_mask_value=None,
check_inner_dims_defined=True):
"""Construct the word level attention mechanism.
Args:
attention_v: The attention v variable.
dec_query_layer: Mapping layer for decoder's query.
enc_query_layer: Mapping layer for utterance-level encoder's query.
memory_layer: Mapping layer for memory.
memory: The memory to query; the output of a bidirectional RNN. This
tensor should be shaped `[batch_size, max_uttr_len, 2*n_hidden_units]`.
memory_sequence_length (optional): Sequence lengths for the batch entries
in memory. If provided, the memory tensor rows are masked with zeros
for values past the respective sequence lengths.
probability_fn: (optional) A `callable`. Converts the score to
probabilities. The default is `tf.nn.softmax`. Other options include
`tf.contrib.seq2seq.hardmax` and `tf.contrib.sparsemax.sparsemax`.
Its signature should be: `probabilities = probability_fn(score)`.
score_mask_value: (optional): The mask value for score before passing into
`probability_fn`. The default is -inf. Only used if
`memory_sequence_length` is not None.
check_inner_dims_defined: Python boolean. If `True`, the `memory`
argument's shape is checked to ensure all but the two outermost
dimensions are fully defined.
"""
# super(WordLevelAttentionMechanism, self).__init__(
# query_layer = None,
# memory_layer = memory_layer,
# memory = memory,
# probability_fn = wrapped_probability_fn,
# memory_sequence_length = memory_sequence_length)
# Use custom initialization due to the probable zero values in memory_sequence_length
if probability_fn is None:
probability_fn = tf.nn.softmax
if (memory_layer is not None
and not isinstance(memory_layer, tf.layers.Layer)):
raise TypeError("memory_layer is not a Layer: %s" %
type(memory_layer).__name__)
self._query_layer = None
self._memory_layer = memory_layer
self.dtype = memory_layer.dtype
if not callable(probability_fn):
raise TypeError("probability_fn must be callable, saw type: %s" %
type(probability_fn).__name__)
if score_mask_value is None:
score_mask_value = tf.dtypes.as_dtype(
self._memory_layer.dtype).as_numpy_dtype(-np.inf)
self._probability_fn = lambda score: _maybe_mask_score_softmax(
probability_fn(
_maybe_mask_score_no_check(score, memory_sequence_length,
score_mask_value)),
memory_sequence_length)
with tf.name_scope(None, "BaseAttentionMechanismInit",
tf.contrib.framework.nest.flatten(memory)):
self._values = _prepare_memory(
memory,
memory_sequence_length,
check_inner_dims_defined=check_inner_dims_defined)
self._keys = (self.memory_layer(self._values)
if self.memory_layer else self._values)
self._batch_size = (self._keys.shape[0].value
or tf.shape(self._keys)[0])
self._alignments_size = (self._keys.shape[1].value
or tf.shape(self._keys)[1])
# Extra initialization
self._dec_query_layer = dec_query_layer
self._enc_query_layer = enc_query_layer
self._attention_v = attention_v