DecoderFeedables)
from neuralmonkey.encoders.transformer import (
TransformerLayer, position_signal)
from neuralmonkey.model.sequence import EmbeddedSequence
from neuralmonkey.logging import log
from neuralmonkey.nn.utils import dropout
from neuralmonkey.vocabulary import (
Vocabulary, PAD_TOKEN_INDEX, END_TOKEN_INDEX)
from neuralmonkey.tf_utils import append_tensor, layer_norm
# pylint: disable=invalid-name
# TODO: handle attention histories
TransformerHistories = extend_namedtuple(
"TransformerHistories",
DecoderHistories,
[("decoded_symbols", tf.Tensor),
# TODO(all) handle these!
# ("self_attention_histories", List[Tuple]),
# ("inter_attention_histories", List[Tuple]),
("input_mask", tf.Tensor)])
# pylint: enable=invalid-name
class TransformerDecoder(AutoregressiveDecoder):
# pylint: disable=too-many-arguments,too-many-locals
def __init__(self,
name: str,
encoder: Attendable,
vocabulary: Vocabulary,
data_id: str,
# TODO infer the default for these three from the encoder
EncoderProjection)
from neuralmonkey.decoders.output_projection import (
OutputProjectionSpec, OutputProjection, nonlinear_output)
from neuralmonkey.decorators import tensor
RNN_CELL_TYPES = {
"NematusGRU": NematusGRUCell,
"GRU": OrthoGRUCell,
"LSTM": tf.contrib.rnn.LSTMCell
}
# pylint: disable=invalid-name
RNNFeedables = extend_namedtuple(
"DecoderFeedables",
DecoderFeedables,
[("prev_rnn_state", tf.Tensor),
("prev_rnn_output", tf.Tensor),
("prev_contexts", List[tf.Tensor])])
RNNHistories = extend_namedtuple(
"RNNHistories",
DecoderHistories,
[("attention_histories", List[Tuple])]) # AttentionLoopStateTA and kids
# pylint: enable=invalid-name
# pylint: disable=too-many-instance-attributes
class Decoder(AutoregressiveDecoder):
"""A class managing parts of the computation graph used during decoding."""
# pylint: disable=too-many-locals
LoopState, extend_namedtuple,
DecoderHistories,
DecoderFeedables)
from neuralmonkey.encoders.transformer import (TransformerLayer,
position_signal)
from neuralmonkey.model.sequence import EmbeddedSequence
from neuralmonkey.logging import log
from neuralmonkey.nn.utils import dropout
from neuralmonkey.vocabulary import (Vocabulary, PAD_TOKEN_INDEX,
END_TOKEN_INDEX)
from neuralmonkey.tf_utils import layer_norm
# pylint: disable=invalid-name
TransformerHistories = extend_namedtuple(
"RNNHistories",
DecoderHistories, [("decoded_symbols", tf.TensorArray),
("self_attention_histories", List[Tuple]),
("inter_attention_histories", List[Tuple]),
("input_mask", tf.TensorArray)])
# pylint: enable=invalid-name
class TransformerDecoder(AutoregressiveDecoder):
# pylint: disable=too-many-arguments,too-many-locals
def __init__(
self,
name: str,
encoder: Attendable,
vocabulary: Vocabulary,
data_id: str,
# TODO infer the default for these three from the encoder