def __init__(self, name: str, n_heads: int, keys_encoder: Attendable, values_encoder: Attendable = None, dropout_keep_prob: float = 1.0, reuse: ModelPart = None, save_checkpoint: str = None, load_checkpoint: str = None, initializers: InitializerSpecs = None) -> None: check_argument_types() BaseAttention.__init__(self, name, reuse, save_checkpoint, load_checkpoint, initializers) self.n_heads = n_heads self.dropout_keep_prob = dropout_keep_prob if self.n_heads <= 0: raise ValueError("Number of heads must be greater than zero.") if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0: raise ValueError("Dropout keep prob must be inside (0,1].") if values_encoder is None: values_encoder = keys_encoder self.attention_keys = get_attention_states(keys_encoder) self.attention_mask = get_attention_mask(keys_encoder) self.attention_values = get_attention_states(values_encoder) self._variable_scope.set_initializer( tf.variance_scaling_initializer(mode="fan_avg", distribution="uniform"))
def __init__(self, name: str, n_heads: int, keys_encoder: Attendable, values_encoder: Attendable = None, dropout_keep_prob: float = 1.0, save_checkpoint: str = None, load_checkpoint: str = None, initializers: InitializerSpecs = None) -> None: check_argument_types() BaseAttention.__init__(self, name, save_checkpoint, load_checkpoint, initializers) self.n_heads = n_heads self.dropout_keep_prob = dropout_keep_prob if self.n_heads <= 0: raise ValueError("Number of heads must be greater than zero.") if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0: raise ValueError("Dropout keep prob must be inside (0,1].") if values_encoder is None: values_encoder = keys_encoder self.attention_keys = get_attention_states(keys_encoder) self.attention_mask = get_attention_mask(keys_encoder) self.attention_values = get_attention_states(values_encoder)
def cross_attention_sublayer(self, queries: tf.Tensor) -> tf.Tensor: assert self.cross_attention_sublayer is not None assert self.n_cross_att_heads is not None assert self.input_for_cross_attention is not None encoder_att_states = get_attention_states( self.input_for_cross_attention) encoder_att_mask = get_attention_mask(self.input_for_cross_attention) # Layer normalization normalized_queries = layer_norm(queries) encoder_context, _ = attention( queries=normalized_queries, keys=encoder_att_states, values=encoder_att_states, keys_mask=encoder_att_mask, num_heads=self.n_cross_att_heads, dropout_callback=lambda x: dropout( x, self.attention_dropout_keep_prob, self.train_mode), use_bias=self.use_att_transform_bias) # Apply dropout encoder_context = dropout( encoder_context, self.dropout_keep_prob, self.train_mode) # Add residual connections return encoder_context + queries
def cross_attention_sublayer(self, queries: tf.Tensor) -> tf.Tensor: assert self.cross_attention_sublayer is not None encoder_att_states = get_attention_states( self.input_for_cross_attention) encoder_att_mask = get_attention_mask(self.input_for_cross_attention) # Layer normalization normalized_queries = layer_norm(queries) encoder_context, _ = attention( queries=normalized_queries, keys=encoder_att_states, values=encoder_att_states, keys_mask=encoder_att_mask, num_heads=self.n_cross_att_heads, dropout_callback=lambda x: dropout( x, self.attention_dropout_keep_prob, self.train_mode), use_bias=self.use_att_transform_bias) # Apply dropout encoder_context = dropout(encoder_context, self.dropout_keep_prob, self.train_mode) # Add residual connections return encoder_context + queries
def __init__(self, name: str, input_sequence: Attendable, hidden_size: int, num_heads: int, output_size: int = None, state_proj_size: int = None, dropout_keep_prob: float = 1.0, save_checkpoint: str = None, load_checkpoint: str = None, initializers: InitializerSpecs = None) -> None: """Initialize an instance of the encoder.""" check_argument_types() ModelPart.__init__(self, name, save_checkpoint, load_checkpoint, initializers) self.input_sequence = input_sequence self.hidden_size = hidden_size self.num_heads = num_heads self.output_size = output_size self.state_proj_size = state_proj_size self.dropout_keep_prob = dropout_keep_prob if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0: raise ValueError("Dropout keep prob must be inside (0,1].") with self.use_scope(): self._attention_states_dropped = dropout( get_attention_states(self.input_sequence), self.dropout_keep_prob, self.train_mode)
def encoder_attention_sublayer(self, queries: tf.Tensor) -> tf.Tensor: """Create the encoder-decoder attention sublayer.""" encoder_att_states = get_attention_states(self.encoder) encoder_att_mask = get_attention_mask(self.encoder) # Layer normalization normalized_queries = layer_norm(queries) # Attend to the encoder # TODO handle histories encoder_context, _ = attention( queries=normalized_queries, keys=encoder_att_states, values=encoder_att_states, keys_mask=encoder_att_mask, num_heads=self.n_heads_enc, dropout_callback=lambda x: dropout( x, self.attention_dropout_keep_prob, self.train_mode), use_bias=self.use_att_transform_bias) # Apply dropout encoder_context = dropout(encoder_context, self.dropout_keep_prob, self.train_mode) # Add residual connections return encoder_context + queries
def __init__(self, name: str, encoders: List[Attendable], attention_state_size: int, share_attn_projections: bool = False, use_sentinels: bool = False, reuse: ModelPart = None, save_checkpoint: str = None, load_checkpoint: str = None, initializers: InitializerSpecs = None) -> None: check_argument_types() MultiAttention.__init__(self, name=name, attention_state_size=attention_state_size, share_attn_projections=share_attn_projections, use_sentinels=use_sentinels, reuse=reuse, save_checkpoint=save_checkpoint, load_checkpoint=load_checkpoint, initializers=initializers) self._encoders = encoders # pylint: disable=protected-access self._encoders_tensors = [ get_attention_states(e) for e in self._encoders ] self._encoders_masks = [get_attention_mask(e) for e in self._encoders] # pylint: enable=protected-access for e_m in self._encoders_masks: assert_shape(e_m, [-1, -1]) for e_t in self._encoders_tensors: assert_shape(e_t, [-1, -1, -1]) with self.use_scope(): self.encoder_projections_for_logits = \ self.get_encoder_projections("logits_projections") self.encoder_attn_biases = [ get_variable(name="attn_bias_{}".format(i), shape=[], initializer=tf.zeros_initializer()) for i in range(len(self._encoders_tensors)) ] if self._share_projections: self.encoder_projections_for_ctx = \ self.encoder_projections_for_logits else: self.encoder_projections_for_ctx = \ self.get_encoder_projections("context_projections") if self._use_sentinels: self._encoders_masks.append( tf.ones([tf.shape(self._encoders_masks[0])[0], 1])) self.masks_concat = tf.concat(self._encoders_masks, 1)
def model_dimension(self) -> int: dim = self.input_sequence.dimension if self.input_for_cross_attention is not None: cross_att_dim = get_attention_states( self.input_for_cross_attention).get_shape()[-1].value if cross_att_dim != dim: raise ValueError( "The input for cross-attention must be of the same " "dimension as the model, was {}.".format(cross_att_dim)) return dim
def model_dimension(self) -> int: dim = self.input_sequence.dimension if self.input_for_cross_attention is not None: cross_att_dim = get_attention_states( self.input_for_cross_attention).get_shape()[-1].value if cross_att_dim != dim: raise ValueError( "The input for cross-attention must be of the same " "dimension as the model, was {}.".format(cross_att_dim)) return dim
def __init__(self, name: str, n_heads: int, keys_encoder: Attendable, values_encoder: Attendable = None, dropout_keep_prob: float = 1.0, save_checkpoint: str = None, load_checkpoint: str = None) -> None: check_argument_types() BaseAttention.__init__(self, name, save_checkpoint, load_checkpoint) self.n_heads = n_heads self.dropout_keep_prob = dropout_keep_prob if self.n_heads <= 0: raise ValueError("Number of heads must be greater than zero.") if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0: raise ValueError("Dropout keep prob must be inside (0,1].") if values_encoder is None: values_encoder = keys_encoder self.attention_keys = get_attention_states(keys_encoder) self.attention_values = get_attention_states(values_encoder) self.attention_mask = get_attention_mask(keys_encoder) self._dimension = self.attention_keys.get_shape()[-1].value if self._dimension % self.n_heads != 0: raise ValueError("Model dimension ({}) must be divisible by the " "number of attention heads ({})".format( self._dimension, self.n_heads)) self._head_dim = int(self._dimension / self.n_heads) self._scaling_factor = 1 / math.sqrt(self._head_dim)
def encoder_attention(self, level: int, queries: tf.Tensor) -> tf.Tensor: with tf.variable_scope("dec_inter_att_level_{}".format(level), reuse=tf.AUTO_REUSE): encoder_att_states = get_attention_states(self.encoder) encoder_att_mask = get_attention_mask(self.encoder) # TODO handle histories encoder_context, _ = attention( queries=queries, keys=encoder_att_states, values=encoder_att_states, keys_mask=encoder_att_mask, num_heads=self.n_heads_enc, dropout_callback=lambda x: dropout( x, self.attention_dropout_keep_prob, self.train_mode)) return dropout( encoder_context, self.dropout_keep_prob, self.train_mode)
def attention_states(self) -> tf.Tensor: return dropout(get_attention_states(self.encoder), self.dropout_keep_prob, self.train_mode)
def _encoders_tensors(self) -> List[tf.Tensor]: tensors = [get_attention_states(e) for e in self._encoders] for e_t in tensors: assert_shape(e_t, [-1, -1, -1]) return tensors
def __init__(self, name: str, input_sequence: TemporalStateful, ff_hidden_size: int, depth: int, n_heads: int, dropout_keep_prob: float = 1.0, attention_dropout_keep_prob: float = 1.0, target_space_id: int = None, use_att_transform_bias: bool = False, use_positional_encoding: bool = True, input_for_cross_attention: Attendable = None, n_cross_att_heads: int = None, save_checkpoint: str = None, load_checkpoint: str = None) -> None: """Create an encoder of the Transformer model. Described in Vaswani et al. (2017), arxiv.org/abs/1706.03762 Arguments: input_sequence: Embedded input sequence. name: Name of the decoder. Should be unique accross all Neural Monkey objects. dropout_keep_prob: Probability of keeping a value during dropout. target_space_id: Specifies the modality of the target space. use_att_transform_bias: Add bias when transforming qkv vectors for attention. use_positional_encoding: If True, position encoding signal is added to the input. Keyword arguments: ff_hidden_size: Size of the feedforward sublayers. n_heads: Number of the self-attention heads. depth: Number of sublayers. attention_dropout_keep_prob: Probability of keeping a value during dropout on the attention output. input_for_cross_attention: An attendable model part that is attended using cross-attention on every layer of the decoder, analogically to how encoder is attended in the decoder. n_cross_att_heads: Number of heads used in the cross-attention. """ check_argument_types() ModelPart.__init__(self, name, save_checkpoint, load_checkpoint) self.input_sequence = input_sequence self.model_dimension = self.input_sequence.dimension self.ff_hidden_size = ff_hidden_size self.depth = depth self.n_heads = n_heads self.dropout_keep_prob = dropout_keep_prob self.attention_dropout_keep_prob = attention_dropout_keep_prob self.target_space_id = target_space_id self.use_att_transform_bias = use_att_transform_bias self.use_positional_encoding = use_positional_encoding self.input_for_cross_attention = input_for_cross_attention self.n_cross_att_heads = n_cross_att_heads if self.depth <= 0: raise ValueError("Depth must be a positive integer.") if self.ff_hidden_size <= 0: raise ValueError("Feed forward hidden size must be a " "positive integer.") if self.dropout_keep_prob <= 0.0 or self.dropout_keep_prob > 1.0: raise ValueError("Dropout keep prob must be inside (0,1].") if (self.attention_dropout_keep_prob <= 0.0 or self.attention_dropout_keep_prob > 1.0): raise ValueError("Dropout keep prob for attn must be in (0,1].") if self.target_space_id is not None and (self.target_space_id >= 32 or self.target_space_id < 0): raise ValueError( "If provided, the target space ID should be between 0 and 31. " "Was: {}".format(self.target_space_id)) if (input_for_cross_attention is None) ^ (n_cross_att_heads is None): raise ValueError( "Either both input_for_cross_attention and n_cross_att_heads " "must be provided or none of them.") if input_for_cross_attention is not None: cross_att_dim = get_attention_states( input_for_cross_attention).get_shape()[-1].value if cross_att_dim != self.model_dimension: raise ValueError( "The input for cross-attention must be of the same " "dimension as the model, was {}.".format(cross_att_dim)) self.train_mode = tf.placeholder(tf.bool, [], "train_mode") log("Output op: {}".format(self.output))
def attention_states(self) -> tf.Tensor: return dropout(get_attention_states(self.encoder), self.dropout_keep_prob, self.train_mode)
def __init__( self, name: str, encoder: Attendable, vocabulary: Vocabulary, data_id: str, # TODO infer the default for these three from the encoder ff_hidden_size: int, n_heads_self: int, n_heads_enc: int, depth: int, max_output_len: int, dropout_keep_prob: float = 1.0, embedding_size: int = None, embeddings_source: EmbeddedSequence = None, tie_embeddings: bool = True, label_smoothing: float = None, attention_dropout_keep_prob: float = 1.0, use_att_transform_bias: bool = False, supress_unk: bool = False, save_checkpoint: str = None, load_checkpoint: str = None) -> None: """Create a decoder of the Transformer model. Described in Vaswani et al. (2017), arxiv.org/abs/1706.03762 Arguments: encoder: Input encoder of the decoder. vocabulary: Target vocabulary. data_id: Target data series. name: Name of the decoder. Should be unique accross all Neural Monkey objects. max_output_len: Maximum length of an output sequence. dropout_keep_prob: Probability of keeping a value during dropout. embedding_size: Size of embedding vectors for target words. embeddings_source: Embedded sequence to take embeddings from. tie_embeddings: Use decoder.embedding_matrix also in place of the output decoding matrix. Keyword arguments: ff_hidden_size: Size of the feedforward sublayers. n_heads_self: Number of the self-attention heads. n_heads_enc: Number of the attention heads over the encoder. depth: Number of sublayers. label_smoothing: A label smoothing parameter for cross entropy loss computation. attention_dropout_keep_prob: Probability of keeping a value during dropout on the attention output. supress_unk: If true, decoder will not produce symbols for unknown tokens. """ check_argument_types() AutoregressiveDecoder.__init__(self, name=name, vocabulary=vocabulary, data_id=data_id, max_output_len=max_output_len, dropout_keep_prob=dropout_keep_prob, embedding_size=embedding_size, embeddings_source=embeddings_source, tie_embeddings=tie_embeddings, label_smoothing=label_smoothing, supress_unk=supress_unk, save_checkpoint=save_checkpoint, load_checkpoint=load_checkpoint) self.encoder = encoder self.ff_hidden_size = ff_hidden_size self.n_heads_self = n_heads_self self.n_heads_enc = n_heads_enc self.depth = depth self.attention_dropout_keep_prob = attention_dropout_keep_prob self.use_att_transform_bias = use_att_transform_bias self.encoder_states = get_attention_states(self.encoder) self.encoder_mask = get_attention_mask(self.encoder) self.dimension = (self.encoder_states.get_shape()[2].value ) # type: ignore if self.embedding_size != self.dimension: raise ValueError("Model dimension and input embedding size" "do not match") self._variable_scope.set_initializer( tf.variance_scaling_initializer(mode="fan_avg", distribution="uniform")) log("Decoder cost op: {}".format(self.cost)) self._variable_scope.reuse_variables() log("Runtime logits: {}".format(self.runtime_logits))
def attention_values(self) -> tf.Tensor: return get_attention_states(self.values_encoder)
def attention_keys(self) -> tf.Tensor: return get_attention_states(self.keys_encoder)
def _encoders_tensors(self) -> List[tf.Tensor]: tensors = [get_attention_states(e) for e in self._encoders] for e_t in tensors: assert_shape(e_t, [-1, -1, -1]) return tensors
def __init__(self, name: str, encoders: List[Attendable], vocabulary: Vocabulary, data_id: str, # TODO infer the default for these three from the encoder ff_hidden_size: int, n_heads_self: int, n_heads_enc: Union[List[int], int], depth: int, max_output_len: int, attention_combination_strategy: str = "serial", n_heads_hier: int = None, dropout_keep_prob: float = 1.0, embedding_size: int = None, embeddings_source: EmbeddedSequence = None, tie_embeddings: bool = True, label_smoothing: float = None, self_attention_dropout_keep_prob: float = 1.0, attention_dropout_keep_prob: Union[float, List[float]] = 1.0, use_att_transform_bias: bool = False, supress_unk: bool = False, reuse: ModelPart = None, save_checkpoint: str = None, load_checkpoint: str = None, initializers: InitializerSpecs = None) -> None: """Create a decoder of the Transformer model. Described in Vaswani et al. (2017), arxiv.org/abs/1706.03762 Arguments: encoders: Input encoders for the decoder. vocabulary: Target vocabulary. data_id: Target data series. name: Name of the decoder. Should be unique accross all Neural Monkey objects. max_output_len: Maximum length of an output sequence. dropout_keep_prob: Probability of keeping a value during dropout. embedding_size: Size of embedding vectors for target words. embeddings_source: Embedded sequence to take embeddings from. tie_embeddings: Use decoder.embedding_matrix also in place of the output decoding matrix. ff_hidden_size: Size of the feedforward sublayers. n_heads_self: Number of the self-attention heads. n_heads_enc: Number of the attention heads over each encoder. Either a list which size must be equal to ``encoders``, or a single integer. In the latter case, the number of heads is equal for all encoders. attention_comnbination_strategy: One of ``serial``, ``parallel``, ``flat``, ``hierarchical``. Controls the attention combination strategy for enc-dec attention. n_heads_hier: Number of the attention heads for the second attention in the ``hierarchical`` attention combination. depth: Number of sublayers. label_smoothing: A label smoothing parameter for cross entropy loss computation. attention_dropout_keep_prob: Probability of keeping a value during dropout on the attention output. supress_unk: If true, decoder will not produce symbols for unknown tokens. reuse: Reuse the variables from the given model part. """ check_argument_types() AutoregressiveDecoder.__init__( self, name=name, vocabulary=vocabulary, data_id=data_id, max_output_len=max_output_len, dropout_keep_prob=dropout_keep_prob, embedding_size=embedding_size, embeddings_source=embeddings_source, tie_embeddings=tie_embeddings, label_smoothing=label_smoothing, supress_unk=supress_unk, reuse=reuse, save_checkpoint=save_checkpoint, load_checkpoint=load_checkpoint) self.encoders = encoders self.ff_hidden_size = ff_hidden_size self.n_heads_self = n_heads_self if isinstance(n_heads_enc, int): if attention_combination_strategy == "flat": self.n_heads_enc = [n_heads_enc] else: self.n_heads_enc = [n_heads_enc for _ in self.encoders] else: self.n_heads_enc = n_heads_enc self.depth = depth if isinstance(attention_dropout_keep_prob, float): self.attention_dropout_keep_prob = [ attention_dropout_keep_prob for _ in encoders] else: self.attention_dropout_keep_prob = attention_dropout_keep_prob self.self_att_dropout_keep_prob = self_attention_dropout_keep_prob self.use_att_transform_bias = use_att_transform_bias self.attention_combination_strategy = attention_combination_strategy self.n_heads_hier = n_heads_hier self.encoder_states = lambda: [get_attention_states(e) for e in self.encoders] self.encoder_masks = lambda: [get_attention_mask(e) for e in self.encoders] if self.attention_combination_strategy not in STRATEGIES: raise ValueError( "Unknown attention combination strategy '{}'. " "Allowed: {}.".format(self.attention_combination_strategy, ", ".join(STRATEGIES))) if (self.attention_combination_strategy == "hierarchical" and self.n_heads_hier is None): raise ValueError( "You must provide n_heads_hier when using the hierarchical " "attention combination strategy.") if (self.attention_combination_strategy != "hierarchical" and self.n_heads_hier is not None): warn("Ignoring n_heads_hier parameter -- use the hierarchical " "attention combination strategy instead.") if (self.attention_combination_strategy == "flat" and len(self.n_heads_enc) != 1): raise ValueError( "For the flat attention combination strategy, only a single " "value is permitted in n_heads_enc.") self._variable_scope.set_initializer(tf.variance_scaling_initializer( mode="fan_avg", distribution="uniform"))
def _attention_states_dropped(self) -> tf.Tensor: return dropout(get_attention_states(self.input_sequence), self.dropout_keep_prob, self.train_mode)
def attention_keys(self) -> tf.Tensor: return get_attention_states(self.keys_encoder)
def attention_values(self) -> tf.Tensor: return get_attention_states(self.values_encoder)
def _attention_states_dropped(self) -> tf.Tensor: return dropout(get_attention_states(self.input_sequence), self.dropout_keep_prob, self.train_mode)