def __init__(self, vocab_size=None, output_layer=None, hparams=None): ModuleBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) # Make the output layer self._output_layer, self._vocab_size = _make_output_layer( output_layer, vocab_size, self._hparams.output_layer_bias, self.variable_scope) # Make attention and poswise networks self.multihead_attentions = {'self_att': [], 'encdec_att': []} self.poswise_networks = [] for i in range(self._hparams.num_blocks): layer_name = 'layer_{}'.format(i) with tf.variable_scope(layer_name): with tf.variable_scope("self_attention"): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attentions['self_att'].append( multihead_attention) if self._hparams.dim != \ multihead_attention.hparams.output_dim: raise ValueError('The output dimenstion of ' 'MultiheadEncoder should be equal ' 'to the dim of TransformerDecoder') with tf.variable_scope('encdec_attention'): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attentions['encdec_att'].append( multihead_attention) if self._hparams.dim != \ multihead_attention.hparams.output_dim: raise ValueError('The output dimenstion of ' 'MultiheadEncoder should be equal ' 'to the dim of TransformerDecoder') pw_net = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) final_dim = pw_net.hparams.layers[-1]['kwargs']['units'] if self._hparams.dim != final_dim: raise ValueError( 'The output dimenstion of ' '"poswise_feedforward" should be equal ' 'to the "dim" of TransformerDecoder.') self.poswise_networks.append(pw_net) # Built in _build() self.context = None self.context_sequence_length = None self.embedding = None self._helper = None self._cache = None self.max_decoding_length = None
def __init__(self, embedding, hparams=None): ModuleBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) if self._hparams.position_embedder_type == 'sinusoids': self.position_embedder = SinusoidsPositionEmbedder( self._hparams.position_embedder_hparams) else: self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embedder_hparams) self._embedding = embedding self._vocab_size = self._embedding.get_shape().as_list()[0] self.output_layer = \ self._build_output_layer(shape_list(self._embedding)[-1]) self.multihead_attentions = {'self_att': [], 'encdec_att': []} self.poswise_networks = [] for i in range(self._hparams.num_blocks): layer_name = 'layer_{}'.format(i) with tf.variable_scope(layer_name): with tf.variable_scope("self_attention"): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attentions['self_att'].append( multihead_attention) # pylint: disable=protected-access if self._hparams.dim != \ multihead_attention._hparams.output_dim: raise ValueError('The output dimenstion of ' 'MultiheadEncoder should be equal ' 'to the dim of TransformerDecoder') with tf.variable_scope('encdec_attention'): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attentions['encdec_att'].append( multihead_attention) if self._hparams.dim != \ multihead_attention._hparams.output_dim: raise ValueError('The output dimenstion of ' 'MultiheadEncoder should be equal ' 'to the dim of TransformerDecoder') poswise_network = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) if self._hparams.dim != \ poswise_network._hparams.layers[-1]['kwargs']['units']: raise ValueError('The output dimenstion of ' 'FeedForwardNetwork should be equal ' 'to the dim of TransformerDecoder') self.poswise_networks.append(poswise_network)
def __init__(self, hparams=None): EncoderBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) self.multihead_attention_list = [] self.poswise_networks = [] for i in range(self._hparams.num_blocks): with tf.variable_scope("layer_{}".format(i)): with tf.variable_scope('attention'): mh_attn = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attention_list.append(mh_attn) if self._hparams.dim != mh_attn.hparams.output_dim: raise ValueError( 'The "dim" in the hparams of ' '"multihead_attention" should be equal to the ' '"dim" of TransformerEncoder') pw_net = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) final_dim = pw_net.hparams.layers[-1]['kwargs']['units'] if self._hparams.dim != final_dim: raise ValueError( 'The output dimenstion of ' '"poswise_feedforward" should be equal ' 'to the "dim" of TransformerEncoder.') self.poswise_networks.append(pw_net)
def __init__(self, hparams=None): EncoderBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) if self._hparams.position_embedder_type == 'sinusoids': self.position_embedder = SinusoidsPositionEmbedder( self._hparams.position_embedder_hparams) else: self.position_embedder = PositionEmbedder( position_size=self._hparams.position_size, hparams=self._hparams.position_embedder_hparams) # pylint: disable=protected-access if self._hparams.dim != \ self.position_embedder._hparams.dim: raise ValueError('"dim" in ' 'TransformerEncoder hparams must be equal ' 'to "dim" in its ' 'position_embedder_hparams.') self.multihead_attention_list = [] self.poswise_networks = [] for i in range(self._hparams.num_blocks): with tf.variable_scope("layer_{}".format(i)): with tf.variable_scope('attention'): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attention_list.append( multihead_attention) # pylint: disable=protected-access if self._hparams.dim != \ multihead_attention._hparams.output_dim: raise ValueError('The "dim" in the hparams of ' 'multihead_attention should be equal ' 'to the "dim" of TransformerEncoder') poswise_network = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) # pylint: disable=protected-access if self._hparams.dim != \ poswise_network._hparams.layers[-1]['kwargs']['units']: # poswise_network._hparams.layers[-1]['units']: raise ValueError('The "units" in the "kwargs" of ' 'FeedForwardNetwork should be equal ' 'to the "dim" of TransformerEncoder') self.poswise_networks.append(poswise_network)
def __init__(self, hparams=None): EncoderBase.__init__(self, hparams) with tf.variable_scope(self.variable_scope): if self._hparams.initializer: tf.get_variable_scope().set_initializer( layers.get_initializer(self._hparams.initializer)) self.position_embedder = \ SinusoidsPositionEmbedder( self._hparams.position_embedder_hparams) self.multihead_attention_list = [] self.poswise_networks = [] for i in range(self._hparams.num_blocks): with tf.variable_scope("layer_{}".format(i)): with tf.variable_scope('self_attention'): multihead_attention = MultiheadAttentionEncoder( self._hparams.multihead_attention) self.multihead_attention_list.append( multihead_attention) # pylint: disable=protected-access if self._hparams.dim != \ multihead_attention._hparams.output_dim: raise ValueError('The output dimenstion of' 'MultiheadEncoder should be equal' 'to the dim of TransformerEncoder') poswise_network = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) # pylint: disable=protected-access if self._hparams.dim != \ poswise_network._hparams.layers[-1]['kwargs']['units']: # poswise_network._hparams.layers[-1]['units']: raise ValueError('The output dimenstion of' 'FeedForwardNetwork should be equal' 'to the dim of TransformerEncoder') self.poswise_networks.append(poswise_network)
def __init__(self, vocab_size: Optional[int] = None, output_layer: Optional[Union[nn.Module, torch.Tensor]] = None, hparams: Optional[HParams] = None): super().__init__( 0, vocab_size, # dummy value for input_size input_time_major=False, output_time_major=False, hparams=hparams) self._input_size = self._hparams.dim self._output_layer, self._vocab_size = _make_output_layer( output_layer, vocab_size, self._input_size, self._hparams.output_layer_bias) self.self_attns = nn.ModuleList() self.self_attn_layer_norm = nn.ModuleList() self.enc_dec_attns = nn.ModuleList() self.end_dec_attn_layer_norm = nn.ModuleList() self.poswise_networks = nn.ModuleList() self.poswise_layer_norm = nn.ModuleList() if self._hparams.use_gpt_config: eps = 1e-5 else: eps = 1e-12 for _ in range(self._hparams.num_blocks): attn_module = MultiheadAttentionEncoder( self._input_size, self._hparams.multihead_attention) if self._hparams.dim != attn_module.output_size: raise ValueError("The output dimension of " "MultiheadEncoder should be equal " "to the dim of TransformerDecoder") self.self_attns.append(attn_module) self.self_attn_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) attn_module = MultiheadAttentionEncoder( self._input_size, self._hparams.multihead_attention) if self._hparams.dim != attn_module.output_size: raise ValueError("The output dimension of " "MultiheadEncoder should be equal " "to the dim of TransformerDecoder") self.enc_dec_attns.append(attn_module) self.end_dec_attn_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) poswise_network = FeedForwardNetwork( hparams=self._hparams.poswise_feedforward) if (poswise_network.hparams.layers[-1]['kwargs']['out_features'] != self._hparams.dim): raise ValueError("The output dimension of " "FeedForwardNetwork should be equal " "to the dim of TransformerDecoder") self.poswise_networks.append(poswise_network) self.poswise_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) self.final_layer_norm = nn.LayerNorm(self._input_size, eps=eps) self.embed_dropout = nn.Dropout(self._hparams.embedding_dropout) self.residual_dropout = nn.Dropout(self._hparams.residual_dropout) if self._hparams.initializer: # TODO: This might be different to what TensorFlow does initialize = layers.get_initializer(self._hparams.initializer) assert initialize is not None # Do not re-initialize LayerNorm modules. for name, param in self.named_parameters(): if name.split( ".")[-1] == "weight" and "layer_norm" not in name: initialize(param)
def __init__(self, hparams=None): EncoderBase.__init__(self, hparams) self._input_size = self._hparams.dim self.self_attns = nn.ModuleList() if not self._hparams.use_bert_config: self.self_attn_layer_norm = nn.ModuleList() self.poswise_networks = nn.ModuleList() self.poswise_layer_norm = nn.ModuleList() self.output_layer_norm = nn.ModuleList() if self._hparams.use_bert_config: # In TensorFlow, eps for LayerNorm is 1e-12 by default. eps = 1e-12 else: # In PyTorch, eps for LayerNorm is 1e-6 by default. eps = 1e-6 for _ in range(self._hparams.num_blocks): mh_attn = MultiheadAttentionEncoder( self._input_size, self._hparams.multihead_attention) self.self_attns.append(mh_attn) if not self._hparams.use_bert_config: self.self_attn_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) if self._hparams.dim != mh_attn.hparams.output_dim: raise ValueError( 'The "dim" in the hparams of ' '"multihead_attention" should be equal to the ' '"dim" of TransformerEncoder') pw_net = FeedForwardNetwork( hparams=self._hparams['poswise_feedforward']) final_dim = pw_net.hparams.layers[-1]['kwargs']['out_features'] if self._hparams.dim != final_dim: raise ValueError('The output dimenstion of ' '"poswise_feedforward" should be equal ' 'to the "dim" of TransformerEncoder.') self.poswise_networks.append(pw_net) self.poswise_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) if self._hparams.use_bert_config: self.output_layer_norm.append( nn.LayerNorm(self._input_size, eps=eps)) self.embed_dropout = nn.Dropout(p=self._hparams.embedding_dropout) self.residual_dropout = nn.Dropout(p=self._hparams.residual_dropout) if self._hparams.use_bert_config: self.input_normalizer = nn.LayerNorm(self._input_size, eps=eps) else: self.final_layer_normalizer = nn.LayerNorm(self._input_size, eps=eps) if self._hparams.initializer: initialize = layers.get_initializer(self._hparams.initializer) assert initialize is not None # Do not re-initialize LayerNorm modules. for name, param in self.named_parameters(): if name.split( '.')[-1] == 'weight' and 'layer_norm' not in name: initialize(param)