def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('attention'):
mh_attn = MultiheadAttentionEncoder(
self._hparams.multihead_attention)
self.multihead_attention_list.append(mh_attn)
if self._hparams.dim != mh_attn.hparams.output_dim:
raise ValueError(
'The "dim" in the hparams of '
'"multihead_attention" should be equal to the '
'"dim" of TransformerEncoder')
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
if self._hparams.dim != final_dim:
raise ValueError(
'The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of TransformerEncoder.')
self.poswise_networks.append(pw_net)
def __init__(self, encoder_major=None, encoder_minor=None, hparams=None):
EncoderBase.__init__(self, hparams)
encoder_major_hparams = utils.get_instance_kwargs(
None, self._hparams.encoder_major_hparams)
encoder_minor_hparams = utils.get_instance_kwargs(
None, self._hparams.encoder_minor_hparams)
if encoder_major is not None:
self._encoder_major = encoder_major
else:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_major'):
self._encoder_major = utils.check_or_get_instance(
self._hparams.encoder_major_type,
encoder_major_hparams,
['texar.modules.encoders', 'texar.custom'])
if encoder_minor is not None:
self._encoder_minor = encoder_minor
elif self._hparams.config_share:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_minor'):
self._encoder_minor = utils.check_or_get_instance(
self._hparams.encoder_major_type,
encoder_major_hparams,
['texar.modules.encoders', 'texar.custom'])
else:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_minor'):
self._encoder_minor = utils.check_or_get_instance(
self._hparams.encoder_minor_type,
encoder_minor_hparams,
['texar.modules.encoders', 'texar.custom'])
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.position_embedder = \
SinusoidsPositionEmbedder(
self._hparams.position_embedder_hparams)
def __init__(self,
vocab_size=None,
output_layer=None,
tau=None,
hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
# Make the output layer
self._output_layer, self._vocab_size = _make_output_layer(
output_layer, vocab_size, self._hparams.output_layer_bias,
self.variable_scope)
# Make attention and poswise networks
self.graph_multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('attention'):
mh_attn = GraphMultiheadAttentionEncoder(
self._hparams.graph_multihead_attention)
self.graph_multihead_attention_list.append(mh_attn)
if self._hparams.dim != mh_attn.hparams.output_dim:
raise ValueError(
'The "dim" in the hparams of '
'"multihead_attention" should be equal to the '
'"dim" of CrossGraphTransformerFixedLengthDecoder'
)
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
if self._hparams.dim != final_dim:
raise ValueError(
'The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of CrossGraphTransformerFixedLengthDecoder.'
)
self.poswise_networks.append(pw_net)
self._helper = None
self._tau = tau
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
if self._hparams.position_embedder_type == 'sinusoids':
self.position_embedder = SinusoidsPositionEmbedder(
self._hparams.position_embedder_hparams)
else:
self.position_embedder = PositionEmbedder(
position_size=self._hparams.position_size,
hparams=self._hparams.position_embedder_hparams)
# pylint: disable=protected-access
if self._hparams.dim != \
self.position_embedder._hparams.dim:
raise ValueError('"dim" in '
'TransformerEncoder hparams must be equal '
'to "dim" in its '
'position_embedder_hparams.')
self.multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('attention'):
multihead_attention = MultiheadAttentionEncoder(
self._hparams.multihead_attention)
self.multihead_attention_list.append(
multihead_attention)
# pylint: disable=protected-access
if self._hparams.dim != \
multihead_attention._hparams.output_dim:
raise ValueError('The "dim" in the hparams of '
'multihead_attention should be equal '
'to the "dim" of TransformerEncoder')
poswise_network = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
# pylint: disable=protected-access
if self._hparams.dim != \
poswise_network._hparams.layers[-1]['kwargs']['units']:
# poswise_network._hparams.layers[-1]['units']:
raise ValueError('The "units" in the "kwargs" of '
'FeedForwardNetwork should be equal '
'to the "dim" of TransformerEncoder')
self.poswise_networks.append(poswise_network)
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.Q_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=False,
name='q')
self.K_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=False,
name='k')
self.V_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=False,
name='v')
self.O_dense = tf.layers.Dense(self._hparams.output_dim,
use_bias=False,
name='o')
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
use_bias = self._hparams.use_bias
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.Q_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='query')
self.K_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='key')
self.V_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='value')
self.O_dense = tf.layers.Dense(self._hparams.output_dim,
use_bias=use_bias,
name='output')
def __init__(self, embedding, vocab_size=None, hparams=None):
EncoderBase.__init__(self, hparams)
self._vocab_size = vocab_size
self._embedding = None
self.enc = None
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
if self._hparams.position_embedder.name == 'sinusoids':
self.position_embedder = \
position_embedders.SinusoidsPositionEmbedder(\
self._hparams.position_embedder.hparams)
if self._hparams.use_embedding:
if isinstance(embedding, tf.Variable):
self._embedding = embedding
embed_dim = self._embedding.get_shape().as_list()[-1]
if self._hparams.zero_pad: # TODO(zhiting): vocab has zero pad
if not self._hparams.bos_pad:
self._embedding = tf.concat(\
(tf.zeros(shape=[1, embed_dim]),
self._embedding[1:, :]), 0)
else:
self._embedding = tf.concat(\
(tf.zeros(shape=[2, embed_dim]),
self._embedding[2:, :]), 0)
if self._vocab_size is None:
self._vocab_size = self._embedding.get_shape().as_list()[0]
with tf.variable_scope(self.variable_scope):
if self._hparams.target_space_id is not None:
space_embedding = tf.get_variable('target_space_embedding', \
[32, embed_dim])
self.target_symbol_embedding = tf.gather(space_embedding, \
self._hparams.target_space_id)
else:
self.target_symbol_embedding = None
self.stack_output = None
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.position_embedder = \
SinusoidsPositionEmbedder(
self._hparams.position_embedder_hparams)
self.multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('self_attention'):
multihead_attention = MultiheadAttentionEncoder(
self._hparams.multihead_attention)
self.multihead_attention_list.append(
multihead_attention)
# pylint: disable=protected-access
if self._hparams.dim != \
multihead_attention._hparams.output_dim:
raise ValueError('The output dimenstion of'
'MultiheadEncoder should be equal'
'to the dim of TransformerEncoder')
poswise_network = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
# pylint: disable=protected-access
if self._hparams.dim != \
poswise_network._hparams.layers[-1]['kwargs']['units']:
# poswise_network._hparams.layers[-1]['units']:
raise ValueError('The output dimenstion of'
'FeedForwardNetwork should be equal'
'to the dim of TransformerEncoder')
self.poswise_networks.append(poswise_network)
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
self._input_size = self._hparams.dim
self.self_attns = nn.ModuleList()
if not self._hparams.use_bert_config:
self.self_attn_layer_norm = nn.ModuleList()
self.poswise_networks = nn.ModuleList()
self.poswise_layer_norm = nn.ModuleList()
self.output_layer_norm = nn.ModuleList()
if self._hparams.use_bert_config:
# In TensorFlow, eps for LayerNorm is 1e-12 by default.
eps = 1e-12
else:
# In PyTorch, eps for LayerNorm is 1e-6 by default.
eps = 1e-6
for _ in range(self._hparams.num_blocks):
mh_attn = MultiheadAttentionEncoder(
self._input_size, self._hparams.multihead_attention)
self.self_attns.append(mh_attn)
if not self._hparams.use_bert_config:
self.self_attn_layer_norm.append(
nn.LayerNorm(self._input_size, eps=eps))
if self._hparams.dim != mh_attn.hparams.output_dim:
raise ValueError(
'The "dim" in the hparams of '
'"multihead_attention" should be equal to the '
'"dim" of TransformerEncoder')
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['out_features']
if self._hparams.dim != final_dim:
raise ValueError('The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of TransformerEncoder.')
self.poswise_networks.append(pw_net)
self.poswise_layer_norm.append(
nn.LayerNorm(self._input_size, eps=eps))
if self._hparams.use_bert_config:
self.output_layer_norm.append(
nn.LayerNorm(self._input_size, eps=eps))
self.embed_dropout = nn.Dropout(p=self._hparams.embedding_dropout)
self.residual_dropout = nn.Dropout(p=self._hparams.residual_dropout)
if self._hparams.use_bert_config:
self.input_normalizer = nn.LayerNorm(self._input_size, eps=eps)
else:
self.final_layer_normalizer = nn.LayerNorm(self._input_size,
eps=eps)
if self._hparams.initializer:
initialize = layers.get_initializer(self._hparams.initializer)
assert initialize is not None
# Do not re-initialize LayerNorm modules.
for name, param in self.named_parameters():
if name.split(
'.')[-1] == 'weight' and 'layer_norm' not in name:
initialize(param)