def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('attention'):
mh_attn = MultiheadAttentionEncoder(
self._hparams.multihead_attention)
self.multihead_attention_list.append(mh_attn)
if self._hparams.dim != mh_attn.hparams.output_dim:
raise ValueError(
'The "dim" in the hparams of '
'"multihead_attention" should be equal to the '
'"dim" of TransformerEncoder')
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
if self._hparams.dim != final_dim:
raise ValueError(
'The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of TransformerEncoder.')
self.poswise_networks.append(pw_net)
def __init__(self, encoder_major=None, encoder_minor=None, hparams=None):
EncoderBase.__init__(self, hparams)
encoder_major_hparams = utils.get_instance_kwargs(
None, self._hparams.encoder_major_hparams)
encoder_minor_hparams = utils.get_instance_kwargs(
None, self._hparams.encoder_minor_hparams)
if encoder_major is not None:
self._encoder_major = encoder_major
else:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_major'):
self._encoder_major = utils.check_or_get_instance(
self._hparams.encoder_major_type,
encoder_major_hparams,
['texar.tf.modules.encoders', 'texar.tf.custom'])
if encoder_minor is not None:
self._encoder_minor = encoder_minor
elif self._hparams.config_share:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_minor'):
self._encoder_minor = utils.check_or_get_instance(
self._hparams.encoder_major_type,
encoder_major_hparams,
['texar.tf.modules.encoders', 'texar.tf.custom'])
else:
with tf.variable_scope(self.variable_scope.name):
with tf.variable_scope('encoder_minor'):
self._encoder_minor = utils.check_or_get_instance(
self._hparams.encoder_minor_type,
encoder_minor_hparams,
['texar.tf.modules.encoders', 'texar.tf.custom'])
def __init__(self,
vocab_size=None,
output_layer=None,
tau=None,
hparams=None):
EncoderBase.__init__(self, hparams)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
# Make the output layer
self._output_layer, self._vocab_size = _make_output_layer(
output_layer, vocab_size, self._hparams.output_layer_bias,
self.variable_scope)
# Make attention and poswise networks
self.graph_multihead_attention_list = []
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
with tf.variable_scope("layer_{}".format(i)):
with tf.variable_scope('attention'):
mh_attn = GraphMultiheadAttentionEncoder(
self._hparams.graph_multihead_attention)
self.graph_multihead_attention_list.append(mh_attn)
if self._hparams.dim != mh_attn.hparams.output_dim:
raise ValueError(
'The "dim" in the hparams of '
'"multihead_attention" should be equal to the '
'"dim" of CrossGraphTransformerFixedLengthDecoder'
)
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
if self._hparams.dim != final_dim:
raise ValueError(
'The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of CrossGraphTransformerFixedLengthDecoder.'
)
self.poswise_networks.append(pw_net)
self._helper = None
self._tau = tau
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)
use_bias = self._hparams.use_bias
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
self.Q_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='query')
self.K_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='key')
self.V_dense = tf.layers.Dense(self._hparams.num_units,
use_bias=use_bias,
name='value')
self.O_dense = tf.layers.Dense(self._hparams.output_dim,
use_bias=use_bias,
name='output')
def default_hparams():
"""Returns a dictionary of hyperparameters with default values.
.. role:: python(code)
:language: python
.. code-block:: python
{
"encoder_major_type": "UnidirectionalRNNEncoder",
"encoder_major_hparams": {},
"encoder_minor_type": "UnidirectionalRNNEncoder",
"encoder_minor_hparams": {},
"config_share": False,
"name": "hierarchical_encoder_wrapper"
}
Here:
"encoder_major_type": str or class or instance
The high-level encoder. Can be a RNN encoder class, its name or
module path, or a class instance.
Ignored if `encoder_major` is given to the encoder constructor.
"encoder_major_hparams": dict
The hyperparameters for the high-level encoder. The high-level
encoder is created with
:python:`encoder_class(hparams=encoder_major_hparams)`.
Ignored if `encoder_major` is given to the encoder constructor,
or if "encoder_major_type" is an encoder instance.
"encoder_minor_type": str or class or instance
The low-level encoder. Can be a RNN encoder class, its name or
module path, or a class instance.
Ignored if `encoder_minor` is given to the encoder constructor,
or if "config_share" is True.
"encoder_minor_hparams": dict
The hyperparameters for the low-level encoder. The high-level
encoder is created with
:python:`encoder_class(hparams=encoder_minor_hparams)`.
Ignored if `encoder_minor` is given to the encoder constructor,
or if "config_share" is True,
or if "encoder_minor_type" is an encoder instance.
"config_share":
Whether to use encoder_major's hyperparameters
to construct encoder_minor.
"name":
Name of the encoder.
"""
hparams = {
"name": "hierarchical_encoder",
"encoder_major_type": "UnidirectionalRNNEncoder",
"encoder_major_hparams": {},
"encoder_minor_type": "UnidirectionalRNNEncoder",
"encoder_minor_hparams": {},
"config_share": False,
"@no_typecheck":
['encoder_major_hparams', 'encoder_minor_hparams']
}
hparams.update(EncoderBase.default_hparams())
return hparams
def __init__(self, hparams=None):
EncoderBase.__init__(self, hparams)