def __init__(self, H=None, hparams=None):
ModuleBase.__init__(self, hparams)
self._H = H
def __init__(self, network=None, network_kwargs=None, hparams=None):
ModuleBase.__init__(self, hparams=hparams)
with tf.variable_scope(self.variable_scope):
self._build_network(network, network_kwargs)
def __init__(self, hparams=None):
ModuleBase.__init__(self, hparams)
def __init__(self, rate=None, hparams=None):
ModuleBase.__init__(self, hparams)
self._rate = rate
def __init__(self, vocab_size=None, output_layer=None, hparams=None):
ModuleBase.__init__(self, hparams)
self._hparams.add_hparam('multihead_attention',
self._hparams.graph_multihead_attention)
with tf.variable_scope(self.variable_scope):
if self._hparams.initializer:
tf.get_variable_scope().set_initializer(
layers.get_initializer(self._hparams.initializer))
# Make the output layer
self._output_layer, self._vocab_size = _make_output_layer(
output_layer, vocab_size, self._hparams.output_layer_bias,
self.variable_scope)
# Make attention and poswise networks
self.multihead_attentions = {'self_att': [], 'encdec_att': []}
self.poswise_networks = []
for i in range(self._hparams.num_blocks):
layer_name = 'layer_{}'.format(i)
with tf.variable_scope(layer_name):
with tf.variable_scope("self_attention"):
multihead_attention = GraphMultiheadAttentionEncoder(
self._hparams.graph_multihead_attention)
self.multihead_attentions['self_att'].append(
multihead_attention)
if self._hparams.dim != \
multihead_attention.hparams.output_dim:
raise ValueError('The output dimenstion of '
'MultiheadEncoder should be equal '
'to the dim of TransformerDecoder')
with tf.variable_scope('encdec_attention'):
multihead_attention = GraphMultiheadAttentionEncoder(
self._hparams.graph_multihead_attention)
self.multihead_attentions['encdec_att'].append(
multihead_attention)
if self._hparams.dim != \
multihead_attention.hparams.output_dim:
raise ValueError('The output dimenstion of '
'MultiheadEncoder should be equal '
'to the dim of TransformerDecoder')
pw_net = FeedForwardNetwork(
hparams=self._hparams['poswise_feedforward'])
final_dim = pw_net.hparams.layers[-1]['kwargs']['units']
if self._hparams.dim != final_dim:
raise ValueError(
'The output dimenstion of '
'"poswise_feedforward" should be equal '
'to the "dim" of TransformerDecoder.')
self.poswise_networks.append(pw_net)
# Built in _build()
self.context = None
self.context_sequence_length = None
self.embedding = None
self._helper = None
self._cache = None
def __init__(self, output_size, hparams=None):
ModuleBase.__init__(self, hparams)
self._output_size = output_size
def __init__(self, num_embeds=None, hparams=None):
ModuleBase.__init__(self, hparams)
self._num_embeds = num_embeds