def __init__(self, params, name=None):
"""Initialize layers to build Transformer model.
Args:
params: hyperparameter object defining layer sizes, dropout values, etc.
name: name of the model.
"""
super(Transformer, self).__init__(name=name)
self.params = params
self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["vocab_size"], params["hidden_size"])
self.encoder_stack = EncoderStack(params)
self.decoder_stack = DecoderStack(params)
def __init__(self, params, name=None):
super(OrderedDecoder, self).__init__(name=name)
self.params = params
self.word_embeddings = embedding_layer.EmbeddingSharedWeights(
tf.cast(params["vocab_size"], tf.int32), params["hidden_size"])
self.tag_embeddings = embedding_layer.EmbeddingSharedWeights(
tf.cast(params["parts_of_speech_size"], tf.int32),
params["hidden_size"])
self.image_layer = tf.keras.layers.Dense(params["hidden_size"],
use_bias=False)
self.merge_embeddings = tf.keras.layers.Dense(params["hidden_size"],
use_bias=False)
self.encoder = transformer.EncoderStack(params)
self.decoder = transformer.DecoderStack(params)
self.pointer_layer = tf.keras.layers.Dense(
1, activation=lambda x: tf.squeeze(x, -1))
self.un_merge_embeddings = tf.keras.layers.Dense(params["hidden_size"],
use_bias=False)
def __init__(self, params, name=None):
"""Initialize layers to build model.
Args:
params: hyperparameter object defining layer sizes, dropout values, etc.
name: name of the model.
"""
super(Reformer, self).__init__(name=name)
self.params = params
self.positional_encoding_strategy = pes = FLAGS.positional_encoding_strategy
self.positional_encoding_concat_dimension = int(
pes[7:]) if pes.startswith('concat-') else 0
self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["vocab_size"],
params["hidden_size"] - self.positional_encoding_concat_dimension)
self.encoder_stack = EncoderStack(params)
self.decoder_stack = DecoderStack(params)
def test_embedding_shared_weights(self):
vocab_size = 50
hidden_size = 64
length = 2
layer = embedding_layer.EmbeddingSharedWeights(vocab_size, hidden_size)
self.assertDictEqual(layer.get_config(), {
"vocab_size": 50,
"hidden_size": 64,
})
idx = tf.ones([1, length], dtype="int32")
y = layer(idx)
self.assertEqual(y.shape, (
1,
length,
hidden_size,
))
x = tf.ones([1, length, hidden_size])
output = layer(x, "linear")
self.assertEqual(output.shape, (
1,
length,
vocab_size,
))
