def __init__(self, params, train):
"""Initialize layers to build Transformer model.
Args:
params: hyperparameter object defining layer sizes, dropout values, etc.
train: boolean indicating whether the model is in training mode. Used to
determine if dropout layers should be added.
"""
self.train = train
self.params = params
self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["vocab_size"],
params["hidden_size"],
method="matmul" if params["tpu"] else "gather")
self.segment_embedding_layer = embedding_layer.SegmentEmbedding(
vocab_size=16,
hidden_size=params["hidden_size"],
method="matmul" if params["tpu"] else "gather")
self.position_embedding_layer = embedding_layer.PositionEmbedding(
vocab_size=512,
hidden_size=params["hidden_size"],
method="matmul" if params["tpu"] else "gather")
self.encoder_stack = EncoderStack(params, train)
self.decoder_stack = DecoderStack(params, train)
self.distribute_layer = DistributeLayer(params, train)
def __init__(self, params, train):
"""Initialize layers to build Transformer model.
Args:
params: hyperparameter object defining layer sizes, dropout values, etc.
train: boolean indicating whether the model is in training mode. Used to
determine if dropout layers should be added.
"""
self.train = train
self.params = params
self.inputs_embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["inputs_vocab_size"], params["hidden_size"], "inputs","gather")
self.targets_embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["targets_vocab_size"], params["hidden_size"], "targets","gather")
self.encoder_stack = EncoderStack(params, train)
self.decoder_stack = DecoderStack(params, train)
def __init__(self, params, train):
"""Initialize layers to build Transformer model.
Args:
params: hyperparameter object defining layer sizes, dropout values, etc.
train: boolean indicating whether the model is in training mode. Used to
determine if dropout layers should be added.
"""
self.train = train
self.params = params
with tf.variable_scope("input_norm"):
self.input_normalization = LayerNormalization(params["hidden_size"])
self.embedding_softmax_layer = embedding_layer.EmbeddingSharedWeights(
params["vocab_size"], params["hidden_size"],
method="matmul" if params["tpu"] else "gather")
self.encoder_stack = EncoderStack(params, train)
self.decoder_stack = DecoderStack(params, train)
def get_embeddings(self, tensor):
embedding_softmax_layer = tf_embedding.EmbeddingSharedWeights(
self.num_chars, self.params["hidden_size"], method="gather")
return embedding_softmax_layer(tensor)