def __init__(self, num_layers, pre_norm=True):
self.embeddings = common_spec.EmbeddingsSpec()
self.position_encodings = PositionEncoderSpec()
self.layer_norm = (common_spec.LayerNormSpec()
if pre_norm else model_spec.OPTIONAL)
self.projection = common_spec.LinearSpec()
self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]
def __init__(
self,
num_layers,
pre_norm=True,
num_source_embeddings=1,
layernorm_embedding=False,
):
self.embeddings = [
common_spec.EmbeddingsSpec() for _ in range(num_source_embeddings)
]
self.scale_embeddings = True
self.position_encodings = PositionEncoderSpec()
self.layer_norm = (common_spec.LayerNormSpec()
if pre_norm else model_spec.OPTIONAL)
self.layernorm_embedding = (common_spec.LayerNormSpec()
if layernorm_embedding else
model_spec.OPTIONAL)
self.layer = [TransformerEncoderLayerSpec() for _ in range(num_layers)]
def __init__(self, num_layers):
self.embeddings = common_spec.EmbeddingsSpec()
self.position_encodings = PositionEncoderSpec()
self.layer_norm = common_spec.LayerNormSpec()
self.projection = common_spec.LinearSpec()
self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]