def __init__(self, num_layers, pre_norm=True, layernorm_embedding=False):
self.embeddings = common_spec.EmbeddingsSpec()
self.scale_embeddings = True
self.position_encodings = PositionEncoderSpec()
self.layer_norm = (common_spec.LayerNormSpec()
if pre_norm else model_spec.OPTIONAL)
self.layernorm_embedding = (common_spec.LayerNormSpec()
if layernorm_embedding else
model_spec.OPTIONAL)
self.projection = common_spec.LinearSpec()
self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]
def __init__(self, self_attention=False):
self.layer_norm = common_spec.LayerNormSpec()
if self_attention:
num_projections = 2
else:
num_projections = 3
self.linear = [common_spec.LinearSpec() for _ in range(num_projections)]
def __init__(self, self_attention=False):
self.layer_norm = common_spec.LayerNormSpec()
if self_attention:
num_projections = 2
else:
num_projections = 3
self.linear = [
common_spec.LinearSpec() for _ in range(num_projections)
]
self.relative_position_keys = model_spec.OPTIONAL
self.relative_position_values = model_spec.OPTIONAL
def __init__(self):
self.layer_norm = common_spec.LayerNormSpec()
self.linear_0 = common_spec.LinearSpec()
self.linear_1 = common_spec.LinearSpec()
def __init__(self, num_layers):
self.embeddings = common_spec.EmbeddingsSpec()
self.position_encodings = PositionEncoderSpec()
self.layer_norm = common_spec.LayerNormSpec()
self.projection = common_spec.LinearSpec()
self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]
def __init__(self, num_layers, pre_norm=True):
self.embeddings = common_spec.EmbeddingsSpec()
self.position_encodings = PositionEncoderSpec()
self.layer_norm = (common_spec.LayerNormSpec()
if pre_norm else model_spec.OPTIONAL)
self.layer = [TransformerEncoderLayerSpec() for _ in range(num_layers)]