def __init__(self, num_layers, pre_norm=True): self.embeddings = common_spec.EmbeddingsSpec() self.position_encodings = PositionEncoderSpec() self.layer_norm = (common_spec.LayerNormSpec() if pre_norm else model_spec.OPTIONAL) self.projection = common_spec.LinearSpec() self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]
def __init__( self, num_layers, pre_norm=True, num_source_embeddings=1, layernorm_embedding=False, ): self.embeddings = [ common_spec.EmbeddingsSpec() for _ in range(num_source_embeddings) ] self.scale_embeddings = True self.position_encodings = PositionEncoderSpec() self.layer_norm = (common_spec.LayerNormSpec() if pre_norm else model_spec.OPTIONAL) self.layernorm_embedding = (common_spec.LayerNormSpec() if layernorm_embedding else model_spec.OPTIONAL) self.layer = [TransformerEncoderLayerSpec() for _ in range(num_layers)]
def __init__(self, num_layers): self.embeddings = common_spec.EmbeddingsSpec() self.position_encodings = PositionEncoderSpec() self.layer_norm = common_spec.LayerNormSpec() self.projection = common_spec.LinearSpec() self.layer = [TransformerDecoderLayerSpec() for _ in range(num_layers)]