def __init__(self, num_embeddings, embedding_dim, padding_idx, learned=True): super().__init__() if learned: self.embeddings = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx) else: self.embeddings = SinusoidalPositionalEmbedding(int(embedding_dim), padding_idx) self.padding_idx = padding_idx
def PositionalEmbedding(num_embeddings: int, embedding_dim: int, padding_idx: int, left_pad: bool) -> nn.Embedding: m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx, left_pad) nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5) nn.init.constant_(m.weight[padding_idx], 0) return m
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad, learned=False): if learned: m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx, left_pad) nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5) nn.init.constant_(m.weight[padding_idx], 0) else: m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad, num_embeddings + padding_idx + 1) return m
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, learned=False): if learned: m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx) nn.init.normal_(m.weight, mean=0, std=0.02) nn.init.constant_(m.weight[padding_idx], 0) else: m = SinusoidalPositionalEmbedding( embedding_dim, padding_idx, init_size=num_embeddings + padding_idx + 1, ) return m
def PositionalEmbeddingCreator( num_embeddings, embedding_dim, padding_idx, left_pad, learned=False ): if learned: m = LearnedPositionalEmbedding( num_embeddings + padding_idx + 1, embedding_dim, padding_idx ) nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5) nn.init.constant_(m.weight[padding_idx], 0) else: # sys.stderr.write(str(type(embedding_dim)) + ",") # sys.stderr.write(str(type(padding_idx)) + ",") # sys.stderr.write(str(type(left_pad)) + ",") # sys.stderr.write(str(type(num_embeddings))) # sys.stderr.write("\n") m = SinusoidalPositionalEmbedding( embedding_dim=embedding_dim, padding_idx=padding_idx, init_size=num_embeddings + padding_idx + 1, ) return m
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx): m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx) nn.init.normal_(m.weight, 0, 0.1) nn.init.constant_(m.weight[padding_idx], 0) return m
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx): m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx) nn.init.normal_(m.weight, mean=0, std=0.02) nn.init.constant_(m.weight[padding_idx], 0) return m
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad): m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad) m.weight.data.normal_(0, 0.1) return m