Beispiel #1
0
 def __init__(self, num_embeddings, embedding_dim, padding_idx, learned=True):
     super().__init__()
     if learned:
         self.embeddings = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx)
     else:
         self.embeddings = SinusoidalPositionalEmbedding(int(embedding_dim), padding_idx)
     self.padding_idx = padding_idx
Beispiel #2
0
def PositionalEmbedding(num_embeddings: int, embedding_dim: int,
                        padding_idx: int, left_pad: bool) -> nn.Embedding:
    m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1,
                                   embedding_dim, padding_idx, left_pad)
    nn.init.normal_(m.weight, mean=0, std=embedding_dim**-0.5)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m
Beispiel #3
0
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad, learned=False):
    if learned:
        m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx, left_pad)
        nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
        nn.init.constant_(m.weight[padding_idx], 0)
    else:
        m = SinusoidalPositionalEmbedding(embedding_dim, padding_idx, left_pad, num_embeddings + padding_idx + 1)
    return m
Beispiel #4
0
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, learned=False):
    if learned:
        m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx)
        nn.init.normal_(m.weight, mean=0, std=0.02)
        nn.init.constant_(m.weight[padding_idx], 0)
    else:
        m = SinusoidalPositionalEmbedding(
            embedding_dim,
            padding_idx,
            init_size=num_embeddings + padding_idx + 1,
        )
    return m
Beispiel #5
0
def PositionalEmbeddingCreator(
    num_embeddings, embedding_dim, padding_idx, left_pad, learned=False
):
    if learned:
        m = LearnedPositionalEmbedding(
            num_embeddings + padding_idx + 1, embedding_dim, padding_idx
        )
        nn.init.normal_(m.weight, mean=0, std=embedding_dim ** -0.5)
        nn.init.constant_(m.weight[padding_idx], 0)
    else:
        # sys.stderr.write(str(type(embedding_dim)) + ",")
        # sys.stderr.write(str(type(padding_idx)) + ",")
        # sys.stderr.write(str(type(left_pad)) + ",")
        # sys.stderr.write(str(type(num_embeddings)))
        # sys.stderr.write("\n")
        m = SinusoidalPositionalEmbedding(
            embedding_dim=embedding_dim,
            padding_idx=padding_idx,
            init_size=num_embeddings + padding_idx + 1,
        )
    return m
Beispiel #6
0
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx):
    m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx)
    nn.init.normal_(m.weight, 0, 0.1)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m
Beispiel #7
0
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx):
    m = LearnedPositionalEmbedding(num_embeddings + padding_idx + 1, embedding_dim, padding_idx)
    nn.init.normal_(m.weight, mean=0, std=0.02)
    nn.init.constant_(m.weight[padding_idx], 0)
    return m
Beispiel #8
0
def PositionalEmbedding(num_embeddings, embedding_dim, padding_idx, left_pad):
    m = LearnedPositionalEmbedding(num_embeddings, embedding_dim, padding_idx,
                                   left_pad)
    m.weight.data.normal_(0, 0.1)
    return m