def __init__(
self,
num_embeddings: int,
embedding_dim: int = 300,
embeddings_weight: Optional[torch.Tensor] = None,
init_range: Optional[List[int]] = None,
init_std: Optional[float] = None,
unk_token_idx: int = 0,
mlp_layer_dims: List[int] = (),
padding_idx: Optional[int] = None,
vocab: Optional[List[str]] = None,
) -> None:
output_embedding_dim = mlp_layer_dims[-1] if mlp_layer_dims else embedding_dim
EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim)
# Create word embedding
self.word_embedding = nn.Embedding(
num_embeddings,
embedding_dim,
_weight=embeddings_weight,
padding_idx=padding_idx,
)
if embeddings_weight is None:
if init_range:
self.word_embedding.weight.data.uniform_(init_range[0], init_range[1])
if init_std:
self.word_embedding.weight.data.normal_(mean=0.0, std=init_std)
# Initialize unk embedding with zeros
# to guard the model against randomized decisions based on unknown words
self.word_embedding.weight.data[unk_token_idx].fill_(0.0)
# Create MLP layers
if mlp_layer_dims is None:
mlp_layer_dims = []
self.mlp = nn.Sequential(
*(
nn.Sequential(nn.Linear(m, n), nn.ReLU())
for m, n in zip([embedding_dim] + list(mlp_layer_dims), mlp_layer_dims)
)
)
self.vocab = vocab
self.padding_idx = padding_idx
log_class_usage(__class__)
def __init__(
self,
embedding_dim: int = 300,
embeddings_weight: Optional[torch.Tensor] = None,
init_range: Optional[List[int]] = None,
init_std: Optional[float] = None,
mlp_layer_dims: List[int] = (),
) -> None:
output_embedding_dim = mlp_layer_dims[
-1] if mlp_layer_dims else embedding_dim
EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim)
# Create MLP layers
if mlp_layer_dims is None:
mlp_layer_dims = []
self.mlp = nn.Sequential(
*(nn.Sequential(nn.Linear(m, n), nn.ReLU())
for m, n in zip([embedding_dim] +
list(mlp_layer_dims), mlp_layer_dims)))
log_class_usage(__class__)
def __init__(
self,
lstm_config: BiLSTM.Config,
num_embeddings: int,
word_embed_dim: int = 300,
embeddings_weight: Optional[torch.Tensor] = None,
init_range: Optional[List[int]] = None,
init_std: Optional[float] = None,
unk_token_idx: int = 0,
padding_idx: Optional[int] = None,
vocab: Optional[List[str]] = None,
) -> None:
lstm = BiLSTM(lstm_config, word_embed_dim)
output_embedding_dim = lstm.representation_dim
EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim)
self.lstm = lstm
self.num_lstm_directions = 2 if lstm_config.bidirectional else 1
# Create word embedding
self.word_embedding = nn.Embedding(
num_embeddings,
word_embed_dim,
_weight=embeddings_weight,
padding_idx=padding_idx,
)
if embeddings_weight is None:
if init_range:
self.word_embedding.weight.data.uniform_(
init_range[0], init_range[1])
if init_std:
self.word_embedding.weight.data.normal_(mean=0.0, std=init_std)
# Initialize unk embedding with zeros
# to guard the model against randomized decisions based on unknown words
self.word_embedding.weight.data[unk_token_idx].fill_(0.0)
self.vocab = vocab
self.padding_idx = padding_idx