def __init__( self, num_embeddings: int, embedding_dim: int = 300, embeddings_weight: Optional[torch.Tensor] = None, init_range: Optional[List[int]] = None, init_std: Optional[float] = None, unk_token_idx: int = 0, mlp_layer_dims: List[int] = (), padding_idx: Optional[int] = None, vocab: Optional[List[str]] = None, ) -> None: output_embedding_dim = mlp_layer_dims[-1] if mlp_layer_dims else embedding_dim EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim) # Create word embedding self.word_embedding = nn.Embedding( num_embeddings, embedding_dim, _weight=embeddings_weight, padding_idx=padding_idx, ) if embeddings_weight is None: if init_range: self.word_embedding.weight.data.uniform_(init_range[0], init_range[1]) if init_std: self.word_embedding.weight.data.normal_(mean=0.0, std=init_std) # Initialize unk embedding with zeros # to guard the model against randomized decisions based on unknown words self.word_embedding.weight.data[unk_token_idx].fill_(0.0) # Create MLP layers if mlp_layer_dims is None: mlp_layer_dims = [] self.mlp = nn.Sequential( *( nn.Sequential(nn.Linear(m, n), nn.ReLU()) for m, n in zip([embedding_dim] + list(mlp_layer_dims), mlp_layer_dims) ) ) self.vocab = vocab self.padding_idx = padding_idx log_class_usage(__class__)
def __init__( self, embedding_dim: int = 300, embeddings_weight: Optional[torch.Tensor] = None, init_range: Optional[List[int]] = None, init_std: Optional[float] = None, mlp_layer_dims: List[int] = (), ) -> None: output_embedding_dim = mlp_layer_dims[ -1] if mlp_layer_dims else embedding_dim EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim) # Create MLP layers if mlp_layer_dims is None: mlp_layer_dims = [] self.mlp = nn.Sequential( *(nn.Sequential(nn.Linear(m, n), nn.ReLU()) for m, n in zip([embedding_dim] + list(mlp_layer_dims), mlp_layer_dims))) log_class_usage(__class__)
def __init__( self, lstm_config: BiLSTM.Config, num_embeddings: int, word_embed_dim: int = 300, embeddings_weight: Optional[torch.Tensor] = None, init_range: Optional[List[int]] = None, init_std: Optional[float] = None, unk_token_idx: int = 0, padding_idx: Optional[int] = None, vocab: Optional[List[str]] = None, ) -> None: lstm = BiLSTM(lstm_config, word_embed_dim) output_embedding_dim = lstm.representation_dim EmbeddingBase.__init__(self, embedding_dim=output_embedding_dim) self.lstm = lstm self.num_lstm_directions = 2 if lstm_config.bidirectional else 1 # Create word embedding self.word_embedding = nn.Embedding( num_embeddings, word_embed_dim, _weight=embeddings_weight, padding_idx=padding_idx, ) if embeddings_weight is None: if init_range: self.word_embedding.weight.data.uniform_( init_range[0], init_range[1]) if init_std: self.word_embedding.weight.data.normal_(mean=0.0, std=init_std) # Initialize unk embedding with zeros # to guard the model against randomized decisions based on unknown words self.word_embedding.weight.data[unk_token_idx].fill_(0.0) self.vocab = vocab self.padding_idx = padding_idx