def __init__(self, embeddings=None, embeddings_path=None, labels_count=1, mlp_dim=512, mlp_layers_count=1, concat='simple', prob='auto'): super().__init__() self.labels_count = labels_count if embeddings: # id to vector mapping is provided as matrix self.embed = nn.Embedding.from_pretrained(embeddings) elif embeddings_path: # load id to vector mapping from file # Load from txt file (in word2vec format) w2v_model = gensim.models.KeyedVectors.load_word2vec_format(embeddings_path) # Convert to PyTorch tensor weights = torch.FloatTensor(w2v_model.vectors) self.embed = nn.Embedding.from_pretrained(weights) else: raise ValueError('Either `embeddings` or `embeddings_path` must be set!') self.embedding_dim = self.embed.embedding_dim # Concat mode self.concat = concat self.concat_func, self.concat_dim = get_concat(concat, self.embedding_dim) self.mlp = get_mlp(input_dim=self.concat_dim, output_dim=self.labels_count, hidden_dim=mlp_dim, hidden_layers_count=mlp_layers_count, activation_cls=nn.ReLU) self.prob = self.get_classification_probability_layer(prob)
def __init__(self, bert_model_path, hidden_dim=768, dropout=0.1, mlp_dim=512, mlp_layers_count=1, labels_count=1, bert_cls=None, concat='simple', prob='auto'): super().__init__() self.labels_count = labels_count self.embedding_dim = hidden_dim # size of bert representations self.bert_cls = bert_cls if bert_cls else BertModel self.bert = self.bert_cls.from_pretrained(bert_model_path, output_hidden_states=False, output_attentions=False) self.dropout = nn.Dropout(dropout) # distilbert self.distil_pooler = nn.Linear(self.embedding_dim, self.embedding_dim) # Concat mode self.concat = concat self.concat_func, self.concat_dim = get_concat(concat, self.embedding_dim) self.mlp = get_mlp(input_dim=self.concat_dim, output_dim=self.labels_count, hidden_dim=mlp_dim, hidden_layers_count=mlp_layers_count, activation_cls=nn.ReLU) # TODO fill linear layers # nn.init.xavier_normal_(self.classifier.weight) # Fills the input Tensor with values according to the method described in “Understanding the difficulty of training deep feedforward neural networks” - Glorot, X. & Bengio, Y. (2010), using a normal distribution. # kaiming_normal_ # Fills the input Tensor with values according to the method described in “Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification” - He, K. et al. (2015), using a normal distribution. self.prob = self.get_classification_probability_layer(prob)
def __init__(self, bert_model_path, hidden_dim=768, dropout=0.1, mlp_dim=512, mlp_layers_count=1, labels_count=1, bert_cls=None, concat='simple', prob='auto'): super().__init__() self.labels_count = labels_count self.embedding_dim = hidden_dim # size of bert representations self.bert_cls = bert_cls if bert_cls else BertModel self.bert = self.bert_cls.from_pretrained(bert_model_path, output_hidden_states=False) self.dropout = nn.Dropout(dropout) self.lstm = LSTM(input_size=hidden_dim, hidden_size=hidden_dim, batch_first=True) # Concat mode self.concat = concat self.concat_func, self.concat_dim = get_concat(concat, self.embedding_dim) self.mlp = get_mlp(input_dim=self.concat_dim, output_dim=self.labels_count, hidden_dim=mlp_dim, hidden_layers_count=mlp_layers_count, activation_cls=nn.ReLU) self.prob = self.get_classification_probability_layer(prob)