def __init__(self, args, word_emb, ent_conf, spo_conf): print('mhs using only char2v+w2v mixed and word_emb is freeze ') super(ERENet, self).__init__() self.max_len = args.max_len self.word_emb = nn.Embedding.from_pretrained(torch.tensor(word_emb, dtype=torch.float32), freeze=True, padding_idx=0) self.char_emb = nn.Embedding(num_embeddings=args.char_vocab_size, embedding_dim=args.char_emb_size, padding_idx=0) self.word_convert_char = nn.Linear(args.word_emb_size, args.char_emb_size, bias=False) self.classes_num = len(spo_conf) self.first_sentence_encoder = SentenceEncoder(args, args.char_emb_size) # self.second_sentence_encoder = SentenceEncoder(args, args.hidden_size) # self.token_entity_emb = nn.Embedding(num_embeddings=2, embedding_dim=config.hidden_size, # padding_idx=0) self.encoder_layer = TransformerEncoderLayer(args.hidden_size * 2, nhead=3) self.transformer_encoder = TransformerEncoder(self.encoder_layer, num_layers=1) self.LayerNorm = ConditionalLayerNorm(args.hidden_size * 2, eps=1e-12) # self.subject_dense = nn.Linear(args.hidden_size * 2, 2) self.ent_emission = nn.Linear(args.hidden_size * 2, len(ent_conf)) self.ent_crf = CRF(len(ent_conf), batch_first=True) self.emission = nn.Linear(args.hidden_size * 2, len(spo_conf)) self.crf = CRF(len(spo_conf), batch_first=True) self.loss_fct = nn.BCEWithLogitsLoss(reduction='none')
def __init__(self, args, word_emb): super(ERENet, self).__init__() print('mhs with w2v') if args.activation.lower() == 'relu': self.activation = nn.ReLU() elif args.activation.lower() == 'tanh': self.activation = nn.Tanh() self.word_emb = nn.Embedding.from_pretrained(torch.tensor( word_emb, dtype=torch.float32), freeze=True, padding_idx=0) self.word_convert_char = nn.Linear(args.word_emb_size, args.char_emb_size, bias=False) self.char_emb = nn.Embedding(num_embeddings=args.char_vocab_size, embedding_dim=args.char_emb_size, padding_idx=0) self.rel_emb = nn.Embedding(num_embeddings=len(BAIDU_RELATION), embedding_dim=args.rel_emb_size) self.ent_emb = nn.Embedding(num_embeddings=len(BAIDU_ENTITY), embedding_dim=args.ent_emb_size) self.sentence_encoder = SentenceEncoder(args, args.char_emb_size) self.emission = nn.Linear(args.hidden_size * 2, len(BAIDU_ENTITY)) self.crf = CRF(len(BAIDU_ENTITY), batch_first=True) self.selection_u = nn.Linear(2 * args.hidden_size + args.ent_emb_size, args.rel_emb_size) self.selection_v = nn.Linear(2 * args.hidden_size + args.ent_emb_size, args.rel_emb_size) self.selection_uv = nn.Linear(2 * args.rel_emb_size, args.rel_emb_size)
def __init__(self, args, model_conf): super(NERNet, self).__init__() char_emb = model_conf['char_emb'] bichar_emb = model_conf['bichar_emb'] embed_size = args.char_emb_dim if char_emb is not None: # self.char_emb = nn.Embedding.from_pretrained(char_emb, freeze=False, padding_idx=0) self.char_emb = nn.Embedding(num_embeddings=char_emb.shape[0], embedding_dim=char_emb.shape[1], padding_idx=0, _weight=char_emb) self.char_emb.weight.requires_grad = True embed_size = char_emb.size()[1] else: vocab_size = len(model_conf['char_vocab']) self.char_emb = nn.Embedding(num_embeddings=vocab_size, embedding_dim=args.char_emb_dim, padding_idx=0) self.bichar_emb = None if bichar_emb is not None: # self.bichar_emb = nn.Embedding.from_pretrained(bichar_emb, freeze=False, padding_idx=0) self.bichar_emb = nn.Embedding(num_embeddings=bichar_emb.shape[0], embedding_dim=bichar_emb.shape[1], padding_idx=0, _weight=bichar_emb) self.bichar_emb.weight.requires_grad = True embed_size += bichar_emb.size()[1] self.drop = nn.Dropout(p=0.5) # self.sentence_encoder = SentenceEncoder(args, embed_size) self.sentence_encoder = nn.LSTM(embed_size, args.hidden_size, num_layers=1, batch_first=True, bidirectional=True) self.emission = nn.Linear(args.hidden_size * 2, len(model_conf['entity_type'])) self.crf = CRF(len(model_conf['entity_type']), batch_first=True)
def __init__(self, data): super(BertNER, self).__init__() self.gpu = data.HP_gpu self.use_bert = data.use_bert self.bertpath = data.bertpath char_feature_dim = 768 print('total char_feature_dim is {}'.format(char_feature_dim)) self.bert_encoder = BertModel.from_pretrained(self.bertpath) self.hidden2tag = nn.Linear(char_feature_dim, data.label_alphabet_size + 2) self.drop = nn.Dropout(p=data.HP_dropout) self.crf = CRF(data.label_alphabet_size, self.gpu) if self.gpu: self.bert_encoder = self.bert_encoder.cuda() self.hidden2tag = self.hidden2tag.cuda() self.crf = self.crf.cuda()
def __init__(self, data): super(GazLSTM, self).__init__() self.gpu = data.HP_gpu self.use_biword = data.use_bigram self.hidden_dim = data.HP_hidden_dim self.word_emb_dim = data.word_emb_dim self.biword_emb_dim = data.biword_emb_dim self.bilstm_flag = data.HP_bilstm self.lstm_layer = data.HP_lstm_layer self.num_layer = data.HP_num_layer self.model_type = data.model_type self.use_bert = data.use_bert self.device = data.device self.word_embedding = nn.Embedding(data.word_alphabet.size(), self.word_emb_dim, padding_idx=0) if data.pretrain_word_embedding is not None: self.word_embedding.weight.data.copy_( torch.from_numpy(data.pretrain_word_embedding)) if self.use_biword: self.biword_embedding = nn.Embedding(data.biword_alphabet.size(), self.biword_emb_dim, padding_idx=0) if data.pretrain_biword_embedding is not None: self.biword_embedding.weight.data.copy_( torch.from_numpy(data.pretrain_biword_embedding)) char_feature_dim = self.word_emb_dim if self.use_biword: char_feature_dim += self.biword_emb_dim if self.use_bert: char_feature_dim = char_feature_dim + 768 * 2 print('total char_feature_dim is {}'.format(char_feature_dim)) ## lstm model if self.model_type == 'lstm': lstm_hidden = self.hidden_dim if self.bilstm_flag: self.hidden_dim *= 2 self.NERmodel = NERmodel(model_type='lstm', input_dim=char_feature_dim, hidden_dim=lstm_hidden, num_layer=self.lstm_layer, biflag=self.bilstm_flag) self.hidden2tag = nn.Linear(self.hidden_dim, data.label_alphabet_size + 2) # ## cnn model # if self.model_type == 'cnn': # self.NERmodel = NERmodel(model_type='cnn', input_dim=char_feature_dim, hidden_dim=self.hidden_dim, # num_layer=self.num_layer, dropout=data.HP_dropout, gpu=self.gpu) # # ## attention model if self.model_type == 'transformer': self.NERmodel = NERmodel(model_type='transformer', input_dim=char_feature_dim, hidden_dim=self.hidden_dim, num_layer=self.num_layer, dropout=data.HP_dropout) self.hidden2tag = nn.Linear(480, data.label_alphabet_size + 2) self.drop = nn.Dropout(p=data.HP_dropout) self.crf = CRF(data.label_alphabet_size, self.gpu, self.device) if self.use_bert: self.bert_encoder_1 = BertModel.from_pretrained( 'transformer_cpt/bert/') self.bert_encoder_2 = BertModel.from_pretrained( 'transformer_cpt/chinese_roberta_wwm_ext_pytorch/') for p in self.bert_encoder_1.parameters(): p.requires_grad = False for p in self.bert_encoder_2.parameters(): p.requires_grad = False if self.gpu: self.word_embedding = self.word_embedding.cuda(self.device) if self.use_biword: self.biword_embedding = self.biword_embedding.cuda(self.device) self.NERmodel = self.NERmodel.cuda(self.device) self.hidden2tag = self.hidden2tag.cuda(self.device) self.crf = self.crf.cuda(self.device) if self.use_bert: self.bert_encoder_1 = self.bert_encoder_1.cuda(self.device) self.bert_encoder_2 = self.bert_encoder_2.cuda(self.device)
def __init__(self, data): super(GazLSTM, self).__init__() self.gpu = data.HP_gpu self.use_biword = data.use_bigram self.hidden_dim = data.HP_hidden_dim self.gaz_alphabet = data.gaz_alphabet self.gaz_emb_dim = data.gaz_emb_dim self.word_emb_dim = data.word_emb_dim self.biword_emb_dim = data.biword_emb_dim self.use_char = data.HP_use_char self.bilstm_flag = data.HP_bilstm self.lstm_layer = data.HP_lstm_layer self.use_count = data.HP_use_count self.num_layer = data.HP_num_layer self.model_type = data.model_type self.use_bert = data.use_bert scale = np.sqrt(3.0 / self.gaz_emb_dim) data.pretrain_gaz_embedding[0, :] = np.random.uniform( -scale, scale, [1, self.gaz_emb_dim]) if self.use_char: scale = np.sqrt(3.0 / self.word_emb_dim) data.pretrain_word_embedding[0, :] = np.random.uniform( -scale, scale, [1, self.word_emb_dim]) self.gaz_embedding = nn.Embedding(data.gaz_alphabet.size(), self.gaz_emb_dim) self.word_embedding = nn.Embedding(data.word_alphabet.size(), self.word_emb_dim) if self.use_biword: self.biword_embedding = nn.Embedding(data.biword_alphabet.size(), self.biword_emb_dim) if data.pretrain_gaz_embedding is not None: self.gaz_embedding.weight.data.copy_( torch.from_numpy(data.pretrain_gaz_embedding)) else: self.gaz_embedding.weight.data.copy_( torch.from_numpy( self.random_embedding(data.gaz_alphabet.size(), self.gaz_emb_dim))) if data.pretrain_word_embedding is not None: self.word_embedding.weight.data.copy_( torch.from_numpy(data.pretrain_word_embedding)) else: self.word_embedding.weight.data.copy_( torch.from_numpy( self.random_embedding(data.word_alphabet.size(), self.word_emb_dim))) if self.use_biword: if data.pretrain_biword_embedding is not None: self.biword_embedding.weight.data.copy_( torch.from_numpy(data.pretrain_biword_embedding)) else: self.biword_embedding.weight.data.copy_( torch.from_numpy( self.random_embedding(data.biword_alphabet.size(), self.word_emb_dim))) char_feature_dim = self.word_emb_dim + 4 * self.gaz_emb_dim if self.use_biword: char_feature_dim += self.biword_emb_dim if self.use_bert: char_feature_dim = char_feature_dim + 768 print('total char_feature_dim {}'.format(char_feature_dim)) ## lstm model if self.model_type == 'lstm': lstm_hidden = self.hidden_dim if self.bilstm_flag: self.hidden_dim *= 2 self.NERmodel = NERmodel(model_type='lstm', input_dim=char_feature_dim, hidden_dim=lstm_hidden, num_layer=self.lstm_layer, biflag=self.bilstm_flag) ## cnn model if self.model_type == 'cnn': self.NERmodel = NERmodel(model_type='cnn', input_dim=char_feature_dim, hidden_dim=self.hidden_dim, num_layer=self.num_layer, dropout=data.HP_dropout, gpu=self.gpu) ## attention model if self.model_type == 'transformer': self.NERmodel = NERmodel(model_type='transformer', input_dim=char_feature_dim, hidden_dim=self.hidden_dim, num_layer=self.num_layer, dropout=data.HP_dropout) self.drop = nn.Dropout(p=data.HP_dropout) self.hidden2tag = nn.Linear(self.hidden_dim, data.label_alphabet_size + 2) self.crf = CRF(data.label_alphabet_size, self.gpu) if self.use_bert: self.bert_encoder = BertModel.from_pretrained( 'transformer_cpt/bert/') for p in self.bert_encoder.parameters(): p.requires_grad = False if self.gpu: self.gaz_embedding = self.gaz_embedding.cuda() self.word_embedding = self.word_embedding.cuda() if self.use_biword: self.biword_embedding = self.biword_embedding.cuda() self.NERmodel = self.NERmodel.cuda() self.hidden2tag = self.hidden2tag.cuda() self.crf = self.crf.cuda() if self.use_bert: self.bert_encoder = self.bert_encoder.cuda()