def __init__(self, args, word_emb, ent_conf, spo_conf):
print('mhs using only char2v+w2v mixed and word_emb is freeze ')
super(ERENet, self).__init__()
self.max_len = args.max_len
self.word_emb = nn.Embedding.from_pretrained(torch.tensor(word_emb, dtype=torch.float32), freeze=True,
padding_idx=0)
self.char_emb = nn.Embedding(num_embeddings=args.char_vocab_size, embedding_dim=args.char_emb_size,
padding_idx=0)
self.word_convert_char = nn.Linear(args.word_emb_size, args.char_emb_size, bias=False)
self.classes_num = len(spo_conf)
self.first_sentence_encoder = SentenceEncoder(args, args.char_emb_size)
# self.second_sentence_encoder = SentenceEncoder(args, args.hidden_size)
# self.token_entity_emb = nn.Embedding(num_embeddings=2, embedding_dim=config.hidden_size,
# padding_idx=0)
self.encoder_layer = TransformerEncoderLayer(args.hidden_size * 2, nhead=3)
self.transformer_encoder = TransformerEncoder(self.encoder_layer, num_layers=1)
self.LayerNorm = ConditionalLayerNorm(args.hidden_size * 2, eps=1e-12)
# self.subject_dense = nn.Linear(args.hidden_size * 2, 2)
self.ent_emission = nn.Linear(args.hidden_size * 2, len(ent_conf))
self.ent_crf = CRF(len(ent_conf), batch_first=True)
self.emission = nn.Linear(args.hidden_size * 2, len(spo_conf))
self.crf = CRF(len(spo_conf), batch_first=True)
self.loss_fct = nn.BCEWithLogitsLoss(reduction='none')
def __init__(self, args, word_emb):
super(ERENet, self).__init__()
print('mhs with w2v')
if args.activation.lower() == 'relu':
self.activation = nn.ReLU()
elif args.activation.lower() == 'tanh':
self.activation = nn.Tanh()
self.word_emb = nn.Embedding.from_pretrained(torch.tensor(
word_emb, dtype=torch.float32),
freeze=True,
padding_idx=0)
self.word_convert_char = nn.Linear(args.word_emb_size,
args.char_emb_size,
bias=False)
self.char_emb = nn.Embedding(num_embeddings=args.char_vocab_size,
embedding_dim=args.char_emb_size,
padding_idx=0)
self.rel_emb = nn.Embedding(num_embeddings=len(BAIDU_RELATION),
embedding_dim=args.rel_emb_size)
self.ent_emb = nn.Embedding(num_embeddings=len(BAIDU_ENTITY),
embedding_dim=args.ent_emb_size)
self.sentence_encoder = SentenceEncoder(args, args.char_emb_size)
self.emission = nn.Linear(args.hidden_size * 2, len(BAIDU_ENTITY))
self.crf = CRF(len(BAIDU_ENTITY), batch_first=True)
self.selection_u = nn.Linear(2 * args.hidden_size + args.ent_emb_size,
args.rel_emb_size)
self.selection_v = nn.Linear(2 * args.hidden_size + args.ent_emb_size,
args.rel_emb_size)
self.selection_uv = nn.Linear(2 * args.rel_emb_size, args.rel_emb_size)
def __init__(self, args, model_conf):
super(NERNet, self).__init__()
char_emb = model_conf['char_emb']
bichar_emb = model_conf['bichar_emb']
embed_size = args.char_emb_dim
if char_emb is not None:
# self.char_emb = nn.Embedding.from_pretrained(char_emb, freeze=False, padding_idx=0)
self.char_emb = nn.Embedding(num_embeddings=char_emb.shape[0],
embedding_dim=char_emb.shape[1],
padding_idx=0,
_weight=char_emb)
self.char_emb.weight.requires_grad = True
embed_size = char_emb.size()[1]
else:
vocab_size = len(model_conf['char_vocab'])
self.char_emb = nn.Embedding(num_embeddings=vocab_size,
embedding_dim=args.char_emb_dim,
padding_idx=0)
self.bichar_emb = None
if bichar_emb is not None:
# self.bichar_emb = nn.Embedding.from_pretrained(bichar_emb, freeze=False, padding_idx=0)
self.bichar_emb = nn.Embedding(num_embeddings=bichar_emb.shape[0],
embedding_dim=bichar_emb.shape[1],
padding_idx=0,
_weight=bichar_emb)
self.bichar_emb.weight.requires_grad = True
embed_size += bichar_emb.size()[1]
self.drop = nn.Dropout(p=0.5)
# self.sentence_encoder = SentenceEncoder(args, embed_size)
self.sentence_encoder = nn.LSTM(embed_size,
args.hidden_size,
num_layers=1,
batch_first=True,
bidirectional=True)
self.emission = nn.Linear(args.hidden_size * 2,
len(model_conf['entity_type']))
self.crf = CRF(len(model_conf['entity_type']), batch_first=True)
def __init__(self, data):
super(BertNER, self).__init__()
self.gpu = data.HP_gpu
self.use_bert = data.use_bert
self.bertpath = data.bertpath
char_feature_dim = 768
print('total char_feature_dim is {}'.format(char_feature_dim))
self.bert_encoder = BertModel.from_pretrained(self.bertpath)
self.hidden2tag = nn.Linear(char_feature_dim,
data.label_alphabet_size + 2)
self.drop = nn.Dropout(p=data.HP_dropout)
self.crf = CRF(data.label_alphabet_size, self.gpu)
if self.gpu:
self.bert_encoder = self.bert_encoder.cuda()
self.hidden2tag = self.hidden2tag.cuda()
self.crf = self.crf.cuda()
def __init__(self, data):
super(GazLSTM, self).__init__()
self.gpu = data.HP_gpu
self.use_biword = data.use_bigram
self.hidden_dim = data.HP_hidden_dim
self.word_emb_dim = data.word_emb_dim
self.biword_emb_dim = data.biword_emb_dim
self.bilstm_flag = data.HP_bilstm
self.lstm_layer = data.HP_lstm_layer
self.num_layer = data.HP_num_layer
self.model_type = data.model_type
self.use_bert = data.use_bert
self.device = data.device
self.word_embedding = nn.Embedding(data.word_alphabet.size(),
self.word_emb_dim,
padding_idx=0)
if data.pretrain_word_embedding is not None:
self.word_embedding.weight.data.copy_(
torch.from_numpy(data.pretrain_word_embedding))
if self.use_biword:
self.biword_embedding = nn.Embedding(data.biword_alphabet.size(),
self.biword_emb_dim,
padding_idx=0)
if data.pretrain_biword_embedding is not None:
self.biword_embedding.weight.data.copy_(
torch.from_numpy(data.pretrain_biword_embedding))
char_feature_dim = self.word_emb_dim
if self.use_biword:
char_feature_dim += self.biword_emb_dim
if self.use_bert:
char_feature_dim = char_feature_dim + 768 * 2
print('total char_feature_dim is {}'.format(char_feature_dim))
## lstm model
if self.model_type == 'lstm':
lstm_hidden = self.hidden_dim
if self.bilstm_flag:
self.hidden_dim *= 2
self.NERmodel = NERmodel(model_type='lstm',
input_dim=char_feature_dim,
hidden_dim=lstm_hidden,
num_layer=self.lstm_layer,
biflag=self.bilstm_flag)
self.hidden2tag = nn.Linear(self.hidden_dim,
data.label_alphabet_size + 2)
# ## cnn model
# if self.model_type == 'cnn':
# self.NERmodel = NERmodel(model_type='cnn', input_dim=char_feature_dim, hidden_dim=self.hidden_dim,
# num_layer=self.num_layer, dropout=data.HP_dropout, gpu=self.gpu)
#
# ## attention model
if self.model_type == 'transformer':
self.NERmodel = NERmodel(model_type='transformer',
input_dim=char_feature_dim,
hidden_dim=self.hidden_dim,
num_layer=self.num_layer,
dropout=data.HP_dropout)
self.hidden2tag = nn.Linear(480, data.label_alphabet_size + 2)
self.drop = nn.Dropout(p=data.HP_dropout)
self.crf = CRF(data.label_alphabet_size, self.gpu, self.device)
if self.use_bert:
self.bert_encoder_1 = BertModel.from_pretrained(
'transformer_cpt/bert/')
self.bert_encoder_2 = BertModel.from_pretrained(
'transformer_cpt/chinese_roberta_wwm_ext_pytorch/')
for p in self.bert_encoder_1.parameters():
p.requires_grad = False
for p in self.bert_encoder_2.parameters():
p.requires_grad = False
if self.gpu:
self.word_embedding = self.word_embedding.cuda(self.device)
if self.use_biword:
self.biword_embedding = self.biword_embedding.cuda(self.device)
self.NERmodel = self.NERmodel.cuda(self.device)
self.hidden2tag = self.hidden2tag.cuda(self.device)
self.crf = self.crf.cuda(self.device)
if self.use_bert:
self.bert_encoder_1 = self.bert_encoder_1.cuda(self.device)
self.bert_encoder_2 = self.bert_encoder_2.cuda(self.device)
def __init__(self, data):
super(GazLSTM, self).__init__()
self.gpu = data.HP_gpu
self.use_biword = data.use_bigram
self.hidden_dim = data.HP_hidden_dim
self.gaz_alphabet = data.gaz_alphabet
self.gaz_emb_dim = data.gaz_emb_dim
self.word_emb_dim = data.word_emb_dim
self.biword_emb_dim = data.biword_emb_dim
self.use_char = data.HP_use_char
self.bilstm_flag = data.HP_bilstm
self.lstm_layer = data.HP_lstm_layer
self.use_count = data.HP_use_count
self.num_layer = data.HP_num_layer
self.model_type = data.model_type
self.use_bert = data.use_bert
scale = np.sqrt(3.0 / self.gaz_emb_dim)
data.pretrain_gaz_embedding[0, :] = np.random.uniform(
-scale, scale, [1, self.gaz_emb_dim])
if self.use_char:
scale = np.sqrt(3.0 / self.word_emb_dim)
data.pretrain_word_embedding[0, :] = np.random.uniform(
-scale, scale, [1, self.word_emb_dim])
self.gaz_embedding = nn.Embedding(data.gaz_alphabet.size(),
self.gaz_emb_dim)
self.word_embedding = nn.Embedding(data.word_alphabet.size(),
self.word_emb_dim)
if self.use_biword:
self.biword_embedding = nn.Embedding(data.biword_alphabet.size(),
self.biword_emb_dim)
if data.pretrain_gaz_embedding is not None:
self.gaz_embedding.weight.data.copy_(
torch.from_numpy(data.pretrain_gaz_embedding))
else:
self.gaz_embedding.weight.data.copy_(
torch.from_numpy(
self.random_embedding(data.gaz_alphabet.size(),
self.gaz_emb_dim)))
if data.pretrain_word_embedding is not None:
self.word_embedding.weight.data.copy_(
torch.from_numpy(data.pretrain_word_embedding))
else:
self.word_embedding.weight.data.copy_(
torch.from_numpy(
self.random_embedding(data.word_alphabet.size(),
self.word_emb_dim)))
if self.use_biword:
if data.pretrain_biword_embedding is not None:
self.biword_embedding.weight.data.copy_(
torch.from_numpy(data.pretrain_biword_embedding))
else:
self.biword_embedding.weight.data.copy_(
torch.from_numpy(
self.random_embedding(data.biword_alphabet.size(),
self.word_emb_dim)))
char_feature_dim = self.word_emb_dim + 4 * self.gaz_emb_dim
if self.use_biword:
char_feature_dim += self.biword_emb_dim
if self.use_bert:
char_feature_dim = char_feature_dim + 768
print('total char_feature_dim {}'.format(char_feature_dim))
## lstm model
if self.model_type == 'lstm':
lstm_hidden = self.hidden_dim
if self.bilstm_flag:
self.hidden_dim *= 2
self.NERmodel = NERmodel(model_type='lstm',
input_dim=char_feature_dim,
hidden_dim=lstm_hidden,
num_layer=self.lstm_layer,
biflag=self.bilstm_flag)
## cnn model
if self.model_type == 'cnn':
self.NERmodel = NERmodel(model_type='cnn',
input_dim=char_feature_dim,
hidden_dim=self.hidden_dim,
num_layer=self.num_layer,
dropout=data.HP_dropout,
gpu=self.gpu)
## attention model
if self.model_type == 'transformer':
self.NERmodel = NERmodel(model_type='transformer',
input_dim=char_feature_dim,
hidden_dim=self.hidden_dim,
num_layer=self.num_layer,
dropout=data.HP_dropout)
self.drop = nn.Dropout(p=data.HP_dropout)
self.hidden2tag = nn.Linear(self.hidden_dim,
data.label_alphabet_size + 2)
self.crf = CRF(data.label_alphabet_size, self.gpu)
if self.use_bert:
self.bert_encoder = BertModel.from_pretrained(
'transformer_cpt/bert/')
for p in self.bert_encoder.parameters():
p.requires_grad = False
if self.gpu:
self.gaz_embedding = self.gaz_embedding.cuda()
self.word_embedding = self.word_embedding.cuda()
if self.use_biword:
self.biword_embedding = self.biword_embedding.cuda()
self.NERmodel = self.NERmodel.cuda()
self.hidden2tag = self.hidden2tag.cuda()
self.crf = self.crf.cuda()
if self.use_bert:
self.bert_encoder = self.bert_encoder.cuda()