def __init__(self, word_seq_indexer, tag_seq_indexer, class_num, batch_size=1, rnn_hidden_dim=100,
freeze_word_embeddings=False, dropout_ratio=0.5, rnn_type='GRU', gpu=-1, useElmo = False, weight_file = '', options_file = ''):
super(TaggerBiRNNCRF, self).__init__(word_seq_indexer, tag_seq_indexer, gpu, batch_size, useElmo, weight_file, options_file)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
print ("init targer BiRNNcfr")
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError('Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim, out_features=class_num + 2)
self.crf_layer = LayerCRF(gpu, states_num=class_num + 2, pad_idx=tag_seq_indexer.pad_idx, sos_idx=class_num + 1,
tag_seq_indexer=tag_seq_indexer)
if gpu >= 0:
self.cuda(device=self.gpu)
def __init__(self, word_seq_indexer, tag_seq_indexer, class_num, batch_size=1, rnn_hidden_dim=100,
freeze_word_embeddings=False, dropout_ratio=0.5, rnn_type='GRU', gpu=-1,
freeze_char_embeddings = False, char_embeddings_dim=25, word_len=20, char_cnn_filter_num=30,
char_window_size=3, emb_type='word'):
super(TaggerBiRNNCNNCRF, self).__init__(word_seq_indexer, tag_seq_indexer, gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
#self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.word_embeddings_layer = EmbeddingsFactory.create(emb_type, word_seq_indexer, gpu, freeze_word_embeddings)
self.char_embeddings_layer = LayerCharEmbeddings(gpu, char_embeddings_dim, freeze_char_embeddings,
word_len, word_seq_indexer.get_unique_characters_list())
self.char_cnn_layer = LayerCharCNN(gpu, char_embeddings_dim, char_cnn_filter_num, char_window_size,
word_len)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError('Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim, out_features=class_num + 2)
self.crf_layer = LayerCRF(gpu, states_num=class_num + 2, pad_idx=tag_seq_indexer.pad_idx, sos_idx=class_num + 1,
tag_seq_indexer=tag_seq_indexer)
self.softmax = nn.Softmax(dim=2)
if gpu >= 0:
self.cuda(device=self.gpu)
def __init__(self,
args,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1,
freeze_char_embeddings=False,
char_embeddings_dim=25,
word_len=20,
char_cnn_filter_num=30,
char_window_size=3):
super(TaggerBiRNNCNNCRF,
self).__init__(word_seq_indexer, tag_seq_indexer, gpu,
batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(
args, word_seq_indexer, gpu, freeze_word_embeddings)
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.args = args
self.if_bert = args.if_bert
self.if_flair = args.if_flair
self.dropout = torch.nn.Dropout(p=dropout_ratio)
# self.bert_embeddings_dim = self.args.bert_embeddings_dim
emb_models_dim = []
print('load embedding...')
if args.if_bert:
# print('PYTORCH_PRETRAINED_BERT_CACHE',PYTORCH_PRETRAINED_BERT_CACHE)
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
# cache_dir =os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),'distributed_{}'.format(args.local_rank))
print('cache_dir', cache_dir)
# cache_dir = '/home/jlfu/.pytorch_pretrained_bert/distributed_-1'
#cache_dir='emb/bert_model_cache/bert_cache.hdf5',
self.bert_embeddings_layer = LayerBertEmbeddings.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=class_num)
# self.bert_embeddings_layer = LayerBertEmbeddings(gpu, freeze_bert_embeddings=True)
reduce_dim = False
if reduce_dim:
self.W_bert = nn.Linear(args.bert_output_dim, 256)
emb_models_dim.append(256)
else:
emb_models_dim.append(args.bert_output_dim)
if args.if_flair:
self.flair_embeddings_layer = LayerFlairEmbeddings(gpu)
reduce_dim = True
if reduce_dim == True:
self.W_flair = nn.Linear(
self.flair_embeddings_layer.output_dim, 256).cuda()
emb_models_dim.append(256)
else:
emb_models_dim.append(self.flair_embeddings_layer.output_dim)
if args.if_elmo:
self.elmo_embeddings_layer = LayerElmoEmbeddings(
args, gpu, args.options_file, args.weight_file,
freeze_char_embeddings, word_len)
elmo_reduce_dim = False
if elmo_reduce_dim:
# self.W_elmo = nn.Linear(self.elmo_embeddings_layer.output_dim,256)
# emb_models_dim.append(256)
self.W_elmo = nn.Linear(self.elmo_embeddings_layer.output_dim,
self.word_embeddings_layer.output_dim)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
else:
emb_models_dim.append(self.elmo_embeddings_layer.output_dim)
# if args.if_glove:
# self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
# emb_models_dim.append(self.word_embeddings_layer.output_dim)
self.if_word = False
if args.if_wordEmbRand == True and args.if_glove == False:
self.word_embeddings_layer = LayerWordEmbeddings_Rand(
word_seq_indexer, gpu, freeze_word_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
print('load random word emb ')
self.if_word = True
elif args.if_wordEmbRand == False and args.if_glove == True:
self.word_embeddings_layer = LayerWordEmbeddings(
args, word_seq_indexer, gpu, freeze_word_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
print('load glove word emb ')
self.if_word = True
else:
print('can only use one word embedding (random or glove)')
self.if_char = False
if args.if_cnnChar == True and args.if_lstmChar == False:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharCNN(gpu, char_embeddings_dim,
char_cnn_filter_num,
char_window_size, word_len)
emb_models_dim.append(self.char_layer.output_dim)
self.if_char = True
elif args.if_cnnChar == False and args.if_lstmChar == True:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharLSTM(gpu, char_embeddings_dim,
self.char_lstm_hidden_dim,
word_len)
emb_models_dim.append(self.char_layer.output_dim)
self.if_char = True
else:
print('can only use one char embedding (cnnChar or lstmChar)')
self.input_dim = sum(emb_models_dim)
if self.args.transformer:
self.n_head = self.args.trans_head
self.emb_dim = int((self.input_dim / self.n_head)) * self.n_head
print('self.emb_dim', self.emb_dim)
print('self.input_dim', self.input_dim)
self.emb_linear = nn.Linear(in_features=self.input_dim,
out_features=self.emb_dim)
self.transEncodeLayer = TransformerEncoderLayer(
d_model=self.emb_dim, nhead=self.n_head)
self.transformer_encoder = TransformerEncoder(
encoder_layer=self.transEncodeLayer, num_layers=6)
self.input_dim = self.emb_dim
self.transClassify_lin = nn.Linear(in_features=self.emb_dim,
out_features=class_num + 1)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif self.rnn_type == 'SATN':
self.birnn_layer = LayerSelfAttn(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif self.rnn_type == 'WCNN':
self.birnn_layer = LayerWCNN(
args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
cnn_layer=args.wcnn_layer,
# wcnn_hidden_dim =args.wcnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim,
out_features=class_num + 2)
self.crf_layer = LayerCRF(gpu,
states_num=class_num + 2,
pad_idx=tag_seq_indexer.pad_idx,
sos_idx=class_num + 1,
tag_seq_indexer=tag_seq_indexer)
self.softmax = nn.Softmax(dim=2)
if gpu >= 0:
self.cuda(device=self.gpu)