def __init__(self, word_seq_indexer, tag_seq_indexer, class_num, batch_size=1, rnn_hidden_dim=100,
freeze_word_embeddings=False, dropout_ratio=0.5, rnn_type='GRU', gpu=-1, useElmo = False, weight_file = '', options_file = ''):
super(TaggerBiRNNCRF, self).__init__(word_seq_indexer, tag_seq_indexer, gpu, batch_size, useElmo, weight_file, options_file)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
print ("init targer BiRNNcfr")
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError('Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim, out_features=class_num + 2)
self.crf_layer = LayerCRF(gpu, states_num=class_num + 2, pad_idx=tag_seq_indexer.pad_idx, sos_idx=class_num + 1,
tag_seq_indexer=tag_seq_indexer)
if gpu >= 0:
self.cuda(device=self.gpu)
def __init__(self,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1):
super(TaggerBiRNN, self).__init__(word_seq_indexer, tag_seq_indexer,
gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
if ((not word_seq_indexer.bert) and (not word_seq_indexer.elmo)):
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
elif (word_seq_indexer.bert):
print("word_seq_indexer.bert gpu", gpu)
self.word_embeddings_layer = LayerContextWordEmbeddingsBert(
word_seq_indexer, gpu, freeze_word_embeddings)
else:
self.word_embeddings_layer = LayerContextWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(
input_dim=self.word_embeddings_layer.output_dim +
self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
# We add an additional class that corresponds to the zero-padded values not to be included to the loss function
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim,
out_features=class_num + 1)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss(
ignore_index=0
) # "0" target values actually are zero-padded parts of sequences
def __init__(self, word_seq_indexer, tag_seq_indexer, class_num, batch_size=1, rnn_hidden_dim=100,
freeze_word_embeddings=False, dropout_ratio=0.5, rnn_type='GRU', gpu=-1,
freeze_char_embeddings = False, char_embeddings_dim=25, word_len=20, char_cnn_filter_num=30,
char_window_size=3):
super(TaggerBiRNNCNNCRF, self).__init__(word_seq_indexer, tag_seq_indexer, gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.char_embeddings_layer = LayerCharEmbeddings(gpu, char_embeddings_dim, freeze_char_embeddings,
word_len, word_seq_indexer.get_unique_characters_list())
self.char_cnn_layer = LayerCharCNN(gpu, char_embeddings_dim, char_cnn_filter_num, char_window_size,
word_len)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(input_dim=self.word_embeddings_layer.output_dim+self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError('Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim, out_features=class_num + 2)
self.crf_layer = LayerCRF(gpu, states_num=class_num + 2, pad_idx=tag_seq_indexer.pad_idx, sos_idx=class_num + 1,
tag_seq_indexer=tag_seq_indexer)
self.softmax = nn.Softmax(dim=2)
if gpu >= 0:
self.cuda(device=self.gpu)
def __init__(self,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=50,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='LSTM',
gpu=-1,
latent_dim=16,
pooling_type='attention'):
super(TaggerBiRNN, self).__init__(word_seq_indexer, tag_seq_indexer,
gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
self.latent_dim = latent_dim
self.birnn_layer = LayerBiLSTM(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
self.pooler = LayerPooler(input_dim=self.birnn_layer.output_dim,
gpu=gpu,
pooling_type=pooling_type)
if latent_dim is not None:
self.dim_red = nn.Sequential(
nn.Linear(in_features=self.pooler.output_dim,
out_features=latent_dim), nn.Sigmoid())
self.dim_red.apply(self.inititialize_random_projection)
lin_layer_in = latent_dim
else:
lin_layer_in = self.pooler.output_dim
self.lin_layer = nn.Linear(in_features=lin_layer_in,
out_features=class_num)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss()
def __init__(self,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1,
freeze_char_embeddings=False,
char_embeddings_dim=25,
word_len=20,
char_cnn_filter_num=30,
char_window_size=3):
super(TaggerBiRNNCNN, self).__init__(word_seq_indexer, tag_seq_indexer,
gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_cnn_layer = LayerCharCNN(gpu, char_embeddings_dim,
char_cnn_filter_num,
char_window_size, word_len)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(
input_dim=self.word_embeddings_layer.output_dim +
self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(
input_dim=self.word_embeddings_layer.output_dim +
self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(
input_dim=self.word_embeddings_layer.output_dim +
self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
# We add an additional class that corresponds to the zero-padded values not to be included to the loss function
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim,
out_features=class_num + 1)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss(
ignore_index=0
) # "0" target values actually are zero-padded parts of sequences
def __init__(self,
args,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1,
freeze_char_embeddings=False,
char_embeddings_dim=25,
word_len=20,
char_cnn_filter_num=30,
char_window_size=3):
super(TaggerBiRNNCNN, self).__init__(word_seq_indexer, tag_seq_indexer,
gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = True
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.if_elmo = args.if_elmo
self.if_bert = args.if_bert
self.if_flair = args.if_flair
if args.if_glove or args.if_wordEmbRand or args.if_twitter_emb:
self.if_word = True
else:
self.if_word = False
if args.if_char_cnn or args.if_char_lstm:
self.if_char = True
else:
self.if_char = False
# self.elmo_embeddings_dim = args.elmo_embeddings_dim
# self.bert_embeddings_dim = args.bert_embeddings_dim
self.bert_mode = 'mean'
self.options_file = args.options_file
self.weight_file = args.weight_file
emb_models_dim = []
if args.if_wordEmbRand:
self.word_embeddings_layer = LayerWordEmbeddings_Rand(
word_seq_indexer, gpu, self.freeze_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
if args.if_glove:
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, self.freeze_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
if args.if_char_lstm:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharLSTM(gpu, char_embeddings_dim,
self.char_lstm_hidden_dim,
word_len)
emb_models_dim.append(self.char_layer.output_dim)
if args.if_char_cnn:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharCNN(gpu, char_embeddings_dim,
char_cnn_filter_num,
char_window_size, word_len)
emb_models_dim.append(self.char_layer.output_dim)
if args.if_elmo:
self.elmo_embeddings_layer = LayerElmoEmbeddings(
args, gpu, self.elmo_embeddings_dim, self.options_file,
self.weight_file, freeze_char_embeddings, word_len)
emb_models_dim.append(self.elmo_embeddings_layer.output_dim)
if args.if_bert:
self.bert_embeddings_layer = LayerBertEmbeddings(
gpu, self.bert_embeddings_dim, self.bert_mode)
emb_models_dim.append(self.bert_embeddings_layer.output_dim)
print('start bert embedding successful...')
if args.if_flair:
self.flair_embeddings_layer = LayerFlairEmbeddings(gpu)
emb_models_dim.append(self.flair_embeddings_layer.output_dim)
self.input_dim = sum(emb_models_dim)
#
# self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_cnn_layer = LayerCharCNN(gpu, char_embeddings_dim,
char_cnn_filter_num,
char_window_size, word_len)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
# We add an additional class that corresponds to the zero-padded values not to be included to the loss function
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim,
out_features=class_num + 1)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss(
ignore_index=0
) # "0" target values actually are zero-padded parts of sequences
def __init__(self,
args,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1,
freeze_char_embeddings=False,
char_embeddings_dim=25,
word_len=20,
char_cnn_filter_num=30,
char_window_size=3):
super(TaggerBiRNNCNN, self).__init__(word_seq_indexer, tag_seq_indexer,
gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.freeze_char_embeddings = freeze_char_embeddings
self.char_embeddings_dim = char_embeddings_dim
self.word_len = word_len
self.char_cnn_filter_num = char_cnn_filter_num
self.char_window_size = char_window_size
self.args = args
self.if_bert = args.if_bert
self.if_flair = args.if_flair
self.dropout = torch.nn.Dropout(p=dropout_ratio)
emb_models_dim = []
print('load embedding...')
if args.if_bert:
# print('PYTORCH_PRETRAINED_BERT_CACHE',PYTORCH_PRETRAINED_BERT_CACHE)
cache_dir = args.cache_dir if args.cache_dir else os.path.join(
str(PYTORCH_PRETRAINED_BERT_CACHE), 'distributed_{}'.format(
args.local_rank))
# cache_dir =os.path.join(str(PYTORCH_PRETRAINED_BERT_CACHE),'distributed_{}'.format(args.local_rank))
print('cache_dir', cache_dir)
# cache_dir = '/home/jlfu/.pytorch_pretrained_bert/distributed_-1'
# cache_dir='emb/bert_model_cache/bert_cache.hdf5',
self.bert_embeddings_layer = LayerBertEmbeddings.from_pretrained(
args.bert_model, cache_dir=cache_dir, num_labels=class_num)
# self.bert_embeddings_layer = LayerBertEmbeddings(gpu, freeze_bert_embeddings=True)
reduce_dim = False
if reduce_dim:
self.W_bert = nn.Linear(args.bert_output_dim, 256)
emb_models_dim.append(256)
else:
emb_models_dim.append(args.bert_output_dim)
if args.if_flair:
self.flair_embeddings_layer = LayerFlairEmbeddings(gpu)
reduce_dim = True
if reduce_dim == True:
self.W_flair = nn.Linear(
self.flair_embeddings_layer.output_dim, 256).cuda()
emb_models_dim.append(256)
else:
emb_models_dim.append(self.flair_embeddings_layer.output_dim)
if args.if_elmo:
self.elmo_embeddings_layer = LayerElmoEmbeddings(
args, gpu, args.options_file, args.weight_file,
freeze_char_embeddings, word_len)
elmo_reduce_dim = False
if elmo_reduce_dim:
# self.W_elmo = nn.Linear(self.elmo_embeddings_layer.output_dim,256)
# emb_models_dim.append(256)
self.W_elmo = nn.Linear(self.elmo_embeddings_layer.output_dim,
self.word_embeddings_layer.output_dim)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
else:
emb_models_dim.append(self.elmo_embeddings_layer.output_dim)
# if args.if_glove:
# self.word_embeddings_layer = LayerWordEmbeddings(word_seq_indexer, gpu, freeze_word_embeddings)
# emb_models_dim.append(self.word_embeddings_layer.output_dim)
self.if_word = False
if args.if_wordEmbRand == True and args.if_glove == False:
self.word_embeddings_layer = LayerWordEmbeddings_Rand(
word_seq_indexer, gpu, freeze_word_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
print('load random word emb ')
self.if_word = True
elif args.if_wordEmbRand == False and args.if_glove == True:
self.word_embeddings_layer = LayerWordEmbeddings(
args, word_seq_indexer, gpu, freeze_word_embeddings)
emb_models_dim.append(self.word_embeddings_layer.output_dim)
print('load glove word emb ')
self.if_word = True
else:
print('can only use one word embedding (random or glove)')
self.if_char = False
if args.if_cnnChar == True and args.if_lstmChar == False:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharCNN(gpu, char_embeddings_dim,
char_cnn_filter_num,
char_window_size, word_len)
emb_models_dim.append(self.char_layer.output_dim)
self.if_char = True
elif args.if_cnnChar == False and args.if_lstmChar == True:
self.char_embeddings_layer = LayerCharEmbeddings(
gpu, char_embeddings_dim, freeze_char_embeddings, word_len,
word_seq_indexer.get_unique_characters_list())
self.char_layer = LayerCharLSTM(gpu, char_embeddings_dim,
self.char_lstm_hidden_dim,
word_len)
emb_models_dim.append(self.char_layer.output_dim)
self.if_char = True
else:
print('can only use one char embedding (cnnChar or lstmChar)')
self.input_dim = sum(emb_models_dim)
if self.args.transformer:
self.n_head = self.args.trans_head
self.emb_dim = int((self.input_dim / self.n_head)) * self.n_head
print('self.emb_dim', self.emb_dim)
print('self.input_dim', self.input_dim)
self.emb_linear = nn.Linear(in_features=self.input_dim,
out_features=self.emb_dim)
self.transEncodeLayer = TransformerEncoderLayer(
d_model=self.emb_dim, nhead=self.n_head)
self.transformer_encoder = TransformerEncoder(
encoder_layer=self.transEncodeLayer, num_layers=6)
self.input_dim = self.emb_dim
self.transClassify_lin = nn.Linear(in_features=self.emb_dim,
out_features=class_num + 1)
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif self.rnn_type == 'SATN':
self.birnn_layer = LayerSelfAttn(args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif self.rnn_type == 'WCNN':
self.birnn_layer = LayerWCNN(
args=args,
input_dim=self.input_dim,
hidden_dim=rnn_hidden_dim,
cnn_layer=args.wcnn_layer,
# wcnn_hidden_dim =args.wcnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
self.lin_layer = nn.Linear(in_features=self.birnn_layer.output_dim,
out_features=class_num + 1)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss(
ignore_index=0
) # "0" target values actually are zero-padded parts of sequences
def __init__(self,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='GRU',
gpu=-1,
num_prototypes_per_class=6,
proto_dim=None,
pretrained_path=None,
max_pool_protos=False,
pooling_type='attention',
similarity_epsilon=1e-4,
hadamard_importance=False,
similarity_function_name='gaussian'):
super(TaggerProtoBiRNNFixed,
self).__init__(word_seq_indexer, tag_seq_indexer, gpu,
batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.dropout = torch.nn.Dropout(p=dropout_ratio)
self.num_prototypes_per_class = num_prototypes_per_class
self.num_prototypes = class_num * num_prototypes_per_class
self.proto_dim = proto_dim
self.max_pool = max_pool_protos
self.hadamard_importance = hadamard_importance
# parameters
self.prototypes_shape = (
self.num_prototypes, self.proto_dim, 1
) # the last dimension is 1 since the prototype vectors are used as a conv1d filter weight
self.prototypes = nn.Parameter(torch.rand(self.prototypes_shape))
# layers
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
self.birnn_layer = LayerBiLSTM(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
self.pooler = LayerPooler(input_dim=self.birnn_layer.output_dim,
gpu=gpu,
pooling_type=pooling_type)
self.proto_layer = LayerProto(
input_dim=self.proto_dim,
prototypes=self.prototypes,
num_classes=class_num,
num_prototypes_per_class=num_prototypes_per_class,
gpu=gpu,
max_pool=max_pool_protos,
similarity_epsilon=similarity_epsilon,
hadamard_importance=hadamard_importance,
similarity_function_name=similarity_function_name)
self.lin_layer = nn.Linear(in_features=self.proto_layer.output_dim,
out_features=class_num,
bias=False)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss()
# init weights and set grad reqs
self._initialize_weights()
self._set_grad_reqs()
def create(emb_type, word_seq_indexer, gpu, freeze_word_embeddings):
if emb_type == 'elmo':
return LayerContextWordEmbeddings(word_seq_indexer, gpu,
freeze_word_embeddings)
return LayerWordEmbeddings(word_seq_indexer, gpu,
freeze_word_embeddings)
def __init__(self,
word_seq_indexer,
tag_seq_indexer,
class_num,
batch_size=1,
rnn_hidden_dim=100,
freeze_word_embeddings=False,
dropout_ratio=0.5,
rnn_type='LSTM',
gpu=-1,
latent_dim=None):
super(TaggerAttentive, self).__init__(word_seq_indexer,
tag_seq_indexer, gpu, batch_size)
self.tag_seq_indexer = tag_seq_indexer
self.class_num = class_num
self.rnn_hidden_dim = rnn_hidden_dim
self.freeze_embeddings = freeze_word_embeddings
self.dropout_ratio = dropout_ratio
self.rnn_type = rnn_type
self.gpu = gpu
self.word_embeddings_layer = LayerWordEmbeddings(
word_seq_indexer, gpu, freeze_word_embeddings)
self.dropout = torch.nn.Dropout(p=dropout_ratio)
self.latent_dim = latent_dim
if rnn_type == 'GRU':
self.birnn_layer = LayerBiGRU(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'LSTM':
self.birnn_layer = LayerBiLSTM(
input_dim=self.word_embeddings_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
elif rnn_type == 'Vanilla':
self.birnn_layer = LayerBiVanilla(
input_dim=self.word_embeddings_layer.output_dim +
self.char_cnn_layer.output_dim,
hidden_dim=rnn_hidden_dim,
gpu=gpu)
else:
raise ValueError(
'Unknown rnn_type = %s, must be either "LSTM" or "GRU"')
# equal weight attention
self.attention = LayerAttention(
input_dim=self.birnn_layer.output_dim,
embedding_dim=self.word_embeddings_layer.output_dim,
output_dim=self.birnn_layer.output_dim,
gpu=gpu)
# dimension reduction
if latent_dim is not None:
self.dim_red = nn.Sequential(
nn.Linear(in_features=self.attention.output_dim +
self.word_embeddings_layer.output_dim,
out_features=latent_dim), nn.Sigmoid())
self.dim_red.apply(self.inititialize_random_projection)
lin_layer_in = latent_dim
else:
lin_layer_in = self.attention.output_dim + self.word_embeddings_layer.output_dim
# We add an additional class that corresponds to the zero-padded values not to be included to the loss function
self.lin_layer = nn.Linear(in_features=lin_layer_in,
out_features=class_num + 1)
self.log_softmax_layer = nn.LogSoftmax(dim=1)
if gpu >= 0:
self.cuda(device=self.gpu)
self.nll_loss = nn.NLLLoss(
ignore_index=0
) # "0" target values actually are zero-padded parts of sequences