def __init__(self,
word_vocab_size,
word_embedding_dim,
word_hidden_dim,
char_vocab_size,
char_embedding_dim,
char_out_channels,
tag_to_id,
cap_input_dim=4,
cap_embedding_dim=0,
pretrained=None):
super(CNN_BiLSTM_CRF_MC, self).__init__()
self.word_vocab_size = word_vocab_size
self.word_embedding_dim = word_embedding_dim
self.word_hidden_dim = word_hidden_dim
self.char_vocab_size = char_vocab_size
self.char_embedding_dim = char_embedding_dim
self.char_out_channels = char_out_channels
self.cap_input_dim = cap_input_dim
self.cap_embedding_dim = cap_embedding_dim
self.tag_to_ix = tag_to_id
self.tagset_size = len(tag_to_id)
self.initializer = Initializer()
if self.cap_embedding_dim:
self.cap_embedder = nn.Embedding(self.cap_input_dim,
self.cap_embedding_dim)
self.initializer.init_embedding(self.cap_embedder.weight)
self.char_encoder = CharEncoderCNN(char_vocab_size,
char_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1)
self.initializer.init_embedding(self.char_encoder.embedding.weight)
self.word_encoder = WordEncoderRNN(word_vocab_size,
word_embedding_dim,
word_hidden_dim,
char_out_channels,
cap_embedding_dim,
input_dropout_p=0.5)
if pretrained is not None:
self.word_encoder.embedding.weight = nn.Parameter(
torch.FloatTensor(pretrained))
self.initializer.init_lstm(self.word_encoder.rnn)
self.decoder = DecoderCRF(word_hidden_dim * 2,
self.tag_to_ix,
input_dropout_p=0.5)
self.initializer.init_linear(self.decoder.hidden2tag)
class CNN_CNN_LSTM_MC(nn.Module):
def __init__(self,
word_vocab_size,
word_embedding_dim,
word_out_channels,
char_vocab_size,
char_embedding_dim,
char_out_channels,
decoder_hidden_units,
tag_to_id,
cap_input_dim=4,
cap_embedding_dim=0,
pretrained=None):
super(CNN_CNN_LSTM_MC, self).__init__()
self.word_vocab_size = word_vocab_size
self.word_embedding_dim = word_embedding_dim
self.word_out_channels = word_out_channels
self.char_vocab_size = char_vocab_size
self.char_embedding_dim = char_embedding_dim
self.char_out_channels = char_out_channels
self.cap_input_dim = cap_input_dim
self.cap_embedding_dim = cap_embedding_dim
self.tag_to_ix = tag_to_id
self.tagset_size = len(tag_to_id)
self.initializer = Initializer()
if self.cap_input_dim and self.cap_embedding_dim:
self.cap_embedder = nn.Embedding(self.cap_input_dim,
self.cap_embedding_dim)
self.initializer.init_embedding(self.cap_embedder.weight)
self.char_encoder = CharEncoderCNN(char_vocab_size,
char_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1)
self.initializer.init_embedding(self.char_encoder.embedding.weight)
self.word_encoder = WordEncoderCNN(word_vocab_size,
word_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1,
input_dropout_p=0.5,
out_channels=word_out_channels)
if pretrained is not None:
self.word_encoder.embedding.weight = nn.Parameter(
torch.FloatTensor(pretrained))
augmented_decoder_inp_size = (word_out_channels + word_embedding_dim +
char_out_channels + cap_embedding_dim)
self.decoder = DecoderRNN(augmented_decoder_inp_size,
decoder_hidden_units,
self.tagset_size,
self.tag_to_ix,
input_dropout_p=0.5)
def forward(self,
words,
tags,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features, word_input_feats = self.word_encoder(
words, char_features, cap_features)
new_word_features = torch.cat((word_features, word_input_feats), 2)
loss = self.decoder(new_word_features, tags, tagsmask, usecuda=usecuda)
return loss
def decode(self,
words,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True,
score_only=False):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features, word_input_feats = self.word_encoder(
words, char_features, cap_features)
new_word_features = torch.cat((word_features, word_input_feats), 2)
if score_only:
score, _ = self.decoder.decode(new_word_features,
tagsmask,
wordslen,
usecuda=usecuda)
return score
score, tag_seq = self.decoder.decode(new_word_features,
tagsmask,
wordslen,
usecuda=usecuda)
return score, tag_seq
def __init__(self,
word_vocab_size,
word_embedding_dim,
word_out_channels,
char_vocab_size,
char_embedding_dim,
char_out_channels,
decoder_hidden_units,
tag_to_id,
cap_input_dim=4,
cap_embedding_dim=0,
pretrained=None):
super(CNN_CNN_LSTM_MC, self).__init__()
self.word_vocab_size = word_vocab_size
self.word_embedding_dim = word_embedding_dim
self.word_out_channels = word_out_channels
self.char_vocab_size = char_vocab_size
self.char_embedding_dim = char_embedding_dim
self.char_out_channels = char_out_channels
self.cap_input_dim = cap_input_dim
self.cap_embedding_dim = cap_embedding_dim
self.tag_to_ix = tag_to_id
self.tagset_size = len(tag_to_id)
self.initializer = Initializer()
if self.cap_input_dim and self.cap_embedding_dim:
self.cap_embedder = nn.Embedding(self.cap_input_dim,
self.cap_embedding_dim)
self.initializer.init_embedding(self.cap_embedder.weight)
self.char_encoder = CharEncoderCNN(char_vocab_size,
char_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1)
self.initializer.init_embedding(self.char_encoder.embedding.weight)
self.word_encoder = WordEncoderCNN(word_vocab_size,
word_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1,
input_dropout_p=0.5,
out_channels=word_out_channels)
if pretrained is not None:
self.word_encoder.embedding.weight = nn.Parameter(
torch.FloatTensor(pretrained))
augmented_decoder_inp_size = (word_out_channels + word_embedding_dim +
char_out_channels + cap_embedding_dim)
self.decoder = DecoderRNN(augmented_decoder_inp_size,
decoder_hidden_units,
self.tagset_size,
self.tag_to_ix,
input_dropout_p=0.5)
class CNN_BiLSTM_CRF_MC(nn.Module):
def __init__(self,
word_vocab_size,
word_embedding_dim,
word_hidden_dim,
char_vocab_size,
char_embedding_dim,
char_out_channels,
tag_to_id,
cap_input_dim=4,
cap_embedding_dim=0,
pretrained=None):
super(CNN_BiLSTM_CRF_MC, self).__init__()
self.word_vocab_size = word_vocab_size
self.word_embedding_dim = word_embedding_dim
self.word_hidden_dim = word_hidden_dim
self.char_vocab_size = char_vocab_size
self.char_embedding_dim = char_embedding_dim
self.char_out_channels = char_out_channels
self.cap_input_dim = cap_input_dim
self.cap_embedding_dim = cap_embedding_dim
self.tag_to_ix = tag_to_id
self.tagset_size = len(tag_to_id)
self.initializer = Initializer()
if self.cap_embedding_dim:
self.cap_embedder = nn.Embedding(self.cap_input_dim,
self.cap_embedding_dim)
self.initializer.init_embedding(self.cap_embedder.weight)
self.char_encoder = CharEncoderCNN(char_vocab_size,
char_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1)
self.initializer.init_embedding(self.char_encoder.embedding.weight)
self.word_encoder = WordEncoderRNN(word_vocab_size,
word_embedding_dim,
word_hidden_dim,
char_out_channels,
cap_embedding_dim,
input_dropout_p=0.5)
if pretrained is not None:
self.word_encoder.embedding.weight = nn.Parameter(
torch.FloatTensor(pretrained))
self.initializer.init_lstm(self.word_encoder.rnn)
self.decoder = DecoderCRF(word_hidden_dim * 2,
self.tag_to_ix,
input_dropout_p=0.5)
self.initializer.init_linear(self.decoder.hidden2tag)
def forward(self,
words,
tags,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features = self.word_encoder(words, char_features, cap_features,
wordslen)
score = self.decoder(word_features, tags, tagsmask, usecuda=usecuda)
return score
def decode(self,
words,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True,
score_only=False):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features = self.word_encoder(words, char_features, cap_features,
wordslen)
if score_only:
score = self.decoder.decode(word_features,
tagsmask,
usecuda=usecuda,
score_only=True)
return score
score, tag_seq = self.decoder.decode(word_features,
tagsmask,
usecuda=usecuda)
return score, tag_seq
class CNN_CNN_LSTM_BB(nn.Module):
def __init__(self,
word_vocab_size,
word_embedding_dim,
word_out_channels,
char_vocab_size,
char_embedding_dim,
char_out_channels,
decoder_hidden_units,
tag_to_id,
sigma_prior,
cap_input_dim=4,
cap_embedding_dim=0,
pretrained=None):
super(CNN_CNN_LSTM_BB, self).__init__()
self.word_vocab_size = word_vocab_size
self.word_embedding_dim = word_embedding_dim
self.word_out_channels = word_out_channels
self.char_vocab_size = char_vocab_size
self.char_embedding_dim = char_embedding_dim
self.char_out_channels = char_out_channels
self.cap_input_dim = cap_input_dim
self.cap_embedding_dim = cap_embedding_dim
self.sigma_prior = sigma_prior
self.tag_to_ix = tag_to_id
self.tagset_size = len(tag_to_id)
self.initializer = Initializer()
if self.cap_input_dim and self.cap_embedding_dim:
self.cap_embedder = nn.Embedding(self.cap_input_dim,
self.cap_embedding_dim)
self.initializer.init_embedding(self.cap_embedder.weight)
self.char_encoder = CharEncoderCNN(char_vocab_size,
char_embedding_dim,
char_out_channels,
kernel_width=3,
pad_width=1)
self.initializer.init_embedding(self.char_encoder.embedding.weight)
self.word_encoder = WordEncoderCNN_BB(word_vocab_size,
word_embedding_dim,
char_out_channels,
sigma_prior=sigma_prior,
kernel_width=3,
pad_width=1,
input_dropout_p=0.5,
out_channels=word_out_channels,
usecuda=self.usecuda)
if pretrained is not None:
self.word_encoder.embedding.weight = nn.Parameter(
torch.FloatTensor(pretrained))
augmented_decoder_inp_size = (word_out_channels + word_embedding_dim +
char_out_channels + cap_embedding_dim)
self.decoder = DecoderRNN(augmented_decoder_inp_size,
decoder_hidden_units,
self.tagset_size,
self.tag_to_ix,
input_dropout_p=0.5)
def forward_pass(self,
words,
tags,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features, word_input_feats = self.word_encoder(
words, char_features, cap_features)
new_word_features = torch.cat((word_features, word_input_feats), 2)
loss = self.decoder(new_word_features, tags, tagsmask, usecuda=usecuda)
return loss
def forward(self,
words,
tags,
chars,
caps,
wordslen,
charslen,
tagsmask,
n_batches,
n_samples=3,
usecuda=True):
batch_size, max_len = words.size()
s_log_pw, s_log_qw, s_log_likelihood = 0., 0., 0.
for _ in range(n_samples):
sample_log_likelihood = -1. * self.forward_pass(words,
tags,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=usecuda)
sample_log_pw, sample_log_qw = self.word_encoder.get_lpw_lqw()
s_log_pw += sample_log_pw
s_log_qw += sample_log_qw
s_log_likelihood += sample_log_likelihood
log_pw, log_qw, log_llh = s_log_pw / n_samples, s_log_qw / n_samples, s_log_likelihood / n_samples
loss = bayes_loss_function(log_pw, log_qw, log_llh, n_batches,
batch_size)
return loss
def decode(self,
words,
chars,
caps,
wordslen,
charslen,
tagsmask,
usecuda=True,
score_only=False):
batch_size, max_len = words.size()
cap_features = self.cap_embedder(
caps) if self.cap_embedding_dim else None
char_features = self.char_encoder(chars)
char_features = char_features.view(batch_size, max_len, -1)
word_features, word_input_feats = self.word_encoder(
words, char_features, cap_features)
new_word_features = torch.cat((word_features, word_input_feats), 2)
if score_only:
score, _ = self.decoder.decode(new_word_features,
tagsmask,
wordslen,
usecuda=usecuda)
return score
score, tag_seq = self.decoder.decode(new_word_features,
tagsmask,
wordslen,
usecuda=usecuda)
return score, tag_seq