def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size, labelset_size, pad_token_idx, bidirectional=True, num_layers=1, dropout=0., device=None):
"""Initialize model."""
super(LSTMTagger_CRF, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.tagset_size = tagset_size
self.labelset_size = labelset_size
self.pad_token_idx = pad_token_idx
self.bidirectional = bidirectional
self.num_layers = num_layers
self.dropout = dropout
self.device = device
self.num_directions = 2 if self.bidirectional else 1
self.word_embeddings = nn.Embedding(self.vocab_size, self.embedding_dim, self.pad_token_idx)
self.dropout_layer = nn.Dropout(p=self.dropout)
# The LSTM takes word embeddings as inputs, and outputs hidden states
# with dimensionality hidden_dim.
self.lstm = nn.LSTM(self.embedding_dim, self.hidden_dim, num_layers=self.num_layers, bidirectional=self.bidirectional, batch_first=True, dropout=self.dropout)
# The linear layer that maps from hidden state space to tag space
self.hidden2tag = nn.Linear(self.num_directions * self.hidden_dim, self.tagset_size + 2)
self.crf_layer = crf.CRF(self.tagset_size, self.device)
#self.init_weights()
self.intent_layer = intent.Intent(self.hidden_dim*self.num_directions, self.labelset_size, self.device)
def __init__(self, config, embedding, word2Idx, label2Idx, char2Idx,
description, exemplar):
super(Bilstm_LabelEmbedding, self).__init__()
self.embed_size = config.embed_size
self.max_char_len = config.max_char_len
self.hidden_size = config.hidden_size
self.num_layers = config.num_layers
self.Bidirectional = config.bidirectional
self.lstm_dropout = config.lstm_dropout
self.dropout_rate = config.dropout
self.use_charEmbedding = config.use_charEmbedding
self.use_crf = config.crf
self.device = config.device
self.word2Idx = word2Idx
self.char2Idx = char2Idx
self.label2Idx = label2Idx
self.embedding = embedding
self.description = description
self.exemplar = exemplar
self.label2Idx = label2Idx
self.config = config
self.TokenEmbedding = nn.Embedding.from_pretrained(
torch.from_numpy(embedding.astype(np.float32)),
padding_idx=word2Idx['<PAD>'])
self.charemb = None
if self.use_charEmbedding:
self.charemb = charEmbedding(eval(config.conv_filter_sizes),
eval(config.conv_filter_nums),
char2Idx, config.char_emb_size,
self.device)
self.LabelEmbedding = self.init_LabelEmbedding(sz=(3, 1206))
LSTMDim = self.embed_size
if self.use_charEmbedding:
LSTMDim += sum(self.charemb.conv_filter_nums)
self.Lstm = nn.LSTM(input_size=LSTMDim,
dropout=self.lstm_dropout,
hidden_size=self.hidden_size,
num_layers=self.num_layers,
bias=True,
batch_first=True,
bidirectional=self.Bidirectional)
self.h_projection = nn.Linear(
2 * self.hidden_size if self.Bidirectional else self.hidden_size,
self.LabelEmbedding.size(1),
bias=True)
self.dropout = nn.Dropout(self.dropout_rate)
self.crf = None
if self.use_crf:
self.crf = crf.CRF(labelEmbedding=self.LabelEmbedding,
num_tags=self.LabelEmbedding.size(0),
batch_first=True)
def __init__(self,
pretrained_model_type,
pretrained_model,
tagset_size,
class_size,
dropout=0.,
device=None,
extFeats_dim=None,
multi_class=False,
task_st='NN',
task_sc='CLS'):
"""Initialize model."""
super(Transformers_joint_slot_and_intent, self).__init__()
self.tagset_size = tagset_size
self.class_size = class_size
self.dropout = dropout
self.device = device
self.extFeats_dim = extFeats_dim
self.multi_class = multi_class
self.task_st = task_st # 'NN', 'NN_crf'
self.task_sc = task_sc # None, 'CLS', 'max', 'CLS_max'
self.dropout_layer = nn.Dropout(p=self.dropout)
self.pretrained_model_type = pretrained_model_type
self.pretrained_model = pretrained_model
if self.pretrained_model_type == 'xlnet':
self.sequence_summary = SequenceSummary(
self.pretrained_model.config)
self.embedding_dim = self.pretrained_model.config.hidden_size
# The LSTM takes word embeddings as inputs, and outputs hidden states
self.append_feature_dim = 0
if self.extFeats_dim:
self.append_feature_dim += self.extFeats_dim
self.extFeats_linear = nn.Linear(self.append_feature_dim,
self.append_feature_dim)
else:
self.extFeats_linear = None
# The linear layer that maps from hidden state space to tag space
if self.task_st == 'NN':
self.hidden2tag = nn.Linear(
self.embedding_dim + self.append_feature_dim, self.tagset_size)
else:
self.hidden2tag = nn.Linear(
self.embedding_dim + self.append_feature_dim,
self.tagset_size + 2)
self.crf_layer = crf.CRF(self.tagset_size, self.device)
if self.task_sc == 'CLS' or self.task_sc == 'max':
self.hidden2class = nn.Linear(self.embedding_dim, self.class_size)
elif self.task_sc == 'CLS_max':
self.hidden2class = nn.Linear(self.embedding_dim * 2,
self.class_size)
else:
pass
def __init__(self, embedding_dim, hidden_dim, vocab_size, tagset_size, bidirectional=True, num_layers=1, dropout=0., device=None, extFeats_dim=None, elmo_model=None, bert_model=None, fix_bert_model=False):
"""Initialize model."""
super(LSTMTagger_CRF, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_dim = hidden_dim
self.vocab_size = vocab_size
self.tagset_size = tagset_size
#self.pad_token_idxs = pad_token_idxs
self.bidirectional = bidirectional
self.num_layers = num_layers
self.dropout = dropout
self.device = device
self.extFeats_dim = extFeats_dim
self.num_directions = 2 if self.bidirectional else 1
self.dropout_layer = nn.Dropout(p=self.dropout)
self.elmo_model = elmo_model
self.bert_model = bert_model
self.fix_bert_model = fix_bert_model
if self.fix_bert_model:
self.number_of_last_hiddens_of_bert = 4
self.weighted_scores_of_last_hiddens = nn.Linear(self.number_of_last_hiddens_of_bert, 1, bias=False)
for weight in self.bert_model.parameters():
weight.requires_grad = False
else:
self.number_of_last_hiddens_of_bert = 1
if self.elmo_model and self.bert_model:
self.embedding_dim = self.elmo_model.get_output_dim() + self.bert_model.config.hidden_size
elif self.elmo_model:
self.embedding_dim = self.elmo_model.get_output_dim()
elif self.bert_model:
self.embedding_dim = self.bert_model.config.hidden_size
else:
self.word_embeddings = nn.Embedding(self.vocab_size, self.embedding_dim)
# The LSTM takes word embeddings as inputs, and outputs hidden states
self.append_feature_dim = 0
if self.extFeats_dim:
self.append_feature_dim += self.extFeats_dim
self.extFeats_linear = nn.Linear(self.append_feature_dim, self.append_feature_dim)
else:
self.extFeats_linear = None
# with dimensionality hidden_dim.
self.lstm = nn.LSTM(self.embedding_dim + self.append_feature_dim, self.hidden_dim, num_layers=self.num_layers, bidirectional=self.bidirectional, batch_first=True, dropout=self.dropout)
# The linear layer that maps from hidden state space to tag space
self.hidden2tag = nn.Linear(self.num_directions * self.hidden_dim, self.tagset_size + 2)
self.crf_layer = crf.CRF(self.tagset_size, self.device)
def __init__(self,
bert_model,
tagset_size,
class_size,
dropout=0.,
device=None,
extFeats_dim=None,
multi_class=False,
task_st='NN',
task_sc='CLS'):
"""Initialize model."""
super(BERT_joint_slot_and_intent, self).__init__()
self.tagset_size = tagset_size
self.class_size = class_size
self.dropout = dropout
self.device = device
self.extFeats_dim = extFeats_dim
self.multi_class = multi_class
self.task_st = task_st # NN, NN_crf
self.task_sc = task_sc # CLS, max, CLS_max
self.dropout_layer = nn.Dropout(p=self.dropout)
self.bert_model = bert_model
self.embedding_dim = self.bert_model.config.hidden_size
# The LSTM takes word embeddings as inputs, and outputs hidden states
self.append_feature_dim = 0
if self.extFeats_dim:
self.append_feature_dim += self.extFeats_dim
self.extFeats_linear = nn.Linear(self.append_feature_dim,
self.append_feature_dim)
else:
self.extFeats_linear = None
# The linear layer that maps from hidden state space to tag space
if self.task_st == 'NN':
self.hidden2tag = nn.Linear(
self.embedding_dim + self.append_feature_dim, self.tagset_size)
else:
self.hidden2tag = nn.Linear(
self.embedding_dim + self.append_feature_dim,
self.tagset_size + 2)
self.crf_layer = crf.CRF(self.tagset_size, self.device)
if self.task_sc == 'CLS' or self.task_sc == 'max':
self.hidden2class = nn.Linear(self.embedding_dim, self.class_size)
else:
self.hidden2class = nn.Linear(self.embedding_dim * 2,
self.class_size)
def __init__(self,
embedding_dim,
max_sen_len,
sen_size,
hidden_dim,
tagset_size,
bidirectional=True,
num_layers=1,
dropout=0.,
device=None):
"""Initialize model."""
super(LSTMTagger_CRF_sen_level, self).__init__()
print('embedding: %s, max_sen_len: %s, sen_size: %s' %
(embedding_dim, max_sen_len, sen_size))
self.embedding_dim = embedding_dim
self.sen_size = sen_size
self.max_sen_len = max_sen_len
self.hidden_dim = hidden_dim
self.tagset_size = tagset_size
#self.pad_token_idxs = pad_token_idxs
self.bidirectional = bidirectional
self.num_layers = num_layers
self.dropout = dropout
self.device = device
self.num_directions = 2 if self.bidirectional else 1
self.dropout_layer = nn.Dropout(p=self.dropout)
self.sen_embeddings = nn.Embedding(
self.sen_size, self.embedding_dim * self.max_sen_len)
# The LSTM takes word embeddings as inputs, and outputs hidden states
# with dimensionality hidden_dim.
self.lstm = nn.LSTM(self.embedding_dim,
self.hidden_dim,
num_layers=self.num_layers,
bidirectional=self.bidirectional,
batch_first=True,
dropout=self.dropout)
# The linear layer that maps from hidden state space to tag space
self.hidden2tag = nn.Linear(self.num_directions * self.hidden_dim,
self.tagset_size + 2)
self.crf_layer = crf.CRF(self.tagset_size, self.device)