def build(self):
# build word embedding
word_ids = Input(batch_shape=(None, None), dtype='int32')
lengths = Input(batch_shape=(None, None), dtype='int32')
inputs = [word_ids]
if self._embeddings is None:
word_embeddings = Embedding(input_dim=self._word_vocab_size,
output_dim=self._word_embedding_dim,
mask_zero=True)(word_ids)
else:
word_embeddings = Embedding(input_dim=self._embeddings.shape[0],
output_dim=self._embeddings.shape[1],
mask_zero=True,
weights=[self._embeddings])(word_ids)
# build character based word embedding
if self._use_char:
char_ids = Input(batch_shape=(None, None, None), dtype='int32')
inputs.append(char_ids)
char_embeddings = Embedding(input_dim=self._char_vocab_size,
output_dim=self._char_embedding_dim,
mask_zero=True)(char_ids)
s = K.shape(char_embeddings)
char_embeddings = Lambda(lambda x: K.reshape(
x, shape=(-1, s[-2], self._char_embedding_dim)))(
char_embeddings)
fwd_state = LSTM(self._char_lstm_size,
return_state=True)(char_embeddings)[-2]
bwd_state = LSTM(self._char_lstm_size,
return_state=True,
go_backwards=True)(char_embeddings)[-2]
char_embeddings = Concatenate(axis=-1)([fwd_state, bwd_state])
# shape = (batch size, max sentence length, char hidden size)
char_embeddings = Lambda(lambda x: K.reshape(
x, shape=[-1, s[1], 2 * self._char_lstm_size]))(
char_embeddings)
# combine characters and word
word_embeddings = Concatenate(axis=-1)(
[word_embeddings, char_embeddings])
inputs.append(lengths)
word_embeddings = Dropout(self._dropout)(word_embeddings)
z = Bidirectional(
LSTM(units=self._word_lstm_size,
return_sequences=True))(word_embeddings)
z = Dropout(self._dropout)(z)
z = Dense(self._fc_dim, activation='tanh')(z)
z = Dense(self._fc_dim, activation='tanh')(z)
if self._use_crf:
crf = CRF(self._num_labels, sparse_target=False)
self._loss = crf.loss_function
pred = crf(z)
else:
self._loss = 'categorical_crossentropy'
pred = Dense(self._num_labels, activation='softmax')(z)
self.model = Model(inputs=inputs, outputs=pred)
def build(self):
# build word embedding
word_ids = Input(batch_shape=(None, None),
dtype='int32',
name='word_input')
inputs = [word_ids]
if self._embeddings is None:
word_embeddings = Embedding(input_dim=self._word_vocab_size,
output_dim=self._word_embedding_dim,
mask_zero=True,
name='word_embedding')(word_ids)
else:
word_embeddings = Embedding(input_dim=self._embeddings.shape[0],
output_dim=self._embeddings.shape[1],
mask_zero=True,
weights=[self._embeddings],
name='word_embedding')(word_ids)
# build character based word embedding
if self._use_char:
char_ids = Input(batch_shape=(None, None, None),
dtype='int32',
name='char_input')
inputs.append(char_ids)
char_embeddings = Embedding(input_dim=self._char_vocab_size,
output_dim=self._char_embedding_dim,
mask_zero=True,
name='char_embedding')(char_ids)
char_embeddings = TimeDistributed(
Bidirectional(LSTM(self._char_lstm_size)))(char_embeddings)
word_embeddings = Concatenate()([word_embeddings, char_embeddings])
word_embeddings = Dropout(self._dropout)(word_embeddings)
z = Bidirectional(
LSTM(units=self._word_lstm_size,
return_sequences=True,
dropout=self._layerdropout,
recurrent_dropout=self._layerdropout))(word_embeddings)
if (self._layer2Flag):
z = Bidirectional(
LSTM(units=self._word_lstm_size,
return_sequences=True,
dropout=self._layerdropout,
recurrent_dropout=self._layerdropout))(z)
z = Dense(self._fc_dim, activation='tanh')(z)
if self._use_crf:
crf = CRF(self._num_labels, sparse_target=False)
loss = crf.loss_function
pred = crf(z)
else:
loss = 'categorical_crossentropy'
pred = Dense(self._num_labels, activation='softmax')(z)
model = Model(inputs=inputs, outputs=pred)
return model, loss
def build(self):
# build word embedding
#in_id = Input(shape=(self._bretMaxLen,), name="input_ids")
#in_mask = Input(shape=(self._bretMaxLen,), name="input_masks")
#in_segment = Input(shape=(self._bretMaxLen,), name="segment_ids")
#inputs = [in_id, in_mask, in_segment]
#word_embeddings = BertLayer(n_fine_tune_layers=3,bert_path=self._bert_path)(inputs)
word_ids = Input(batch_shape=(None, None),
dtype='int32',
name='word_input')
word_embeddings = BertEmbedding(
model='bert_24_1024_16',
dataset_name='book_corpus_wiki_en_cased')(word_ids)
# build character based word embedding
# if self._use_char:
# print("char Embedding layer On")
# char_ids = Input(batch_shape=(None, None, None), dtype='int32', name='char_input')
# inputs.append(char_ids)
# char_embeddings = Embedding(input_dim=self._char_vocab_size,
# output_dim=self._char_embedding_dim,
# mask_zero=True,
# name='char_embedding')(char_ids)
# char_embeddings = TimeDistributed(Bidirectional(LSTM(self._char_lstm_size)))(char_embeddings)
# word_embeddings = Concatenate()([word_embeddings, char_embeddings])
#
# word_embeddings = Dropout(self._dropout)(word_embeddings)
z = Bidirectional(
LSTM(units=self._word_lstm_size,
return_sequences=True,
dropout=self._layerdropout,
recurrent_dropout=self._layerdropout))(word_embeddings)
if (self._layer2Flag):
z = Bidirectional(
LSTM(units=self._word_lstm_size,
return_sequences=True,
dropout=self._layerdropout,
recurrent_dropout=self._layerdropout))(z)
z = Dense(self._fc_dim, activation='tanh')(z)
if self._use_crf:
crf = CRF(self._num_labels, sparse_target=False)
loss = crf.loss_function
pred = crf(z)
else:
loss = 'categorical_crossentropy'
pred = Dense(self._num_labels, activation='softmax')(z)
model = Model(inputs=inputs, outputs=pred)
return model, loss
def build(self):
# build word embedding
word_ids = Input(batch_shape=(None, None),
dtype='int32',
name='word_input')
if self._embeddings is None:
word_embeddings = Embedding(input_dim=self._word_vocab_size,
output_dim=self._word_embedding_dim,
mask_zero=True,
name='word_embedding')(word_ids)
else:
word_embeddings = Embedding(input_dim=self._embeddings.shape[0],
output_dim=self._embeddings.shape[1],
mask_zero=False,
trainable=False,
weights=[self._embeddings],
name='word_embedding')(word_ids)
# build character based word embedding
char_ids = Input(batch_shape=(None, None, None),
dtype='int32',
name='char_input')
char_embeddings = Embedding(input_dim=self._char_vocab_size,
output_dim=self._char_embedding_dim,
mask_zero=False,
trainable=False,
name='char_embedding')(char_ids)
char_embeddings = TimeDistributed(
Bidirectional(CuDNNLSTM(self._char_lstm_size)))(char_embeddings)
elmo_embeddings = Input(shape=(None, 1024), dtype='float32')
word_embeddings = Concatenate(trainable=False)(
[word_embeddings, char_embeddings, elmo_embeddings])
word_embeddings = Dropout(self._dropout)(word_embeddings)
z = Bidirectional(
CuDNNLSTM(units=self._word_lstm_size,
return_sequences=True))(word_embeddings)
z = Dense(self._fc_dim, activation='tanh')(z)
z = Dropout(self._dropout)(z)
crf = CRF(self._num_labels, sparse_target=False)
loss = crf.loss_function
pred = crf(z)
model = Model(inputs=[word_ids, char_ids, elmo_embeddings],
outputs=pred)
return model, loss
def get_model(max_len, max_features, embed_size, embedding_matrix, num_tags):
sequence_input = Input(shape=(max_len, ))
embedding = Embedding(max_features,
embed_size,
weights=[embedding_matrix],
trainable=config.train_embeddings,
mask_zero=True)(sequence_input)
x = Dropout(0.5)(embedding)
x1 = Bidirectional(LSTM(100, return_sequences=True))(x)
td = Dense(100, activation='tanh')(x1)
crf = CRF(num_tags, sparse_target=False) # CRF layer
out = crf(td) # output
model = Model(sequence_input, out)
return model, crf
def build_model(self): """ Builds Bidirectional LSTM-CRF model. """ # Word embedding - TODO #word_embeddings = Dropout(self.dropout) # Model model = Sequential() # Input #input = Input(shape=(maxlen,), dtype='int32') # Embeddings embeddings = self.create_embeddings_layer()#(input) model.add(embeddings) #model.add(Embedding(max_features, self.word_lstm_size * 2, input_length=maxlen)) # Bidirectional LSTM #bi_lstm = Bidirectional(LSTM(units=self.word_lstm_size, return_sequences=True)) model.add(Bidirectional(LSTM(units=self.word_lstm_size, return_sequences=True))) #bi_lstm = Dense(self.fc_dim, activation='tanh')(bi_lstm) model.add(Dense(self.fc_dim, activation='tanh')) #model.add(bi_lstm) model.add(Dropout(self.dropout)) # TODO # CRF if self.use_crf: crf = CRF(self.labels_cnt, sparse_target=False) loss = crf.loss_function model.add(crf) else: loss = 'categorical_crossentropy' model.add(Dense(self.labels_cnt, activation='softmax')) return model, loss
def build(self):
# build word embedding
word_ids = Input(batch_shape=(None, None),
dtype='int32',
name='word_input')
inputs = [word_ids]
if self._embeddings is None:
word_embeddings = Embedding(input_dim=self._word_vocab_size,
output_dim=self._word_embedding_dim,
mask_zero=True,
name='word_embedding')(word_ids)
else:
word_embeddings = Embedding(input_dim=self._embeddings.shape[0],
output_dim=self._embeddings.shape[1],
mask_zero=False,
trainable=False,
weights=[self._embeddings],
name='word_embedding')(word_ids)
# build character based word embedding
if self._use_char:
char_ids = Input(batch_shape=(None, None, None),
dtype='int32',
name='char_input')
inputs.append(char_ids)
char_embeddings = Embedding(input_dim=self._char_vocab_size,
output_dim=self._char_embedding_dim,
mask_zero=False,
trainable=True,
name='char_embedding')(char_ids)
# dropout = Dropout(self._dropout)(char_embeddings)
# conv1d_out = TimeDistributed(
# Conv1D(kernel_size=3, filters=50, padding='same', activation='tanh', strides=1))(dropout)
# maxpool_out = TimeDistributed(MaxPooling1D(self._char_embedding_dim))(conv1d_out)
# char = TimeDistributed(Flatten())(maxpool_out)
# char_embeddings = Dropout(self._dropout)(char)
char_embeddings = TimeDistributed(
Bidirectional(CuDNNLSTM(
self._char_lstm_size)))(char_embeddings)
# chars = TimeDistributed(Conv1D(self._char_lstm_size, 3, padding='same'), name="char_cnn")(char_embeddings)
# chars = TimeDistributed(GlobalMaxPooling1D(), name="char_pooling")(chars)
# char_embeddings = Dropout(self._dropout)(char_embeddings)
word_embeddings = Concatenate()([word_embeddings, char_embeddings])
word_embeddings = Dropout(self._dropout)(word_embeddings)
z = Bidirectional(
CuDNNLSTM(units=self._word_lstm_size,
return_sequences=True))(word_embeddings)
z = Bidirectional(CuDNNLSTM(units=100, return_sequences=True))(z)
# z = Dropout(self._dropout)(z)
z = Dense(self._fc_dim, activation='tanh')(z)
z = Dropout(self._dropout)(z)
if self._use_crf:
crf = CRF(self._num_labels, sparse_target=False)
loss = crf.loss_function
pred = crf(z)
else:
loss = 'categorical_crossentropy'
pred = Dense(self._num_labels, activation='softmax')(z)
model = Model(inputs=inputs, outputs=pred)
return model, loss
