def test_shape_rnn(self):
inputs, embd_layer = get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=5,
word_embd_dim=150,
char_embd_dim=25,
char_hidden_dim=75,
)
self.assertEqual(len(inputs), 2)
self.assertEqual(inputs[0]._keras_shape, (None, None))
self.assertEqual(inputs[1]._keras_shape, (None, None, 5))
self.assertEqual(embd_layer._keras_shape, (None, None, 300))
inputs, embd_layer = get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=7,
word_embd_dim=300,
char_embd_dim=50,
char_hidden_dim=150,
char_hidden_layer_type='gru',
)
self.assertEqual(len(inputs), 2)
self.assertEqual(inputs[0]._keras_shape, (None, None))
self.assertEqual(inputs[1]._keras_shape, (None, None, 7))
self.assertEqual(embd_layer._keras_shape, (None, None, 600))
def test_not_implemented(self):
with self.assertRaises(NotImplementedError):
get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=7,
char_hidden_layer_type='Jack',
)
def char_embd_wrong_shape():
get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=5,
word_embd_dim=150,
char_embd_dim=25,
char_hidden_dim=75,
word_embd_weights=numpy.random.random((3, 150)),
char_embd_weights=numpy.random.random((7, 25)),
)
def build_model(rnn_num,
rnn_units,
word_dict_len,
char_dict_len,
max_word_len,
output_dim,
word_dim=100,
char_dim=100,
char_embd_dim=100,
lmbd=0.01,
word_embd_weights=None):
"""Build model for NER.
:param rnn_num: Number of parallel RNNs.
:param rnn_units: Unit size for each RNN.
:param word_dict_len: The number of words in the dictionary.
:param char_dict_len: The numbers of characters in the dictionary.
:param max_word_len: The maximum length of a word in the dictionary.
:param output_dim: The output dimension / number of NER types.
:param word_dim: The dimension of word embedding.
:param char_dim: The final dimension of character embedding.
:param char_embd_dim: The embedding dimension of characters before bidirectional RNN.
:param lmbd: A constant controlling the weights of losses.
:param word_embd_weights: Pre-trained embeddings for words.
:return model: The built model.
"""
inputs, embd_layer = get_embedding_layer(
word_dict_len=word_dict_len,
char_dict_len=char_dict_len,
max_word_len=max_word_len,
word_embd_dim=word_dim,
char_hidden_dim=char_dim // 2,
char_embd_dim=char_embd_dim,
word_embd_weights=word_embd_weights,
)
rnns, layers = [], []
for i in range(rnn_num):
lstm_layer = keras.layers.LSTM(
units=rnn_units,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True,
)
bi_lstm_layer = keras.layers.Bidirectional(
layer=lstm_layer,
name='LSTM_%d' % (i + 1),
)
layers.append((bi_lstm_layer.forward_layer, bi_lstm_layer.backward_layer))
rnns.append(bi_lstm_layer(embd_layer))
concat_layer = keras.layers.Concatenate(name='Concatenation')(rnns)
dense_layer = keras.layers.Dense(units=output_dim, activation='softmax', name='Dense')(concat_layer)
model = keras.models.Model(inputs=inputs, outputs=dense_layer)
loss = _loss(layers, rnn_units, lmbd=lmbd)
model.compile(
optimizer=keras.optimizers.Adam(),
loss=loss,
metrics=[keras.metrics.categorical_accuracy],
)
return model
def build_model(rnn_num,
rnn_units,
word_dict_len,
char_dict_len,
max_word_len,
output_dim,
word_dim=100,
char_dim=100,
char_embd_dim=100,
lmbd=0.01,
word_embd_weights=None):
"""Build model for NER.
:param rnn_num: Number of parallel RNNs.
:param rnn_units: Unit size for each RNN.
:param word_dict_len: The number of words in the dictionary.
:param char_dict_len: The numbers of characters in the dictionary.
:param max_word_len: The maximum length of a word in the dictionary.
:param output_dim: The output dimension / number of NER types.
:param word_dim: The dimension of word embedding.
:param char_dim: The final dimension of character embedding.
:param char_embd_dim: The embedding dimension of characters before bidirectional RNN.
:param lmbd: A constant controlling the weights of losses.
:param word_embd_weights: Pre-trained embeddings for words.
:return model: The built model.
"""
inputs, embd_layer = get_embedding_layer(
word_dict_len=word_dict_len,
char_dict_len=char_dict_len,
max_word_len=max_word_len,
word_embd_dim=word_dim,
char_hidden_dim=char_dim // 2,
char_embd_dim=char_embd_dim,
word_embd_weights=word_embd_weights,
)
rnns, layers = [], []
for i in range(rnn_num):
lstm_layer = keras.layers.LSTM(
units=rnn_units,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True,
)
bi_lstm_layer = keras.layers.Bidirectional(
layer=lstm_layer,
name='LSTM_%d' % (i + 1),
)
layers.append((bi_lstm_layer.forward_layer, bi_lstm_layer.backward_layer))
rnns.append(bi_lstm_layer(embd_layer))
concat_layer = keras.layers.Concatenate(name='Concatenation')(rnns)
dense_layer = keras.layers.Dense(units=output_dim, activation='softmax', name='Dense')(concat_layer)
model = keras.models.Model(inputs=inputs, outputs=dense_layer)
loss = _loss(layers, rnn_units, lmbd=lmbd)
model.compile(
optimizer=keras.optimizers.Adam(),
loss=loss,
metrics=[keras.metrics.categorical_accuracy],
)
return model
def test_custom(self):
layers = [
keras.layers.Conv1D(
filters=16,
kernel_size=3,
activation='relu',
),
keras.layers.Conv1D(
filters=16,
kernel_size=3,
activation='relu',
),
keras.layers.Flatten(),
keras.layers.Dense(
units=50,
name='Dense_Char',
),
]
inputs, embd_layer = get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=7,
word_embd_dim=300,
char_embd_dim=50,
char_hidden_layer_type=layers,
char_mask_zero=False,
)
self.assertEqual(len(inputs), 2)
self.assertEqual(inputs[0]._keras_shape, (None, None))
self.assertEqual(inputs[1]._keras_shape, (None, None, 7))
self.assertEqual(embd_layer._keras_shape, (None, None, 350))
inputs, embd_layer = get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=7,
word_embd_dim=300,
char_embd_dim=50,
char_hidden_layer_type=keras.layers.GRU(units=30),
char_mask_zero=False,
)
self.assertEqual(len(inputs), 2)
self.assertEqual(inputs[0]._keras_shape, (None, None))
self.assertEqual(inputs[1]._keras_shape, (None, None, 7))
self.assertEqual(embd_layer._keras_shape, (None, None, 330))
def test_pre_trained_shape(self):
get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=5,
word_embd_dim=150,
char_embd_dim=25,
char_hidden_dim=75,
word_embd_weights=numpy.random.random((3, 150)),
char_embd_weights=numpy.random.random((5, 25)),
)
def word_embd_wrong_shape():
get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=5,
word_embd_dim=150,
char_embd_dim=25,
char_hidden_dim=75,
word_embd_weights=numpy.random.random((3, 100)),
char_embd_weights=numpy.random.random((5, 25)),
)
self.assertRaises(ValueError, word_embd_wrong_shape)
def char_embd_wrong_shape():
get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=5,
word_embd_dim=150,
char_embd_dim=25,
char_hidden_dim=75,
word_embd_weights=numpy.random.random((3, 150)),
char_embd_weights=numpy.random.random((7, 25)),
)
self.assertRaises(ValueError, char_embd_wrong_shape)
def test_shape_cnn(self):
inputs, embd_layer = get_embedding_layer(
word_dict_len=3,
char_dict_len=5,
max_word_len=7,
word_embd_dim=300,
char_embd_dim=50,
char_hidden_dim=150,
char_hidden_layer_type='cnn',
char_mask_zero=False,
)
self.assertEqual(len(inputs), 2)
self.assertEqual(inputs[0]._keras_shape, (None, None))
self.assertEqual(inputs[1]._keras_shape, (None, None, 7))
self.assertEqual(embd_layer._keras_shape, (None, None, 450))
word = word.lower()
if word in pre_trained:
weights[index] = pre_trained[word]
else:
weights[index] = np.random.random((embd_dim, )).tolist()
return np.asarray(weights)
word_embd_weights = get_embedding_weights_from_file(
word_dict, MODEL_FILE, ignore_case=True) #return the word embedding matrix
inputs, embd_layer = get_embedding_layer( #inputs: [word_input_layer, char_input_layer]
#embd_layer: concatenate
word_dict_len=len(word_dict),
char_dict_len=len(char_dict),
max_word_len=max_word_len,
word_embd_dim=EMBEDDING_DIM,
char_embd_dim=EMBEDDING_DIM,
char_hidden_dim=25,
word_embd_weights=word_embd_weights,
rnn='lstm',
)
#--- Build Model
print('Building model...')
lstm_layer = Bidirectional(
LSTM(units=100),
name='Bi-LSTM',
)(embd_layer)
dense_layer = Dense(
units=5,
activation='softmax',
def build_model(word_dict_len,
char_dict_len,
max_word_len,
output_dim,
bi_lm_model,
rnn_1_dim=150,
rnn_2_dim=150,
rnn_type='gru',
word_dim=300,
char_dim=80,
char_embd_dim=25,
word_embd_weights=None):
"""Build model for NER.
:param word_dict_len: The number of words in the dictionary.
:param char_dict_len: The numbers of characters in the dictionary.
:param max_word_len: The maximum length of a word in the dictionary.
:param output_dim: The output dimension / number of NER types.
:param bi_lm_model: The trained BiLM model.
:param word_dim: The dimension of word embedding.
:param rnn_1_dim: The dimension of RNN after word/char embedding.
:param rnn_2_dim: The dimension of RNN after embedding and bidirectional language model.
:param rnn_type: The type of the two RNN layers.
:param char_dim: The final dimension of character embedding.
:param char_embd_dim: The embedding dimension of characters before bidirectional RNN.
:param word_embd_weights: Pre-trained embeddings for words.
:return model: The built model.
"""
inputs, embd_layer = get_embedding_layer(
word_dict_len=word_dict_len,
char_dict_len=char_dict_len,
max_word_len=max_word_len,
word_embd_dim=word_dim,
char_hidden_dim=char_dim // 2,
char_embd_dim=char_embd_dim,
word_embd_weights=word_embd_weights,
)
if rnn_type == 'gru':
rnn = keras.layers.GRU
else:
rnn = keras.layers.LSTM
dropout_layer_1 = keras.layers.Dropout(rate=0.25, name='Dropout-1')(embd_layer)
bi_rnn_layer_1 = keras.layers.Bidirectional(
layer=rnn(
units=rnn_1_dim,
dropout=0.0,
recurrent_dropout=0.0,
return_sequences=True,
),
name='Bi-RNN-1',
)(dropout_layer_1)
lm_layer = bi_lm_model.get_feature_layers(input_layer=inputs[0])
embd_lm_layer = keras.layers.Concatenate(name='Embd-Bi-LM-Feature')([bi_rnn_layer_1, lm_layer])
dropout_layer_2 = keras.layers.Dropout(rate=0.25, name='Dropout-2')(embd_lm_layer)
bi_rnn_layer_2 = keras.layers.Bidirectional(
layer=rnn(
units=rnn_2_dim,
dropout=0.0,
recurrent_dropout=0.0,
return_sequences=True,
),
name='Bi-RNN-2',
)(dropout_layer_2)
dense_layer = keras.layers.Dense(units=output_dim, name='Dense')(bi_rnn_layer_2)
crf_layer = CRF(
units=output_dim,
sparse_target=True,
name='CRF',
)
model = keras.models.Model(inputs=inputs, outputs=crf_layer(dense_layer))
model.compile(
optimizer=keras.optimizers.Adam(),
loss=crf_layer.loss_function,
metrics=[crf_layer.accuracy],
)
return model
