def test_shape_rnn(self): inputs, embd_layer = get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=5, word_embd_dim=150, char_embd_dim=25, char_hidden_dim=75, ) self.assertEqual(len(inputs), 2) self.assertEqual(inputs[0]._keras_shape, (None, None)) self.assertEqual(inputs[1]._keras_shape, (None, None, 5)) self.assertEqual(embd_layer._keras_shape, (None, None, 300)) inputs, embd_layer = get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=7, word_embd_dim=300, char_embd_dim=50, char_hidden_dim=150, char_hidden_layer_type='gru', ) self.assertEqual(len(inputs), 2) self.assertEqual(inputs[0]._keras_shape, (None, None)) self.assertEqual(inputs[1]._keras_shape, (None, None, 7)) self.assertEqual(embd_layer._keras_shape, (None, None, 600))
def test_not_implemented(self): with self.assertRaises(NotImplementedError): get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=7, char_hidden_layer_type='Jack', )
def char_embd_wrong_shape(): get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=5, word_embd_dim=150, char_embd_dim=25, char_hidden_dim=75, word_embd_weights=numpy.random.random((3, 150)), char_embd_weights=numpy.random.random((7, 25)), )
def build_model(rnn_num, rnn_units, word_dict_len, char_dict_len, max_word_len, output_dim, word_dim=100, char_dim=100, char_embd_dim=100, lmbd=0.01, word_embd_weights=None): """Build model for NER. :param rnn_num: Number of parallel RNNs. :param rnn_units: Unit size for each RNN. :param word_dict_len: The number of words in the dictionary. :param char_dict_len: The numbers of characters in the dictionary. :param max_word_len: The maximum length of a word in the dictionary. :param output_dim: The output dimension / number of NER types. :param word_dim: The dimension of word embedding. :param char_dim: The final dimension of character embedding. :param char_embd_dim: The embedding dimension of characters before bidirectional RNN. :param lmbd: A constant controlling the weights of losses. :param word_embd_weights: Pre-trained embeddings for words. :return model: The built model. """ inputs, embd_layer = get_embedding_layer( word_dict_len=word_dict_len, char_dict_len=char_dict_len, max_word_len=max_word_len, word_embd_dim=word_dim, char_hidden_dim=char_dim // 2, char_embd_dim=char_embd_dim, word_embd_weights=word_embd_weights, ) rnns, layers = [], [] for i in range(rnn_num): lstm_layer = keras.layers.LSTM( units=rnn_units, dropout=0.1, recurrent_dropout=0.1, return_sequences=True, ) bi_lstm_layer = keras.layers.Bidirectional( layer=lstm_layer, name='LSTM_%d' % (i + 1), ) layers.append((bi_lstm_layer.forward_layer, bi_lstm_layer.backward_layer)) rnns.append(bi_lstm_layer(embd_layer)) concat_layer = keras.layers.Concatenate(name='Concatenation')(rnns) dense_layer = keras.layers.Dense(units=output_dim, activation='softmax', name='Dense')(concat_layer) model = keras.models.Model(inputs=inputs, outputs=dense_layer) loss = _loss(layers, rnn_units, lmbd=lmbd) model.compile( optimizer=keras.optimizers.Adam(), loss=loss, metrics=[keras.metrics.categorical_accuracy], ) return model
def test_custom(self): layers = [ keras.layers.Conv1D( filters=16, kernel_size=3, activation='relu', ), keras.layers.Conv1D( filters=16, kernel_size=3, activation='relu', ), keras.layers.Flatten(), keras.layers.Dense( units=50, name='Dense_Char', ), ] inputs, embd_layer = get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=7, word_embd_dim=300, char_embd_dim=50, char_hidden_layer_type=layers, char_mask_zero=False, ) self.assertEqual(len(inputs), 2) self.assertEqual(inputs[0]._keras_shape, (None, None)) self.assertEqual(inputs[1]._keras_shape, (None, None, 7)) self.assertEqual(embd_layer._keras_shape, (None, None, 350)) inputs, embd_layer = get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=7, word_embd_dim=300, char_embd_dim=50, char_hidden_layer_type=keras.layers.GRU(units=30), char_mask_zero=False, ) self.assertEqual(len(inputs), 2) self.assertEqual(inputs[0]._keras_shape, (None, None)) self.assertEqual(inputs[1]._keras_shape, (None, None, 7)) self.assertEqual(embd_layer._keras_shape, (None, None, 330))
def test_pre_trained_shape(self): get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=5, word_embd_dim=150, char_embd_dim=25, char_hidden_dim=75, word_embd_weights=numpy.random.random((3, 150)), char_embd_weights=numpy.random.random((5, 25)), ) def word_embd_wrong_shape(): get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=5, word_embd_dim=150, char_embd_dim=25, char_hidden_dim=75, word_embd_weights=numpy.random.random((3, 100)), char_embd_weights=numpy.random.random((5, 25)), ) self.assertRaises(ValueError, word_embd_wrong_shape) def char_embd_wrong_shape(): get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=5, word_embd_dim=150, char_embd_dim=25, char_hidden_dim=75, word_embd_weights=numpy.random.random((3, 150)), char_embd_weights=numpy.random.random((7, 25)), ) self.assertRaises(ValueError, char_embd_wrong_shape)
def test_shape_cnn(self): inputs, embd_layer = get_embedding_layer( word_dict_len=3, char_dict_len=5, max_word_len=7, word_embd_dim=300, char_embd_dim=50, char_hidden_dim=150, char_hidden_layer_type='cnn', char_mask_zero=False, ) self.assertEqual(len(inputs), 2) self.assertEqual(inputs[0]._keras_shape, (None, None)) self.assertEqual(inputs[1]._keras_shape, (None, None, 7)) self.assertEqual(embd_layer._keras_shape, (None, None, 450))
word = word.lower() if word in pre_trained: weights[index] = pre_trained[word] else: weights[index] = np.random.random((embd_dim, )).tolist() return np.asarray(weights) word_embd_weights = get_embedding_weights_from_file( word_dict, MODEL_FILE, ignore_case=True) #return the word embedding matrix inputs, embd_layer = get_embedding_layer( #inputs: [word_input_layer, char_input_layer] #embd_layer: concatenate word_dict_len=len(word_dict), char_dict_len=len(char_dict), max_word_len=max_word_len, word_embd_dim=EMBEDDING_DIM, char_embd_dim=EMBEDDING_DIM, char_hidden_dim=25, word_embd_weights=word_embd_weights, rnn='lstm', ) #--- Build Model print('Building model...') lstm_layer = Bidirectional( LSTM(units=100), name='Bi-LSTM', )(embd_layer) dense_layer = Dense( units=5, activation='softmax',
def build_model(word_dict_len, char_dict_len, max_word_len, output_dim, bi_lm_model, rnn_1_dim=150, rnn_2_dim=150, rnn_type='gru', word_dim=300, char_dim=80, char_embd_dim=25, word_embd_weights=None): """Build model for NER. :param word_dict_len: The number of words in the dictionary. :param char_dict_len: The numbers of characters in the dictionary. :param max_word_len: The maximum length of a word in the dictionary. :param output_dim: The output dimension / number of NER types. :param bi_lm_model: The trained BiLM model. :param word_dim: The dimension of word embedding. :param rnn_1_dim: The dimension of RNN after word/char embedding. :param rnn_2_dim: The dimension of RNN after embedding and bidirectional language model. :param rnn_type: The type of the two RNN layers. :param char_dim: The final dimension of character embedding. :param char_embd_dim: The embedding dimension of characters before bidirectional RNN. :param word_embd_weights: Pre-trained embeddings for words. :return model: The built model. """ inputs, embd_layer = get_embedding_layer( word_dict_len=word_dict_len, char_dict_len=char_dict_len, max_word_len=max_word_len, word_embd_dim=word_dim, char_hidden_dim=char_dim // 2, char_embd_dim=char_embd_dim, word_embd_weights=word_embd_weights, ) if rnn_type == 'gru': rnn = keras.layers.GRU else: rnn = keras.layers.LSTM dropout_layer_1 = keras.layers.Dropout(rate=0.25, name='Dropout-1')(embd_layer) bi_rnn_layer_1 = keras.layers.Bidirectional( layer=rnn( units=rnn_1_dim, dropout=0.0, recurrent_dropout=0.0, return_sequences=True, ), name='Bi-RNN-1', )(dropout_layer_1) lm_layer = bi_lm_model.get_feature_layers(input_layer=inputs[0]) embd_lm_layer = keras.layers.Concatenate(name='Embd-Bi-LM-Feature')([bi_rnn_layer_1, lm_layer]) dropout_layer_2 = keras.layers.Dropout(rate=0.25, name='Dropout-2')(embd_lm_layer) bi_rnn_layer_2 = keras.layers.Bidirectional( layer=rnn( units=rnn_2_dim, dropout=0.0, recurrent_dropout=0.0, return_sequences=True, ), name='Bi-RNN-2', )(dropout_layer_2) dense_layer = keras.layers.Dense(units=output_dim, name='Dense')(bi_rnn_layer_2) crf_layer = CRF( units=output_dim, sparse_target=True, name='CRF', ) model = keras.models.Model(inputs=inputs, outputs=crf_layer(dense_layer)) model.compile( optimizer=keras.optimizers.Adam(), loss=crf_layer.loss_function, metrics=[crf_layer.accuracy], ) return model