def test_ner_data(self):
train_x, train_y = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data(
'train')
assert len(train_x) == len(train_y)
assert len(train_x) > 0
test_x, test_y = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data(
'test')
assert len(test_x) == len(test_y)
assert len(test_x) > 0
blstm_layer = Bidirectional(LSTM(**self.hyper_parameters['lstm_layer']))(base_model.output)
dense_layer = Dense(128, activation='tanh')(blstm_layer)
crf = CRF(len(self.label2idx), sparse_target=False)
crf_layer = crf(dense_layer)
model = Model(base_model.inputs, crf_layer)
model.compile(loss=crf_loss,
optimizer='adam',
metrics=[crf_accuracy])
self.model = model
self.model.summary()
if __name__ == "__main__":
print("Hello world")
from kashgari.utils.logger import init_logger
init_logger()
from kashgari.corpus import ChinaPeoplesDailyNerCorpus
init_logger()
x_train, y_train = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data()
x_validate, y_validate = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data(data_type='validate')
x_test, y_test = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data(data_type='test')
tagger = BLSTMCRFModel()
tagger.fit(x_train, y_train, epochs=2)
tagger.evaluate(x_validate, y_validate)
tagger.evaluate(x_test, y_test, debug_info=True)
model = BLSTMCRFModel.load_model('/Users/brikerman/Downloads/KashgariNER.output/model')
model.evaluate(x_test, y_test, debug_info=True)
def test_tagging_data(self):
x, y = ChinaPeoplesDailyNerCorpus.get_sequence_tagging_data()
self.assertGreater(len(x), 0)
self.assertEqual(len(x), len(y))