def test_get_classification_data(self):
train_x, train_y = TencentDingdangSLUCorpus.get_classification_data(
'train')
assert len(train_x) == len(train_y)
assert len(train_x) > 0
test_x, test_y = TencentDingdangSLUCorpus.get_classification_data(
'test')
assert len(test_x) == len(test_y)
base_model.output)
max_pool_layer = MaxPooling1D(
**self.hyper_parameters['max_pool_layer'])(conv_layer)
lstm_layer = LSTM(
**self.hyper_parameters['lstm_layer'])(max_pool_layer)
dense_layer = Dense(len(self.label2idx),
activation='sigmoid')(lstm_layer)
output_layers = [dense_layer]
model = Model(base_model.inputs, output_layers)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
self.model = model
self.model.summary()
if __name__ == "__main__":
from kashgari.utils.logger import init_logger
from kashgari.corpus import TencentDingdangSLUCorpus
init_logger()
x_data, y_data = TencentDingdangSLUCorpus.get_classification_data()
classifier = CNNLSTMModel()
classifier.fit(x_data, y_data, epochs=1)
classifier.save('./classifier_saved2')
model = ClassificationModel.load_model('./classifier_saved2')
logging.info(model.predict('我要听音乐'))
average = GlobalAveragePooling1D()(rnn_1)
all_views = concatenate([last, maxpool, attn, average],
**self.hyper_parameters['all_views'])
output = Dropout(**self.hyper_parameters['dropout_0'])(all_views)
output = Dense(**self.hyper_parameters['dense'])(output)
output = Dropout(**self.hyper_parameters['dropout_1'])(output)
output = Dense(len(self.label2idx),
**self.hyper_parameters['activation_layer'])(output)
self.model = Model(base_model.inputs, output)
def _compile_model(self):
optimizer = getattr(eval(self.hyper_parameters['optimizer']['module']),
self.hyper_parameters['optimizer']['name'])(
**self.hyper_parameters['optimizer']['params'])
self.model.compile(optimizer=optimizer,
**self.hyper_parameters['compile_params'])
if __name__ == '__main__':
from kashgari.corpus import TencentDingdangSLUCorpus
from kashgari.embeddings import WordEmbeddings, BERTEmbedding
train_x, train_y = TencentDingdangSLUCorpus.get_classification_data()
w2v = WordEmbeddings('sgns.weibo.bigram', sequence_length=15, limit=5000)
bert = BERTEmbedding('bert-base-chinese', sequence_length=15)
t_model = CNNModel(bert)
t_model.fit(train_x, train_y, epochs=1)
def test_get_sequence_tagging_data(self):
train_x, train_y = TencentDingdangSLUCorpus.get_sequence_tagging_data(
is_test=False)
assert len(train_x) == len(train_y)
assert len(train_x) > 0
def test_tencent_dingdang(self):
x, y = TencentDingdangSLUCorpus.get_classification_data()
self.assertGreater(len(x), 0)
self.assertEqual(len(x), len(y))