def train_ner(x_train, y_train, x_valid, y_valid, x_test, y_test,
sequence_length, epoch, batch_size, bert_model_path,
model_save_path):
"""
BERT-BiLSTM-CRF 模型训练，提取症状内部特征
"""
bert_embedding = BERTEmbedding(bert_model_path,
task=kashgari.LABELING,
sequence_length=sequence_length)
model = BiLSTM_CRF_Model(bert_embedding)
eval_callback_val = EvalCallBack(kash_model=model,
valid_x=x_valid,
valid_y=y_valid,
step=1)
eval_callback_test = EvalCallBack(kash_model=model,
valid_x=x_test,
valid_y=y_test,
step=1)
model.fit(x_train,
y_train,
x_validate=x_valid,
y_validate=y_valid,
epochs=epoch,
batch_size=batch_size,
callbacks=[eval_callback_val, eval_callback_test])
model.save(model_save_path)
model.evaluate(x_test, y_test)
return model
def main():
# train_x, train_y = ChineseDailyNerCorpus.load_data("train")
# valid_x, valid_y = ChineseDailyNerCorpus.load_data("validate")
ChineseDailyNerCorpus.__zip_file__name
test_x, test_y = ChineseDailyNerCorpus.load_data("test")
# print(f"train data count: {len(train_x)}")
# print(f"validate data count: {len(valid_x)}")
print(f"test data count: {len(test_x)}")
bert_embed = BERTEmbedding("models/chinese_L-12_H-768_A-12",
task=kashgari.LABELING,
sequence_length=100)
model = BiLSTM_CRF_Model(bert_embed)
# model.fit(
# train_x,
# train_y,
# x_validate=valid_x,
# y_validate=valid_y,
# epochs=1,
# batch_size=512,
# )
model.save("models/ner.h5")
model.evaluate(test_x, test_y)
predictions = model.predict_classes(test_x)
print(predictions)
def train_BERT_BiLSTM_CRF(
train_test_devide=0.9,
epoch=20,
path='/home/peitian_zhang/data/corpus/labeled_train.txt'):
train_x, train_y = getTrain(path)
x = train_x[:int(len(train_x) * train_test_devide) + 1]
y = train_y[:int(len(train_x) * train_test_devide) + 1]
bert = BERTEmbedding(
model_folder='/home/peitian_zhang/data/chinese_L-12_H-768_A-12',
sequence_length=400,
task=kashgari.LABELING)
model = BiLSTM_CRF_Model(bert)
model.fit(x, y, x, y, epochs=epoch, batch_size=64)
print('---------evaluate on train---------\n{}'.format(
model.evaluate(train_x, train_y)))
print('---------evaluate on test----------\n{}'.format(
model.evaluate(train_x[int(len(train_x) * train_test_devide) + 1:],
train_y[int(len(train_x) * train_test_devide) + 1:])))
try:
model.save('/home/peitian_zhang/models/bert_epoch_{}'.format(epoch))
print('Success in saving!')
except:
pass
return model
def train_it2(train_path, checkpoint_filepath, model_path, start, span):
data_generator = BIODataGenerator(train_path, 100000000)
Xs, ys = data_generator.forfit().__next__()
train_x, train_y = [], []
valid_x, valid_y = [], []
rng = np.random.RandomState(0)
k = 0
for x, y in zip(Xs, ys):
# x = [str(i, 'utf-8') for i in x]
# y = [str(i, 'utf-8') for i in y]
rnum = rng.rand()
k += 1
if rnum < start or rnum >= start + span:
train_x += [x]
train_y += [y]
else:
valid_x += [x]
valid_y += [y]
# dataset = dataset.batch(32)
print('====' * 8)
print('total = ', k)
print('start , span = ', (start, span))
print('len train = ', len(train_x))
# checkpoint_filepath = './checkpoint'
if not os.path.exists(os.path.dirname(checkpoint_filepath)):
os.mkdir(os.path.dirname(checkpoint_filepath))
# train_x, train_y = ChineseDailyNerCorpus.load_data('train')
# test_x, test_y = ChineseDailyNerCorpus.load_data('test')
# valid_x, valid_y = ChineseDailyNerCorpus.load_data('valid')
# model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
# filepath=checkpoint_filepath,
# save_weights_only=True,
# monitor='val_accuracy',
# mode='max',
# save_best_only=True)
#train_x, train_y = train_x[:1000], train_y[:1000]
#valid_x, valid_y = valid_x[:200], valid_y[:200]
model = BiLSTM_CRF_Model(bert_embed, sequence_length=128)
eval_callback = Evaluator(model, checkpoint_filepath, valid_x, valid_y)
early_stop = keras.callbacks.EarlyStopping(patience=10)
reduse_lr_callback = keras.callbacks.ReduceLROnPlateau(factor=0.1,
patience=5)
# eval_callback = EvalCallBack(kash_model=model,
# x_data=valid_x,
# y_data=valid_y,
# step=1)
model.fit(train_x,
train_y,
valid_x,
valid_y,
batch_size=64,
epochs=20,
callbacks=[early_stop, eval_callback, reduse_lr_callback])
model.save(model_path)
def train_it(train_path, checkpoint_filepath, model_path, start, span):
dataset = build_dataset(train_path)
train_x, train_y = [], []
valid_x, valid_y = [], []
rng = np.random.RandomState(0)
k = 0
for x, y in dataset.as_numpy_iterator():
x = [str(i, 'utf-8') for i in x]
y = [str(i, 'utf-8') for i in y]
rnum = rng.rand()
k += 1
if rnum < start or rnum >= start + span:
train_x += [x]
train_y += [y]
else:
valid_x += [x]
valid_y += [y]
# dataset = dataset.batch(32)
print('====' * 8)
print('total = ', k)
print('start , span = ', (start, span))
print('len train = ', len(train_x))
# checkpoint_filepath = './checkpoint'
if not os.path.exists(os.path.dirname(checkpoint_filepath)):
os.mkdir(os.path.dirname(checkpoint_filepath))
# model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
# filepath=checkpoint_filepath,
# save_weights_only=True,
# monitor='val_accuracy',
# mode='max',
# save_best_only=True)
model = BiLSTM_CRF_Model(bert_embed, sequence_length=100)
evaluator = Evaluator(model, checkpoint_filepath, valid_x, valid_y)
model.fit(train_x,
train_y,
valid_x,
valid_y,
batch_size=64,
epochs=20,
callbacks=[evaluator])
model.save(model_path)
def train_BiLSTM_CRF(train_test_devide=0.9,
epoch=100,
path='/home/peitian_zhang/data/corpus/labeled_train.txt'):
train_x, train_y = getTrain(path)
model = BiLSTM_CRF_Model()
x = train_x[:int(len(train_x) * train_test_devide) + 1]
y = train_y[:int(len(train_x) * train_test_devide) + 1]
model.fit(x, y, x, y, epochs=epoch, batch_size=64)
print('---------evaluate on train---------\n{}'.format(
model.evaluate(train_x, train_y)))
print('---------evaluate on test----------\n{}'.format(
model.evaluate(train_x[int(len(train_x) * train_test_devide) + 1:],
train_y[int(len(train_x) * train_test_devide) + 1:])))
try:
model.save('/home/peitian_zhang/models/bert_epoch_{}'.format(epoch))
print('Success in saving!')
except:
pass
return model
tf_board_callback = keras.callbacks_v1.TensorBoard(log_dir='.\\logs',
update_freq=1000)
eval_callback = EvalCallBack(kash_model=model,
valid_x=test_x,
valid_y=test_y,
step=4)
model.fit(train_x,
train_y,
test_x,
test_y,
batch_size=20,
epochs=4,
callbacks=[eval_callback, tf_board_callback])
model.evaluate(test_x, test_y)
model.save('./model_8')
# 预测结果
df_out = pd.DataFrame(columns=['原文', '肿瘤原发部位', '原发病灶大小', '转移部位'])
loaded_model = kashgari.utils.load_model('model_8')
df = pd.read_excel("./data/test_no_ner.xlsx")
for index, row in df.iterrows():
data = row['原文']
''' Word2Vec '''
Y_str, S_str, Z_str = predi_output(data, 'W2V', loaded_model)
''' Bert '''
#Y_str, S_str, Z_str = predi_output(data, 'Bert', loaded_model)
print(index, Y_str, S_str, Z_str)
df_out = df_out.append(
import kashgari
from kashgari.embeddings import BERTEmbedding
from kashgari.corpus import ChineseDailyNerCorpus
from kashgari.tasks.labeling import BiLSTM_CRF_Model
train_x, train_y = ChineseDailyNerCorpus.load_data('./data/train.txt')
valid_x, valid_y = ChineseDailyNerCorpus.load_data('./data/dev.txt')
test_x, test_y = ChineseDailyNerCorpus.load_data('./data/test.txt')
bert_embed = BERTEmbedding('./chinese_L-12_H-768_A-12',
task=kashgari.LABELING,
sequence_length=100)
# 还可以选择 `CNN_LSTM_Model`, `BiLSTM_Model`, `BiGRU_Model` 或 `BiGRU_CRF_Model`
model = BiLSTM_CRF_Model(bert_embed)
model.fit(train_x,
train_y,
x_validate=valid_x,
y_validate=valid_y,
epochs=20,
batch_size=512)
model.save('saved_ner_model')
else:
x.append(rows[0])
y.append(rows[1])
return data_x, data_y
train_x, train_y = get_sequenct_tagging_data(train_path)
dev_x, dev_y = get_sequenct_tagging_data(dev_path)
test_x, test_y = get_sequenct_tagging_data(test_path)
print(f"train data count: {len(train_x)}")
print(f"validate data count: {len(dev_x)}")
print(f"test data count: {len(test_x)}")
bert_embed = BERTEmbedding(bert_path,
task=kashgari.LABELING,
sequence_length=100)
# 创建模型并训练
model = BiLSTM_CRF_Model(bert_embed)
model.fit(train_x,
train_y,
x_validate=dev_x,
y_validate=dev_y,
epochs=20,
batch_size=512)
model.save(model_path)
# 模型评估
model.evaluate(test_x, test_y)
words, labels = [], []
count = 0
for data, label in zip(datafile, labelfile):
count += 1
s1 = data.strip().split(' ')
s2 = label.strip().split(' ')
words.append(s1)
labels.append(s2)
train_x, test_x, train_y, test_y = train_test_split(words, labels, test_size=0.5, random_state=50)
bert_embed = BERTEmbedding('uncased_L-12_H-768_A-12',
trainable=False,
task=kashgari.LABELING,
sequence_length=20,
)
model = BiLSTM_CRF_Model(bert_embed)
model.fit(train_x,
train_y,
x_validate=test_x,
y_validate=test_y,
epochs=35,
batch_size=256)
model.save('model_bilstm_crf_35_256_64')
model.evaluate(x_data=test_x,y_data=test_y,batch_size=64,debug_info=True)
train_x, train_y = ChineseDailyNerCorpus.load_data('train')
valid_x, valid_y = ChineseDailyNerCorpus.load_data('validate')
test_x, test_y = ChineseDailyNerCorpus.load_data('test')
print(f"train data count: {len(train_x)}")
print(f"validate data count: {len(valid_x)}")
print(f"test data count: {len(test_x)}", test_x[0], test_y[0])
import kashgari
from kashgari.embeddings import BERTEmbedding
bert_embed = BERTEmbedding('chinese_wwm_ext_L-12_H-768_A-12',
task=kashgari.LABELING,
sequence_length=100)
from kashgari.tasks.labeling import BiLSTM_CRF_Model
# 还可以选择 `CNN_LSTM_Model`, `BiLSTM_Model`, `BiGRU_Model` 或 `BiGRU_CRF_Model`
model = BiLSTM_CRF_Model(bert_embed)
model.fit(train_x,
train_y,
x_validate=valid_x,
y_validate=valid_y,
epochs=20,
batch_size=512)
model.save('models/org_loc_per_ner.h5')
model.evaluate(test_x, test_y)
# model.evaluate(test_x, test_y)
import matplotlib.pyplot as plt
history = logs.history
plt.plot(history['accuracy'])
plt.plot(history['val_accuracy'])
plt.legend(['val_loss'])
plt.title('accuracy')
model.evaluate(test_x, test_y)
# 驗證模型印出 precision、recall、f1
# model.evaluate(test_x, test_y)
# 保存模型到 `use_this_model` 目錄下
model.save('recall85_rbt3')
# 加載保存模型
# loaded_model = kashgari.utils.load_model('ner_model')
# # 使用模進行預測
# loaded.predict(test_x[0])
# import pandas as pd
# import kashgari
# import re
# import glob
# #使用這個目錄下的模型
# loaded_model = kashgari.utils.load_model('drive/Mydrive/final_model')
import pickle
import kashgari
from kashgari.embeddings import BertEmbedding
from kashgari.tasks.labeling import BiLSTM_CRF_Model
import tensorflow as tf
with open('data.pickle', 'rb') as f:
data_dic = pickle.load(f)
x_train = data_dic[0]
x_validation = data_dic[1]
y_train = data_dic[2]
y_validation = data_dic[3]
embedding = BertEmbedding('bert-base-chinese',
sequence_length = 128)
model = BiLSTM_CRF_Model(embedding)
model.fit( x_train = x_train,
x_validate = x_validation,
y_train = y_train,
y_validate = y_validation,
epochs=5,
batch_size=32,
)
model.save('Model')
model.evaluate(x_data=x_validation,y_data=y_validation)
# 下面我们用 Bi_LSTM 模型实现一个命名实体识别任务：
from kashgari.corpus import ChineseDailyNerCorpus
from kashgari.tasks.labeling import BiLSTM_Model, BiLSTM_CRF_Model
# 加载内置数据集，此处可以替换成自己的数据集，保证格式一致即可
train_x, train_y = ChineseDailyNerCorpus.load_data('train')
test_x, test_y = ChineseDailyNerCorpus.load_data('test')
valid_x, valid_y = ChineseDailyNerCorpus.load_data('valid')
model = BiLSTM_CRF_Model()
model.fit(train_x, train_y, valid_x, valid_y, epochs=1)
model.save("BiLSTM_CRF_Model")
# -*- coding: utf-8 -*-
'''
训练包含：TIME的中文NER任务模型
'''
import kashgari
print(kashgari.__version__)
from kashgari.corpus import DataReader
from kashgari.embeddings import BERTEmbedding
from kashgari.tasks.labeling import BiLSTM_CRF_Model
train_x, train_y = DataReader().read_conll_format_file(
'data/data_all/time.train')
valid_x, valid_y = DataReader().read_conll_format_file(
'data/data_all/time.dev')
test_x, test_y = DataReader().read_conll_format_file('data/data_all/time.test')
bert_embedding = BERTEmbedding('chinese_wwm_ext_L-12_H-768_A-12',
task=kashgari.LABELING)
model = BiLSTM_CRF_Model(bert_embedding)
model.fit(train_x, train_y, valid_x, valid_y, batch_size=64, epochs=5)
model.save('models/time_ner.h5')
model.evaluate(test_x, test_y)
import kashgari
from kashgari.corpus import DataReader
from kashgari.embeddings import BERTEmbedding
from kashgari.tasks.labeling import BiLSTM_CRF_Model
from kashgari import utils
kashgari.config.use_cudnn_cell = False
train_x, train_y = DataReader().read_conll_format_file('data/data_all/example.train')
valid_x, valid_y = DataReader().read_conll_format_file('data/data_all/example.dev')
test_x, test_y = DataReader().read_conll_format_file('data/data_all/example.test')
train_x, train_y = utils.unison_shuffled_copies(train_x, train_y)
valid_x, valid_y = utils.unison_shuffled_copies(valid_x, valid_y)
test_x, test_y = utils.unison_shuffled_copies(test_x, test_y)
print(f"train data count: {len(train_x)}")
print(f"validate data count: {len(valid_x)}")
print(f"test data count: {len(test_x)}", test_x[0], test_y[0])
bert_embedding = BERTEmbedding('chinese_wwm_ext_L-12_H-768_A-12',
task=kashgari.LABELING,
sequence_length=100)
model = BiLSTM_CRF_Model(bert_embedding)
model.fit(train_x, train_y, valid_x, valid_y, batch_size=512, epochs=20)
model.save('models/all_ner.h5')
model.evaluate(test_x, test_y)
model = BiLSTM_CRF_Model(bert_embed)
model.fit(train_x,
train_y,
x_validate=valid_x,
y_validate=valid_y,
callbacks=[stop_callback, save_callback],
batch_size=250,
epochs=25)
# 验证模型，此方法将打印出详细的验证报告
model.evaluate(test_x, test_y)
# 保存模型到 `model_name` 目录下
model.save('5_29_1')
# 加载保存模型
loaded_model = kashgari.utils.load_model('5_29_1')
# 使用模型进行预测
loaded_model.predict(test_x[:100])
# print(len(all_sen), len(aspectTerm_t), len(all_ans))
# # 存成txt檔
# with open("/content/drive/My Drive/tibame/ans.txt","w") as f:
# f.write(str(all_ans))
# with open("/content/drive/My Drive/tibame/sen.txt","w") as f:
# -*- coding: utf-8 -*-
# time: 2019-08-09 16:47
# place: Zhichunlu Beijing
import kashgari
from kashgari.corpus import DataReader
from kashgari.embeddings import BERTEmbedding
from kashgari.tasks.labeling import BiLSTM_CRF_Model
train_x, train_y = DataReader().read_conll_format_file('./data/time.train')
valid_x, valid_y = DataReader().read_conll_format_file('./data/time.dev')
test_x, test_y = DataReader().read_conll_format_file('./data/time.test')
bert_embedding = BERTEmbedding('chinese_wwm_ext_L-12_H-768_A-12',
task=kashgari.LABELING,
sequence_length=128)
model = BiLSTM_CRF_Model(bert_embedding)
model.fit(train_x, train_y, valid_x, valid_y, batch_size=16, epochs=10)
model.save('time_ner.h5')
model.evaluate(test_x, test_y)
with open("data_test.pkl", "rb") as f:
x_test, y_test = pickle.load(f)
x_train, y_train = list(map(list, x_train)), list(map(list, y_train))
x_valid, y_valid = list(map(list, x_valid)), list(map(list, y_valid))
x_test, y_test = list(map(list, x_test)), list(map(list, y_test))
# Skip testing for now
x_train, y_train = x_train + x_test, y_train + y_test
model_dir = 'bert_tagger'
log_dir = os.path.join(model_dir, 'logs')
weights_path = os.path.join(log_dir, 'weights.h5')
BERT_PATH = '/mnt/DATA/data/embeddings/uncased_L-12_H-768_A-12'
EARLY_STOP = 10
bert_embed = BERTEmbedding(BERT_PATH, task=kashgari.LABELING)
model = BiLSTM_CRF_Model(bert_embed)
model.fit(x_train,
y_train,
x_valid,
y_valid,
epochs=10,
batch_size=64,
callbacks=[
TensorBoard(log_dir=log_dir, write_graph=False),
ModelCheckpoint(weights_path, save_weights_only=True),
ReduceLROnPlateau()
])
print('Saving the model...')
model.save(model_dir)
from kashgari.utils import load_model