def training():
if debug:
# sess = K.get_session()
# sess = tfdbg.TensorBoardDebugWrapperSession(sess, "0.0.0.0:9009")
# K.set_session(sess)
K.set_session(
tfdbg.TensorBoardDebugWrapperSession(tf.Session(),
"chaowei-SYS-7048GR-TR:9010"))
if USE_MULTI_INPUT:
model, (train_x, train_y), (test_x,
test_y) = bilsm_crf_model.create_model2()
# train model
model.fit([train_x, train_x],
train_y,
batch_size=batch_size,
epochs=EPOCHS,
validation_data=[[test_x, test_x], test_y],
callbacks=gen_callbacks())
model.save('model/crf_multi.h5')
else:
model, (train_x, train_y), (test_x,
test_y) = bilsm_crf_model.create_model()
# train model
model.fit(train_x,
train_y,
batch_size=batch_size,
epochs=EPOCHS,
validation_data=[test_x, test_y],
callbacks=gen_callbacks())
model.save('model/crf.h5')
import bilsm_crf_model
import process_data
import numpy as np
model, (vocab, chunk_tags) = bilsm_crf_model.create_model(train=False)
predict_text = '中华人民共和国国务院总理周恩来在外交部长陈毅的陪同下，连续访问了埃塞俄比亚等非洲10国以及阿尔巴尼亚'
# predict_text = '樊大志同志1987年8月参加工作。先后在东北财经大学、北京国际信托投资公司、北京市境外融投资管理中心、北京市国有资产经营有限责任公司、北京证券有限责任公司、北京首都创业集团有限公司、华夏银行股份有限公司工作。'
(train_x, train_y), (test_x, test_y), (vocab1,
chunk_tags1) = process_data.load_data()
model.load_weights('model/crf.h5')
# str, length = process_data.process_data(predict_text, vocab)
# raw = model.predict(str)[0][-length:]
# print(raw)
pre_l = model.predict(test_x)
raw = [[np.argmax(row) for row in l] for l in pre_l]
tpre_nump, pre_nump = 0, 0
tpre_numl, pre_numl = 0, 0
tpre_numo, pre_numo = 0, 0
for l, r in zip(test_y, raw):
for s, i in zip(l, r):
if s == 1 and i == 1:
tpre_nump += 1
elif s == 3 and i == 3:
tpre_numl += 1
elif s == 5 and i == 5:
tpre_numo += 1
for l in raw:
import bilsm_crf_model
import process_data
import numpy as np
EPOCHS = 3
model, (train_x, train_y) = bilsm_crf_model.create_model()
model.fit(train_x[:10000], train_y[:10000],batch_size=64,epochs=EPOCHS)
model.save('model/crf.h5')
import numpy as np
import bilsm_crf_model
EPOCHS = 10
model, (train_x, train_y), (test_x,
test_y) = bilsm_crf_model.create_model(train=True)
# train model
print(train_x.shape, train_y.shape)
model.fit(train_x, train_y, batch_size=16)
model.save('model/crf.h5')
import pickle
import keras
import numpy as np
from sklearn_crfsuite.metrics import flat_classification_report
import bilsm_crf_model
import process_data
EPOCHS = 10
model, (train_x, train_y,
_), (test_x, test_y,
length), (vocab, chunk_tags) = bilsm_crf_model.create_model()
dev_x, dev_y, dev_length = process_data.load_data(use_dev=True)
# train model
# split = 7000
# define the grid search parameters
# batch_size = [10, 20, 40, 60, 80, 100]
# epochs = [16, 32, 64, 100]
# param_grid = dict(batch_size=batch_size, nb_epoch=epochs)
# grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1)
# grid_result = grid.fit(train_x[:split], train_y[:split], validation_data=[train_x[split:], train_y[split:]])
#
# print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))
# for params, mean_score, scores in grid_result.grid_scores_:
# print("%f (%f) with: %r" % (scores.mean(), scores.std(), params))
history = model.fit(
train_x,
train_y,
