def chatbot_api(infos):
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
batch_size = 1
tf.reset_default_graph()
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='decode',
**model_config)
# 创建session的时候允许显存增长
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
model.load(sess, save_path)
while True:
q = infos
if q is None or q.strip() == '':
return "请输入聊天信息"
continue
q = q.strip()
indexs = du.transform_sentence(q)
x = np.asarray(indexs).reshape((1, -1))
xl = np.asarray(len(indexs)).reshape((1, ))
pred = model.predict(sess, np.array(x), np.array(xl))
result = du.transform_indexs(pred[0])
return result
def predict():
"""预测模块"""
du = DataUnits(pm.data_path, pm.word2id_path, 'decode')
batch_size = 1
model = Seq2Seq(batch_size=batch_size, mode='decode')
# 加载模型
session = tf.Session()
session.run(tf.global_variables_initializer())
save_path = tf.train.latest_checkpoint(
pm.save_path) # latest_checkpoint() 方法查找最新保存的模型
saver = tf.train.Saver()
saver.restore(sess=session, save_path=save_path)
while True:
predict_contents = input('请输入对话, "E" 结束对话： ')
predict_contents = predict_contents.replace(' ', '')
if predict_contents == 'E':
print('对话结束！')
break
elif not predict_contents:
continue
contents2id = du.sequence2id(predict_contents)
x = np.asarray(contents2id).reshape((1, -1))
xl = np.asarray(len(contents2id)).reshape((1, ))
pred = model.predict(session, np.array(x), np.array(xl))[0]
res = du.id2sequence(pred) # 将id转化为句子
print(f'机器人回答：{res}\n')
def train():
"""
训练模型
:return:
"""
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
steps = int(len(du) / batch_size) + 1
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='train',
**model_config)
init = tf.global_variables_initializer()
writer = tf.summary.FileWriter('./graph/nlp', sess.graph)
sess.run(init)
if continue_train:
model.load(sess, save_path)
model.export(sess)
'''
def train():
"""
训练模型
:return:
"""
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
trainset, _, _, _ = pickle.load(open('dataset.pkl', 'rb'))
steps = int(len(list(trainset)) / batch_size) + 1
print(trainset.shape)
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='train',
**model_config)
init = tf.global_variables_initializer()
sess.run(init)
if continue_train:
model.load(sess, save_path)
loss_list = []
for epoch in range(1, n_epoch + 1):
costs = []
bar = tqdm(range(steps),
total=steps,
desc='epoch {}, loss=0.000000'.format(epoch))
for time in bar:
x, xl, y, yl = du.next_batch(batch_size, list(trainset))
max_len = np.max(yl)
y = y[:, 0:max_len]
cost, lr = model.train(sess, x, xl, y, yl, keep_prob)
costs.append(cost)
if epoch == 1 and time == 0:
loss_list.append(np.mean(costs))
if time == steps - 1:
loss_list.append(np.mean(costs))
bar.set_description(
'epoch {} loss={:.6f} lr={:.6f}'.format(
epoch, np.mean(costs), lr))
epoch_list = list(range(0, n_epoch + 1))
plt.xlabel('Epoch Number')
plt.ylabel('Loss Value')
plt.plot(epoch_list, loss_list)
plt.show()
model.save(sess, save_path=save_path)
def test():
trainset, validateset, _, du = pickle.load(open('dataset.pkl', 'rb'))
q, r = getSampleQA(trainset)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
batch_size = 1
steps = int(len(list(validateset)) / batch_size) + 1
tf.reset_default_graph()
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='decode',
**model_config)
# 创建session的时候允许显存增长
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
init = tf.global_variables_initializer()
sess.run(init)
model.load(sess, save_path)
bar = tqdm(range(steps), total=steps, desc='time:0')
cGram1 = []
cGram2 = []
cGram3 = []
cGram4 = []
for time in bar:
x, xl, references = du.next_batch_test(batch_size, list(q),
list(r), time)
# print(xl[0:2])
# print(x[0:2])
pred = model.predict(sess, np.array(x), np.array(xl))
result = du.transform_indexs(pred[0])
# print(result)
# print("references {}".format(references))
candidates = list(result)
# print("candidate:{}".format(candidates))
cGram1.append(
sentence_bleu(references, candidates, weights=(1, 0, 0, 0)))
cGram2.append(
sentence_bleu(references, candidates,
weights=(0.5, 0.5, 0, 0)))
cGram3.append(
sentence_bleu(references,
candidates,
weights=(0.33, 0.33, 0.33, 0)))
cGram4.append(
sentence_bleu(references,
candidates,
weights=(0.25, 0.25, 0.25, 0.25)))
bar.set_description('time={}'.format(time))
print('Cumulative 1-gram: %f' % np.mean(cGram1))
print('Cumulative 2-gram: %f' % np.mean(cGram2))
print('Cumulative 3-gram: %f' % np.mean(cGram3))
print('Cumulative 4-gram: %f' % np.mean(cGram4))
def train():
"""
训练模型
:return:
"""
du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
steps = int(len(du) / batch_size) + 1 # len(du)处理后的语料库中问答对的数量
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=du.vocab_size,
decoder_vocab_size=du.vocab_size,
mode='train',
**model_config)
init = tf.global_variables_initializer()
writer = tf.summary.FileWriter('./graph/nlp', sess.graph)
sess.run(init)
if continue_train:
model.load(sess, save_path)
emb = pickle.load(open('data/emb.pkl', 'rb'))
model.feed_embedding(sess, encoder=emb, decoder=emb)
for epoch in range(1, n_epoch + 1):
costs = []
bar = tqdm(range(steps),
total=steps,
desc='epoch {}, loss=0.000000'.format(epoch)) #进度条
for _ in bar:
x, xl, y, yl = du.next_batch(
batch_size
) # x为question,xl为question实际长度；y为answer,yl为answer实际长度
max_len = np.max(yl) # 实际最长句子的长度
y = y[:, 0:max_len] # 表示所有行的第0:max_len列
cost, lr = model.train(sess, x, xl, y, yl, keep_prob)
costs.append(cost)
bar.set_description(
'epoch {} loss={:.6f} lr={:.6f}'.format(
epoch, np.mean(costs), lr))
model.save(sess, save_path=save_path)
def train():
"""训练模型"""
save_path = os.path.join(pm.save_path, 'best_validation')
# 获取数据
du = DataUnits(pm.data_path, pm.word2id_path, 'train')
# 创建session的时候设置显存根据需要动态申请
tf.reset_default_graph()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = False
with tf.Graph().as_default():
with tf.Session(config=config) as sess:
# 定义模型
model = Seq2Seq(batch_size=pm.batch_size, mode='train')
init = tf.global_variables_initializer()
writer = tf.summary.FileWriter(pm.graph, sess.graph)
sess.run(init)
if continue_train:
model.load(sess, save_path=pm.save_path)
for epoch in range(1, pm.num_epochs + 1):
print(f'epoch: {epoch}')
start_time = time.time()
costs = []
for x, xl, y, yl in du.next_batch(batch_size=pm.batch_size):
# x为question,xl为question实际长度；y为answer,yl为answer实际长度
max_len = np.max(yl) # 实际最长句子的长度
y = y[:, 0:max_len] # 表示所有行的第0:max_len列
cost, lr, global_step = model.train(sess, x, xl, y, yl)
costs.append(cost)
if global_step % 30 == 0:
model.save(sess, global_step, save_path=save_path)
print('ModelSave...')
print(
f'global_step: {global_step}, cost: {round(cost, 6)}, lr: {round(lr, 8)}'
)
# bar.set_description('epoch {} loss={:.6f} lr={:.6f}'.format(epoch, np.mean(costs), lr))
print(
f'--------------------------------\nmean_loss: {np.mean(costs)}\n--------------------------------'
)
end_time = time.time()
print(f'epoch: {epoch}-->时间：{round(end_time-start_time, 2)}秒')
def __init__(self):
print("init111")
self.du = DataProcessing.DataUnit(**data_config)
save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME)
batch_size = 1
tf.reset_default_graph()
self.model = Seq2Seq(batch_size=batch_size,
encoder_vocab_size=self.du.vocab_size,
decoder_vocab_size=self.du.vocab_size,
mode='decode',
**model_config)
# 创建session的时候允许显存增长
#config = tf.ConfigProto()
#config.gpu_options.allow_growth = True
config = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
self.sess = tf.Session(config=config)
init = tf.global_variables_initializer()
self.sess.run(init)
self.model.load(self.sess, save_path)