def chatbot_api(infos): du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) batch_size = 1 tf.reset_default_graph() model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='decode', **model_config) # 创建session的时候允许显存增长 config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config=config) as sess: init = tf.global_variables_initializer() sess.run(init) model.load(sess, save_path) while True: q = infos if q is None or q.strip() == '': return "请输入聊天信息" continue q = q.strip() indexs = du.transform_sentence(q) x = np.asarray(indexs).reshape((1, -1)) xl = np.asarray(len(indexs)).reshape((1, )) pred = model.predict(sess, np.array(x), np.array(xl)) result = du.transform_indexs(pred[0]) return result
def train(): """ 训练模型 :return: """ du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) steps = int(len(du) / batch_size) + 1 # 创建session的时候设置显存根据需要动态申请 tf.reset_default_graph() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Graph().as_default(): with tf.Session(config=config) as sess: # 定义模型 model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='train', **model_config) init = tf.global_variables_initializer() writer = tf.summary.FileWriter('./graph/nlp', sess.graph) sess.run(init) if continue_train: model.load(sess, save_path) model.export(sess) '''
def train(): """ 训练模型 :return: """ du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) trainset, _, _, _ = pickle.load(open('dataset.pkl', 'rb')) steps = int(len(list(trainset)) / batch_size) + 1 print(trainset.shape) # 创建session的时候设置显存根据需要动态申请 tf.reset_default_graph() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Graph().as_default(): with tf.Session(config=config) as sess: # 定义模型 model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='train', **model_config) init = tf.global_variables_initializer() sess.run(init) if continue_train: model.load(sess, save_path) loss_list = [] for epoch in range(1, n_epoch + 1): costs = [] bar = tqdm(range(steps), total=steps, desc='epoch {}, loss=0.000000'.format(epoch)) for time in bar: x, xl, y, yl = du.next_batch(batch_size, list(trainset)) max_len = np.max(yl) y = y[:, 0:max_len] cost, lr = model.train(sess, x, xl, y, yl, keep_prob) costs.append(cost) if epoch == 1 and time == 0: loss_list.append(np.mean(costs)) if time == steps - 1: loss_list.append(np.mean(costs)) bar.set_description( 'epoch {} loss={:.6f} lr={:.6f}'.format( epoch, np.mean(costs), lr)) epoch_list = list(range(0, n_epoch + 1)) plt.xlabel('Epoch Number') plt.ylabel('Loss Value') plt.plot(epoch_list, loss_list) plt.show() model.save(sess, save_path=save_path)
def train(): """ 训练模型 :return: """ du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) steps = int(len(du) / batch_size) + 1 # len(du)处理后的语料库中问答对的数量 # 创建session的时候设置显存根据需要动态申请 tf.reset_default_graph() config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = 0.9 config.gpu_options.allow_growth = True with tf.Graph().as_default(): with tf.Session(config=config) as sess: # 定义模型 model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=du.vocab_size, decoder_vocab_size=du.vocab_size, mode='train', **model_config) init = tf.global_variables_initializer() writer = tf.summary.FileWriter('./graph/nlp', sess.graph) sess.run(init) if continue_train: model.load(sess, save_path) emb = pickle.load(open('data/emb.pkl', 'rb')) model.feed_embedding(sess, encoder=emb, decoder=emb) for epoch in range(1, n_epoch + 1): costs = [] bar = tqdm(range(steps), total=steps, desc='epoch {}, loss=0.000000'.format(epoch)) #进度条 for _ in bar: x, xl, y, yl = du.next_batch( batch_size ) # x为question,xl为question实际长度;y为answer,yl为answer实际长度 max_len = np.max(yl) # 实际最长句子的长度 y = y[:, 0:max_len] # 表示所有行的第0:max_len列 cost, lr = model.train(sess, x, xl, y, yl, keep_prob) costs.append(cost) bar.set_description( 'epoch {} loss={:.6f} lr={:.6f}'.format( epoch, np.mean(costs), lr)) model.save(sess, save_path=save_path)
def handle_data(): du = DataProcessing.DataUnit(**data_config) fullset = np.array(du.data) index_test = np.random.choice(fullset.shape[0], 13503, replace=False) testset = fullset[index_test] index_full = np.arange(fullset.shape[0]) index_rest = np.delete(index_full, index_test) restset = fullset[index_rest] index_validate = np.random.choice(restset.shape[0], 13503, replace=False) validateset = restset[index_validate] index_rest_full = np.arange(restset.shape[0]) index_rest_rest = np.delete(index_rest_full, index_validate) trainset = restset[index_rest_rest] # in_corpus_data = random.sample(list(trainset),13503) # print(in_corpus_data) print(testset.shape) print(validateset.shape) print(trainset.shape) pickle.dump((trainset, validateset, testset, du), open('dataset.pkl', 'wb'))
def __init__(self): print("init111") self.du = DataProcessing.DataUnit(**data_config) save_path = os.path.join(BASE_MODEL_DIR, MODEL_NAME) batch_size = 1 tf.reset_default_graph() self.model = Seq2Seq(batch_size=batch_size, encoder_vocab_size=self.du.vocab_size, decoder_vocab_size=self.du.vocab_size, mode='decode', **model_config) # 创建session的时候允许显存增长 #config = tf.ConfigProto() #config.gpu_options.allow_growth = True config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) self.sess = tf.Session(config=config) init = tf.global_variables_initializer() self.sess.run(init) self.model.load(self.sess, save_path)