def __init__(self,
flags,
mode='train',
beam_search=False,
max_gradient_norm=5.0):
self.flags = flags
word2id, id2word, trainingSamples = loadDataset(flags.dataset_filename)
self.word2id = word2id
self.id2word = id2word
self.trainingSamples = trainingSamples
self.vocab_size = len(word2id)
self.goToken = flags.goToken
self.endToken = flags.endToken
# hyper parameters
self.rnn_size = flags.rnn_size # Number of hidden units in each layer
# self.batch_size = flags.batch_size
self.beam_search = beam_search
self.beam_size = flags.beam_size
self.mode = mode
self.learning_rate = flags.learning_rate
self.max_gradient_norm = max_gradient_norm
self.num_layers = flags.num_layers # Number of layers in each encoder and decoder
self.embedding_size = flags.embedding_size # Embedding matrix size [vocab_size,embedding_size]
# model
self.batch_size = None
self.keep_prob_dropout = None
self.encoder_inputs = None
self.encoder_inputs_length = None
self.decoder_targets = None
self.decoder_targets_length = None
self.max_target_sequence_length = None
self.mask = None
self.decoder_logits_train = None
self.decoder_predict_train = None
self.loss = None
self.summary_op = None
self.train_op = None
self.decoder_predict_decoder = None
self.saver = None
self.build_model()
import os
tf.app.flags.DEFINE_integer('rnn_size', 1024, 'Number of hidden units in each layer')
tf.app.flags.DEFINE_integer('num_layers', 2, 'Number of layers in each encoder and decoder')
tf.app.flags.DEFINE_integer('embedding_size', 1024, 'Embedding dimensions of encoder and decoder inputs')
tf.app.flags.DEFINE_float('learning_rate', 0.01, 'Learning rate')
tf.app.flags.DEFINE_integer('batch_size', 128, 'Batch size')
tf.app.flags.DEFINE_integer('numEpochs', 3, 'Maximum # of training epochs')
tf.app.flags.DEFINE_integer('steps_per_checkpoint', 100, 'Save model checkpoint every this iteration')
tf.app.flags.DEFINE_string('model_dir', 'ckpt/', 'Path to save model checkpoints')
tf.app.flags.DEFINE_string('model_name', 'chatbot.ckpt', 'File name used for model checkpoints')
FLAGS = tf.app.flags.FLAGS
data_path = '/Users/shengwan/Desktop/seq2seq_chatbot_new-master/data/dataset-cornell-length10-filter1-vocabSize40000.pkl'
word2id, id2word, trainingSamples = loadDataset(data_path)
with tf.Session() as sess:
model = Seq2SeqModel(FLAGS.rnn_size, FLAGS.num_layers, FLAGS.embedding_size, FLAGS.learning_rate, word2id,
mode='train', use_attention=True, beam_search=False, beam_size=5, max_gradient_norm=5.0)
ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir)
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
print('Reloading model parameters from ', ckpt.model_checkpoint_path)
sess = model.restore_last_session()
else:
print('Created new model parameters..')
sess.run(tf.global_variables_initializer())
current_step = 0
summary_writer = tf.summary.FileWriter(FLAGS.model_dir, graph=sess.graph)
for e in range(FLAGS.numEpochs):
print("----- Epoch {}/{} -----".format(e + 1, FLAGS.numEpochs))
'Number of layers in each encoder and decoder')
tf.flags.DEFINE_integer('embedding_size', 1024,
'Embedding dimensions of encoder and decoder inputs')
tf.flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate')
tf.flags.DEFINE_integer('batch_size', 128, 'Batch size')
tf.flags.DEFINE_integer('min_freq', 0, 'Minimum # of word frequency')
tf.flags.DEFINE_integer('numEpochs', 30, 'Maximum # of training epochs')
tf.flags.DEFINE_integer('steps_per_checkpoint', 100,
'Save model checkpoint every this iteration')
tf.flags.DEFINE_string('model_dir', 'model/', 'Path to save model checkpoints')
tf.flags.DEFINE_string('model_name', 'chatbot.ckpt',
'File name used for model checkpoints')
tf.flags.DEFINE_string('data_path', 'test1.txt', 'data_dir')
FLAGS = tf.flags.FLAGS
word2id, id2word, trainingSamples = loadDataset(FLAGS.data_path,
FLAGS.min_freq)
def predict_ids_to_seq(predict_ids, id2word, beam_szie):
'''
将beam_search返回的结果转化为字符串
:param predict_ids: 列表，长度为batch_size，每个元素都是decode_len*beam_size的数组
:param id2word: vocab字典
:return:
'''
for single_predict in predict_ids:
for i in range(beam_szie):
predict_list = np.ndarray.tolist(single_predict[:, :, i])
predict_seq = [id2word[idx] for idx in predict_list[0]]
print(" ".join(predict_seq))
import tensorflow as tf
import numpy as np
import os
import data_helpers
from tensorflow.contrib import learn
from sklearn import metrics
from model.bert import Model, BertConfig
# CHANGE THIS: Load data. Load your own data here
config_parm = BertConfig()
if config_parm.test_file:
x_text, y, y_onehot = data_helpers.load_data_and_labels(config_parm,
is_training=False)
word2id, id2word = data_helpers.loadDataset(config_parm)
y_test = y
y_test = list(y_test)
else:
print("please set a test_file path to test model")
x_text = ""
y_test = ""
# Map data into vocabulary
print("\nEvaluating...\n")
# reset graph
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #不全部占满显存, 按需分配
tf.reset_default_graph()
graph_1 = tf.Graph()
tf.app.flags.DEFINE_integer('rnn_size', 1024, 'Number of hidden units in each layer')
tf.app.flags.DEFINE_integer('num_layers', 2, 'Number of layers in each encoder and decoder')
tf.app.flags.DEFINE_integer('embedding_size', 1024, 'Embedding dimensions of encoder and decoder inputs')
tf.app.flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate')
tf.app.flags.DEFINE_integer('batch_size', 128, 'Batch size')
tf.app.flags.DEFINE_integer('numEpochs', 30, 'Maximum # of training epochs')
tf.app.flags.DEFINE_integer('steps_per_checkpoint', 100, 'Save model checkpoint every this iteration')
tf.app.flags.DEFINE_string('model_dir', 'model/', 'Path to save model checkpoints')
tf.app.flags.DEFINE_string('model_name', 'chatbot.ckpt', 'File name used for model checkpoints')
FLAGS = tf.app.flags.FLAGS
data_path = 'data/dataset-cornell-length10-filter1-vocabSize40000.pkl'
word2id, id2word, trainingSamples = loadDataset(data_path)
def predict_ids_to_seq(predict_ids, id2word, beam_szie):
'''
将beam_search返回的结果转化为字符串
:param predict_ids: 列表，长度为batch_size，每个元素都是decode_len*beam_size的数组
:param id2word: vocab字典
:return:
'''
for single_predict in predict_ids:
for i in range(beam_szie):
predict_list = np.ndarray.tolist(single_predict[:, :, i])
predict_seq = [id2word[idx] for idx in predict_list[0]]
print(" ".join(predict_seq))
with tf.Session() as sess:
