def __init__(self, flags, mode='train', beam_search=False, max_gradient_norm=5.0): self.flags = flags word2id, id2word, trainingSamples = loadDataset(flags.dataset_filename) self.word2id = word2id self.id2word = id2word self.trainingSamples = trainingSamples self.vocab_size = len(word2id) self.goToken = flags.goToken self.endToken = flags.endToken # hyper parameters self.rnn_size = flags.rnn_size # Number of hidden units in each layer # self.batch_size = flags.batch_size self.beam_search = beam_search self.beam_size = flags.beam_size self.mode = mode self.learning_rate = flags.learning_rate self.max_gradient_norm = max_gradient_norm self.num_layers = flags.num_layers # Number of layers in each encoder and decoder self.embedding_size = flags.embedding_size # Embedding matrix size [vocab_size,embedding_size] # model self.batch_size = None self.keep_prob_dropout = None self.encoder_inputs = None self.encoder_inputs_length = None self.decoder_targets = None self.decoder_targets_length = None self.max_target_sequence_length = None self.mask = None self.decoder_logits_train = None self.decoder_predict_train = None self.loss = None self.summary_op = None self.train_op = None self.decoder_predict_decoder = None self.saver = None self.build_model()
import os tf.app.flags.DEFINE_integer('rnn_size', 1024, 'Number of hidden units in each layer') tf.app.flags.DEFINE_integer('num_layers', 2, 'Number of layers in each encoder and decoder') tf.app.flags.DEFINE_integer('embedding_size', 1024, 'Embedding dimensions of encoder and decoder inputs') tf.app.flags.DEFINE_float('learning_rate', 0.01, 'Learning rate') tf.app.flags.DEFINE_integer('batch_size', 128, 'Batch size') tf.app.flags.DEFINE_integer('numEpochs', 3, 'Maximum # of training epochs') tf.app.flags.DEFINE_integer('steps_per_checkpoint', 100, 'Save model checkpoint every this iteration') tf.app.flags.DEFINE_string('model_dir', 'ckpt/', 'Path to save model checkpoints') tf.app.flags.DEFINE_string('model_name', 'chatbot.ckpt', 'File name used for model checkpoints') FLAGS = tf.app.flags.FLAGS data_path = '/Users/shengwan/Desktop/seq2seq_chatbot_new-master/data/dataset-cornell-length10-filter1-vocabSize40000.pkl' word2id, id2word, trainingSamples = loadDataset(data_path) with tf.Session() as sess: model = Seq2SeqModel(FLAGS.rnn_size, FLAGS.num_layers, FLAGS.embedding_size, FLAGS.learning_rate, word2id, mode='train', use_attention=True, beam_search=False, beam_size=5, max_gradient_norm=5.0) ckpt = tf.train.get_checkpoint_state(FLAGS.model_dir) if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path): print('Reloading model parameters from ', ckpt.model_checkpoint_path) sess = model.restore_last_session() else: print('Created new model parameters..') sess.run(tf.global_variables_initializer()) current_step = 0 summary_writer = tf.summary.FileWriter(FLAGS.model_dir, graph=sess.graph) for e in range(FLAGS.numEpochs): print("----- Epoch {}/{} -----".format(e + 1, FLAGS.numEpochs))
'Number of layers in each encoder and decoder') tf.flags.DEFINE_integer('embedding_size', 1024, 'Embedding dimensions of encoder and decoder inputs') tf.flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate') tf.flags.DEFINE_integer('batch_size', 128, 'Batch size') tf.flags.DEFINE_integer('min_freq', 0, 'Minimum # of word frequency') tf.flags.DEFINE_integer('numEpochs', 30, 'Maximum # of training epochs') tf.flags.DEFINE_integer('steps_per_checkpoint', 100, 'Save model checkpoint every this iteration') tf.flags.DEFINE_string('model_dir', 'model/', 'Path to save model checkpoints') tf.flags.DEFINE_string('model_name', 'chatbot.ckpt', 'File name used for model checkpoints') tf.flags.DEFINE_string('data_path', 'test1.txt', 'data_dir') FLAGS = tf.flags.FLAGS word2id, id2word, trainingSamples = loadDataset(FLAGS.data_path, FLAGS.min_freq) def predict_ids_to_seq(predict_ids, id2word, beam_szie): ''' 将beam_search返回的结果转化为字符串 :param predict_ids: 列表,长度为batch_size,每个元素都是decode_len*beam_size的数组 :param id2word: vocab字典 :return: ''' for single_predict in predict_ids: for i in range(beam_szie): predict_list = np.ndarray.tolist(single_predict[:, :, i]) predict_seq = [id2word[idx] for idx in predict_list[0]] print(" ".join(predict_seq))
import tensorflow as tf import numpy as np import os import data_helpers from tensorflow.contrib import learn from sklearn import metrics from model.bert import Model, BertConfig # CHANGE THIS: Load data. Load your own data here config_parm = BertConfig() if config_parm.test_file: x_text, y, y_onehot = data_helpers.load_data_and_labels(config_parm, is_training=False) word2id, id2word = data_helpers.loadDataset(config_parm) y_test = y y_test = list(y_test) else: print("please set a test_file path to test model") x_text = "" y_test = "" # Map data into vocabulary print("\nEvaluating...\n") # reset graph config = tf.ConfigProto() config.gpu_options.allow_growth = True #不全部占满显存, 按需分配 tf.reset_default_graph() graph_1 = tf.Graph()
tf.app.flags.DEFINE_integer('rnn_size', 1024, 'Number of hidden units in each layer') tf.app.flags.DEFINE_integer('num_layers', 2, 'Number of layers in each encoder and decoder') tf.app.flags.DEFINE_integer('embedding_size', 1024, 'Embedding dimensions of encoder and decoder inputs') tf.app.flags.DEFINE_float('learning_rate', 0.0001, 'Learning rate') tf.app.flags.DEFINE_integer('batch_size', 128, 'Batch size') tf.app.flags.DEFINE_integer('numEpochs', 30, 'Maximum # of training epochs') tf.app.flags.DEFINE_integer('steps_per_checkpoint', 100, 'Save model checkpoint every this iteration') tf.app.flags.DEFINE_string('model_dir', 'model/', 'Path to save model checkpoints') tf.app.flags.DEFINE_string('model_name', 'chatbot.ckpt', 'File name used for model checkpoints') FLAGS = tf.app.flags.FLAGS data_path = 'data/dataset-cornell-length10-filter1-vocabSize40000.pkl' word2id, id2word, trainingSamples = loadDataset(data_path) def predict_ids_to_seq(predict_ids, id2word, beam_szie): ''' 将beam_search返回的结果转化为字符串 :param predict_ids: 列表,长度为batch_size,每个元素都是decode_len*beam_size的数组 :param id2word: vocab字典 :return: ''' for single_predict in predict_ids: for i in range(beam_szie): predict_list = np.ndarray.tolist(single_predict[:, :, i]) predict_seq = [id2word[idx] for idx in predict_list[0]] print(" ".join(predict_seq)) with tf.Session() as sess: