def __init__(self, model_path, vocab_path, learning_rate=0.01, batch_size=1): self.word_to_idx = load_word_dict(vocab_path) self.idx_to_word = {v: k for k, v in self.word_to_idx.items()} self.learning_rate = learning_rate self.batch_size = batch_size t1 = time.time() tf.reset_default_graph() self.input_data = tf.placeholder(tf.int32, [batch_size, None]) self.output_targets = tf.placeholder(tf.int32, [batch_size, None]) # init model self.model = rnn_model(model='lstm', input_data=self.input_data, output_data=self.output_targets, vocab_size=len(self.word_to_idx), rnn_size=128, num_layers=2, batch_size=batch_size, learning_rate=learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) self.sess = tf.Session() # init op self.sess.run(init_op) checkpoint = tf.train.latest_checkpoint(model_path) saver.restore(self.sess, checkpoint)
def ppl(sentence_list): ppl_list = [] # load data dict word_to_int = load_word_dict(conf.word_dict_path) # init params batch_size = 1 tf.reset_default_graph() input_data = tf.placeholder(tf.int32, [batch_size, None]) output_targets = tf.placeholder(tf.int32, [batch_size, None]) # init model end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(word_to_int), rnn_size=128, num_layers=2, batch_size=batch_size, learning_rate=conf.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # init op sess.run(init_op) checkpoint = tf.train.latest_checkpoint(conf.model_dir) saver.restore(sess, checkpoint) print("loading model from the checkpoint {0}".format(checkpoint)) # infer each sentence for sentence in sentence_list: ppl = 0 # data idx x = [ word_to_int[c] if c in word_to_int else word_to_int[UNK_TOKEN] for c in sentence ] x = [word_to_int[START_TOKEN]] + x + [word_to_int[END_TOKEN]] print('x:', x) # reshape y = np.array(x[1:]).reshape((-1, batch_size)) x = np.array(x[:-1]).reshape((-1, batch_size)) print(x.shape) print(y.shape) # get each word perplexity word_count = x.shape[0] for i in range(word_count): perplexity = sess.run(end_points['perplexity'], feed_dict={ input_data: x[i:i + 1, :], output_targets: y[i:i + 1, :] }) print('{0} -> {1}, perplexity: {2}'.format( x[i:i + 1, :], y[i:i + 1, :], perplexity)) if i == 0 or i == word_count: continue ppl += perplexity ppl /= (word_count - 2) print('perplexity:' + str(ppl)) ppl_list.append(ppl) return ppl_list
def ppl(sentence_list): ppl_list = [] # load data dict word_to_int = load_word_dict(conf.word_dict_path) # init params batch_size = 1 tf.reset_default_graph() input_data = tf.placeholder(tf.int32, [batch_size, None]) output_targets = tf.placeholder(tf.int32, [batch_size, None]) # init model end_points = rnn_model(model='lstm', input_data=input_data, output_data=output_targets, vocab_size=len(word_to_int), rnn_size=128, num_layers=2, batch_size=batch_size, learning_rate=conf.learning_rate) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: # init op sess.run(init_op) checkpoint = tf.train.latest_checkpoint(conf.model_dir) saver.restore(sess, checkpoint) print("loading model from the checkpoint {0}".format(checkpoint)) # infer each sentence for sentence in sentence_list: ppl = 0 # data idx x = [word_to_int[c] if c in word_to_int else word_to_int[UNK_TOKEN] for c in sentence] x = [word_to_int[START_TOKEN]] + x + [word_to_int[END_TOKEN]] print('x:', x) # reshape y = np.array(x[1:]).reshape((-1, batch_size)) x = np.array(x[:-1]).reshape((-1, batch_size)) print(x.shape) print(y.shape) # get each word perplexity word_count = x.shape[0] for i in range(word_count): perplexity = sess.run(end_points['perplexity'], feed_dict={input_data: x[i:i + 1, :], output_targets: y[i:i + 1, :]}) print('{0} -> {1}, perplexity: {2}'.format(x[i:i + 1, :], y[i:i + 1, :], perplexity)) if i == 0 or i == word_count: continue ppl += perplexity ppl /= (word_count - 2) print('perplexity:' + str(ppl)) ppl_list.append(ppl) return ppl_list
def generate(begin_word): batch_size = 1 word_to_idx = load_word_dict(config.word_dict_path) vocabularies = [k for k, v in word_to_idx.items()] tf.reset_default_graph() input_data = tf.placeholder(tf.int32, [batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len(word_to_idx), rnn_size=128, num_layers=2, batch_size=64, learning_rate=0.0002) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) checkpoint = tf.train.latest_checkpoint(config.model_dir) saver.restore(sess, checkpoint) print("loading model from the checkpoint {0}".format(checkpoint)) x = np.array([list(map(word_to_idx.get, START_TOKEN))]) [predict, last_state ] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x}) if begin_word: word = begin_word else: word = to_word(predict, vocabularies) sentence = '' i = 0 while word != END_TOKEN and word != START_TOKEN and word != UNK_TOKEN: sentence += word i += 1 if i >= 24: break x = np.zeros((1, 1)) try: x[0, 0] = word_to_idx[word] except KeyError: print("please enter a chinese char again.") break [predict, last_state ] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={ input_data: x, end_points['initial_state']: last_state }) word = to_word(predict, vocabularies) return sentence
def generate(begin_word): batch_size = 1 word_to_int = load_word_dict(conf.word_dict_path) vocabularies = [k for k, v in word_to_int.items()] tf.reset_default_graph() input_data = tf.placeholder(tf.int32, [batch_size, None]) end_points = rnn_model(model='lstm', input_data=input_data, output_data=None, vocab_size=len(word_to_int), rnn_size=128, num_layers=2, batch_size=64, learning_rate=0.0002) saver = tf.train.Saver(tf.global_variables()) init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) with tf.Session() as sess: sess.run(init_op) checkpoint = tf.train.latest_checkpoint(conf.model_dir) saver.restore(sess, checkpoint) print("loading model from the checkpoint {0}".format(checkpoint)) x = np.array([list(map(word_to_int.get, START_TOKEN))]) [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x}) if begin_word: word = begin_word else: word = to_word(predict, vocabularies) sentence = '' i = 0 while word != END_TOKEN and word != START_TOKEN and word != UNK_TOKEN: sentence += word i += 1 if i >= 24: break x = np.zeros((1, 1)) try: x[0, 0] = word_to_int[word] except KeyError: print("please enter a chinese char again.") break [predict, last_state] = sess.run([end_points['prediction'], end_points['last_state']], feed_dict={input_data: x, end_points['initial_state']: last_state}) word = to_word(predict, vocabularies) return sentence