def main():
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data)
train_batch_len = train_data_len // TRAIN_BATCH_SIZE
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
initializer = tf.random_uniform_initializer(-0.05, 0.05)
with tf.variable_scope("language_model",
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
with tf.variable_scope("language_model",
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
# 训练模型。
with tf.Session() as session:
tf.global_variables_initializer().run()
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
eval_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
for i in range(NUM_EPOCH):
print("In iteration: %d" % (i + 1))
run_epoch(session, train_model, train_queue, train_model.train_op,
True, train_epoch_size)
valid_perplexity = run_epoch(session, eval_model, eval_queue,
tf.no_op(), False, valid_epoch_size)
print("Epoch: %d Validation Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, eval_model, test_queue,
tf.no_op(), False, test_epoch_size)
print("Test Perplexity: %.3f" % test_perplexity)
coord.request_stop()
coord.join(threads)
cost = vals["cost"]
state = vals["final_state"]
costs += cost
iters += model.input.num_steps
if verbose and step % (model.input.epoch_size // 10) == 10:
print("%.3f perplexity: %.3f speed: %.0f wps" %
(step * 1.0 / model.input.epoch_size, np.exp(
costs / iters), iters * model.input.batch_size /
(time.time() - start_time)))
return np.exp(costs / iters)
raw_data = reader.ptb_raw_data('simple-examples/data/')
train_data, valid_data, test_data, _ = raw_data
config = Smallconfig()
eval_congfig = Smallconfig()
eval_congfig.batch_size = 1
eval_congfig.num_steps = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.name_scope("Train"):
train_input = PTBInput(config=config,
data=train_data,
name="TrainInput")
feed_dict[c], feed_dict[h] = state_list[i * 2:(i + 1) * 2]
# fetch_list에 담긴 final_state의 결과를 state_list로
cost, *state_list, _ = session.run(fetch_list, feed_dict)
costs += cost
iters += m.num_steps
if is_training and step % (epoch_size // 10) == 10:
print("%.3f perplecity: %.3f speed: %.0f wps" %
(step * 1.0 / epoch_size, np.exp(costs / iters),
iters * m.batch_size / (time.time() - start_time)))
return np.exp(costs / iters)
raw_data = reader.ptb_raw_data('../simple-examples/data')
train_data, valid_data, test_data, _ = raw_data
with tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
# 학습, 검증, 테스트 모델
with tf.variable_scope("model", reuse=None, initializer=initializer):
m = PTBModel(config, is_training=True)
with tf.variable_scope("model", reuse=True, initializer=initializer):
mvalid = PTBModel(config)
mtest = PTBModel(eval_config)
tf.initialize_all_variables().run()
embedding_size,
rnn_size,
batch_size,
learning_rate,
training_seq_len,
vocab_size,
infer_sample=False):
self.embedding_size = embedding_size
self.rnn_size = rnn_size
self.vocab_size = vocab_size
self.infer_sample = infer_sample
self.learning_rate = learning_rate
if infer_sample:
self.batch_size = 1
self.training_seq_len = 1
else:
self.batch_size = batch_size
self.training_seq_len = training_seq_len
self.lstm_cell = tf.contrib.rnn.B
if __name__ == "__main__":
file = "./data/pg100.txt"
s_text = open(file, mode="r").read().strip()
s_text = s_text.replace('\r\n', '')
s_text = s_text.replace('\n', '')
print(len(s_text))
train, valid, test, _ = reader.ptb_raw_data("./data/ptb/")
print(_)
state = vals["final_state"]
costs += cost # 累加cost到costs
iters += model.input.num_steps # 累加num_steps到iters
if verbose and step % (model.input.epoch_size // 10) == 10: # 每完成约10%的epoch，就进行一次结果的展示
# 一次展示当前epoch的进度，perplexity和训练速度
# perplexity(即平均cost的自然常数指数，是语言模型中用来比较模型性能的重要指标，越低表示模型输出的概率分布在预测样本上越好)
print("%.3f perplexity: %.3f speed: %.0f wps" %
(step * 1.0 / model.input.epoch_size, np.exp(costs / iters),
iters * model.input.batch_size / (time.time() - start_time)))
return np.exp(costs / iters) # 返回perplexity作为函数结果
raw_data = reader.ptb_raw_data('data/') # 直接读取的数据
train_data, valid_data, test_data, _ = raw_data # 得到训练数据，验证数据和测试数据
config = SmallConfig() # 定义训练模型的配置为SmallConfig
eval_config = SmallConfig() # 测试配置需和训练配置一致
eval_config.batch_size = 1 # 将测试配置的batch_size和num_steps修改为1
eval_config.num_steps = 1
# 创建默认的Graph
with tf.Graph().as_default():
# 设置参数的初始化器，令参数范围在[-init_scale,init_scale]之间
initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale)
'''使用PTBInput和PTBModel创建一个用来训练的模型m,以及用来验证的模型mvalid和测试的模型mtest，
其中训练和验证模型直接使用前面的config，测试模型使用前面的测试配置eval_config'''
# 训练模型m
elif FLAGS.model == 'medium':
return MediumConfig
elif FLAGS.model == 'large':
return LargeConfig
elif FLAGS.mosel == 'test':
return TestConfig
else:
raise ValueError('Invalid model: %s', FLAGS.model)
if __name__ == '__main__':
if not FLAGS.data_path:
raise ValueError('Must set --data_path to PTB data directory')
print(FLAGS.data_path)
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default(), tf.Session() as session:
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.variable_scope('model', reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config) #训练模型
with tf.variable_scope('model', reuse=True, initializer=initializer):
mvalid = PTBModel(is_training=False, config=config) #交叉验证模型和测试模型
mtest = PTBModel(is_training=False, config=eval_config)
cost = vals['cost']
state = vals['final_state']
costs += cost
iters += model.input.num_steps
if (verbose and (step % (model.input.epoch_size // 10) == 0)):
print('%.3f perplexity: %.3f speed: %.0f wps' % \
(step/model.input.epoch_size,np.exp(costs/iters),\
iters*model.input.batch_size/(time.time()-start_time)))
return (np.exp(costs / iters)) #cost是平均过batch_size的.
##############解压PTB数据.
raw_data = reader.ptb_raw_data(r'E:\tensorflow\RNN\simple-examples\data')
train_data, valid_data, test_data, _ = raw_data
config = SmallConfig()
eval_config = SmallConfig()
eval_config.batch_size = 1
eval_config.num_steps = 1
###########创建默认Graph.
with tf.Graph().as_default():
initializer=tf.random_uniform_initializer(-config.init_scale,\
config.init_scale)
####
with tf.name_scope('Train'):
def main(_):
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
gpus = [
x.name for x in device_lib.list_local_devices()
if x.device_type == "GPU"
]
if FLAGS.num_gpus > len(gpus):
raise ValueError("Your machine has only %d gpus "
"which is less than the requested --num_gpus=%d." %
(len(gpus), FLAGS.num_gpus))
raw_data = reader.ptb_raw_data(FLAGS.data_path)
train_data, valid_data, test_data, _ = raw_data
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
with tf.name_scope("Train"):
train_input = PTBInput(config=config,
data=train_data,
name="TrainInput")
with tf.variable_scope("Model",
reuse=None,
initializer=initializer):
m = PTBModel(is_training=True,
config=config,
input_=train_input)
tf.summary.scalar("Training Loss", m.cost)
tf.summary.scalar("Learning Rate", m.lr)
with tf.name_scope("Valid"):
valid_input = PTBInput(config=config,
data=valid_data,
name="ValidInput")
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
mvalid = PTBModel(is_training=False,
config=config,
input_=valid_input)
tf.summary.scalar("Validation Loss", mvalid.cost)
with tf.name_scope("Test"):
test_input = PTBInput(config=eval_config,
data=test_data,
name="TestInput")
with tf.variable_scope("Model",
reuse=True,
initializer=initializer):
mtest = PTBModel(is_training=False,
config=eval_config,
input_=test_input)
models = {"Train": m, "Valid": mvalid, "Test": mtest}
for name, model in models.items():
model.export_ops(name)
metagraph = tf.train.export_meta_graph()
if tf.__version__ < "1.1.0" and FLAGS.num_gpus > 1:
raise ValueError(
"num_gpus > 1 is not supported for TensorFlow versions "
"below 1.1.0")
soft_placement = False
if FLAGS.num_gpus > 1:
soft_placement = True
util.auto_parallel(metagraph, m)
with tf.Graph().as_default():
tf.train.import_meta_graph(metagraph)
for model in models.values():
model.import_ops()
sv = tf.train.Supervisor(logdir=FLAGS.save_path)
config_proto = tf.ConfigProto(allow_soft_placement=soft_placement)
with sv.managed_session(config=config_proto) as session:
for i in range(config.max_max_epoch):
lr_decay = config.lr_decay**max(i + 1 - config.max_epoch, 0.0)
m.assign_lr(session, config.learning_rate * lr_decay)
print("Epoch: %d Learning rate: %.3f" %
(i + 1, session.run(m.lr)))
train_perplexity = run_epoch(session,
m,
eval_op=m.train_op,
verbose=True)
print("Epoch: %d Train Perplexity: %.3f" %
(i + 1, train_perplexity))
valid_perplexity = run_epoch(session, mvalid)
print("Epoch: %d Valid Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, mtest)
print("Test Perplexity: %.3f" % test_perplexity)
if FLAGS.save_path:
print("Saving model to %s." % FLAGS.save_path)
sv.saver.save(session,
FLAGS.save_path,
global_step=sv.global_step)
def get_all_raw_data():
raw_data = reader.ptb_raw_data('ptb_data')
return raw_data
def main(_):
# 原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data) # 数据集的大小
train_batch_len = train_data_len // TRAIN_BATCH_SIZE # batch的个数
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP # 该epoch的训练次数
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
# 定义初始化函数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
# 定义训练用的模型
with tf.variable_scope('language_model',
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
# 定义评估用的模型
with tf.variable_scope('language_model',
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
# 生成数据队列，必须放在开启多线程之前
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
valid_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
with tf.Session() as sess:
tf.global_variables_initializer().run()
# 开启多线程从而支持ptb_producer()使用tf.train.range_input_producer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# 使用训练数据训练模型
for i in range(NUM_EPOCH):
print('In iteration: %d' % (i + 1))
run_epoch(sess, train_model, train_queue, train_model.train_op,
True, train_epoch_size) # 训练模型
valid_perplexity = run_epoch(sess, eval_model, valid_queue,
tf.no_op(), False,
valid_epoch_size) # 使用验证数据评估模型
print('Epoch: %d Validation Perplexity: %.3f' %
(i + 1, valid_perplexity))
# 使用测试数据测试模型
test_perplexity = run_epoch(sess, eval_model, test_queue, tf.no_op(),
False, test_epoch_size)
print('Test Perplexity: %.3f' % test_perplexity)
saver = tf.train.Saver()
saver.save(sess, "save_path/model2")
# 停止所有线程
coord.request_stop()
coord.join(threads)
feed_dict[c] = state[i].c
feed_dict[h] = state[i].h
vals = session.run(fetches, feed_dict)
cost = vals["cost"]
state = vals["final_state"]
costs += cost
iters += model.input.num_steps
if verbose and step % (model.input.epoch_size // 10) == 10:
print("%.3f perplexity: %.3f speed: %.0f wps" % (step * 1.0 / model.input.epoch_size, np.exp(costs / iters),
iters * model.input.batch_size / (
time.time() - start_time)))
return np.exp(costs / iters)
raw_data = reader.ptb_raw_data(r'G:/PycharmWorkspace/tf/src/PTB_data/simple-examples/data/')
train_data, valid_data, test_data, _ = raw_data
config = SmallConfig()
eval_config = SmallConfig()
eval_config.batch_size = 1
eval_config.num_steps = 1
with tf.Graph().as_default():
initializer = tf.random_uniform_initializer(-config.init_scale, config.init_scale)
with tf.name_scope("Train"):
train_input = PTBInput(config=config, data=train_data, name="TrainInput")
with tf.variable_scope("Model", reuse=None, initializer=initializer):
m = PTBModel(is_training=True, config=config, input_=train_input)
with tf.name_scope("Valid"):
else:
raise ValueError("Invalid model: %s", FLAGS.model)
# def main(_):
if __name__ == '__main__':
if not FLAGS.data_path:
raise ValueError("Must set --data_path to PTB data directory")
print(FLAGS.data_path)
config = get_config()
eval_config = get_config()
eval_config.batch_size = 1
eval_config.num_steps = 1
# 原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(FLAGS.data_path)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data) # 数据集的大小
train_batch_len = train_data_len // config.batch_size # batch的个数
train_epoch_size = (train_batch_len - 1) // config.num_steps # 该epoch的训练次数
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // eval_config.batch_size
valid_epoch_size = (valid_batch_len - 1) // eval_config.num_steps
test_data_len = len(test_data)
test_batch_len = test_data_len // eval_config.batch_size
test_epoch_size = (test_batch_len - 1) // eval_config.num_steps
# 生成数据队列，必须放在开启多线程之前
costs += cost # 累加cost到costs
iters += model.input.num_steps # 累加num_steps到iters
if verbose and step % (model.input.epoch_size //
10) == 10: # 每完成约10%的epoch，就进行一次结果的展示
# 一次展示当前epoch的进度，perplexity和训练速度
# perplexity(即平均cost的自然常数指数，是语言模型中用来比较模型性能的重要指标，越低表示模型输出的概率分布在预测样本上越好)
print("%.3f perplexity: %.3f speed: %.0f wps" %
(step * 1.0 / model.input.epoch_size, np.exp(
costs / iters), iters * model.input.batch_size /
(time.time() - start_time)))
return np.exp(costs / iters) # 返回perplexity作为函数结果
raw_data = reader.ptb_raw_data(
'F://研究生资料/data/simple-examples/data/') # 直接读取解压后的数据
train_data, valid_data, test_data, _ = raw_data # 得到训练数据，验证数据和测试数据
config = SmallConfig() # 定义训练模型的配置为SmallConfig
eval_config = SmallConfig() # 测试配置需和训练配置一致
eval_config.batch_size = 1 # 将测试配置的batch_size和num_steps修改为1
eval_config.num_steps = 1
# 创建默认的Graph
with tf.Graph().as_default():
# 设置参数的初始化器，令参数范围在[-init_scale,init_scale]之间
initializer = tf.random_uniform_initializer(-config.init_scale,
config.init_scale)
'''使用PTBInput和PTBModel创建一个用来训练的模型m,以及用来验证的模型mvalid和测试的模型mtest，
其中训练和验证模型直接使用前面的config，测试模型使用前面的测试配置eval_config'''
# 训练模型m
