def run_epoch(session, model, data, train_op, output_log):
total_cost = 0.0
iters = 0
state = session.run(model.initial_state)
print(
session.run(
reader.ptb_producer(data, model.batch_size, model.num_steps)))
for step, (x, y) in enumerate(
reader.ptb_producer(data, model.batch_size, model.num_steps)
# ptb_iterator
):
cost, state, _ = session.run([model.cost, model.final_state, train_op],
{
model.input_data: x,
model.targets: y,
model.initial_state: state
})
total_cost += cost
iters += model.num_steps
if output_log and step % 100 == 0:
print("After %d steps, perplexity is %.3f" %
(step, np.exp(total_cost / iters)))
return np.exp(total_cost / iters)
def get_config():
logger.auto_set_dir()
data3, wd2id = get_PennTreeBank()
global VOCAB_SIZE
VOCAB_SIZE = len(wd2id)
steps_per_epoch = (data3[0].shape[0] // BATCH - 1) // SEQ_LEN
train_data = TensorInput(lambda: ptb_producer(data3[0], BATCH, SEQ_LEN),
steps_per_epoch)
val_data = TensorInput(lambda: ptb_producer(data3[1], BATCH, SEQ_LEN),
(data3[1].shape[0] // BATCH - 1) // SEQ_LEN)
M = Model()
return TrainConfig(
data=train_data,
model=M,
callbacks=[
ModelSaver(),
HyperParamSetterWithFunc('learning_rate', lambda e, x: x * 0.80
if e > 6 else x),
RunOp(lambda: M.reset_lstm_state()),
FeedfreeInferenceRunner(val_data, [ScalarStats(['cost'])]),
CallbackFactory(
trigger_epoch=lambda self: self.trainer.add_scalar_summary(
'validation_perplexity',
np.exp(
self.trainer.stat_holder.get_stat_now('validation_cost'
) / SEQ_LEN))),
RunOp(lambda: M.reset_lstm_state()),
],
max_epoch=70,
)
def main():
#获取原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data)
train_batch_len = train_data_len // TRAIN_BATCH_SIZE
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
#定义初始化参数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
#定义训练用的RNN模型
with tf.variable_scope("language_model",
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
#定义评测用的RNN模型
with tf.variable_scope("language_model",
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
# 训练模型。
with tf.Session() as session:
tf.global_variables_initializer().run()
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
eval_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
for i in range(NUM_EPOCH):
print("In iteration: %d" % (i + 1))
run_epoch(session, train_model, train_queue, train_model.train_op,
True, train_epoch_size)
valid_perplexity = run_epoch(session, eval_model, eval_queue,
tf.no_op(), False, valid_epoch_size)
print("Epoch: %d Validation Perplexity: %.3f" %
(i + 1, valid_perplexity))
test_perplexity = run_epoch(session, eval_model, test_queue,
tf.no_op(), False, test_epoch_size)
print("Test Perplexity: %.3f" % test_perplexity)
coord.request_stop()
coord.join(threads)
def main(_):
# 获取原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
train_epoch_size = (len(train_data) // TRAIN_BATCH_SIZE) // TRAIN_NUM_STEP
valid_epoch_size = (len(valid_data) // EVAL_BATCH_SIZE) // EVAL_NUM_STEP
test_epoch_size = (len(test_data) // EVAL_BATCH_SIZE) // EVAL_NUM_STEP
# 定义初始化函数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
# 定义训练用的循环神经网络模型
with tf.variable_scope('language_model',
reuse=None,
initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
# 定义评测用的循环神经网络模型
with tf.variable_scope('language_model',
reuse=True,
initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
#首先生成数据队列，必须要将数据队列的声明放在启动多线程之前，
# 不然会出现队列一直等待出队的状态，
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
valid_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
with tf.Session() as session:
tf.global_variables_initializer().run()
#tf.local_variables_initializer().run()
# 开启多线程从而支持ptb_producer()使用tf.train.range_input_producer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=session, coord=coord)
# 使用训练数据训练模型
for i in range(NUM_EPOCH):
print('In iteration: %d' % (i + 1))
# 训练
run_epoch(session, train_model, train_queue, train_model.train_op,
True, train_epoch_size)
# 验证数据评测
# tf.no_op(): Does nothing. Only useful as a placeholder for control edges.
valid_perplexity = run_epoch(session, eval_model, valid_queue,
tf.no_op(), False, valid_epoch_size)
print('Epoch: %d Validation Perplexity: %.3f' %
(i + 1, valid_perplexity))
# 测试数据测试
test_perplexity = run_epoch(session, eval_model, test_queue,
tf.no_op(), False, test_epoch_size)
print('Test Perplexity: %.3f' % test_perplexity)
coord.request_stop()
coord.join(threads)
def main():
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
train_data_len = len(train_data)
train_batch_len = train_data_len // TRAIN_BATCH_SIZE
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
initializer = tf.random_uniform_initializer(-0.05, 0.05)
perpelexity_hist = []
with tf.variable_scope('language_model', reuse = None, initializer = initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
with tf.variable_scope('language_model', reuse = True, initializer = initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as session:
tf.global_variables_initializer().run()
train_queue = reader.ptb_producer(train_data, train_model.batch_size, train_model.num_steps)
eval_queue = reader.ptb_producer(valid_data, eval_model.batch_size, eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size, eval_model.num_steps)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess = session, coord = coord)
for i in range(NUM_EPOCH):
print('In iteration: {}'.format(i + 1))
perpelexity_hist.append(
run_epoch(session, train_model, train_queue, train_model.train_op, True, train_epoch_size)
)
valid_perpelexity = run_epoch(session, eval_model, eval_queue, tf.no_op(), False, valid_epoch_size)
print('Epoch: {} Validation Perplexity: {}'.format(i + 1, valid_perpelexity))
test_perplexity = run_epoch(session, eval_model, test_queue, tf.no_op(), False, test_epoch_size)
print('Test Perplexity: {}'.format(test_perplexity))
coord.request_stop()
coord.join(threads)
print(perpelexity_hist)
def get_config():
data3, wd2id = get_PennTreeBank(args.datadir)
args.vocab_size = len(wd2id)
steps_per_epoch = (data3[0].shape[0] // args.batch_size -
1) // args.seq_len
train_data = TensorInput(
lambda: ptb_producer(data3[0], args.batch_size, args.seq_len),
steps_per_epoch)
val_data = TensorInput(
lambda: ptb_producer(data3[1], args.batch_size, args.seq_len),
(data3[1].shape[0] // args.batch_size - 1) // args.seq_len)
test_data = TensorInput(
lambda: ptb_producer(data3[2], args.batch_size, args.seq_len),
(data3[2].shape[0] // args.batch_size - 1) // args.seq_len)
M = Model()
return TrainConfig(
data=train_data,
model=M,
callbacks=[
# ModelSaver(),
HyperParamSetterWithFunc(
'learning_rate',
# lambda e, lr: lr_schedule(e, lr)),
# lambda e, _: args.init_lr * (e/args.warmup_epochs) if e < args.warmup_epochs else args.init_lr * ((1 - (e / float(args.epochs))) ** 2.0)),
lambda e, x: x * 0.8 if e > 6 else x),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(val_data, [ScalarStats(['cost'])]),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(test_data, [ScalarStats(['cost'], prefix='test')],
tower_name='InferenceTowerTest'),
RunOp(lambda: M.reset_lstm_state()),
CallbackFactory(trigger=lambda self: [
self.trainer.monitors.put_scalar(
'validation_perplexity',
np.exp(
self.trainer.monitors.get_latest('validation_cost') /
args.seq_len)),
self.trainer.monitors.put_scalar(
'test_perplexity',
np.exp(
self.trainer.monitors.get_latest('test_cost') / args.
seq_len))
]),
],
max_epoch=args.epochs,
)
def __init__(self, batch_size,num_steps, data, name=None):
self.batch_size = batch_size # Size of the batches
self.num_steps = num_steps # Number of elements of the chain
# Num steps should be given by the input chains and could be different for every chain
# In the initial case I think what happened is that they use this size to chop the file.
# TODO: The way is implemented is this the only way to do it, the sum_steps has to be the same for all chaines ?
# Number of epochs ! The number of times we run the entire data ?
# Why is it constrained by the number of steps ?
self.epoch_size = ((len(data) // self.batch_size) - 1) // self.num_steps
# This reader formats the data in the desired way somehow.
"""
IMPORTANT:
In this architecture we do not need to feed data to the model.
What will happen is, when we fetch the output, it will depend on this
self.input_data object, calling the reader.ptb_producer function.
Which will output a different batch anytime we call it, since it references them with
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
"""
self.input_data, self.targets = reader.ptb_producer(data, self.batch_size, self.num_steps, name=name)
def __init__(self, config, data, name=None):
#debug('data: {0}\n'.format(data))
if 'per_sentence' in config:
data = [word for sentence in data
for word in sentence] # flatten list of lists
self.batch_size = batch_size = config['batch_size']
self.num_steps = num_steps = config['num_steps']
data_len = len(data)
batch_len = data_len // batch_size
# if processing per sentence: we don't want to cut in the middle of a sentence,
# so make sure that batch_len is a multiple of num_steps (= length of each padded sentence)
if 'per_sentence' in config:
num_sentences = batch_len // (num_steps + 1)
batch_len = num_sentences * (num_steps + 1)
if num_steps == 1:
self.epoch_size = (batch_len - 1) // num_steps
else:
self.epoch_size = (batch_len - 1) // (num_steps + 1)
#debug('len(data) (wordInput): {0}\n'.format(len(data)))
#debug('batch_len (wordInput): {0}\n'.format(batch_len))
#debug('batch_size (wordInput): {0}\n'.format(self.batch_size))
#debug('num_steps (wordInput): {0}\n'.format(self.num_steps))
#debug('epoch_size (wordInput): {0}\n'.format(self.epoch_size))
# input_data = Tensor of size batch_size x num_steps, same for targets (but shifted 1 step to the right)
self.input_data, self.targets = reader.ptb_producer(data,
config,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size= config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
#print(data[0:10],batch_size,num_steps)
self.input_data, self.targets = reader.ptb_producer(
data, batch_size, num_steps)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
# // "来表示整数除法，返回不大于结果的一个最大的整数
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(
data, batch_size, num_steps, name=name)
def __init__(self, config, data, name=None):
self.epoch_size = (
(len(data) // config.batch_size) - 1) // config.num_steps
self.input_data, self.targets = reader.ptb_producer(data,
config.batch_size,
config.num_steps,
name=name)
def __init__(self, batch_size, num_steps, data, name=None):
self.batch_size = batch_size # Size of the batches
self.num_steps = num_steps # Number of elements of the chain
# Num steps should be given by the input chains and could be different for every chain
# In the initial case I think what happened is that they use this size to chop the file.
# TODO: The way is implemented is this the only way to do it, the sum_steps has to be the same for all chaines ?
# Number of epochs ! The number of times we run the entire data ?
# Why is it constrained by the number of steps ?
self.epoch_size = (
(len(data) // self.batch_size) - 1) // self.num_steps
# This reader formats the data in the desired way somehow.
"""
IMPORTANT:
In this architecture we do not need to feed data to the model.
What will happen is, when we fetch the output, it will depend on this
self.input_data object, calling the reader.ptb_producer function.
Which will output a different batch anytime we call it, since it references them with
i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
"""
self.input_data, self.targets = reader.ptb_producer(data,
self.batch_size,
self.num_steps,
name=name)
def get_config():
logger.auto_set_dir()
data3, wd2id = get_PennTreeBank()
global VOCAB_SIZE
VOCAB_SIZE = len(wd2id)
steps_per_epoch = (data3[0].shape[0] // BATCH - 1) // SEQ_LEN
train_data = TensorInput(
lambda: ptb_producer(data3[0], BATCH, SEQ_LEN),
steps_per_epoch)
val_data = TensorInput(
lambda: ptb_producer(data3[1], BATCH, SEQ_LEN),
(data3[1].shape[0] // BATCH - 1) // SEQ_LEN)
test_data = TensorInput(
lambda: ptb_producer(data3[2], BATCH, SEQ_LEN),
(data3[2].shape[0] // BATCH - 1) // SEQ_LEN)
M = Model()
return TrainConfig(
data=train_data,
model=M,
callbacks=[
ModelSaver(),
HyperParamSetterWithFunc(
'learning_rate',
lambda e, x: x * 0.80 if e > 6 else x),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(val_data, [ScalarStats(['cost'])]),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(
test_data,
[ScalarStats(['cost'], prefix='test')], tower_name='InferenceTowerTest'),
RunOp(lambda: M.reset_lstm_state()),
CallbackFactory(
trigger=lambda self:
[self.trainer.monitors.put_scalar(
'validation_perplexity',
np.exp(self.trainer.monitors.get_latest('validation_cost') / SEQ_LEN)),
self.trainer.monitors.put_scalar(
'test_perplexity',
np.exp(self.trainer.monitors.get_latest('test_cost') / SEQ_LEN))]
),
],
max_epoch=70,
)
def get_config():
logger.auto_set_dir()
data3, wd2id = get_PennTreeBank()
global VOCAB_SIZE
VOCAB_SIZE = len(wd2id)
steps_per_epoch = (data3[0].shape[0] // BATCH - 1) // SEQ_LEN
train_data = TensorInput(
lambda: ptb_producer(data3[0], BATCH, SEQ_LEN),
steps_per_epoch)
val_data = TensorInput(
lambda: ptb_producer(data3[1], BATCH, SEQ_LEN),
(data3[1].shape[0] // BATCH - 1) // SEQ_LEN)
test_data = TensorInput(
lambda: ptb_producer(data3[2], BATCH, SEQ_LEN),
(data3[2].shape[0] // BATCH - 1) // SEQ_LEN)
M = Model()
return TrainConfig(
data=train_data,
model=M,
callbacks=[
ModelSaver(),
HyperParamSetterWithFunc(
'learning_rate',
lambda e, x: x * 0.80 if e > 6 else x),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(val_data, [ScalarStats(['cost'])]),
RunOp(lambda: M.reset_lstm_state()),
InferenceRunner(
test_data,
[ScalarStats(['cost'], prefix='test')], tower_name='InferenceTowerTest'),
RunOp(lambda: M.reset_lstm_state()),
CallbackFactory(
trigger=lambda self:
[self.trainer.monitors.put_scalar(
'validation_perplexity',
np.exp(self.trainer.monitors.get_latest('validation_cost') / SEQ_LEN)),
self.trainer.monitors.put_scalar(
'test_perplexity',
np.exp(self.trainer.monitors.get_latest('test_cost') / SEQ_LEN))]
),
],
max_epoch=70,
)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
# ptb_producer: Iterate on the raw PTB data.
# Each time, it produce a pair of Tensors, each shaped [batch_size, num_steps].
# The second element of the tuple is the same data time-shifted to the right by one.
self.input_data, self.targets = reader.ptb_producer(data, batch_size, num_steps, name=name)
def __init__(self, data, name=None):
self.batch_size = batch_size = FLAGS.batch_size
self.num_steps = num_steps = FLAGS.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def handle_data():
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
print len(train_data)
print train_data[:100]
x, y = reader.ptb_producer(train_data, 4, 5)
print "X: ",x
print "Y: ",y
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
#self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.epoch_size = 1
self.input_data, self.targets = reader.ptb_producer(
data, batch_size, num_steps, name=name)
print(self.input_data)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size # 读取config中的batch_size，num_steps到本地变量
self.num_steps = num_steps = config.num_steps
self.epoch_size = (
(len(data) // batch_size) - 1) // num_steps # 全部样本被训练的次数
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, predicting=None, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
if predicting == True:
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size # The size of one mini-batch
self.num_steps = num_steps = config.num_steps # The length of one sentence in one batch
self.epoch_size = ((len(data) // batch_size) -
1) // num_steps # The number of sentences
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name) # x, y
def __init__(self, data_path, batch_size, num_steps, name=None):
raw_data = ptb_raw_data(data_path=data_path)
train_data, valid_data, test_data, _ = raw_data
self._dataset = ptb_producer(train_data, batch_size, num_steps, name=name)
self._iterater = self._dataset.make_initializable_iterator()
self.input_data, self.target = self._iterater.get_next()
self.init = self._iterater.initializer
self.batch_size = batch_size
self.num_steps = num_steps
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = (
(len(data) // batch_size) -
1) // num_steps # Q: why minus 1? A: for generating targets
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
# LSTM的展开步数（unrolled steps of LSTM）
self.num_steps = num_steps = config.num_steps
# 每一个epoch内需要的多少轮迭代
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
# 获取特征数据input_data，label数据targets，每次执行获取一个batch的数据。
#
# self.targets的shape是[batch_size,num_steps]
self.input_data, self.targets = reader.ptb_producer(data, batch_size, num_steps, name=name)
def __init__(self, config, data, name=None):
#number of batches
self.batch_size = batch_size = config.batch_size
self.num_step = num_steps = config.num_steps
#number of epochs each batch
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
# self.epoch_size 决定了data的个数的最小值，若len(data) 小于 batch_size*(num_steps+1)，则无法训练。
# 若要在数据量很小的时候继续训练，则需要减小 batch_size 或 num_steps， 建议减小 batch_size
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
# ｌｓｔｍ展开的步长
self.num_steps = num_steps = config.num_steps
# 每轮迭代的需要训练的次数
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def _load_dataset(self, datadir, batch_size):
train, valid, test, self.vocab_size = reader.ptb_raw_data(str(datadir))
self._train, self.train_epoch_size = reader.ptb_producer(
train, batch_size, num_steps=NUM_STEPS, repeat=True)
self._test, self.test_epoch_size = reader.ptb_producer(
test, batch_size, num_steps=NUM_STEPS, repeat=False)
self._valid, self.valid_epoch_size = reader.ptb_producer(
valid, batch_size, num_steps=NUM_STEPS, repeat=False)
self._handle = tf.placeholder(tf.string,
shape=[],
name='dataset_handle')
iterator = tf.data.Iterator.from_string_handle(
self._handle, self._train.output_types, self._train.output_shapes)
self.inputs, self.labels = iterator.get_next()
self.is_training = tf.placeholder(tf.bool)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
#计算一个epoch训练的迭代次数
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
#返回一个batch batch_size = [batch_size, num_steps] 得到一个num_steps所需的训练集
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps # num_steps: 20
# epoch_size 表示批次总数。也就是说，需要向session喂这么多批数据
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
# input_data:[batch_size,num_steps], targets:[batch_size,num_steps]
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
'''
input_data,个targets是[batch_size,num_steps]的tensor
int32类型
'''
self.input_data, self.targets = reader.ptb_producer(
data, batch_size, num_steps, name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
#LSTM展开步数
self.num_steps = num_steps = config.num_steps
#计算每个epoch的size，即每个epoch内需要多少轮训练的迭代
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
#获取特征数据input_data以及label数据targets
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
# ptb_producer: Iterate on the raw PTB data.
# Each time, it produce a pair of Tensors, each shaped [batch_size, num_steps].
# The second element of the tuple is the same data time-shifted to the right by one.
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def __init__(self, config, data, name=None):
'''
num_steps: the number of timesteps (or unrolled steps)
'''
self.batch_size = batch_size = config.batch_size
self.num_steps = num_steps = config.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data, self.targets = reader.ptb_producer(data,
batch_size,
num_steps,
name=name)
def testPtbProducer(self):
raw_data = [4, 3, 2, 1, 0, 5, 6, 1, 1, 1, 1, 0, 3, 4, 1]
batch_size = 3
num_steps = 2
x, y = reader.ptb_producer(raw_data, batch_size, num_steps)
with self.test_session() as session:
coord = tf.train.Coordinator()
tf.train.start_queue_runners(session, coord=coord)
try:
xval, yval = session.run([x, y])
self.assertAllEqual(xval, [[4, 3], [5, 6], [1, 0]])
self.assertAllEqual(yval, [[3, 2], [6, 1], [0, 3]])
xval, yval = session.run([x, y])
self.assertAllEqual(xval, [[2, 1], [1, 1], [3, 4]])
self.assertAllEqual(yval, [[1, 0], [1, 1], [4, 1]])
finally:
coord.request_stop()
coord.join()
def testPtbProducer(self):
raw_data = [4, 3, 2, 1, 0, 5, 6, 1, 1, 1, 1, 0, 3, 4, 1]
batch_size = 3
num_steps = 2
x, y = reader.ptb_producer(raw_data, batch_size, num_steps)
with self.test_session() as session:
coord = tf.train.Coordinator()
tf.train.start_queue_runners(session, coord=coord)
try:
data_len = tf.size(raw_data)
batch_len = data_len // batch_size
data = tf.reshape(tf.slice(raw_data, [0],[batch_size * batch_len]),[batch_size, batch_len])
epoch_size = (batch_len - 1) // num_steps
epoch_size = tf.identity(epoch_size, name="epoch_size")
# i = tf.train.range_input_producer(epoch_size, shuffle=False).dequeue()
#
x1 = tf.strided_slice(data, [0, 1 * num_steps],
[batch_size, (1 + 1) * num_steps])
# x.set_shape([batch_size, num_steps])
# y = tf.strided_slice(data, [0, i * num_steps + 1],
# [batch_size, (i + 1) * num_steps + 1])
# y.set_shape([batch_size, num_steps])
print('data_len' + str(session.run(data_len)))
print('batch_len' + str(session.run(batch_len)))
print('data' + str(session.run(data)))
print('epoch_size' + str(session.run(epoch_size)))
print('x1' + str(session.run(x1)))
print('data' + str(session.run(data)))
# print('data' + str(session.run(data)))
xval, yval = session.run([x, y])
# print(str((xval, yval)))
self.assertAllEqual(xval, [[4, 3], [5, 6], [1, 0]])
self.assertAllEqual(yval, [[3, 2], [6, 1], [0, 3]])
xval, yval = session.run([x, y])
# print(str((xval, yval)))
self.assertAllEqual(xval, [[2, 1], [1, 1], [3, 4]])
self.assertAllEqual(yval, [[1, 0], [1, 1], [4, 1]])
finally:
coord.request_stop()
coord.join()
import reader
import tensorflow as tf
# 数据路径
DATA_PATH = 'simple-examples/data/'
# 读取原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 将数据组织成batch大小为4，截断长度为5的数据组，要放在开启多线程之前
batch = reader.ptb_producer(train_data, 4, 5)
with tf.Session() as sess:
tf.global_variables_initializer().run()
# 开启多线程
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# 读取前两个batch，其中包括每个时刻的输入和对应的答案，ptb_producer()会自动迭代
for i in range(2):
x, y = sess.run(batch)
print('x:', x)
print('y:', y)
# 关闭多线程
coord.request_stop()
coord.join(threads)
def main(_):
# 原始数据
train_data, valid_data, test_data, _ = reader.ptb_raw_data(DATA_PATH)
# 计算一个epoch需要训练的次数
train_data_len = len(train_data) # 数据集的大小
train_batch_len = train_data_len // TRAIN_BATCH_SIZE # batch的个数
train_epoch_size = (train_batch_len - 1) // TRAIN_NUM_STEP # 该epoch的训练次数
valid_data_len = len(valid_data)
valid_batch_len = valid_data_len // EVAL_BATCH_SIZE
valid_epoch_size = (valid_batch_len - 1) // EVAL_NUM_STEP
test_data_len = len(test_data)
test_batch_len = test_data_len // EVAL_BATCH_SIZE
test_epoch_size = (test_batch_len - 1) // EVAL_NUM_STEP
# 生成数据队列，必须放在开启多线程之前
train_queue = reader.ptb_producer(train_data, train_model.batch_size,
train_model.num_steps)
valid_queue = reader.ptb_producer(valid_data, eval_model.batch_size,
eval_model.num_steps)
test_queue = reader.ptb_producer(test_data, eval_model.batch_size,
eval_model.num_steps)
# 定义初始化函数
initializer = tf.random_uniform_initializer(-0.05, 0.05)
# 定义训练用的模型
with tf.variable_scope(
'language_model', reuse=None, initializer=initializer):
train_model = PTBModel(True, TRAIN_BATCH_SIZE, TRAIN_NUM_STEP)
# 定义评估用的模型
with tf.variable_scope(
'language_model', reuse=True, initializer=initializer):
eval_model = PTBModel(False, EVAL_BATCH_SIZE, EVAL_NUM_STEP)
with tf.Session() as sess:
tf.global_variables_initializer().run()
# 开启多线程从而支持ptb_producer()使用tf.train.range_input_producer()
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# 使用训练数据训练模型
for i in range(NUM_EPOCH):
print('In iteration: %d' % (i + 1))
run_epoch(sess, train_model, train_queue, train_model.train_op,
True, train_epoch_size) # 训练模型
valid_perplexity = run_epoch(sess, eval_model, valid_queue,
tf.no_op(), False,
valid_epoch_size) # 使用验证数据评估模型
print('Epoch: %d Validation Perplexity: %.3f' % (i + 1,
valid_perplexity))
# 使用测试数据测试模型
test_perplexity = run_epoch(sess, eval_model, test_queue,
tf.no_op(), False, test_epoch_size)
print('Test Perplexity: %.3f' % test_perplexity)
# 停止所有线程
coord.request_stop()
coord.join(threads)
def __init__(self, params, data, name=None):
self.batch_size = batch_size = params.batch_size
self.num_steps = num_steps = params.num_steps
self.epoch_size = ((len(data) // batch_size) - 1) // num_steps
self.input_data = lambda: reader.ptb_producer(
data, params.batch_size, params.num_steps, name)