def backward():
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
train_num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
img_batch, label_batch = mnist_generateds.get_tfrecord(BATCH_SIZE,
isTrain=True)
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
#实现断点续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(STEPS):
xs, ys = sess.run([img_batch, label_batch])
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: xs,
y_: ys
})
if i % 1000 == 0:
print(
"After %d training step(s),loss on training batch is %g." %
(step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
coord.regust_stop()
coord.join(threads)
def test():
with tf.Graph().as_default() as g:
x=tf.placeholder(tf.float32,shape=[None,mnist_forward.INPUT_NODE])
y_=tf.placeholder(tf.float32,shape=[None,mnist_forward.OUTPUT_NODE])
y=mnist_forward.forward(x,None)
ema=tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY)
ema_restore=ema.variables_to_restore()
saver=tf.train.Saver(ema_restore)
correct_prediction=tf.equal(tf.argmax(y,1),tf.argmax(y_,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
img_batch,label_batch=mnist_generateds.get_tfrecord(TEST_NUM,isTrain=False)
while True:
with tf.Session() as sess:
ckpt=tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess,ckpt.model_checkpoint_path)
global_step=ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
coord=tf.train.Coordinator()
threads=tf.train.start_queue_runners(sess=sess,coord=coord)
xs,ys=sess.run([img_batch,label_batch])
accuracy_score=sess.run(accuracy,feed_dict={x:xs,y_:ys})
print('After %s steps,test accuracy is %g'%(global_step,accuracy_score))
coord.request_stop()
coord.join(threads)
else:
print('No checkpoints file found')
return
time.sleep(TEST_INTERVAL_SECS)
def test(mnist):
#复现计算图
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, None)
#恢复过后，数据会得到各自的滑动平均值
ema = tf.train.ExponentialMovingAverage(
mnist_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#拿到数据，因为是测试集中取出数据，所以istrain为False
img_batch, label_batch = mnist_generateds.get_tfrecord(TEST_NUM,
isTrain=False)
while True:
with tf.Session() as sess:
#滑动平均值赋值给各个参数
ckpt = tf.train.get_checkpoint_state(
mnist_backward.MODEL_SAVE_PATH)
#判断是否有模型
if ckpt and ckpt.model_checkpoint_path:
#恢复模型到当前会话
saver.restore(sess, ckpt.model_checkpoint_path)
#恢复global_step值
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
#县城协调器
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,
coord=coord)
xs, ys = sess.run([img_batch, label_batch])
#执行准确率计算
accuracy_score = sess.run(accuracy,
feed_dict={
x: xs,
y_: ys
})
#打印准确率
print("after %s training steps , test accuracy = %g" %
(global_step, accuracy_score))
coord.request_stop()
coord.join(threads)
else:
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def test(mnist):
# 复现计算图
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
# 前向传播计算出y的值
y = mnist_forward.forward(x, None)
# 实例化带滑动平均的saver对象，赋值为各自的滑动平均值
ema = tf.train.ExponentialMovingAverage(
mnist_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
# 计算准确率
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
img_batch, label_batch = mnist_generateds.get_tfrecord(TEST_NUM,
isTrain=False)
while True:
with tf.Session() as sess:
# 加载ckpt
ckpt = tf.train.get_checkpoint_state(
mnist_backward.MODEL_SAVE_PATH)
# 先判断是否已经由模型，如果有，恢复模型到当前会话，恢复参数
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess,
coord=coord)
accuracy_score = sess.run(accuracy,
feed_dict={
x: mnist.test.images,
y_: mnist.test.labels
})
print("After %s training step(s), test accuracy = %g" %
(global_step, accuracy_score))
coord.request_stop()
coord.join(threads)
else:
# 提示模型没有找到
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def test():
with tf.Graph().as_default() as g:
#其内定义的节点在计算图g中
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, None)
ema = tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY)
#
ema_restore = ema.variables_to_restore()
#实例化可还原滑动平均值的saver对象,此后在会话中恢复模型后,所有参数将使用其滑动平均值
saver = tf.train.Saver(ema_restore)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#获取单位 batch 组的测试数据
img_batch, label_batch = mnist_generateds.get_tfrecord(TEST_NUM, isTrain=False)
while True:
with tf.Session() as sess:
#尝试加载ckpt模型
ckpt = tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
# 恢复模型到当前会话
saver.restore(sess, ckpt.model_checkpoint_path)
# 获取原训练轮数
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
#开启线程协调器
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
xs, ys = sess.run([img_batch, label_batch])
accuracy_score = sess.run(accuracy, feed_dict={x: xs, y_: ys})
#accuracy_score = sess.run(accuracy, feed_dict={x: mnist.test.images, y_: mnist.test.labels})
print("After %s training steps, test accuracy = %g" % (global_step, accuracy_score))
#关闭线程协调器
coord.request_stop()
coord.join(threads)
else:
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def test():
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, None)
ema = tf.train.ExponentialMovingAverage(mnist_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
img_batch, label_batch = mnist_generateds.get_tfrecord(TEST_NUM, isTrain=False)#2
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(mnist_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
coord = tf.train.Coordinator()#3
threads = tf.train.start_queue_runners(sess=sess, coord=coord)#4
xs, ys = sess.run([img_batch, label_batch])#5
accuracy_score = sess.run(accuracy, feed_dict={x: xs, y_: ys})
print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score))
coord.request_stop()#6
coord.join(threads)#7
else:
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def backward():
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
y = mnist_forward.forward(x, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
# 定义损失函数
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
# 指数衰减学习率
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
train_num_examples /
BATCH_SIZE, # mnist.train.num_examples/BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
# 定义反向传播算法
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
# 滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
# 用于保存模型
saver = tf.train.Saver()
# 获取batch_size张图片和标签
img_batch, label_batch = mnist_generateds.get_tfrecord(
BATCH_SIZE, isTrain=True) # 不同点3 用函数get_tfrecord，一次批获取batch_size张图片和标签
with tf.Session() as sess:
# 初始化参数
init_op = tf.global_variables_initializer()
sess.run(init_op)
#断点续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
# 开启线程协调器
coord = tf.train.Coordinator() # 不同点4
threads = tf.train.start_queue_runners(
sess=sess,
coord=coord) # 不同点5 该句将会启动输入队列的线程，填充训练样本到队列中，以便出队操作可以从队列中拿到样本。
for i in range(STEPS):
xs, ys = sess.run([img_batch, label_batch]) #不同点6 执行图片和标签的批获取
# xs,ys=mnist.train.next_batch(BATCH_SIZE) #直接读出图片和标签
# 开始训练
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: xs,
y_: ys
})
if i % 1000 == 0:
print(
"After %d training step(s), loss on training batch is %g."
% (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
# 关闭线程协调器
coord.request_stop() # 不同点7
coord.join(threads) # 不同点8
def backward():#parameter type :class mnist
#define placeholder x,which act as input image
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
#define placeholder y_,which is output result
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
#define y as the computed result which will feed the parameter y_ below
y = mnist_forward.forward(x, REGULARIZER)
#define variable golbal_step to count the step where the model run and set it untrainable
global_step = tf.Variable(0, trainable = False)
#cross entropy,to calculate the distance between the standard probability
#distribution and the nerual network calculated probability distribution
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = y,
labels = tf.argmax(y_, 1))
#compute average ce
cem = tf.reduce_mean(ce)
#compute total loss with cross entropy
loss = cem + tf.add_n(tf.get_collection('losses'))
#set learning_rate with exponential decay(staircase option on)
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
train_num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase = True)
#use gradient descent optimizer to train model
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step = global_step)
#compute exponential moving average(ema) of all tainable variabels
#create class ema,prepare to be computed
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
#aplly ema to all trainable variables
ema_op = ema.apply(tf.trainable_variables())
#bind operation train_step & ema_op together to realize two operations at time
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name = 'train')
#create class saver to save the session below
saver = tf.train.Saver()
img_batch, lable_batch = mnist_generateds.get_tfrecord(BATCH_SIZE, isTrain=True) #3
#run the compute graph below
with tf.Session() as sess:
#initialize all global variables
sess.run(tf.global_variables_initializer())
#restore module
#fetch the checkpoint from path "./model"
ckpt = tf.train.get_checkpoint_state("./model")
#if checkpoint and it’s path exist,do restore()
if ckpt and ckpt.model_checkpoint_path:
#restore model to current neural network
saver.restore(sess, ckpt.model_checkpoint_path)
#end of restore
coord = tf.train.Coordinator()#4
threads = tf.train.start_queue_runners(sess=sess, coord=coord)#5
for i in range(STEPS):
xs, ys = sess.run([img_batch, lable_batch]) #6
#fetch dataset to be trained,assign train image to xs,train labels to ys
##xs, ys = mnist.train.next_batch(BATCH_SIZE)
#calculate node train_op, loss,global_step and return the result to _,loss_value, step
#'_' means an anonymous variable which will not in use any more
_, loss_value, step = sess.run([train_op, loss,global_step],
feed_dict = {x : xs, y_ : ys})
#save current neural network with a 1000-step frequency,
if i % 1000 == 0:
print("after %d training step(s), loss on training batch is %g." % (step, loss_value))
#save the global_step neural network(current NN)to specified path(which is MODEL_SAVE_PATH + MODEL_NAME)
# and append the step number(3rd argument) to the checkpoint name(2rd argument )
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME),global_step = global_step)
coord.request_stop() #7
coord.join(threads) #8
def backward(mnist):
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
#调用前向传播的程序，输出y
y = mnist_forward.forward(x, REGULARIZER)
#轮数计数器
global_step = tf.Variable(0, trainable=False)
#交叉熵：表明两个概率分布之间的距离
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
#定义损失函数
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
train_num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
#定义训练过程：这个是梯度下降优化程序
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
#定义滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
#每一次运行这一句，所有待优化参数求滑动平均
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
#自己写的图片和标签 因为从训练集中取出，所以istrain为true
img_batch, label_batch = mnist_generateds.get_tfrecord(BATCH_SIZE,
isTrain=true)
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
#加载ckpt模型,实现断点续训 #给所有的w和b赋值保存再ckpt中的值,实现断点续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
#加载完成
#开启线程协调器
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(STEPS):
#每次读入BATCH_SIZE组数据
xs, ys = sess.run([img_batch, label_batch
]) # xs,ys = mnist.train.next_batch(BATCH_SIZE)
#喂入神经网络，执行训练过程
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: xs,
y_: ys
})
if i % 1000 == 0:
print("after %d training steps,loss on training batch is %g" %
(step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
#关闭线程协调器
coord.request_stop()
coord.join(threads)
pass
def test():
#creates a new graph and places everything (declared inside its scope) into this graph.
with tf.Graph().as_default() as g:
#define placeholder x,which act as input image
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
#define y as the computed result which will feed the parameter y_ below
y_ = tf.placeholder(tf.float32, [None, mnist_forward.OUTPUT_NODE])
#execute forward propagation without regularization,and return it's outcome to y
y = mnist_forward.forward(x, None)
#Create an ExponentialMovingAverage object ema
ema = tf.train.ExponentialMovingAverage(
mnist_backward.MOVING_AVERAGE_DECAY)
#method variable_to_restore() return a dict ({ema_variables : variables})
ema_restore = ema.variables_to_restore()
#Create a saver that loads variables from their saved shadow values.
saver = tf.train.Saver(ema_restore)
#if tf.argmax(y, 1) equals to tf.argmax(y_, 1),correct_prediction will be set True
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
#cast the type of corrent_prediction from boolean to float and compute it's average
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
img_batch, label_batch = mnist_generateds.get_tfrecord(
TEST_NUM, isTrain=False) #2
#load trained model in the loop
while True:
#create session to manage the context
with tf.Session() as sess:
#fetch chechpoint from sepcified path
ckpt = tf.train.get_checkpoint_state(
mnist_backward.MODEL_SAVE_PATH)
#if got the checkpoint sucessfully do things below
if ckpt and ckpt.model_checkpoint_path:
#restore the model to current neural network
saver.restore(sess, ckpt.model_checkpoint_path)
#fetch the step from the string ckpt.model_checkpoint and extract
#the last integer via charactor "/" & "-"
global_step = ckpt.model_checkpoint_path.split(
'/')[-1].split('-')[-1]
coord = tf.train.Coordinator() #3
threads = tf.train.start_queue_runners(sess=sess,
coord=coord) #4
xs, ys = sess.run([img_batch, label_batch]) #5
#compute accuracy_score via test data set
accuracy_score = sess.run(accuracy,
feed_dict={
x: xs,
y_: ys
})
#print the predict result
print("after %s training step(s), test accuracy = %g" %
(global_step, accuracy_score))
coord.request_stop() #6
coord.join(threads) #7
else: #can not get checkpoint file ,print error infomation
print("No checkpoint file found")
#exit this moudle
return
#set interval time to wait for the checkpoint file which the backward function produce
time.sleep(INTERVAL_TIME)
