def test(mnist):
with tf.Graph().as_default() as g:
x = tf.placeholder(tf.float32,[
mnist.test.num_examples,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x,False,None)
ema = tf.train.ExponentialMovingAverage(mnist_lenet5_backward.MOVING_AVERAGE_DECAY)
ema_restore = ema.variables_to_restore()
saver = tf.train.Saver(ema_restore)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
while True:
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(mnist_lenet5_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
reshaped_x = np.reshape(mnist.test.images,(
mnist.test.num_examples,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
accuracy_score = sess.run(accuracy, feed_dict={x:reshaped_x,y_:mnist.test.labels})
print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score))
else:
print('No checkpoint file found')
return
time.sleep(TEST_INTERVAL_SECS)
def backward(mnist):
x = tf.placeholder(tf.float32,[
BATCH_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x,True, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
reshaped_xs = np.reshape(xs,(
BATCH_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys})
if i % 100 == 0:
print("After %d training step(s), loss on training batch is %g." % (step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist):
x = tf.placeholder(tf.float32,shape=[BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS])
y_ = tf.placeholder(tf.float32,[None,mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x,True,REGULARIZER)
global_step = tf.Variable(0,trainable=False)
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples/BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase = True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step,ema_op]):
train_op = tf.no_op(name ='train')
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.save(sess,ckpt.model_checkpoint_path)
for i in range(STEPS):
xs,ys = mnist.train.next_batch(BATCH_SIZE)
reshaped_xs = np.reshape(xs,(BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS))
_,loss_val,step = sess.run([train_op,loss,global_step],feed_dict={x:reshaped_xs,y_:ys})
if i % 1000 == 0:
print('After %d training step(s),loss on training batch is %s'%(step,loss_val))
saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),global_step=global_step)
backward(mnist)
def restore_model(testPicArr):
with tf.Graph().as_default() as tg:
x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE])
y = mnist_forward.forward(x, None)
preValue = tf.argmax(y, 1)
variable_averages = tf.train.ExponentialMovingAverage(
mnist_backward.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(
mnist_backward.MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
preValue = sess.run(preValue, feed_dict={x: testPicArr})
return preValue
else:
print("No checkpoint file found")
return -1
def restore_model(testPicArr):
# 复现计算图
with tf.Graph().as_default() as tg:
# 搭建整个计算图
# x = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.INPUT_NODE])
x = tf.placeholder(
tf.float32,
[
test_Pic, # 一次喂入神经网络的数量
mnist_lenet5_forward.IMAGE_SIZE, # 行分辨率
mnist_lenet5_forward.IMAGE_SIZE, # 列分辨率
mnist_lenet5_forward.NUM_CHANNELS # 输入的通道数
])
y = mnist_lenet5_forward.forward(x, False,
mnist_lenet5_backward.REGULARIZER)
preValue = tf.argmax(y, 1)
# 实例化带有滑动平均的saver
variable_averages = tf.train.ExponentialMovingAverage(
mnist_lenet5_backward.MOVING_AVERAGE_DECAY)
variable_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variable_to_restore)
with tf.Session() as sess:
# 恢复ckpt
ckpt = tf.train.get_checkpoint_state(
mnist_lenet5_backward.MODEL_SAVE_PATH)
# 如果ckpt存在，则将ckpt恢复到当前会话
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
# 将准备好的图片喂入神经网络
preValue = sess.run(preValue, feed_dict={x: testPicArr})
return preValue
else:
print("No checkpoint file found")
return -1
def backward(mnist):
x = tf.placeholder(
tf.float32,
[
BATCH_SIZE, # 一次喂入神经网络的数量
mnist_lenet5_forward.IMAGE_SIZE, # 行分辨率
mnist_lenet5_forward.IMAGE_SIZE, # 列分辨率
mnist_lenet5_forward.NUM_CHANNELS # 输入的通道数
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
# 正向传输推算出模型
y = mnist_lenet5_forward.forward(x, True, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
# 下面算出包括正则化的损失函数
# logits为神经网络输出层的输出
# 传入的label为一个一维的vector，长度等于batch_size，每一个值的取值区间必须是[0，num_classes)，其实每一个值就是代表了batch中对应样本的类别
# tf.argmax(vector, 1)：返回的是vector中的最大值的索引号
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
# 矩阵中所有元素求平均值
cem = tf.reduce_mean(ce)
# tf.get_collection(‘losses’)：返回名称为losses的列表
# tf.add_n(list)：将列表元素相加并返回
loss = cem + tf.add_n(tf.get_collection('losses'))
# 定义指数衰减学习率
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
# 定义训练过程
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
# 定义滑动平均
# tf.train.ExponentialMovingAverage()这个函数用于更新参数，就是采用滑动平均的方法更新参数
# MOVING_AVERAGE_DECAY是衰减率,用于控制模型的更新速度,设置为接近1的值比较合理
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
# apply()方法添加了训练变量的影子副本
# 返回值：ExponentialMovingAverage对象，通过对象调用apply方法可以通过滑动平均模型来更新参数。
# tf.trainable_variables返回的是需要训练的变量列表
# tf.all_variables返回的是所有变量的列表
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
# 里面的内容需要在将train_step、ema_op执行完后才能执行
# tf.no_op()表示执行完 train_step, ema_op 操作之后什么都不做
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
with tf.Session() as sess:
# 初始化所有变量
init_op = tf.global_variables_initializer()
sess.run(init_op)
# 通过设置ckpt实现断点续训的功能
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
# 对数据集中的xs进行reshape操作
reshaped_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshaped_xs,
y_: ys
})
if i % 100 == 0:
print(
"After %d training step(s) , loss on training batch is %g."
% (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def backward(mnist):
#输入x，y_进行占位
#卷积层输入为四阶张量:第一阶表示每轮喂入的图片数量，第二阶和第三阶分别表示图片的行分辨率和列分辨率，第四阶表示通道数
x = tf.placeholder(
tf.float32,
[
BATCH_SIZE, #喂入图片数量
mnist_lenet5_forward.IMAGE_SIZE, #行分辨率
mnist_lenet5_forward.IMAGE_SIZE, #列分辨率
mnist_lenet5_forward.NUM_CHANNELS
]) #通道数
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
'''首先进行前向传播'''
y = mnist_lenet5_forward.forward(x, True,
REGULARIZER) #调用前向传播算法，正则化参数为0.0001
'''轮数计数器赋值0，设定为不可训练，即不会在训练的时候更新它的值'''
global_step = tf.Variable(0, trainable=False)
'''定义损失函数，将softmax和交叉商协同使用，包含正则化'''
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=y, labels=tf.argmax(y_, 1)) #把前向传播结果变成概率分布，再与训练集中的标准答案做对比，求出交叉熵
cem = tf.reduce_mean(ce) #对向量求均值
loss = cem + tf.add_n(
tf.get_collection('losses')) #将参数w的正规化加入到总loss中，防止过拟合
'''函数实现指数衰减学习率'''
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE, #初始的学习率
global_step,
mnist.train.num_examples / BATCH_SIZE, #训练这么多轮之后开始衰减
LEARNING_RATE_DECAY, #衰减指数
staircase=True)
'''训练函数，使用随机梯度下降算法，使梯度沿着梯度的反方向（总损失减少的方向移动），实现参数更新'''
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
'''采用滑动平均的方法进行参数的更新'''
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
'''保存模型'''
saver = tf.train.Saver() #实例化saver对象
'''训练过程'''
with tf.Session() as sess:
#初始化所有参数
init_op = tf.global_variables_initializer()
sess.run(init_op)
#断点续训 breakpoint_continue.py
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
#恢复当前会话，讲ckpt中的值赋给w和b
saver.restore(sess, ckpt.model_checkpoint_path)
#循环迭代：
for i in range(STEPS):
#讲训练集中batch_size的数据和标签赋给左边变量
xs, ys = mnist.train.next_batch(BATCH_SIZE)
#修改喂入神经网络的参数
reshaped_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
#喂入神经网络，执行训练过程train_step
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshaped_xs,
y_: ys
})
if i % 100 == 0:
#每1000轮打印出当前的loss值
print(
"After %d training step(s), loss on training batch is %g."
% (step, loss_value))
#循环1000轮保存模型到当前会话
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def backward(mnist):
# x,y_是定义的占位符,需要指定参数的类型,维度(要和网络的输入与输出维度一致),类似于函数的形参,运行时必须输入的参数
x = tf.placeholder(tf.float32,
[BATCH_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
# 调用前向传播网络得到维度为10的tensor
y = mnist_lenet5_forward.forward(x, True, REGULARIZER)
# 声明一个全局计数器,并输出化为0 并说明是不参与训练过程的
global_step = tf.Variable(0, trainable=False)
# 先是对网络最后一层的输出y做softmax,通常是求取输出属于某一类的概率,其实就是num_classes的大小的向量
# 再将此向量和实际标签值做交叉熵,需要说明的是该函数的返回是一个向量
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1))
# 再对得到的向量求均值就得到loss
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses')) # 添加正则化中的losses
# 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解，又可以保证模型在训练后期不会有太大波动
# 计算公式:decayed_learning_rate = learning_rate*decay_rate^(global_step/decay_steps)
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True) # True(global_step/decay_steps)取整,阶梯 False 平滑
# 传入学习率,构造一个实现梯度下降算法的优化器,再通过使用minimize更新存储要训练的变量列表来减少loss
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step)
# 实现滑动平均模型,参数MOVING_AVERAGE_DECAY用于控制模型更新速度.训练过程中会对每一个变量维护一个影子变量,这个影子变量的初始值
# 就是相应变量的初始值,每次变量更新时,影子变量就会随之更新
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]): # 将train_step和eam_op两个训练操作绑定到train_op上
train_op = tf.no_op(name='train')
saver = tf.train.Saver() # 实例化一个保存和恢复变量的saver
# 创建一个会话,并通过python中的上下文管理器来管理这个会话
with tf.Session() as sess:
init_op = tf.global_variables_initializer() # 初始化计算图中的变量
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) # 通过checkpoint文件定位到最新保存的模型
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path) # 加载最新的模型
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE) # 读取一个batch的数据
reshaped_xs = np.reshape( # 将输入数据xs转换成与网络输入相同形状的矩阵
xs,
(BATCH_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
# 喂入训练图像&标签, 开始训练
_, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys})
if i % 100 == 0: # 每迭代100次打印loss信息,并保存最新的模型
print("After %d training step(s), loss on training batch is %g." % (step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
# mnist_lenet5_forward.IMAGE_SIZE,
# mnist_lenet5_forward.NUM_CHANNELS])
# y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
# y = mnist_lenet5_forward.forward(x,False,None)
#
# ema = tf.train.ExponentialMovingAverage(mnist_lenet5_backward.MOVING_AVERAGE_DECAY)
# ema_restore = ema.variables_to_restore()
# saver = tf.train.Saver(ema_restore)
#
# correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
# accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#image = tf.image.decode_png('D:/workspace/machine-learning/mnist/img/origin-2.png')
# image = tf.cast(image, tf.float32)
y_conv = mnist_lenet5_forward.forward(x, False, None)
#eva = mnist_lenet5_forward.forward([image],False,None)
#prediction = tf.argmax(y,1)
saver = tf.train.Saver()
with tf.Session(graph=g) as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(
mnist_lenet5_backward.MODEL_SAVE_PATH)
saver.restore(sess, ckpt.model_checkpoint_path)
reshaped_xs = np.reshape([result], (1, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
# reshaped_x = np.reshape([ipt],(
# [ipt],
# mnist_lenet5_forward.IMAGE_SIZE,
def backward(mnist):
# 首先利用placeholder给x和y_占位
x = tf.placeholder(
tf.float32,
[ # 只增加了对输入数据的形状调整,
BATCH_SIZE, # 由于卷积的输入要求是4阶张量
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x, True, REGULARIZER) # 调用前向传播的程序计算输出y
global_step = tf.Variable(0, trainable=False) # 给轮数计数器赋初值并设置为不可训练
# 调用包含正则化的损失函数losses
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
# 定义指数衰减学习率
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
# 定义训练过程
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
#定义滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
# 实例化saver
saver = tf.train.Saver()
# 在with结构中初始化所有变量
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
# ckpt = tf.train.get_checkpoint_state('F:\PyCharm\myAI\model')
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpont_path)
#用for循环迭代steps轮
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE) # 每次读入batch_size组数据和标签
reshaped_xs = np.reshape(
xs,
(
BATCH_SIZE, # 由于卷积的输入要求是4阶张量
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshaped_xs,
y: ys
}) # 把他们喂入神经网络，
# 执行训练过程
if i % 1000 == 0: # 每一千轮打印出当前的loss值，要在sess.run运行后才会有结果
print(
'After %d training steps(s)，loss_in_training batch is %g.'
% (step, loss_value))
# 保存模型和当前会话
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def backward(mnist):
x = tf.placeholder(tf.float32, [
BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUT_NODE])
y = mnist_lenet5_forward.forward(x, True, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
#衰减速率（min(decay,1+step/10+step），可以初期使用较小的，后期使用较大一点的，目的是维持一个影子变量，这个影子变量本次的取值与
#上一次的影子变量值与本次的参数值有关，目的是初期让影子变化快一点（参数不太准确），后期倾向于稳定变化，随着迭代一直更新不影响训练过程
#参与趋于稳定时候，影子影响大一点，更新慢一点，参数前期不稳定，应该更新快一点，所以decay应该随着步数变大
#影子更新不影响参数，但是参数更新会影响影子
#华东平均可以每走一步更新一次影子，也可以做很多步，更新一次影子
#开始影子更新快一点，后期影子更新慢一点，每一步都对影子进行更新，也可以隔好多步对影子进行更新
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODER_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
reshape_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshape_xs,
y_: ys
})
if i % 2000 == 0:
print(step, loss_value)
saver.save(sess,
os.path.join(MODER_SAVE_PATH, MODER_NAME),
global_step=global_step)
def backward(mnist):
x = tf.placeholder(tf.float32, [
BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x, True, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
# 先是对网络最后一层的输出 y 做 softmax,通常是求取输出属于某一类的概率,其实就是一个num_classes 大小的向量,
# 再将此向量和实际标签值做交叉熵,需要说明的是该函数返回的是一个向量
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce) # 再对得到的向量求均值就得到 loss
loss = cem + tf.add_n(tf.get_collection('losses')) # 添加正则化中的 losses
# 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解,又可以保证模型在训练后期不会有太大波动
# 计算公式:decayed_learning_rate=learining_rate*decay_rate^(global_step/decay_steps)
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
# 就是相应变量的初始值,每次变量更新时,影子变量就会随之更新
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
reshaped_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshaped_xs,
y_: ys
})
if i % 100 == 0:
print(
"After %d training step(s), loss on training batch is %g."
% (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def backward(mnist):
x = tf.placeholder(tf.float32, [
BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x, True,
REGULARIZER) # 调用前向传播网络得到维度为10 的 tensor
global_step = tf.Variable(0, trainable=False) # 声明一个全局计数器,并输出化为 0
# 先是对网络最后一层的输出 y 做 softmax,通常是求取输出属于某一类的概率,其实就是一个num_classes 大小的向量,
# 再将此向量和实际标签值做交叉熵,需要说明的是该函数返回的是一个向量
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce) # 再对得到的向量求均值就得到 loss
loss = cem + tf.add_n(tf.get_collection('losses')) #损失函数
# 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解,又可以保证模型在训练后期不会有太大波动
# 计算公式:decayed_learning_rate=learining_rate*decay_rate^(global_step/decay_steps)
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True
) ## 当 staircase=True 时,(global_step/decay_steps)则被转化为整数,以此来选择不同的衰减方式
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step) #梯度下降法权重更新
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,
global_step) #定义滑动平均
#tf.train.exponential_decay(learning_rate, global_, decay_steps, decay_rate, staircase=True/False)
ema_op = ema.apply(tf.trainable_variables()) #对所有变量实行滑动平均
#确保train_step,ema_op按顺序都执行
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver() ## 实例化一个保存和恢复变量的 saver
with tf.Session() as sess: # 创建一个会话,并通过 python 中的上下文管理器来管理这个会话
init_op = tf.global_variables_initializer() # 初始化计算图中的变量
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(
MODEL_SAVE_PATH) # 通过 checkpoint 文件定位到最新保存的模型
if ckpt and ckpt.model_checkpoint_path: #断点续训功能
saver.restore(sess, ckpt.model_checkpoint_path) # 加载最新的模型
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE)
reshape_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
#喂入训练图像和标签,开始训练
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshape_xs,
y_: ys
})
if i % 1000 == 0: # 每迭代 1000 次打印 loss 信息,并保存最新的模型
print('After %d trining step(s),loss on trining batch is %g' %
(step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step) #保存训练模型
def backward(mnist):
x = tf.placeholder(tf.float32, [
BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS
])
y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE])
y = mnist_lenet5_forward.forward(x, True, REGULARIZER)
global_step = tf.Variable(0, trainable=False)
#前向传播输出经过softmax函数，以获得输出分类的概率分布，与标准答案求出交叉熵，加上正则化权重得损失函数
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,
labels=tf.argmax(
y_, 1))
cem = tf.reduce_mean(ce)
loss = cem + tf.add_n(tf.get_collection("losses"))
#学习率定为指数衰减型
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
mnist.train.num_examples /
BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
#采取梯度下降优化
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
#滑动平均更新新的神经网络参数，保留上一次的参数影响，给予其合适的权重影响新参数
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
with tf.control_dependencies([train_step, ema_op]):
train_op = tf.no_op(name='train')
#实例化saver对象
saver = tf.train.Saver()
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
#加载保存的会话
saver.restore(sess, ckpt.model_checkpoint_path)
for i in range(STEPS):
xs, ys = mnist.train.next_batch(BATCH_SIZE) #训练参数
reshaped_xs = np.reshape(
xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: reshaped_xs,
y_: ys
}) #参数喂入得到相应的评估参数
if i % 100 == 0:
print(
"After %d training step(s),loss on training batch is %g." %
(step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step
) #将当前sess保存至指定路径，并命名mnist_model_(global_step)
def backward(mnist):
tf.reset_default_graph() #清除默认图的堆栈，并设置全局图为默认图
x = tf.placeholder(tf.float32,shape=[BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,
mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS])
y_ = tf.placeholder(tf.float32, shape=[None,mnist_lenet5_forward.OUTPUT_NODE])
# X,Y_,Y_c = generator.generator()
y= mnist_lenet5_forward.forward(x,True,REGULARIZER)
global_step = tf.Variable(0,trainable=False) #步数记录
variable_averages= tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,global_step)
variable_averages_op = variable_averages.apply(tf.trainable_variables())
#将各参数滑动平均取值单独加入到tf.variables中
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE
,LEARNING_RATE_DECAY,staircase = True)
#指数学习率衰减 学习率初始指 学习率衰减率 starcase=ture 学习率下降为阶梯状下降 取整数 否则为平滑曲线
#定义损失函数
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1))
cem = tf.reduce_mean(ce)
loss_total = cem + tf.add_n(tf.get_collection('losses')) #交叉商
#loss_mse = tf.reduce_mean(tf.square(y-y_))
#loss_total = loss_mse + tf.add_n(tf.get_collection('losses'))
#定义反向传播算法 包含正则化
#train_step=tf.train.AdamOptimizer(learning_rate).minimize(loss_total)
#train_step=tf.train.MomentumOptimizer(0.001,0.9).minimize(loss)
train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_total,global_step=global_step)
with tf.control_dependencies([train_step,variable_averages_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
#获得图片和标签
with tf.Session() as sess:
init_op = tf.global_variables_initializer()
sess.run(init_op)
#sess.run(tf.group(tf.global_variables_initializer(),tf.local_variables_initializer()))
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)#存储路径
# 如果已有 ckpt 模型则恢复
if ckpt and ckpt.model_checkpoint_path:
# 恢复会话
saver.restore(sess, ckpt.model_checkpoint_path)
#ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
# if ckpt and ckpt.model_checkpoint_path: #该函数表示如果断点文件夹中包含有效断点状态文件，则返回该文件
#saver.restore(sess,ckpt.model_checkpoint_path)
#参数说明： tf.train.get_checkpoint_state(checkpoint_dir,lastst_filename=None)checkpoint_dir 表示存储断点文件的目录
#lastst_filename=None 断点文件的可选名称 默认为“check——point
# saver.restore(sess,ckpt.model_checkpoint_path)” sess 当前会话 model_checkpoint_path 模型存储路径 最新的模型
for i in range(STEPS):
xs,ys = mnist.train.next_batch(BATCH_SIZE)
reshape_xs = np.reshape(xs,(BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS))
_, loss_value, step = sess.run([train_op,loss_total,global_step],feed_dict={x: reshape_xs , y_:ys})
if i%100 ==0:
print("After %d training step(s),loss on training batch is %g."%(step,loss_value))
saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),global_step=global_step)
