def test(mnist): with tf.Graph().as_default() as g: x = tf.placeholder(tf.float32,[ mnist.test.num_examples, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x,False,None) ema = tf.train.ExponentialMovingAverage(mnist_lenet5_backward.MOVING_AVERAGE_DECAY) ema_restore = ema.variables_to_restore() saver = tf.train.Saver(ema_restore) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) while True: with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state(mnist_lenet5_backward.MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] reshaped_x = np.reshape(mnist.test.images,( mnist.test.num_examples, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) accuracy_score = sess.run(accuracy, feed_dict={x:reshaped_x,y_:mnist.test.labels}) print("After %s training step(s), test accuracy = %g" % (global_step, accuracy_score)) else: print('No checkpoint file found') return time.sleep(TEST_INTERVAL_SECS)
def backward(mnist): x = tf.placeholder(tf.float32,[ BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x,True, REGULARIZER) global_step = tf.Variable(0, trainable=False) ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step) ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) reshaped_xs = np.reshape(xs,( BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys}) if i % 100 == 0: print("After %d training step(s), loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist): x = tf.placeholder(tf.float32,shape=[BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS]) y_ = tf.placeholder(tf.float32,[None,mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x,True,REGULARIZER) global_step = tf.Variable(0,trainable=False) ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, global_step, mnist.train.num_examples/BATCH_SIZE, LEARNING_RATE_DECAY, staircase = True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss,global_step=global_step) ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step,ema_op]): train_op = tf.no_op(name ='train') saver = tf.train.Saver() with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.save(sess,ckpt.model_checkpoint_path) for i in range(STEPS): xs,ys = mnist.train.next_batch(BATCH_SIZE) reshaped_xs = np.reshape(xs,(BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS)) _,loss_val,step = sess.run([train_op,loss,global_step],feed_dict={x:reshaped_xs,y_:ys}) if i % 1000 == 0: print('After %d training step(s),loss on training batch is %s'%(step,loss_val)) saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),global_step=global_step) backward(mnist)
def restore_model(testPicArr): with tf.Graph().as_default() as tg: x = tf.placeholder(tf.float32, [None, mnist_forward.INPUT_NODE]) y = mnist_forward.forward(x, None) preValue = tf.argmax(y, 1) variable_averages = tf.train.ExponentialMovingAverage( mnist_backward.MOVING_AVERAGE_DECAY) variables_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variables_to_restore) with tf.Session() as sess: ckpt = tf.train.get_checkpoint_state( mnist_backward.MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) preValue = sess.run(preValue, feed_dict={x: testPicArr}) return preValue else: print("No checkpoint file found") return -1
def restore_model(testPicArr): # 复现计算图 with tf.Graph().as_default() as tg: # 搭建整个计算图 # x = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.INPUT_NODE]) x = tf.placeholder( tf.float32, [ test_Pic, # 一次喂入神经网络的数量 mnist_lenet5_forward.IMAGE_SIZE, # 行分辨率 mnist_lenet5_forward.IMAGE_SIZE, # 列分辨率 mnist_lenet5_forward.NUM_CHANNELS # 输入的通道数 ]) y = mnist_lenet5_forward.forward(x, False, mnist_lenet5_backward.REGULARIZER) preValue = tf.argmax(y, 1) # 实例化带有滑动平均的saver variable_averages = tf.train.ExponentialMovingAverage( mnist_lenet5_backward.MOVING_AVERAGE_DECAY) variable_to_restore = variable_averages.variables_to_restore() saver = tf.train.Saver(variable_to_restore) with tf.Session() as sess: # 恢复ckpt ckpt = tf.train.get_checkpoint_state( mnist_lenet5_backward.MODEL_SAVE_PATH) # 如果ckpt存在,则将ckpt恢复到当前会话 if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) # 将准备好的图片喂入神经网络 preValue = sess.run(preValue, feed_dict={x: testPicArr}) return preValue else: print("No checkpoint file found") return -1
def backward(mnist): x = tf.placeholder( tf.float32, [ BATCH_SIZE, # 一次喂入神经网络的数量 mnist_lenet5_forward.IMAGE_SIZE, # 行分辨率 mnist_lenet5_forward.IMAGE_SIZE, # 列分辨率 mnist_lenet5_forward.NUM_CHANNELS # 输入的通道数 ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) # 正向传输推算出模型 y = mnist_lenet5_forward.forward(x, True, REGULARIZER) global_step = tf.Variable(0, trainable=False) # 下面算出包括正则化的损失函数 # logits为神经网络输出层的输出 # 传入的label为一个一维的vector,长度等于batch_size,每一个值的取值区间必须是[0,num_classes),其实每一个值就是代表了batch中对应样本的类别 # tf.argmax(vector, 1):返回的是vector中的最大值的索引号 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) # 矩阵中所有元素求平均值 cem = tf.reduce_mean(ce) # tf.get_collection(‘losses’):返回名称为losses的列表 # tf.add_n(list):将列表元素相加并返回 loss = cem + tf.add_n(tf.get_collection('losses')) # 定义指数衰减学习率 learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) # 定义训练过程 train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) # 定义滑动平均 # tf.train.ExponentialMovingAverage()这个函数用于更新参数,就是采用滑动平均的方法更新参数 # MOVING_AVERAGE_DECAY是衰减率,用于控制模型的更新速度,设置为接近1的值比较合理 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) # apply()方法添加了训练变量的影子副本 # 返回值:ExponentialMovingAverage对象,通过对象调用apply方法可以通过滑动平均模型来更新参数。 # tf.trainable_variables返回的是需要训练的变量列表 # tf.all_variables返回的是所有变量的列表 ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): # 里面的内容需要在将train_step、ema_op执行完后才能执行 # tf.no_op()表示执行完 train_step, ema_op 操作之后什么都不做 train_op = tf.no_op(name='train') saver = tf.train.Saver() with tf.Session() as sess: # 初始化所有变量 init_op = tf.global_variables_initializer() sess.run(init_op) # 通过设置ckpt实现断点续训的功能 ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) # 对数据集中的xs进行reshape操作 reshaped_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshaped_xs, y_: ys }) if i % 100 == 0: print( "After %d training step(s) , loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist): #输入x,y_进行占位 #卷积层输入为四阶张量:第一阶表示每轮喂入的图片数量,第二阶和第三阶分别表示图片的行分辨率和列分辨率,第四阶表示通道数 x = tf.placeholder( tf.float32, [ BATCH_SIZE, #喂入图片数量 mnist_lenet5_forward.IMAGE_SIZE, #行分辨率 mnist_lenet5_forward.IMAGE_SIZE, #列分辨率 mnist_lenet5_forward.NUM_CHANNELS ]) #通道数 y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) '''首先进行前向传播''' y = mnist_lenet5_forward.forward(x, True, REGULARIZER) #调用前向传播算法,正则化参数为0.0001 '''轮数计数器赋值0,设定为不可训练,即不会在训练的时候更新它的值''' global_step = tf.Variable(0, trainable=False) '''定义损失函数,将softmax和交叉商协同使用,包含正则化''' ce = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=y, labels=tf.argmax(y_, 1)) #把前向传播结果变成概率分布,再与训练集中的标准答案做对比,求出交叉熵 cem = tf.reduce_mean(ce) #对向量求均值 loss = cem + tf.add_n( tf.get_collection('losses')) #将参数w的正规化加入到总loss中,防止过拟合 '''函数实现指数衰减学习率''' learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, #初始的学习率 global_step, mnist.train.num_examples / BATCH_SIZE, #训练这么多轮之后开始衰减 LEARNING_RATE_DECAY, #衰减指数 staircase=True) '''训练函数,使用随机梯度下降算法,使梯度沿着梯度的反方向(总损失减少的方向移动),实现参数更新''' train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) '''采用滑动平均的方法进行参数的更新''' ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') '''保存模型''' saver = tf.train.Saver() #实例化saver对象 '''训练过程''' with tf.Session() as sess: #初始化所有参数 init_op = tf.global_variables_initializer() sess.run(init_op) #断点续训 breakpoint_continue.py ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: #恢复当前会话,讲ckpt中的值赋给w和b saver.restore(sess, ckpt.model_checkpoint_path) #循环迭代: for i in range(STEPS): #讲训练集中batch_size的数据和标签赋给左边变量 xs, ys = mnist.train.next_batch(BATCH_SIZE) #修改喂入神经网络的参数 reshaped_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) #喂入神经网络,执行训练过程train_step _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshaped_xs, y_: ys }) if i % 100 == 0: #每1000轮打印出当前的loss值 print( "After %d training step(s), loss on training batch is %g." % (step, loss_value)) #循环1000轮保存模型到当前会话 saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist): # x,y_是定义的占位符,需要指定参数的类型,维度(要和网络的输入与输出维度一致),类似于函数的形参,运行时必须输入的参数 x = tf.placeholder(tf.float32, [BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) # 调用前向传播网络得到维度为10的tensor y = mnist_lenet5_forward.forward(x, True, REGULARIZER) # 声明一个全局计数器,并输出化为0 并说明是不参与训练过程的 global_step = tf.Variable(0, trainable=False) # 先是对网络最后一层的输出y做softmax,通常是求取输出属于某一类的概率,其实就是num_classes的大小的向量 # 再将此向量和实际标签值做交叉熵,需要说明的是该函数的返回是一个向量 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax(y_, 1)) # 再对得到的向量求均值就得到loss cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) # 添加正则化中的losses # 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解,又可以保证模型在训练后期不会有太大波动 # 计算公式:decayed_learning_rate = learning_rate*decay_rate^(global_step/decay_steps) learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) # True(global_step/decay_steps)取整,阶梯 False 平滑 # 传入学习率,构造一个实现梯度下降算法的优化器,再通过使用minimize更新存储要训练的变量列表来减少loss train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss, global_step=global_step) # 实现滑动平均模型,参数MOVING_AVERAGE_DECAY用于控制模型更新速度.训练过程中会对每一个变量维护一个影子变量,这个影子变量的初始值 # 就是相应变量的初始值,每次变量更新时,影子变量就会随之更新 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): # 将train_step和eam_op两个训练操作绑定到train_op上 train_op = tf.no_op(name='train') saver = tf.train.Saver() # 实例化一个保存和恢复变量的saver # 创建一个会话,并通过python中的上下文管理器来管理这个会话 with tf.Session() as sess: init_op = tf.global_variables_initializer() # 初始化计算图中的变量 sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) # 通过checkpoint文件定位到最新保存的模型 if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) # 加载最新的模型 for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) # 读取一个batch的数据 reshaped_xs = np.reshape( # 将输入数据xs转换成与网络输入相同形状的矩阵 xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) # 喂入训练图像&标签, 开始训练 _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={x: reshaped_xs, y_: ys}) if i % 100 == 0: # 每迭代100次打印loss信息,并保存最新的模型 print("After %d training step(s), loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
# mnist_lenet5_forward.IMAGE_SIZE, # mnist_lenet5_forward.NUM_CHANNELS]) # y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) # y = mnist_lenet5_forward.forward(x,False,None) # # ema = tf.train.ExponentialMovingAverage(mnist_lenet5_backward.MOVING_AVERAGE_DECAY) # ema_restore = ema.variables_to_restore() # saver = tf.train.Saver(ema_restore) # # correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #image = tf.image.decode_png('D:/workspace/machine-learning/mnist/img/origin-2.png') # image = tf.cast(image, tf.float32) y_conv = mnist_lenet5_forward.forward(x, False, None) #eva = mnist_lenet5_forward.forward([image],False,None) #prediction = tf.argmax(y,1) saver = tf.train.Saver() with tf.Session(graph=g) as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state( mnist_lenet5_backward.MODEL_SAVE_PATH) saver.restore(sess, ckpt.model_checkpoint_path) reshaped_xs = np.reshape([result], (1, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) # reshaped_x = np.reshape([ipt],( # [ipt], # mnist_lenet5_forward.IMAGE_SIZE,
def backward(mnist): # 首先利用placeholder给x和y_占位 x = tf.placeholder( tf.float32, [ # 只增加了对输入数据的形状调整, BATCH_SIZE, # 由于卷积的输入要求是4阶张量 mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x, True, REGULARIZER) # 调用前向传播的程序计算输出y global_step = tf.Variable(0, trainable=False) # 给轮数计数器赋初值并设置为不可训练 # 调用包含正则化的损失函数losses ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) # 定义指数衰减学习率 learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) # 定义训练过程 train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) #定义滑动平均 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') # 实例化saver saver = tf.train.Saver() # 在with结构中初始化所有变量 with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) # ckpt = tf.train.get_checkpoint_state('F:\PyCharm\myAI\model') if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpont_path) #用for循环迭代steps轮 for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) # 每次读入batch_size组数据和标签 reshaped_xs = np.reshape( xs, ( BATCH_SIZE, # 由于卷积的输入要求是4阶张量 mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshaped_xs, y: ys }) # 把他们喂入神经网络, # 执行训练过程 if i % 1000 == 0: # 每一千轮打印出当前的loss值,要在sess.run运行后才会有结果 print( 'After %d training steps(s),loss_in_training batch is %g.' % (step, loss_value)) # 保存模型和当前会话 saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist): x = tf.placeholder(tf.float32, [ BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUT_NODE]) y = mnist_lenet5_forward.forward(x, True, REGULARIZER) global_step = tf.Variable(0, trainable=False) ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection('losses')) learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) #衰减速率(min(decay,1+step/10+step),可以初期使用较小的,后期使用较大一点的,目的是维持一个影子变量,这个影子变量本次的取值与 #上一次的影子变量值与本次的参数值有关,目的是初期让影子变化快一点(参数不太准确),后期倾向于稳定变化,随着迭代一直更新不影响训练过程 #参与趋于稳定时候,影子影响大一点,更新慢一点,参数前期不稳定,应该更新快一点,所以decay应该随着步数变大 #影子更新不影响参数,但是参数更新会影响影子 #华东平均可以每走一步更新一次影子,也可以做很多步,更新一次影子 #开始影子更新快一点,后期影子更新慢一点,每一步都对影子进行更新,也可以隔好多步对影子进行更新 with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODER_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) reshape_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshape_xs, y_: ys }) if i % 2000 == 0: print(step, loss_value) saver.save(sess, os.path.join(MODER_SAVE_PATH, MODER_NAME), global_step=global_step)
def backward(mnist): x = tf.placeholder(tf.float32, [ BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x, True, REGULARIZER) global_step = tf.Variable(0, trainable=False) # 先是对网络最后一层的输出 y 做 softmax,通常是求取输出属于某一类的概率,其实就是一个num_classes 大小的向量, # 再将此向量和实际标签值做交叉熵,需要说明的是该函数返回的是一个向量 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) cem = tf.reduce_mean(ce) # 再对得到的向量求均值就得到 loss loss = cem + tf.add_n(tf.get_collection('losses')) # 添加正则化中的 losses # 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解,又可以保证模型在训练后期不会有太大波动 # 计算公式:decayed_learning_rate=learining_rate*decay_rate^(global_step/decay_steps) learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) # 就是相应变量的初始值,每次变量更新时,影子变量就会随之更新 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) reshaped_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshaped_xs, y_: ys }) if i % 100 == 0: print( "After %d training step(s), loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def backward(mnist): x = tf.placeholder(tf.float32, [ BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x, True, REGULARIZER) # 调用前向传播网络得到维度为10 的 tensor global_step = tf.Variable(0, trainable=False) # 声明一个全局计数器,并输出化为 0 # 先是对网络最后一层的输出 y 做 softmax,通常是求取输出属于某一类的概率,其实就是一个num_classes 大小的向量, # 再将此向量和实际标签值做交叉熵,需要说明的是该函数返回的是一个向量 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) cem = tf.reduce_mean(ce) # 再对得到的向量求均值就得到 loss loss = cem + tf.add_n(tf.get_collection('losses')) #损失函数 # 实现指数级的减小学习率,可以让模型在训练的前期快速接近较优解,又可以保证模型在训练后期不会有太大波动 # 计算公式:decayed_learning_rate=learining_rate*decay_rate^(global_step/decay_steps) learning_rate = tf.train.exponential_decay( LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True ) ## 当 staircase=True 时,(global_step/decay_steps)则被转化为整数,以此来选择不同的衰减方式 train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) #梯度下降法权重更新 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) #定义滑动平均 #tf.train.exponential_decay(learning_rate, global_, decay_steps, decay_rate, staircase=True/False) ema_op = ema.apply(tf.trainable_variables()) #对所有变量实行滑动平均 #确保train_step,ema_op按顺序都执行 with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() ## 实例化一个保存和恢复变量的 saver with tf.Session() as sess: # 创建一个会话,并通过 python 中的上下文管理器来管理这个会话 init_op = tf.global_variables_initializer() # 初始化计算图中的变量 sess.run(init_op) ckpt = tf.train.get_checkpoint_state( MODEL_SAVE_PATH) # 通过 checkpoint 文件定位到最新保存的模型 if ckpt and ckpt.model_checkpoint_path: #断点续训功能 saver.restore(sess, ckpt.model_checkpoint_path) # 加载最新的模型 for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) reshape_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) #喂入训练图像和标签,开始训练 _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshape_xs, y_: ys }) if i % 1000 == 0: # 每迭代 1000 次打印 loss 信息,并保存最新的模型 print('After %d trining step(s),loss on trining batch is %g' % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step) #保存训练模型
def backward(mnist): x = tf.placeholder(tf.float32, [ BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS ]) y_ = tf.placeholder(tf.float32, [None, mnist_lenet5_forward.OUTPUT_NODE]) y = mnist_lenet5_forward.forward(x, True, REGULARIZER) global_step = tf.Variable(0, trainable=False) #前向传播输出经过softmax函数,以获得输出分类的概率分布,与标准答案求出交叉熵,加上正则化权重得损失函数 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y, labels=tf.argmax( y_, 1)) cem = tf.reduce_mean(ce) loss = cem + tf.add_n(tf.get_collection("losses")) #学习率定为指数衰减型 learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, mnist.train.num_examples / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) #采取梯度下降优化 train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) #滑动平均更新新的神经网络参数,保留上一次的参数影响,给予其合适的权重影响新参数 ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step) ema_op = ema.apply(tf.trainable_variables()) with tf.control_dependencies([train_step, ema_op]): train_op = tf.no_op(name='train') #实例化saver对象 saver = tf.train.Saver() with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) if ckpt and ckpt.model_checkpoint_path: #加载保存的会话 saver.restore(sess, ckpt.model_checkpoint_path) for i in range(STEPS): xs, ys = mnist.train.next_batch(BATCH_SIZE) #训练参数 reshaped_xs = np.reshape( xs, (BATCH_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: reshaped_xs, y_: ys }) #参数喂入得到相应的评估参数 if i % 100 == 0: print( "After %d training step(s),loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step ) #将当前sess保存至指定路径,并命名mnist_model_(global_step)
def backward(mnist): tf.reset_default_graph() #清除默认图的堆栈,并设置全局图为默认图 x = tf.placeholder(tf.float32,shape=[BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE, mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS]) y_ = tf.placeholder(tf.float32, shape=[None,mnist_lenet5_forward.OUTPUT_NODE]) # X,Y_,Y_c = generator.generator() y= mnist_lenet5_forward.forward(x,True,REGULARIZER) global_step = tf.Variable(0,trainable=False) #步数记录 variable_averages= tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY,global_step) variable_averages_op = variable_averages.apply(tf.trainable_variables()) #将各参数滑动平均取值单独加入到tf.variables中 learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,global_step,mnist.train.num_examples/BATCH_SIZE ,LEARNING_RATE_DECAY,staircase = True) #指数学习率衰减 学习率初始指 学习率衰减率 starcase=ture 学习率下降为阶梯状下降 取整数 否则为平滑曲线 #定义损失函数 ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=y,labels=tf.argmax(y_,1)) cem = tf.reduce_mean(ce) loss_total = cem + tf.add_n(tf.get_collection('losses')) #交叉商 #loss_mse = tf.reduce_mean(tf.square(y-y_)) #loss_total = loss_mse + tf.add_n(tf.get_collection('losses')) #定义反向传播算法 包含正则化 #train_step=tf.train.AdamOptimizer(learning_rate).minimize(loss_total) #train_step=tf.train.MomentumOptimizer(0.001,0.9).minimize(loss) train_step=tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_total,global_step=global_step) with tf.control_dependencies([train_step,variable_averages_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() #获得图片和标签 with tf.Session() as sess: init_op = tf.global_variables_initializer() sess.run(init_op) #sess.run(tf.group(tf.global_variables_initializer(),tf.local_variables_initializer())) ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)#存储路径 # 如果已有 ckpt 模型则恢复 if ckpt and ckpt.model_checkpoint_path: # 恢复会话 saver.restore(sess, ckpt.model_checkpoint_path) #ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) # if ckpt and ckpt.model_checkpoint_path: #该函数表示如果断点文件夹中包含有效断点状态文件,则返回该文件 #saver.restore(sess,ckpt.model_checkpoint_path) #参数说明: tf.train.get_checkpoint_state(checkpoint_dir,lastst_filename=None)checkpoint_dir 表示存储断点文件的目录 #lastst_filename=None 断点文件的可选名称 默认为“check——point # saver.restore(sess,ckpt.model_checkpoint_path)” sess 当前会话 model_checkpoint_path 模型存储路径 最新的模型 for i in range(STEPS): xs,ys = mnist.train.next_batch(BATCH_SIZE) reshape_xs = np.reshape(xs,(BATCH_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.IMAGE_SIZE,mnist_lenet5_forward.NUM_CHANNELS)) _, loss_value, step = sess.run([train_op,loss_total,global_step],feed_dict={x: reshape_xs , y_:ys}) if i%100 ==0: print("After %d training step(s),loss on training batch is %g."%(step,loss_value)) saver.save(sess,os.path.join(MODEL_SAVE_PATH,MODEL_NAME),global_step=global_step)