def backward():
#加载图片及图片batch张量
cifar10.maybe_download_and_extract()
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=DATA_DIR, batch_size=BATCH_SIZE)
#构建反馈
x = tf.placeholder(tf.float32, shape = [None,24,24,3])
y_ = tf.placeholder(tf.float32, shape = [None])
keep_prob = tf.placeholder(tf.float32)
y = cnn_forward.forward(x,keep_prob,REGULARIZER)
global_step = tf.Variable(0, trainable = False)
learning_rate = tf.train.exponential_decay(
LEARNING_RATE_BASE,
global_step,
20000/BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase = True)
#计算loss
ce = tf.nn.sparse_softmax_cross_entropy_with_logits(logits = y, labels = tf.cast(y_, tf.int64))
loss_ce = tf.reduce_mean(ce)
loss_total = loss_ce + tf.add_n(tf.get_collection('losses'))
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss_total, global_step = global_step)
#滑动平均
ema = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
ema_op = ema.apply(tf.trainable_variables())
#训练模型
with tf.control_dependencies([train_step, ema_op]) :
train_op = tf.no_op(name = 'train')
saver = tf.train.Saver()
sess = tf.InteractiveSession()
init_op = tf.global_variables_initializer()
sess.run(init_op)
#续训
ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
#启用多线程队列加载图片
tf.train.start_queue_runners()
for i in range(STEPS):
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value, step = sess.run([train_op, loss_total, global_step], feed_dict = {x: image_batch, y_: label_batch, keep_prob:0.5})
if i % 10 == 0:
print("After %d steps, loss is: %f" %(step, loss_value))
saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step = global_step)
sess.close()
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
return cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=FLAGS.batch_size)
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
return cifar10_input.distorted_inputs(data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size)
def distorted_inputs():
""" Construct distorted input for CIFAR training using the Reader ops.
Those are data generate by distort original picture in order to augment data.
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a directory.')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=FLAGS.batch_size)
# if use float16 just cast
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
# return data
return images, labels
def get_distorted_train_batch(data_dir,batch_size):
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,batch_size=batch_size)
return images,labels
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
data_dir = os.path.join(dest_directory, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=batch_size)
return images, labels
def distorted_inputs(batch_size=FLAGS.batch_size):
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
# print ('global batch_size: %d' % batch_size)
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
return cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=batch_size)
def distorted_inputs():
"""Construct distorted input for COCO training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 3D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE] size.
Raises:
ValueError: If no data_dir
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
# data_dir = os.path.join(FLAGS.data_dir, 'test_records.tfrecords')
data_dir = FLAGS.data_dir
return cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=FLAGS.batch_size)
def train(self,
epochs,
learning_rate=.01,
batch_size=32,
early_stop=False):
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
num_train = 50000
steps_per_epoch = num_train // batch_size
learning_rate = tf.train.exponential_decay(learning_rate,
global_step,
steps_per_epoch * 350,
0.1,
staircase=True)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
checkpoint = 'checkpoints/model.ckpt'
sess = tf.Session()
sess.run(init)
with tqdm(range(epochs)) as t:
# training data QueueRunner that returns (32, 24, 24, 3) training batches
# 50000/10000 train/test split, no validation
# Random crop, flip, standardization, no random brightness/contrast
images, labels = cifar10_input.distorted_inputs(
'cifar-10-batches-bin', batch_size)
best = 0
for epoch in t:
for step in range(steps_per_epoch):
train_op = self.step(images, labels, global_step,
learning_rate, .9, .004)
a_train, c_train, _ = sess.run(train_op)
# if epoch % 10 == 0:
# a_validation, c_validation, _ = sess.run([accuracy, loss, optim],
# feed_dict={X: data.validation.images, Y: data.validation.labels, keep_prob: 1.0})
# if a_validation >= best:
# saver.save(sess, checkpoint)
t.set_postfix(t_acc=a_train,
t_loss=c_train,
v_acc=a_validation,
v_loss=c_validation)
if (early_stop):
saver.restore(sess, checkpoint)
sess.close
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not tfFLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(tfFLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=tfFLAGS.batch_size)
if tfFLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def obtain_input(self):
'''
获取CNN网络的输入样本
Returns:
train_x: 训练集
train_y: 训练集标记
test_x: 测试集
test_y: 测试集标记
'''
data_dir = '/tmp/cifar10_data/cifar-10-batches-bin'
cifar10.maybe_download_and_extract()
train_x, train_y = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=self.batch_size)
test_x, test_y = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=self.batch_size)
return train_x, train_y, test_x, test_y
def distorted_inputs():
"""调用cifar10_input.py中的distorted_input()对CIFAR数据集进行变形处理
Returns:
images: Images. 4D 张量 [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] .
labels: Labels. 1D 张量 [batch_size].
Raises:
ValueError: 没有data_dir将报错
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def distorted_inputs():
"""使用Reader op为CIFAR训练构造distorted input
Returns:
image: Images. 4维tensor，shape[batch_size, IMAGE_SIZE, IMAGE_SIZE, 3]
labels: Labels. 1维tensor，shape[batch_size]
Raises:
ValueError: 没有data_dir参数
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(data_dir, batch_size=FLAGS.batch_size)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
# if not FLAGS.data_dir:
# raise ValueError('Please supply a data_dir')
config = network_config.getConfig()
data_dir = config['data_dir']
batch_size = config['batch_size']
data_dir = os.path.join(data_dir, 'cifar-10-batches-bin')
return cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=batch_size)
def distorted_inputs(self):
"""Construct distorted input for a given dataset using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not self.data_dir:
raise ValueError('Please supply a data_dir')
if self.dataset == 'cifar10':
data_dir = os.path.join(self.data_dir, 'cifar-10-batches-bin')
images, labels = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=self.batch_size, image_size=self.image_size)
elif self.dataset == 'imagenet':
images, labels = imagenet_input.distorted_inputs()
if self.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def train():
with tf.Graph().as_default():
global_step = tf.train.get_or_create_global_step()
#Make use of CPU memory to avoid GPU memory allocation problem
with tf.device('/cpu:0'):
#images, labels = cifar10_input.inputs(eval_data=False,
# data_dir=FLAGS.data_dir,
# batch_size=FLAGS.batch_size)
images, labels = cifar10_input.distorted_inputs(
data_dir=FLAGS.data_dir, batch_size=FLAGS.batch_size)
test_images, test_labels = cifar10_input.inputs(
eval_data=True,
data_dir=FLAGS.data_dir,
batch_size=cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_EVAL)
logits = build_vgg_model(images)
prediction = build_vgg_model(test_images)
correct_prediction = tf.equal(tf.argmax(prediction, 1),
tf.cast(test_labels, tf.int64))
test_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', test_accuracy)
summary_op = tf.summary.merge_all()
# Training
loss_op = loss(logits, labels)
train_op = train_step(loss_op, global_step)
summary_writer = tf.summary.FileWriter(FLAGS.log_dir)
with tf.train.MonitoredSession() as sess:
for i in range(FLAGS.max_steps + 1):
_, loss_val = sess.run([train_op, loss_op])
if i % FLAGS.log_frequency == 0:
test_acc, summary = sess.run([test_accuracy, summary_op])
summary_writer.add_summary(summary, i)
print("Step:%4d, Loss:%f Test Accuracy:%f" %
(i, loss_val, test_acc))
summary_writer.close()
def distorted_inputs(batch_size, Delta2, epsilon2, W_conv1Noise):
"""Construct distorted input for CIFAR training using the Reader ops.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
Raises:
ValueError: If no data_dir
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
beta = redistributeNoise(os.getcwd() + '/LRP_0_25_v12.txt')
images, labels = cifar10_input.distorted_inputs(beta = beta, data_dir=data_dir,
batch_size=batch_size, Delta2 = Delta2, epsilon2 = epsilon2, W_conv1Noise = W_conv1Noise)
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def train():
max_steps = 30000
batch_size = 128
data_dir = './cifar-10-binary/cifar-10-batches-bin'
'''
使用cifar10_input类中的disorted_inputs函数产生训练数据,产生的数据是已经封装好的Tensor,每次会产生batch_size个
这个函数已经对图片数据做了增强操作(随机水平翻转/剪切/随机对比度等)
同时因为对图像处理需要耗费大量计算资源,该函数使用了16个独立的线程来加速任务,
函数内部会产生线程池,使用会通过TensorFlow queue进行调度
'''
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(eval_data=True, data_dir=data_dir, batch_size=batch_size)
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
# 第一层 卷积-->池化-->lrn
# 不带L2正则项(wl=0)的64个5x5x3的滤波器,
# 使用lrn是从局部多个卷积核的响应中挑选比较大的反馈变得相对最大,并抑制其他反馈小的,增加模型泛化能力
weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=5e-2, wl=0.0)
kernel1 = tf.nn.conv2d(image_holder, weight1, strides=[1, 1, 1, 1], padding='SAME')
bias1 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv1 = tf.nn.relu(tf.nn.bias_add(kernel1, bias1))
pool1 = tf.nn.max_pool(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
# 第二层 卷积-->lrn-->池化
weight2 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel2 = tf.nn.conv2d(norm1, weight2, strides=[1, 1, 1, 1], padding='SAME')
bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
conv2 = tf.nn.relu(tf.nn.bias_add(kernel2, bias2))
norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第三层 使用全连接层 reshape后获取长度并创建FC1层的权重(带L2正则化)
reshape = tf.reshape(pool2, [batch_size, -1])
dim = reshape.get_shape()[1].value
weight3 = variable_with_weight_loss(shape=[dim, 384], stddev=0.04, wl=0.004)
bias3 = tf.Variable(tf.constant(0.1, shape=[384]))
local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3)
# 第四层 FC2层 节点数减半 依旧带L2正则
weight4 = variable_with_weight_loss(shape=[384, 192], stddev=0.04, wl=0.004)
bias4 = tf.Variable(tf.constant(0.1, shape=[192]))
local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4)
# 最后一层 这层的weight设为正态分布标准差设为上一个FC层的节点数的倒数
# 这里我们不计算softmax,把softmax放到后面计算
weight5 = variable_with_weight_loss(shape=[192, 10], stddev=1 / 192.0, wl=0.0)
bias5 = tf.Variable(tf.constant(0.0, shape=[10]))
logits = tf.add(tf.matmul(local4, weight5), bias5)
# 损失函数为两个带L2正则的FC层和最后的转换层
# 优化器依旧是AdamOptimizer,学习率是1e-3
losses = loss(logits, label_holder)
train_op = tf.train.AdamOptimizer(1e-3).minimize(losses)
# in_top_k函数求出输出结果中top k的准确率,这里选择输出top1
top_k_op = tf.nn.in_top_k(logits, label_holder, 1)
# 创建默认session,初始化变量
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# 启动图片增强线程队列
tf.train.start_queue_runners()
# 训练
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value = sess.run([train_op, losses], feed_dict={image_holder: image_batch,
label_holder: label_batch})
duration = time.time() - start_time
if step % 10 == 0:
examples_per_sec = batch_size / duration
sec_per_batch = float(duration)
format_str = ('step %d,loss=%.2f (%.1f examples/sec; %.3f sec/batch)')
print(format_str % (step, loss_value, examples_per_sec, sec_per_batch))
# 评估模型的准确率,测试集一共有10000个样本
# 我们先计算大概要多少个batch能测试完所有样本
num_examples = 10000
import math
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size # 除去不够一个batch的
step = 0
while step < num_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch,
label_holder: label_batch})
true_count += np.sum(predictions) # 利用top_k_op计算输出结果
step += 1
precision = true_count / total_sample_count
print('precision @ 1=%.3f' % precision) # 这里如果输出为0.00 是因为整数/整数 记得要导入float除法
def distorted_inputs():
data_dir = os.path.join('data/cifar10_data', 'cifar-10-batches-bin')
return cifar10_input.distorted_inputs(data_dir=data_dir,
batch_size=batch_size)
# In[5]:
# CIFAR-10 data
if dataset == 2:
num_imgs = 6000 # images per class
num_classes = 10
img_size = 32 # pixel size
# from tensorflow.models.image.cifar10 import cifar10
import cifar10_input as cifar10
url = 'https://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz'
# data_filename = download('cifar-10-binary.tar.gz')
folder = untar_cifar('data/cifar-10-binary.tar.gz')
dataset, labels = cifar10.distorted_inputs('data/cifar-10-binary/cifar-10-batches-bin', 128)
# folder = untar_cifar('data/cifar-10-python.tar.gz')
# data_folders = unpickle(folder[0])
# In[ ]:
# notMNIST data
if dataset == 3:
num_imgs = 1500 # images per class
num_classes = 10
img_size = 28 # pixel size
url = 'http://yaroslavvb.com/upload/notMNIST/'
def train():
print('[Dataset Configuration]')
print('\tCIFAR-10 dir: %s' % FLAGS.data_dir)
print('\tNumber of classes: %d' % FLAGS.num_classes)
print('\tNumber of training images: %d' % FLAGS.num_train_instance)
print('\tNumber of test images: %d' % FLAGS.num_test_instance)
print('[Network Configuration]')
print('\tBatch size: %d' % FLAGS.batch_size)
#print('\tResidual blocks per group: %d' % FLAGS.num_residual_units)
#print('\tNetwork width multiplier: %d' % FLAGS.k)
print('[Optimization Configuration]')
#print('\tL2 loss weight: %f' % FLAGS.l2_weight)
print('\tThe momentum optimizer: %f' % FLAGS.momentum)
print('\tInitial learning rate: %f' % FLAGS.initial_lr)
print('\tEpochs per lr step: %f' % FLAGS.lr_step_epoch)
print('\tLearning rate decay: %f' % FLAGS.lr_decay)
print('[Training Configuration]')
print('\tTrain dir: %s' % FLAGS.train_dir)
print('\tTraining max steps: %d' % FLAGS.max_steps)
print('\tSteps per displaying info: %d' % FLAGS.display)
print('\tSteps per testing: %d' % FLAGS.test_interval)
print('\tSteps during testing: %d' % FLAGS.test_iter)
print('\tSteps per saving checkpoints: %d' % FLAGS.checkpoint_interval)
print('\tGPU memory fraction: %f' % FLAGS.gpu_fraction)
print('\tLog device placement: %d' % FLAGS.log_device_placement)
with tf.Graph().as_default():
init_step = 0
global_step = tf.Variable(0, trainable=False, name='global_step')
# Get images and labels of CIFAR-100
with tf.variable_scope('train_image'):
train_images, train_labels = data_input.distorted_inputs(
FLAGS.data_dir, FLAGS.batch_size)
with tf.variable_scope('test_image'):
test_images, test_labels = data_input.inputs(
True, FLAGS.data_dir, FLAGS.batch_size)
# Build a Graph that computes the predictions from the inference model.
images = tf.placeholder(tf.float32, [
FLAGS.batch_size, data_input.IMAGE_SIZE, data_input.IMAGE_SIZE, 3
])
labels = tf.placeholder(tf.int32, [FLAGS.batch_size])
# Build model
decay_step = FLAGS.lr_step_epoch * FLAGS.num_train_instance / FLAGS.batch_size
hp = wrinc.HParams(batch_size=FLAGS.batch_size,
num_classes=FLAGS.num_classes,
initial_lr=FLAGS.initial_lr,
decay_step=decay_step,
lr_decay=FLAGS.lr_decay,
momentum=FLAGS.momentum)
network = wrinc.wrinc(hp, images, labels, global_step)
network.build_model()
network.build_train_op()
# Summaries(training)
train_summary_op = tf.merge_all_summaries()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
gpu_options=tf.GPUOptions(
per_process_gpu_memory_fraction=FLAGS.gpu_fraction),
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
# Create a saver.
saver = tf.train.Saver(tf.all_variables(), max_to_keep=10000)
ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
#pdb.set_trace()
if ckpt and ckpt.model_checkpoint_path:
print('\tRestore from %s' % ckpt.model_checkpoint_path)
# Restores from checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
init_step = int(
ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1])
else:
print('No checkpoint file found. Start from the scratch.')
# Start queue runners & summary_writer
tf.train.start_queue_runners(sess=sess)
if not os.path.exists(FLAGS.train_dir):
os.mkdir(FLAGS.train_dir)
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph)
# Training!
test_best_acc = 0.0
for step in xrange(init_step, FLAGS.max_steps):
# Test
if step % FLAGS.test_interval == 0:
test_loss, test_acc = 0.0, 0.0
for i in range(FLAGS.test_iter):
test_images_val, test_labels_val = sess.run(
[test_images, test_labels])
loss_value, acc_value = sess.run(
[network.loss, network.acc],
feed_dict={
network.is_train: False,
images: test_images_val,
labels: test_labels_val
})
test_loss += loss_value
test_acc += acc_value
test_loss /= FLAGS.test_iter
test_acc /= FLAGS.test_iter
test_best_acc = max(test_best_acc, test_acc)
format_str = ('%s: (Test) step %d, loss=%.4f, acc=%.4f')
print(format_str % (datetime.now(), step, test_loss, test_acc))
# test_summary = tf.Summary()
# test_summary.value.add(tag='test/loss', simple_value=test_loss)
# test_summary.value.add(tag='test/acc', simple_value=test_acc)
# test_summary.value.add(tag='test/best_acc', simple_value=test_best_acc)
# summary_writer.add_summary(test_summary, step)
# test_loss_summary = tf.Summary()
# test_loss_summary.value.add(tag='test/loss', simple_value=test_loss)
# summary_writer.add_summary(test_loss_summary, step)
# test_acc_summary = tf.Summary()
# test_acc_summary.value.add(tag='test/acc', simple_value=test_acc)
# summary_writer.add_summary(test_acc_summary, step)
# test_best_acc_summary = tf.Summary()
# test_best_acc_summary.value.add(tag='test/best_acc', simple_value=test_best_acc)
# summary_writer.add_summary(test_best_acc_summary, step)
# summary_writer.flush()
# Train
start_time = time.time()
train_images_val, train_labels_val = sess.run(
[train_images, train_labels])
_, lr_value, loss_value, acc_value, = \
sess.run([network.train_op, network.lr, network.loss, network.acc],
feed_dict={network.is_train:True, images:train_images_val, labels:train_labels_val})
duration = time.time() - start_time
assert not np.isnan(loss_value)
# Display & Summary(training)
if step % FLAGS.display == 0:
num_examples_per_step = FLAGS.batch_size
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = (
'%s: (Training) step %d, loss=%.4f, acc=%.4f, lr=%f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str %
(datetime.now(), step, loss_value, acc_value, lr_value,
examples_per_sec, sec_per_batch))
# summary_writer.add_summary(train_summary_str, step)
# Save the model checkpoint periodically.
if (step > init_step and step % FLAGS.checkpoint_interval
== 0) or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
def main():
with tf.Graph().as_default(), tf.device('/cpu:0'):
# _, train = get_input.get_cifar10_input(DATA_PATH)
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=cifar10_data_path, batch_size=BATCH_SIZE)
# image_test, labels_test = cifar10_input.distorted_inputs(eval_data=True, data_dir=cifar10_data_path,
# batch_size=BATCH_SIZE)
# image_holder = tf.placeholder(tf.float32, [BATCH_SIZE, 24, 24, 3])
# label_holder = tf.placeholder(tf.int32, [BATCH_SIZE])
# resized_x = tf.image.resize_images(x, IN_NET_SIZE, method=0)
# regu = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
regu = None
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
# learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step,
# TRAINING_STEPS / math.log(0.005, LEARNING_RATE_DECAY),
# LEARNING_RATE_DECAY)
num_batcher_per_epoch = 50000 / BATCH_SIZE
decay_step = int(num_batcher_per_epoch * 350.0)
learning_rate = tf.train.exponential_decay(0.01, global_step,
decay_step, 0.1)
opt = tf.train.GradientDescentOptimizer(learning_rate)
cur_loss = []
tower_grads = []
reuse_variables = False
for i in N_GPU:
with tf.device('/gpu:%d' % i):
with tf.name_scope('GPU_%d' % i) as scope:
loss = tower_loss(scope, images_train, labels_train,
reuse_variables)
tf.get_variable_scope().reuse_variables()
# cur_loss = get_loss(images_train, labels_train, regu, 10, scope, reuse_variables)
# reuse_variables = True
# # tf.get_variable_scope().reuse_variables()
grads = opt.compute_gradients(loss)
tower_grads.append(grads)
grads = average_gradients(tower_grads)
for grad, var in grads:
if grad is not None:
tf.summary.histogram('gradients_on_average/%s' % var.op.name,
grad)
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
for var in tf.trainable_variables():
tf.summary.histogram(var.op.name, var)
# variable_averages = tf.train.ExponentialMovingAverage(MOVING_AVERAGE_DECAY, global_step)
# variables_to_average = (tf.trainable_variables() + tf.moving_average_variables())
# variables_averages_op = variable_averages.apply(variables_to_average)
# train_op = tf.group(apply_gradient_op, variables_averages_op)
saver = tf.train.Saver(tf.all_variables())
summary_op = tf.summary.merge_all()
# top_k_op = tf.nn.in_top_k()
init = (tf.global_variables_initializer(),
tf.local_variables_initializer())
gpu_options = tf.GPUOptions(allow_growth=True)
config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
with tf.Session(config=config) as sess:
sess.run(init)
# if os.path.exists(MODEL_SAVE_PATH):
# ckpt = tf.train.get_checkpoint_state(MODEL_SAVE_PATH) # net_train.MODEL_SAVE_PATH)
# # num_str = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
# saver_goon = tf.train.Saver()
# saver_goon.restore(sess, ckpt.model_checkpoint_path)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
# tf.train.start_queue_runners()
summary_writer = tf.summary.FileWriter(MODEL_SAVE_PATH, sess.graph)
for step in range(TRAINING_STEPS):
# sta = (step * BATCH_SIZE) % 50000
# end = (step * BATCH_SIZE + BATCH_SIZE) % 50000
# if sta < end:
# xs = train[b'data'][sta:end]
# ys = train[b'labels'][sta:end]
# else:
# xs = np.row_stack((train[b'data'][sta:], train[b'data'][:end]))
# ys = train[b'labels'][sta:] + train[b'labels'][:end]
# ys = np.array(ys)
# print("*****************************")
# print(type(ys))
# print("*****************************")
start_time = time.time()
# image_batch, label_batch = sess.run([images_train, labels_train])
# print(image_batch, image_batch.dtype, image_batch.shape)
# print(label_batch, label_batch.dtype, label_batch.shape)
_, loss_value = sess.run([apply_gradient_op,
loss]) #, train_op, cur_loss
#feed_dict={image_holder: image_batch, label_holder: label_batch})
duration = time.time() - start_time
if step != 0 and step % 10 == 0:
num_examples_per_step = BATCH_SIZE * len(N_GPU)
examples_per_sec = num_examples_per_step / duration
sec_per_batch = duration / len(N_GPU)
format_str = '%s: step %d, loss = %.2f (%.1f examples/sec; %.3f sec/batch)'
print(format_str % (datetime.now(), step, loss_value,
examples_per_sec, sec_per_batch))
summary = sess.run(summary_op)
summary_writer.add_summary(summary, step)
if step % 1000 == 0 or (step + 1) == TRAINING_STEPS:
checkpoint_path = os.path.join(MODEL_SAVE_PATH, MODEL_NAME)
saver.save(sess, checkpoint_path, global_step=step)
# a,c = sess.run([image_batch, label_batch])
# #print(a,a.shape)
# print(c,c.shape,c.dtype)
coord.request_stop()
coord.join(threads)
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
issync = FLAGS.issync
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index,
cluster=cluster)):
global_step = tf.Variable(0, name='global_step', trainable=False)
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(
eval_data=True, data_dir=data_dir, batch_size=batch_size)
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.float32, [batch_size])
# 第一层
# 输入为 batch*24*24*3 卷积后为b*24*24*64 池化后为b*14*14*64 b 是batch的简写
weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64],
stddev=0.05,
w1=0)
kernel1 = tf.nn.conv2d(image_holder,
weight1, [1, 1, 1, 1],
padding='SAME')
bias1 = tf.Variable(tf.constant(0.0, shape=[64]))
# conv1 = tf.nn.relu(tf.nn.bias_add(kernel1,bias1))
conv1 = tf.nn.relu(kernel1 + bias1)
pool1 = tf.nn.max_pool(conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
# 第二层 输入b*14*14*64 卷积后卫 b*14*14*64 池化后卫为 b*7*7*64
weight2 = variable_with_weight_loss(shape=[5, 5, 64, 64],
stddev=0.05,
w1=0.0)
kernel2 = tf.nn.conv2d(norm1,
weight2, [1, 1, 1, 1],
padding='SAME')
bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
conv2 = tf.nn.relu(kernel2 + bias1)
norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool2 = tf.nn.max_pool(norm2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
# 第三层 自己添加一层 Atrous Convolution卷积层 空洞卷积（膨胀卷积）
weight3 = variable_with_weight_loss(shape=[5, 5, 64, 128],
stddev=0.05,
w1=0.0)
"""
rate：
要求是一个int型的正数，正常的卷积操作应该会有stride（即卷积核的滑动步长），
但是空洞卷积是没有stride参数的，这一点尤其要注意。取而代之，它使用了新的rate参数，
那么rate参数有什么用呢？它定义为我们在输入图像上卷积时的采样间隔，
你可以理解为卷积核当中穿插了（rate-1）数量的“0”，把原来的卷积核插出了很多“洞洞”，
这样做卷积时就相当于对原图像的采样间隔变大了。具体怎么插得，可以看后面更加详细的描述。
此时我们很容易得出rate=1时，就没有0插入，此时这个函数就变成了普通卷积。
"""
# 输入为 b*7*7*64 空洞卷积缺少strides参数，所以仍然卷积之后是 b*7*7*128
kernel3 = tf.nn.atrous_conv2d(
pool2, weight3, rate=2,
padding='SAME') # 空洞卷积会把上面的卷积核编程 10*10的大小
bias3 = tf.Variable(tf.constant(0.1, shape=[128])) # 输出维度128
conv3 = tf.nn.relu(kernel3 + bias3) # 输出为b*7*7*128
norm3 = tf.nn.lrn(conv3, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool3 = tf.nn.max_pool(
norm3,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME') # strides的步长上下都是2，输出为b*4*4*128
# 全链接
reshape = tf.reshape(pool3, [batch_size, -1]) # 分开成为
dim = reshape.get_shape()[1].value #
weight3 = variable_with_weight_loss([dim, 384],
stddev=0.04,
w1=0.004)
bias3 = tf.Variable(tf.constant(0.1, shape=[384]))
local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3)
# 全连接层
weight4 = variable_with_weight_loss([384, 192],
stddev=0.04,
w1=0.004)
bias4 = tf.Variable(tf.constant(0.1, shape=[192]))
local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4)
# 输出层
weight5 = variable_with_weight_loss([192, 10],
stddev=1 / 192,
w1=0.0)
bias5 = tf.Variable(tf.constant(0.0, shape=[10]))
logits = tf.matmul(local4, weight5) + bias5
def loss(logits, labels):
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=labels,
name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
return tf.add_n(tf.get_collection('losses'), name='total_loss')
loss_value = loss(logits, label_holder)
optimizer = tf.train.AdamOptimizer()
grads_and_vars = optimizer.compute_gradients(loss_value)
top_k_op = tf.nn.in_top_k(logits, tf.cast(label_holder, tf.int64),
1)
if issync == 1:
# 同步模式计算更新梯度
rep_op = tf.train.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=6,
# replica_id=FLAGS.task_index,
total_num_replicas=6,
use_locking=True)
train_op = rep_op.apply_gradients(grads_and_vars,
global_step=global_step)
init_token_op = rep_op.get_init_tokens_op()
chief_queue_runner = rep_op.get_chief_queue_runner()
else:
# 异步模式计算更新梯度
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
init_op = tf.initialize_all_variables()
# saver = tf.train.Saver()
tf.summary.scalar('cost', loss_value)
summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(
is_chief=(FLAGS.task_index == 0),
# logdir="./checkpoint/",
init_op=init_op,
summary_op=None,
# saver=saver,
global_step=global_step,
# save_model_secs=60
)
with sv.prepare_or_wait_for_session(server.target) as sess:
# 如果是同步模式
if FLAGS.task_index == 0 and issync == 1:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_token_op)
# sess = tf.InteractiveSession()
# tf.global_variables_initializer().run()
# tf.train.start_queue_runners()
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run(
[images_train, labels_train])
_, loss = sess.run([train_op, loss_value],
feed_dict={
image_holder: image_batch,
label_holder: label_batch
})
duration = time.time() - start_time
if step % 10 == 0:
examples_per_sec = batch_size / duration
sec_per_batch = float(duration)
format_str = (
'step %d,loss=%.2f (%.1f examples/sec;%.3f sec/batch)')
print(format_str %
(step, loss, examples_per_sec, sec_per_batch))
num_examples = 10000
import math
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size
step = 0
# with tf.Session() as sess:
while step < num_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op],
feed_dict={
image_holder: image_batch,
label_holder: label_batch
})
true_count += np.sum(predictions)
step += 1
if step % 10 == 0:
print(true_count)
end = time.clock() # 计算程序结束时间
out = (end - start)
precision = float(true_count) / total_sample_count
print('precision @ 1 =%.3f' % precision)
print("running time is", out, "s")
import cifar10_input
import numpy as np
import time
import tensorflow as tf
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir='../../../_dataset/cifar-10-batches-bin', batch_size=128)
images_test, labels_test = cifar10_input.inputs(
eval_data=True,
data_dir='../../../_dataset/cifar-10-batches-bin',
batch_size=128)
x_input = tf.placeholder(tf.float32, [None, 24, 24, 3])
y_input = tf.placeholder(tf.float32, [None])
def predict(x_input):
conv1 = tf.layers.conv2d(x_input,
64,
5,
padding='SAME',
activation=tf.nn.relu)
pool1 = tf.layers.max_pooling2d(conv1, 3, 2, padding='SAME')
norm1 = tf.nn.lrn(pool1, 4, bias=1, alpha=0.001 / 9.0, beta=0.75)
conv2 = tf.layers.conv2d(norm1,
64,
5,
padding='SAME',
activation=tf.nn.relu)
def train():
"""Train CIFAR-10 for a number of steps."""
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
# Get images and labels for CIFAR-10.
#images, labels = cifar10.distorted_inputs()
#images, labels = cifar10.inputs(eval_data=False, data_dir=FLAGS.data_dir,
# batch_size=FLAGS.batch_size)
images, labels = cifar10_input.distorted_inputs(data_dir=FLAGS.data_dir,
batch_size=FLAGS.batch_size)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate loss.
loss = cifar10.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = cifar10.train(loss, global_step)
# Create a saver.
saver = tf.train.Saver(tf.all_variables())
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Build an initialization operation to run below.
init = tf.initialize_all_variables()
# Start running operations on the Graph.
sess = tf.Session(config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement))
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph)
for step in xrange(FLAGS.max_steps):
start_time = time.time()
_, loss_value = sess.run([train_op, loss])
duration = time.time() - start_time
assert not np.isnan(loss_value), 'Model diverged with loss = NaN'
if step % 100 == 0:
num_examples_per_step = FLAGS.batch_size
examples_per_sec = num_examples_per_step / duration
sec_per_batch = float(duration)
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), step, loss_value,
examples_per_sec, sec_per_batch))
if step % 100 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
# Save the model checkpoint periodically.
if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
def train():
batch_size = 128
data_dir = '/Users/macbookair/iCloud/Desktop/Daily/Second Major/CS/Machine Learning/hw2/CIFAR/cifar-10-batches-bin'
'''
使用cifar10_input类中的disorted_inputs函数产生训练数据,产生的数据是已经封装好的Tensor,每次会产生batch_size个
这个函数已经对图片数据做了增强操作(随机水平翻转/剪切/随机对比度等)
同时因为对图像处理需要耗费大量计算资源,该函数使用了16个独立的线程来加速任务,
函数内部会产生线程池,使用会通过TensorFlow queue进行调度
'''
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(eval_data=True, data_dir=data_dir, batch_size=batch_size)
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
keep_proportion_conv1 = tf.placeholder(tf.float32)
keep_proportion_conv2 = tf.placeholder(tf.float32)
keep_proportion_conv3 = tf.placeholder(tf.float32)
keep_proportion_conv4 = tf.placeholder(tf.float32)
keep_proportion_conv5 = tf.placeholder(tf.float32)
keep_proportion_conv6 = tf.placeholder(tf.float32)
keep_proportion_conv7 = tf.placeholder(tf.float32)
keep_proportion_fc = tf.placeholder(tf.float32)
# 第一层 卷积-->池化-->lrn
# 不带L2正则项(wl=0)的64个5x5x3的滤波器,
# 使用lrn是从局部多个卷积核的响应中挑选比较大的反馈变得相对最大,并抑制其他反馈小的,增加模型泛化能力
weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=5e-2, wl=0.0)
kernel1 = tf.nn.conv2d(image_holder, weight1, strides=[1, 1, 1, 1], padding='SAME')
bias1 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv1 = leaky_relu(tf.nn.bias_add(kernel1, bias1))
# drop1 = tf.nn.dropout(conv1, keep_proportion_conv1) # dropout in CNN !
norm1 = tf.nn.lrn(conv1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool1 = tf.nn.max_pool(norm1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第二层 卷积-->池化-->lrn
weight2 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel2 = tf.nn.conv2d(pool1, weight2, strides=[1, 1, 1, 1], padding='SAME')
bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
conv2 = leaky_relu(tf.nn.bias_add(kernel2, bias2))
# drop2 = tf.nn.dropout(conv2, keep_proportion_conv2)
norm2 = tf.nn.lrn(conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool2 = tf.nn.max_pool(norm2, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第三层 卷积-->池化-->lrn
weight3 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel3 = tf.nn.conv2d(pool2, weight3, strides=[1, 1, 1, 1], padding='SAME')
bias3 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv3 = leaky_relu(tf.nn.bias_add(kernel3, bias3))
# drop3 = tf.nn.dropout(conv3, keep_proportion_conv3) # dropout in CNN !
norm3 = tf.nn.lrn(conv3, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool3 = tf.nn.max_pool(norm3, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第四层 卷积-->池化-->lrn
weight4 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel4 = tf.nn.conv2d(pool3, weight4, strides=[1, 1, 1, 1], padding='SAME')
bias4 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv4 = leaky_relu(tf.nn.bias_add(kernel4, bias4))
# drop4 = tf.nn.dropout(conv4, keep_proportion_conv4) # dropout in CNN !
norm4 = tf.nn.lrn(conv4, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool4 = tf.nn.max_pool(norm4, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第五层 卷积-->池化-->lrn
weight5 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel5 = tf.nn.conv2d(pool4, weight5, strides=[1, 1, 1, 1], padding='SAME')
bias5 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv5 = leaky_relu(tf.nn.bias_add(kernel5, bias5))
# drop5 = tf.nn.dropout(conv5, keep_proportion_conv5) # dropout in CNN !
norm5 = tf.nn.lrn(conv5, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool5 = tf.nn.max_pool(norm5, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第六层 卷积-->池化-->lrn
weight6 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel6 = tf.nn.conv2d(pool5, weight6, strides=[1, 1, 1, 1], padding='SAME')
bias6 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv6 = leaky_relu(tf.nn.bias_add(kernel6, bias6))
# drop6 = tf.nn.dropout(conv6, keep_proportion_conv6) # dropout in CNN !
norm6 = tf.nn.lrn(conv6, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool6 = tf.nn.max_pool(norm6, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第七层 卷积-->lrn-->池化
weight7 = variable_with_weight_loss(shape=[5, 5, 64, 64], stddev=5e-2, wl=0.0)
kernel7 = tf.nn.conv2d(pool6, weight7, strides=[1, 1, 1, 1], padding='SAME')
bias7 = tf.Variable(tf.constant(0.0, shape=[64])) # bias直接初始化为0
conv7 = leaky_relu(tf.nn.bias_add(kernel7, bias7))
# drop7 = tf.nn.dropout(conv7, keep_proportion_conv7) # dropout in CNN !
norm7 = tf.nn.lrn(conv7, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75)
pool7 = tf.nn.max_pool(norm7, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME')
# 第八层 使用全连接层 reshape后获取长度并创建FC1层的权重(带L2正则化)
pool7_reshaped = tf.reshape(pool7, [batch_size, -1])
dim = pool7_reshaped.get_shape()[1].value
weight8 = variable_with_weight_loss(shape=[dim, 384], stddev=0.04, wl=0.004)
bias8 = tf.Variable(tf.constant(0.1, shape=[384]))
local8 = leaky_relu(tf.matmul(pool7_reshaped, weight8) + bias8)
fc8 = tf.nn.dropout(local8, keep_proportion_fc)
# 第九层 FC2层 节点数减半 依旧带L2正则
weight9 = variable_with_weight_loss(shape=[384, 192], stddev=0.04, wl=0.004)
bias9 = tf.Variable(tf.constant(0.1, shape=[192]))
local9 = leaky_relu(tf.matmul(fc8, weight9) + bias9)
fc9 = tf.nn.dropout(local9, keep_proportion_fc)
# 最后一层 这层的weight设为正态分布标准差设为上一个FC层的节点数的倒数
# 这里我们不计算softmax,把softmax放到后面计算
weight10 = variable_with_weight_loss(shape=[192, 10], stddev=1 / 192.0, wl=0.0)
bias10 = tf.Variable(tf.constant(0.0, shape=[10]))
logits = tf.add(tf.matmul(fc9, weight10), bias10)
# 损失函数为两个带L2正则的FC层和最后的转换层
# 优化器依旧是AdamOptimizer,学习率是1e-3
losses = loss(logits, label_holder)
global_step = tf.Variable(0) # 相当于global_step,是一个全局变量,在训练完一个批次后自动增加1
learning_rate = tf.train.exponential_decay(learning_rate=0.00007, global_step=global_step,
decay_steps=230, decay_rate=0.37, staircase=True) # 每200轮之后乘以0.9
optimizer = tf.train.AdamOptimizer(learning_rate=1e-3).minimize(loss=losses)
add_global = global_step.assign_add(1)
# in_top_k函数求出输出结果中top k的准确率,这里选择输出top1
top_k_op = tf.nn.in_top_k(logits, label_holder, 1)
# 创建默认session,初始化变量
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# 启动图片增强线程队列
tf.train.start_queue_runners()
max_steps = 23000
# 训练
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value = sess.run([optimizer, losses],
feed_dict={image_holder: image_batch,
label_holder: label_batch,
keep_proportion_conv1: 1.0,
keep_proportion_conv2: 1.0,
keep_proportion_conv3: 1.0,
keep_proportion_conv4: 1.0,
keep_proportion_conv5: 1.0,
keep_proportion_conv6: 1.0,
keep_proportion_conv7: 1.0,
keep_proportion_fc: 0.7})
duration = time.time() - start_time
if step % 10 == 0:
examples_per_sec = batch_size / duration
sec_per_batch = float(duration)
format_str = 'step %d,loss=%.2f (%.1f examples/sec; %.3f sec/batch)'
print(format_str % (step, loss_value, examples_per_sec, sec_per_batch))
# 评估模型的准确率,测试集一共有10000个样本
# 我们先计算大概要多少个batch能测试完所有样本
num_examples = 10000
import math
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size # 除去不够一个batch的
step = 0
while step < num_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch,
label_holder: label_batch,
keep_proportion_conv1: 1.0,
keep_proportion_conv2: 1.0,
keep_proportion_conv3: 1.0,
keep_proportion_conv4: 1.0,
keep_proportion_conv5: 1.0,
keep_proportion_conv6: 1.0,
keep_proportion_conv7: 1.0,
keep_proportion_fc: 1.0})
true_count += np.sum(predictions) # 利用top_k_op计算输出结果
step += 1
precision = true_count / total_sample_count
print('precision @ 1=%.3f' % precision) # 这里如果输出为0.00 是因为整数/整数 记得要导入float除法
def main():
cifar10_image, cifar10_labels = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=BATCH_SIZE)
backward(cifar10_image, cifar10_labels)
def main():
"""
主函数
"""
# 下载cifar10数据集并解压
cifar10.maybe_download_and_extract()
# distorted_inputs产生训练数据
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=dataset_dir,
batch_size=batch_size)
# 产生测试数据
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=dataset_dir,
batch_size=batch_size)
# 为特征和label创建placeholder
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.int32, [batch_size])
# 创建CNN网络，并得到输出
logitis = build_cnn_network(image_holder)
# 计算loss
total_loss = get_total_loss(logitis, label_holder)
# 设置优化算法
train_op = tf.train.AdamOptimizer(1e-3).minimize(total_loss)
top_k_op = tf.nn.in_top_k(logitis, label_holder, 1)
# 创建session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
# 启动线程操作，因为cifar10_input.distorted_inputs需要线程操作
tf.train.start_queue_runners()
# 训练模型
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value = sess.run([train_op, total_loss],
feed_dict={image_holder: image_batch,
label_holder: label_batch})
duration = time.time() - start_time
if step % disp_step == 0:
sample_per_sec = batch_size / duration
sec_per_batch = float(duration)
print('step %d, loss=%.2f (%.1f sample/sec; %.3f sec/batch)' % (
step, loss_value, sample_per_sec, sec_per_batch
))
# 测试模型
n_test_samples = 10000
n_iter = int(math.ceil(n_test_samples / batch_size))
true_count = 0
total_sample_count = n_iter * batch_size
step = 0
while step < n_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op], feed_dict={image_holder: image_batch,
label_holder: label_batch})
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count
print('top 1 precision: %.3f' % precision)
def run_training():
"""Train model for a number of steps."""
# Get the sets of images and labels for training, validation, and
# test on MNIST.
# Tell TensorFlow that the model will be built into the default Graph.
train_dir = os.path.join(FLAGS.model_dir,FLAGS.name)
with tf.Graph().as_default():
global_step = tf.Variable(0, trainable=False)
with tf.name_scope('Input'):
image_batch, label_batch = data_input.distorted_inputs(FLAGS.data_dir, FLAGS.batch_size)
# Generate placeholders for the images and labels.
keep_prob = utils.placeholder_inputs(FLAGS.batch_size)
# Build a Graph that computes predictions from the inference model.
logits = model.inference(image_batch, keep_prob)
# Add to the Graph the Ops for loss calculation.
loss = model.loss(logits, label_batch)
# Add to the Graph the Ops that calculate and apply gradients.
train_op = model.training(loss, global_step=global_step, learning_rate=FLAGS.learning_rate)
# Add the Op to compare the logits to the labels during evaluation.
eval_correct = model.evaluation(logits, label_batch)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
# Create a saver for writing training checkpoints.
saver = tf.train.Saver()
# Create a session for running Ops on the Graph.
sess = tf.Session()
# Run the Op to initialize the variables.
init = tf.initialize_all_variables()
sess.run(init)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
# Instantiate a SummaryWriter to output summaries and the Graph.
summary_writer = tf.train.SummaryWriter(train_dir,
graph_def=sess.graph_def)
# And then after everything is built, start the training loop.
for step in xrange(FLAGS.max_steps):
start_time = time.time()
# Fill a feed dictionary with the actual set of images and labels
# for this particular training step.
feed_dict = utils.fill_feed_dict(keep_prob,
train = True)
# Run one step of the model. The return values are the activations
# from the `train_op` (which is discarded) and the `loss` Op. To
# inspect the values of your Ops or variables, you may include them
# in the list passed to sess.run() and the value tensors will be
# returned in the tuple from the call.
_, loss_value = sess.run([train_op, loss],
feed_dict=feed_dict)
# Write the summaries and print an overview fairly often.
if step % 100 == 0:
# Print status to stdout.
duration = time.time() - start_time
examples_per_sec = FLAGS.batch_size / duration
sec_per_batch = float(duration)
print('Step %d: loss = %.2f ( %.3f sec (per Batch); %.1f examples/sec;)' % (step, loss_value,
sec_per_batch, examples_per_sec))
# Update the events file.
summary_str = sess.run(summary_op, feed_dict=feed_dict)
summary_writer.add_summary(summary_str, step)
# Save a checkpoint and evaluate the model periodically.
if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:
checkpoint_path = os.path.join(train_dir, 'model.ckpt')
saver.save(sess, checkpoint_path , global_step=step)
# Evaluate against the training set.
if (step + 1) % 5000 == 0 or (step + 1) == FLAGS.max_steps:
print('Training Data Eval:')
utils.do_eval(sess,
eval_correct,
keep_prob,
data_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN)
:param shape: 变量尺寸
:param stddev: 方差
:param w1: 权重
:return:
"""
var = tf.Variable(tf.truncated_normal(shape=shape, stddev=stddev))
if w1 is not None:
weight_loss = tf.multiply(tf.nn.l2_loss(var), w1, name='weight_loss')
tf.add_to_collection('losses', weight_loss)
return var
cifar10.maybe_download_and_extract()
# 获取数据
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size)
# 输入数据的placeholder
image_holder = tf.placeholder(tf.float32,
[batch_size, 24, 24, 3]) # 图片规格24x24x3
label_holder = tf.placeholder(tf.float32, [batch_size])
# 卷积层1
weight1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=0.05,
w1=0.0) # 设置初始化函数标准差为0.05，不进行正则
kernel1 = tf.nn.conv2d(image_holder, weight1, [1, 1, 1, 1],
padding='SAME') # 卷积操作
bias1 = tf.Variable(tf.constant(0.0, shape=[64]))
W_fc2 = init_Weights([1024, 10])
b_fc2 = init_Biases([10])
prediction = fc_layer(fc1, W_fc2, b_fc2, tf.nn.softmax)
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=label, logits=prediction))
# a high learning rate could not learn anything.
optimizer = tf.train.AdamOptimizer(1e-4)
train = optimizer.minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(label, 1), tf.argmax(prediction, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
batch_xs_tensor, batch_ys_tensor = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
tf.train.start_queue_runners()
for i in range(epoch):
batch_xs, batch_ys = sess.run([batch_xs_tensor, batch_ys_tensor])
sess.run(train,
feed_dict={
image: batch_xs,
label_holder: batch_ys,
training: True
})
if i % 50 == 0:
images = []
acc = 0
def variable_with_weight_loss(shape, stddev, wl=None):
'''创建变量
创建变量，并为变量添加L2约束'''
var = tf.Variable(tf.truncated_normal(shape, stddev=stddev))
if wl is not None:
weight_loss = tf.multiply(tf.nn.l2_loss(var), wl, name='weight_loss')
tf.add_to_collection('losses', weight_loss)
return var
##下载数据集
cifar10.maybe_download_and_extract()
##生成增强(distorted)过的训练数据
images_train, labels_train = cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=batch_size)
##生成测试数据，只裁剪，无增强
images_test, labels_test = cifar10_input.inputs(eval_data=True, data_dir=data_dir, batch_size=batch_size)
##创建输入数据的占位符
images_holder = tf.placeholder(dtype=tf.float32, shape=[batch_size, 24, 24, 3])
labels_holder = tf.placeholder(dtype=tf.int32, shape=[batch_size])
#================================↓开始构建正向网络↓==================================
#======================================
# 创建第一个卷积层
#======================================
w_conv1 = variable_with_weight_loss(shape=[5, 5, 3, 64], stddev=0.05)
b_conv1 = tf.Variable(tf.constant(0.0, shape=[64]))
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
issync = FLAGS.issync
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
with tf.device(
tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index,
cluster=cluster)):
global_step = tf.Variable(0, name='global_step', trainable=False)
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size)
images_test, labels_test = cifar10_input.inputs(
eval_data=True, data_dir=data_dir, batch_size=batch_size)
image_holder = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
label_holder = tf.placeholder(tf.float32, [batch_size])
#第一层
#输入为 24*24*3
weight1 = variable_with_weight_loss(shape=[7, 7, 3, 64],
stddev=0.05,
w1=0)
kernel1 = tf.nn.conv2d(image_holder,
weight1, [1, 2, 2, 1],
padding='SAME')
bias1 = tf.Variable(tf.constant(0.0, shape=[64]))
# conv1 = tf.nn.relu(tf.nn.bias_add(kernel1,bias1))
conv1 = tf.nn.relu(kernel1 + bias1) #12*12*64
pool1 = tf.nn.max_pool(conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
norm1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0,
beta=0.75) #6*6*64
#第二层 输入6*6*64
weight2 = variable_with_weight_loss(shape=[1, 1, 64, 64],
stddev=0.05,
w1=0.0)
kernel2 = tf.nn.conv2d(norm1,
weight2, [1, 1, 1, 1],
padding='SAME')
bias2 = tf.Variable(tf.constant(0.1, shape=[64]))
conv2 = tf.nn.relu(kernel2 + bias2) #6*6*64
#第三层 输入6*6*128
weight3 = variable_with_weight_loss(shape=[3, 3, 64, 64],
stddev=0.05,
w1=0.0)
kernel3 = tf.nn.conv2d(conv2,
weight3, [1, 1, 1, 1],
padding='SAME')
bias3 = tf.Variable(tf.constant(0.1, shape=[64]))
conv3 = tf.nn.relu(kernel3 + bias3) #6*6*64
#第四层
weight4 = variable_with_weight_loss(shape=[1, 1, 64, 256],
stddev=0.05,
w1=0.0)
kernel4 = tf.nn.conv2d(conv3,
weight4, [1, 1, 1, 1],
padding='SAME')
bias4 = tf.Variable(tf.constant(0.1, shape=[256]))
conv4 = tf.nn.relu(kernel4 + bias4) #6*6*256
#第五层 第二次第二层
weight5 = variable_with_weight_loss(shape=[1, 1, 256, 64],
stddev=0.05,
w1=0.0)
kernel5 = tf.nn.conv2d(conv4,
weight5, [1, 1, 1, 1],
padding='SAME')
bias5 = tf.Variable(tf.constant(0.1, shape=[64]))
conv5 = tf.nn.relu(kernel5 + bias5) #6*6*64
#第六层 第二次第三层
weight6 = variable_with_weight_loss(shape=[3, 3, 64, 64],
stddev=0.05,
w1=0.0)
kernel6 = tf.nn.conv2d(conv5,
weight6, [1, 1, 1, 1],
padding='SAME')
bias6 = tf.Variable(tf.constant(0.1, shape=[64]))
conv6 = tf.nn.relu(kernel6 + bias6) #6*6*64
#第七层 第二次第四层
weight7 = variable_with_weight_loss(shape=[1, 1, 64, 256],
stddev=0.05,
w1=0.0)
kernel7 = tf.nn.conv2d(conv6,
weight7, [1, 1, 1, 1],
padding='SAME')
bias7 = tf.Variable(tf.constant(0.1, shape=[256]))
conv7 = tf.nn.relu(kernel7 + bias7) #6*6*256
#第八层 第三次第二层
weight8 = variable_with_weight_loss(shape=[1, 1, 256, 64],
stddev=0.05,
w1=0.0)
kernel8 = tf.nn.conv2d(conv7,
weight8, [1, 1, 1, 1],
padding='SAME')
bias8 = tf.Variable(tf.constant(0.1, shape=[64]))
conv8 = tf.nn.relu(kernel8 + bias8) #6*6*64
#第八层 第三次第三层
weight9 = variable_with_weight_loss(shape=[3, 3, 64, 64],
stddev=0.05,
w1=0.0)
kernel9 = tf.nn.conv2d(conv8,
weight9, [1, 1, 1, 1],
padding='SAME')
bias9 = tf.Variable(tf.constant(0.1, shape=[64]))
conv9 = tf.nn.relu(kernel9 + bias9) #6*6*64
#第八层 第三次第四层
weight10 = variable_with_weight_loss(shape=[1, 1, 64, 256],
stddev=0.05,
w1=0.0)
kernel10 = tf.nn.conv2d(conv9,
weight10, [1, 1, 1, 1],
padding='SAME')
bias10 = tf.Variable(tf.constant(0.1, shape=[256]))
conv10 = tf.nn.relu(kernel10 + bias10) #6*6*256
#常规卷积
def con_wy(input, shape_weight, shape_bias, strides):
weight = variable_with_weight_loss(shape_weight,
stddev=0.05,
w1=0.0)
kernel = tf.nn.conv2d(input, weight, strides, padding='SAME')
bias = tf.Variable(tf.constant(0.1, shape=[shape_bias]))
conv = tf.nn.relu(kernel + bias)
return conv
#空洞卷积
def atrous_conv_wy(input, shape_weight, shape_bias, rate):
weight = variable_with_weight_loss(shape_weight,
stddev=0.05,
w1=0.0)
kernel = tf.nn.atrous_conv2d(
input, weight, rate=rate,
padding='SAME') # 空洞卷积会把上面的卷积核编程 10*10的大小
bias = tf.Variable(tf.constant(0.1,
shape=[shape_bias])) # 输出维度128
conv = tf.nn.relu(kernel + bias) # 输出为b*7*7*128
return conv
#第九到十七层
conv11 = con_wy(input=conv10,
shape_weight=[1, 1, 256, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv12 = con_wy(input=conv11,
shape_weight=[3, 3, 128, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv13 = con_wy(input=conv12,
shape_weight=[1, 1, 128, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv14 = con_wy(input=conv13,
shape_weight=[1, 1, 512, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv15 = con_wy(input=conv14,
shape_weight=[3, 3, 128, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv16 = con_wy(input=conv15,
shape_weight=[1, 1, 128, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv17 = con_wy(input=conv16,
shape_weight=[1, 1, 512, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv18 = con_wy(input=conv17,
shape_weight=[3, 3, 128, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv19 = con_wy(input=conv18,
shape_weight=[1, 1, 128, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv20 = con_wy(input=conv19,
shape_weight=[1, 1, 512, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv21 = con_wy(input=conv20,
shape_weight=[3, 3, 128, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv22 = con_wy(input=conv21,
shape_weight=[1, 1, 128, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv23 = con_wy(input=conv22,
shape_weight=[1, 1, 512, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv24 = con_wy(input=conv23,
shape_weight=[3, 3, 128, 128],
shape_bias=128,
strides=[1, 1, 1, 1]) #6*6*128
conv25 = con_wy(input=conv24,
shape_weight=[1, 1, 128, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
#第十八到
# def atrous_conv_wy(input,shape_weight,shape_bias,rate):
# weight = variable_with_weight_loss(shape_weight, stddev=0.05, w1=0.0)
# kernel = tf.nn.atrous_conv2d(input, weight, rate=rate, padding='SAME') # 空洞卷积会把上面的卷积核编程 10*10的大小
# bias = tf.Variable(tf.constant(0.1, shape=[shape_bias])) # 输出维度128
# conv = tf.nn.relu(kernel + bias) # 输出为b*7*7*128
# return conv
#第十八层到三十七层
conv26 = con_wy(input=conv25,
shape_weight=[1, 1, 512, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv27 = atrous_conv_wy(input=conv26,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv28 = con_wy(input=conv27,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv29 = con_wy(input=conv28,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv30 = atrous_conv_wy(input=conv29,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv31 = con_wy(input=conv30,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv32 = con_wy(input=conv31,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv33 = atrous_conv_wy(input=conv32,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv34 = con_wy(input=conv33,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv35 = con_wy(input=conv34,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv36 = atrous_conv_wy(input=conv35,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv37 = con_wy(input=conv36,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv38 = con_wy(input=conv37,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv39 = atrous_conv_wy(input=conv38,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv40 = con_wy(input=conv39,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv41 = con_wy(input=conv40,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv42 = atrous_conv_wy(input=conv41,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv43 = con_wy(input=conv42,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
conv44 = con_wy(input=conv43,
shape_weight=[1, 1, 1024, 256],
shape_bias=256,
strides=[1, 1, 1, 1]) #6*6*256
conv45 = atrous_conv_wy(input=conv44,
shape_weight=[3, 3, 256, 256],
shape_bias=256,
rate=2) #6*6*256
conv46 = con_wy(input=conv45,
shape_weight=[1, 1, 256, 1024],
shape_bias=1024,
strides=[1, 1, 1, 1]) #6*6*1024
#第三十八到四十八层
conv47 = con_wy(input=conv46,
shape_weight=[1, 1, 1024, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv48 = atrous_conv_wy(input=conv47,
shape_weight=[3, 3, 512, 512],
shape_bias=512,
rate=4) #6*6*512
conv49 = con_wy(input=conv48,
shape_weight=[1, 1, 512, 2048],
shape_bias=2048,
strides=[1, 1, 1, 1]) #6*6*2048
conv50 = con_wy(input=conv49,
shape_weight=[1, 1, 2048, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv51 = atrous_conv_wy(input=conv50,
shape_weight=[3, 3, 512, 512],
shape_bias=512,
rate=4) #6*6*512
conv52 = con_wy(input=conv51,
shape_weight=[1, 1, 512, 2048],
shape_bias=2048,
strides=[1, 1, 1, 1]) #6*6*2048
conv53 = con_wy(input=conv52,
shape_weight=[1, 1, 2048, 512],
shape_bias=512,
strides=[1, 1, 1, 1]) #6*6*512
conv54 = atrous_conv_wy(input=conv53,
shape_weight=[3, 3, 512, 512],
shape_bias=512,
rate=4) #6*6*512
conv55 = con_wy(input=conv54,
shape_weight=[1, 1, 512, 2048],
shape_bias=2048,
strides=[1, 1, 1, 1]) #6*6*2048
#全链接
reshape = tf.reshape(conv11, [batch_size, -1]) #分开成为
dim = reshape.get_shape()[1].value #
weight3 = variable_with_weight_loss([dim, 384],
stddev=0.04,
w1=0.004)
bias3 = tf.Variable(tf.constant(0.1, shape=[384]))
local3 = tf.nn.relu(tf.matmul(reshape, weight3) + bias3)
#全连接层
weight4 = variable_with_weight_loss([384, 192],
stddev=0.04,
w1=0.004)
bias4 = tf.Variable(tf.constant(0.1, shape=[192]))
local4 = tf.nn.relu(tf.matmul(local3, weight4) + bias4)
#输出层
weight5 = variable_with_weight_loss([192, 10],
stddev=1 / 192,
w1=0.0)
bias5 = tf.Variable(tf.constant(0.0, shape=[10]))
logits = tf.matmul(local4, weight5) + bias5
def loss(logits, labels):
labels = tf.cast(labels, tf.int64)
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits,
labels=labels,
name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy,
name='cross_entropy')
tf.add_to_collection('losses', cross_entropy_mean)
return tf.add_n(tf.get_collection('losses'), name='total_loss')
loss_value = loss(logits, label_holder)
optimizer = tf.train.AdamOptimizer()
grads_and_vars = optimizer.compute_gradients(loss_value)
top_k_op = tf.nn.in_top_k(logits, tf.cast(label_holder, tf.int64),
1)
if issync == 1:
# 同步模式计算更新梯度
rep_op = tf.train.SyncReplicasOptimizer(
optimizer,
replicas_to_aggregate=1,
# replica_id=FLAGS.task_index,
total_num_replicas=1,
use_locking=True)
train_op = rep_op.apply_gradients(grads_and_vars,
global_step=global_step)
init_token_op = rep_op.get_init_tokens_op()
chief_queue_runner = rep_op.get_chief_queue_runner()
else:
# 异步模式计算更新梯度
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
init_op = tf.initialize_all_variables()
# saver = tf.train.Saver()
tf.summary.scalar('cost', loss_value)
summary_op = tf.summary.merge_all()
sv = tf.train.Supervisor(
is_chief=(FLAGS.task_index == 0),
# logdir="./checkpoint/",
init_op=init_op,
summary_op=None,
# saver=saver,
global_step=global_step,
# save_model_secs=60
)
with sv.prepare_or_wait_for_session(server.target) as sess:
# 如果是同步模式
if FLAGS.task_index == 0 and issync == 1:
sv.start_queue_runners(sess, [chief_queue_runner])
sess.run(init_token_op)
# sess = tf.InteractiveSession()
# tf.global_variables_initializer().run()
# tf.train.start_queue_runners()
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run(
[images_train, labels_train])
_, loss = sess.run([train_op, loss_value],
feed_dict={
image_holder: image_batch,
label_holder: label_batch
})
duration = time.time() - start_time
if step % 10 == 0:
examples_per_sec = batch_size / duration
sec_per_batch = float(duration)
format_str = (
'step %d,loss=%.2f (%.1f examples/sec;%.3f sec/batch)')
print(format_str %
(step, loss, examples_per_sec, sec_per_batch))
num_examples = 10000
import math
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size
step = 0
# with tf.Session() as sess:
while step < num_iter:
image_batch, label_batch = sess.run([images_test, labels_test])
predictions = sess.run([top_k_op],
feed_dict={
image_holder: image_batch,
label_holder: label_batch
})
true_count += np.sum(predictions)
step += 1
if step % 10 == 0:
print(true_count)
end = time.clock() # 计算程序结束时间
out = (end - start)
precision = float(true_count) / total_sample_count
print('precision @ 1 =%.3f' % precision)
print("running time is", out, "s")
def distorted_inputs():
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
data_dir = os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin')
return cifar10_input.distorted_inputs(data_dir=data_dir, batch_size=FLAGS.batch_size)
def train():
x_input = tf.placeholder(tf.float32, [BATCH_SIZE, 24, 24, 3],
name="x-input")
y_input = tf.placeholder(tf.float32, [None, 10], name="y-input")
train_images, train_labels = cifar10_input.distorted_inputs(
batch_size=BATCH_SIZE, data_dir=data_dir)
#l2正则化
regularizer = tf.contrib.layers.l2_regularizer(REGULARAZTION_RATE)
#计算前向传播
y = inference.inference(x_input, True, regularizer)
#记录步数，代表训练轮数，设为不可训练
global_step = tf.Variable(0, trainable=False)
#滑动平均模型，加快开始时的训练速度
variable_averages = tf.train.ExponentialMovingAverage(
MOVING_AVERAGE_DECAY, global_step)
#对所有可训练参数应用滑动平均模型
variable_averages_op = variable_averages.apply(tf.trainable_variables())
#计算损失函数
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=y_input,
logits=y)
cross_entropy_mean = tf.reduce_mean(cross_entropy)
#加上正则化损失，tf.add_n实现的是列表元素相加
loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
#指数衰减学习率，使得开始时学习率较大，后期趋向稳定
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step,
50000 / BATCH_SIZE,
LEARNING_RATE_DECAY)
#优化器可以使global_step自增
train_step = tf.train.AdamOptimizer(learning_rate).minimize(
loss, global_step=global_step)
#同时计算后向传播的参数和每一个参数的滑动平均值
with tf.control_dependencies([train_step, variable_averages_op]):
train_op = tf.no_op(name="train")
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_input, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#持久化类
saver = tf.train.Saver()
#建立会话，开始训练
with tf.Session() as sess:
tf.global_variables_initializer().run()
tf.train.start_queue_runners(sess=sess)
for i in range(TRAINING_STEPS):
#小批量随机梯度下降
xs, ys = sess.run([train_images, train_labels])
ys = one_hot(ys, 10)
#后向传播，损失值,当前训练轮数
_, loss_value, step, label = sess.run(
[train_op, loss, global_step, y],
feed_dict={
x_input: xs,
y_input: ys
})
if i % 1000 == 0:
accuracy_score = sess.run(accuracy,
feed_dict={
y: label,
y_input: ys
})
print("After %s training steps,validation accuracy = %g" %
(step, accuracy_score))
print("After %d training steps,loss on training batch is %g" %
(step, loss_value))
#持久化当前模型，保存文件会带有当前训练轮数
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def main():
# 读取图片并预处理
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir, batch_size=batch_size, image_size=24)
images_test, labels_test = cifar10_input.inputs(eval_data=True,
data_dir=data_dir,
batch_size=batch_size,
image_size=24)
# print(images_test, labels_test)
# 输入：24*24的RGB三色通道图片
image_holder = tf.placeholder(tf.float32, [None, 24, 24, 3])
# print(image_holder)
label_holder = tf.placeholder(tf.int32, [None])
training_holder = tf.placeholder(tf.bool, [])
logits = inference(image_holder, training_holder)
loss = loss_fn(logits, label_holder)
global_step = tf.get_variable('global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
# num_batches_per_epoch = (cifar10_input.NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN /
# batch_size)
# decay_steps = int(num_batches_per_epoch * 350.0)
lr = tf.train.exponential_decay(1e-3,
global_step,
300,
0.96,
staircase=True)
tf.summary.scalar("learning rate", lr)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
# minimize需要接收global_step才会更新lr
train_op = tf.train.AdamOptimizer(lr).minimize(loss,
global_step=global_step)
# 输出结果top_k准确率，默认为1
top_k_op = tf.nn.in_top_k(logits, label_holder, 1)
def test_cifar10(session):
# 测试部分
num_examples = 10000
num_iter = int(math.ceil(num_examples / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size
steps = 0
while steps < num_iter:
images, labels = session.run([images_test, labels_test])
predictions = session.run(top_k_op,
feed_dict={
image_holder: images,
label_holder: labels,
training_holder: True
})
true_count += np.sum(predictions)
steps += 1
accuracy = true_count / total_sample_count
print("[*] test accuracy is {:.3f}".format(accuracy))
# 训练部分
summary = tf.summary.merge_all()
sess = tf.InteractiveSession(config=config)
tf.global_variables_initializer().run()
# 启动数据增强队列
tf.train.start_queue_runners()
saver = tf.train.Saver()
writer = tf.summary.FileWriter("./logs", sess.graph)
if not os.path.exists("./logs/model"):
os.makedirs("./logs/model")
ckpt = tf.train.get_checkpoint_state("./logs/model")
if ckpt is not None:
print("[*] Success to read {}".format(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("[*] Failed to find a checkpoint")
for step in range(max_steps):
start_time = time.time()
image_batch, label_batch = sess.run([images_train, labels_train])
_, loss_value, summary_str = sess.run(
[train_op, loss, summary],
feed_dict={
image_holder: image_batch,
label_holder: label_batch,
training_holder: False
})
writer.add_summary(summary_str, step)
duration = time.time() - start_time
if step % 10 == 0:
examples_per_sec = batch_size / duration
sec_per_batch = float(duration)
format_str = "[*] step %d, loss=%.2f (%.1f examples/sec; %.3f sec/batch)"
print(format_str %
(step, loss_value, examples_per_sec, sec_per_batch))
if step % 100 == 0:
test_cifar10(sess)
if step % 500 == 0 and step != 0:
saver.save(sess, "./logs/model/DCGAN.model", global_step=step)
sess.close()
#损失函数和优化器
cross_entropy= tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=labels)
cost = tf.reduce_mean(cross_entropy,name='Train_Cost')
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)
# 用来评估测试数据的准确率
# 数据labels没有使用one-hot编码格式,labels是int32
def accuracy(labels, output):
labels = tf.to_int64(labels)
pred_result = tf.equal(labels, tf.argmax(output, 1))
accu = tf.reduce_mean(tf.cast(pred_result, tf.float32))
return accu
# 加载训练batch_size大小的数据，经过增强处理，剪裁，反转，等等
train_images, train_labels = cifar10_input.distorted_inputs(batch_size= batch_size, data_dir= data_dir)
#Filling queue with 20000 CIFAR images before starting to train. This will take a few minutes.
# 加载测试数据，batch_size大小，不进行增强处理
test_images, test_labels = cifar10_input.inputs(batch_size= batch_size, data_dir= data_dir,eval_data= True)
# Training
def training(sess, max_steps, s_times, keeprob, display):
Cost = []
for i in range(max_steps):
for j in range(s_times):
start = time.time()
batch_images, batch_labels = sess.run([train_images, train_labels])
opt = sess.run(optimizer, feed_dict={images:batch_images, labels:batch_labels,
logits=logits, labels=labels, name='cross_entropy_per_example')
cross_entropy_mean = tf.reduce_mean(cross_entropy, name="cross_entropy")
tf.add_to_collection('losses', cross_entropy_mean)
return tf.add_n(tf.get_collection('losses'),
name='total_loss_cross_entropy_and_l2loss')
'''Args:列表元素相加
inputs:
A list of Tensor objects, each with same shape and type.
name: A name for the operation (optional).
Returns:
A Tensor of same shape and type as the elements of inputs.'''
images_train, labels_train = cifar10_input.distorted_inputs(
data_dir=data_dir,
batch_size=batch_size) #数据增强，水平翻转，随机剪贴24*24，随机亮度和对比度，数据标准化 ,多线程加速
images_test, labels_test = cifar10_input.inputs(
eval_data=True, data_dir=data_dir, batch_size=batch_size) #裁剪中间24*24 数据标准化
data_image = tf.placeholder(tf.float32, [batch_size, 24, 24, 3])
data_label = tf.placeholder(tf.int32, [batch_size])
#convolutional layer
with tf.name_scope("conv1_pool1_lrn_1"):
filter1 = weight_with_loss([5, 5, 3, 64], stddev=0.05)
bias1 = bias([64])
conv1 = tf.nn.relu(conv_op(data_image, filter1) + bias1)
pool1 = pool_op(conv1, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1])
lrn1 = tf.nn.lrn(pool1, 4, bias=1.0, alpha=0.001 / 9.0,
beta=0.75) #lrn......
