def testSimple(self):
labels = [9, 3, 0]
records = [self._record(labels[0], 0, 128, 255),
self._record(labels[1], 255, 0, 1),
self._record(labels[2], 254, 255, 0)]
contents = b"".join([record for record, _ in records])
expected = [expected for _, expected in records]
filename = os.path.join(self.get_temp_dir(), "cifar.jpg")
# img = cv2.imread(filename)
# cv2.imshow("image", img)
open(filename, "wb").write(contents)
with self.test_session() as sess:
q = tf.FIFOQueue(99, [tf.string], shapes=())
q.enqueue([filename]).run()
q.close().run()
result = cifar10_input.read_cifar10(q)
for i in range(3):
key, label, uint8image = sess.run([
result.key, result.label, result.uint8image])
#self.assertEqual("%s:%d" % (filename, i), tf.compat.as_text(key))
self.assertEqual("%s:%d" % (filename, i), cp.as_text(key))
self.assertEqual(labels[i], label)
self.assertAllEqual(expected[i], uint8image)
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run([result.key, result.uint8image])
def inputs_origin(data_dir):
# filenames一共5个，从data_batch_1.bin到data_batch_5.bin
# 读入的都是训练图像
filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
for i in xrange(1, 6)]
# 判断文件是否存在
"""
if not tf.gfile.Exists(dest_directory):
gfile.MakeDirs(dest_directory)
与下述功能相同
if not os.path.exists(dest_directory):
os.makedirs(dest_directory)
"""
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# 将文件名的list包装成TensorFlow中queue的形式
filename_queue = tf.train.string_input_producer(filenames)
# cifar10_input.read_cifar10是事先写好的从queue中读取文件的函数
# 返回的结果read_input的属性uint8image就是图像的Tensor
read_input = cifar10_input.read_cifar10(filename_queue)
# 将图片转换为实数形式
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
# 返回的reshaped_image是一张图片的tensor
# 我们应当这样理解reshaped_image：每次使用sess.run(reshaped_image)，就会取出一张图片
return reshaped_image
def evaluate(timestamp, batch_size=BATCH_SIZE * 64, n_batch=10):
'''
:param timestamp: It will retrieve model from the folder named the timestamp.
:param n_batch: The number of batches to evaluate model.
:return:
'''
with tf.Session() as sess:
vgg, ckpt_path = restore(sess, timestamp)
if ckpt_path is None:
print("Checkpoint not found")
return
test_batch_xs, test_batch_ys = cifar10_input.read_cifar10(
is_train=False, batch_size=batch_size, shuffle=False)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
accs = np.zeros(n_batch)
for step in range(n_batch):
test_xs, test_ys = sess.run([test_batch_xs, test_batch_ys])
acc = sess.run(vgg.accuracy,
feed_dict={
vgg.xs: test_xs,
vgg.ys: test_ys
})
print("Accuracy : %.4f%%" % (acc * 100))
accs[step] = acc
coord.request_stop()
coord.join(threads)
return np.mean(accs)
def testSimple(self):
labels = [9, 3, 0]
records = [self._record(labels[0], 0, 128, 255),
self._record(labels[1], 255, 0, 1),
self._record(labels[2], 254, 255, 0)]
contents = b"".join([record for record, _ in records])
expected = [expected for _, expected in records]
filename = os.path.join(self.get_temp_dir(), "cifar")
open(filename, "wb").write(contents)
with self.test_session() as sess:
q = tf.FIFOQueue(99, [tf.string], shapes=())
q.enqueue([filename]).run()
q.close().run()
result = cifar10_input.read_cifar10(q)
for i in range(3):
key, label, uint8image = sess.run([
result.key, result.label, result.uint8image])
self.assertEqual("%s:%d" % (filename, i), tf.compat.as_text(key))
self.assertEqual(labels[i], label)
self.assertAllEqual(expected[i], uint8image)
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run([result.key, result.uint8image])
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Raises:
ValueError: if no data_dir
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
filenames = [os.path.join(FLAGS.data_dir, 'acacia-10-batches-bin',
'data_batch_%d.bin' % i)
for i in xrange(0, 3)]
for f in filenames:
if not gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
# Read examples from files in the filename queue.
read_input = cifar10_input.read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# # Image processing for training the network. Note the many random
# distortions applied to the image.
# Randomly crop a [height, width] section of the image.
distorted_image = tf.image.random_crop(reshaped_image, [height, width])
# Randomly flip the image horizontally.
distorted_image = tf.image.random_flip_left_right(distorted_image)
# Because these operations are not commutative, consider randomizing
# randomize the order their operation.
distorted_image = tf.image.random_brightness(distorted_image,
max_delta=40) #era 63
distorted_image = tf.image.random_contrast(distorted_image,
lower=0.6, upper=1.2) #era 0.2 e 1.8
# distorted_image = reshaped_image
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_whitening(distorted_image)
# Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue = MIN_FRACTION_OF_EXAMPLES_QUEUE # 0.4
min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN *
min_fraction_of_examples_in_queue)
print ('Filling queue with LALALALALu %d CIFAR images before starting to train. '
'This will take a few minutes.' % min_queue_examples)
# Generate a batch of images and labels by building up a queue of examples.
print "I have read", read_input.label, "for this batch"
return _generate_image_and_label_batch(float_image, read_input.label,
min_queue_examples)
def distorted_inputs():
"""Construct distorted input for CIFAR training using the Reader ops.
Raises:
ValueError: if no data_dir
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
filenames = [os.path.join(FLAGS.data_dir, 'acacia-10-batches-bin',
'data_batch_%d.bin' % i)
for i in xrange(0, 3)]
for f in filenames:
if not gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
# Read examples from files in the filename queue.
read_input = cifar10_input.read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# # Image processing for training the network. Note the many random
# distortions applied to the image.
# Randomly crop a [height, width] section of the image.
distorted_image = tf.image.random_crop(reshaped_image, [height, width])
# Randomly flip the image horizontally.
distorted_image = tf.image.random_flip_left_right(distorted_image)
# Because these operations are not commutative, consider randomizing
# randomize the order their operation.
distorted_image = tf.image.random_brightness(distorted_image,
max_delta=40) #era 63
distorted_image = tf.image.random_contrast(distorted_image,
lower=0.6, upper=1.2) #era 0.2 e 1.8
# distorted_image = reshaped_image
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_whitening(distorted_image)
# Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue = MIN_FRACTION_OF_EXAMPLES_QUEUE # 0.4
min_queue_examples = int(NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN *
min_fraction_of_examples_in_queue)
print ('Filling queue with LALALALALu %d CIFAR images before starting to train. '
'This will take a few minutes.' % min_queue_examples)
# Generate a batch of images and labels by building up a queue of examples.
print "I have read", read_input.label, "for this batch"
return _generate_image_and_label_batch(float_image, read_input.label,
min_queue_examples)
def inputs(eval_data):
"""Construct input for CIFAR evaluation using the Reader ops.
Args:
eval_data: bool, indicating if one should use the train or eval data set.
Raises:
ValueError: if no data_dir
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
if not eval_data:
filenames = [
os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin',
'data_batch_%d.bin' % i) for i in xrange(1, 5)
]
num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
else:
filenames = [
os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin',
'test_batch.bin')
]
num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_EVAL
for f in filenames:
if not gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
# Read examples from files in the filename queue.
read_input = cifar10_input.read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# Image processing for evaluation.
# Crop the central [height, width] of the image.
resized_image = tf.image.resize_image_with_crop_or_pad(
reshaped_image, width, height)
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_whitening(resized_image)
# Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue = 0.4
min_queue_examples = int(num_examples_per_epoch *
min_fraction_of_examples_in_queue)
# Generate a batch of images and labels by building up a queue of examples.
return _generate_image_and_label_batch(float_image, read_input.label,
min_queue_examples)
def train():
my_global_step = tf.Variable(0, name='global_step', trainable=False)
data_dir = './data/cifar-10-batches-bin/'
log_dir = './logs/train/'
images, labels = cifar10_input.read_cifar10(data_dir=data_dir,
is_train=True,
batch_size= BATCH_SIZE,
shuffle=True)
logits = cifar10_model.inference(images, BATCH_SIZE, n_classes=N_CLASSES)
loss = cifar10_model.losses(logits, labels)
optimizer = tf.train.GradientDescentOptimizer(learning_rate) # 定义优化器
train_op = optimizer.minimize(loss, global_step= my_global_step) # 运行优化
saver = tf.train.Saver(tf.global_variables())
summary_op = tf.summary.merge_all()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
try:
for step in np.arange(MAX_STEP):
if coord.should_stop():
break
_, loss_value = sess.run([train_op, loss])
if step % 50 == 0:
print ('Step: %d, loss: %.4f' % (step, loss_value))
if step % 100 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
if step % 2000 == 0 or (step + 1) == MAX_STEP:
checkpoint_path = os.path.join(log_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
#%%
def train():
my_global_step = tf.Variable(0, name='global_step', trainable=False)
data_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
log_dir = '/home/kevin/tensorflow/CIFAR10/logs234/'
images, labels = cifar10_input.read_cifar10(data_dir=data_dir,
is_train=True,
batch_size= BATCH_SIZE,
shuffle=True)
logits = inference(images)
loss = losses(logits, labels)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss, global_step= my_global_step)
saver = tf.train.Saver(tf.global_variables())
summary_op = tf.summary.merge_all()
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
summary_writer = tf.summary.FileWriter(log_dir, sess.graph)
try:
for step in np.arange(MAX_STEP):
if coord.should_stop():
break
_, loss_value = sess.run([train_op, loss])
if step % 50 == 0:
print ('Step: %d, loss: %.4f' % (step, loss_value))
if step % 100 == 0:
summary_str = sess.run(summary_op)
summary_writer.add_summary(summary_str, step)
if step % 2000 == 0 or (step + 1) == MAX_STEP:
checkpoint_path = os.path.join(log_dir, 'model.ckpt')
saver.save(sess, checkpoint_path, global_step=step)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def inputs(eval_data):
"""Construct input for CIFAR evaluation using the Reader ops.
Args:
eval_data: bool, indicating if one should use the train or eval data set.
Raises:
ValueError: if no data_dir
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
if not FLAGS.data_dir:
raise ValueError('Please supply a data_dir')
if not eval_data:
filenames = [os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin',
'data_batch_%d.bin' % i)
for i in xrange(1, 5)]
num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
else:
filenames = [os.path.join(FLAGS.data_dir, 'cifar-10-batches-bin',
'test_batch.bin')]
num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_EVAL
for f in filenames:
if not gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
# Read examples from files in the filename queue.
read_input = cifar10_input.read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# Image processing for evaluation.
# Crop the central [height, width] of the image.
resized_image = tf.image.resize_image_with_crop_or_pad(reshaped_image,
width, height)
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_whitening(resized_image)
# Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue = 0.4
min_queue_examples = int(num_examples_per_epoch *
min_fraction_of_examples_in_queue)
# Generate a batch of images and labels by building up a queue of examples.
return _generate_image_and_label_batch(float_image, read_input.label,
min_queue_examples)
def inputs_origin(data_dir):
filenames = [os.path.join(data_dir,'data_batch_%d.bin' % i) for i in range(1,6)]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
filename_queue = tf.train.string_input_producer(filenames)
read_input = cifar10_input.read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image,tf.float32)
return reshaped_image
def evaluate():
with tf.Graph().as_default():
log_dir = './logs/train/'
test_dir = './data/cifar-10-batches-bin/'
n_test = 10000
# reading test data
images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
is_train=False,
batch_size= BATCH_SIZE,
shuffle=False)
logits = cifar10_model.inference(images, BATCH_SIZE, N_CLASSES)
# 比较真实label和预测值
top_k_op = tf.nn.in_top_k(logits, labels, 1)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
# 读取模型文件
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess = sess, coord = coord)
try:
num_iter = int(math.ceil(n_test / BATCH_SIZE)) # 10000/64 取整
true_count = 0
total_sample_count = num_iter * BATCH_SIZE # 我们测试的数量
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count
print('precision = %.3f' % precision)
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
#%%
def inputs_origin(data_dir):
# 读取训练图像：filenames一共5个，从data_batch_1.bin到data_batch_5.bin，
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in range(1, 6)
]
# 判断文件是否存在
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
filename_queue = tf.train.string_input_producer(filenames) # 包装成queue
read_input = cifar10_input.read_cifar10(filename_queue) # 读取queue
reshaped_image = tf.cast(read_input.uint8image, tf.float32) # 将图片转换为实数形式
return reshaped_image # 返回的reshaped_image是一张图片的tensor
def inputs_origin(data_dir):
filenames = [
os.path.join(data_dir, 'data_batch_%d' % i1) for i1 in range(1, 6)
]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
filename_queue = tf.train.string_input_producer(filenames)
# cifar10_input.read_cifar10是事先写好的从queue中读取文件的函数
# 返回的结果read_input的属性uint8image就是图像的Tensor
read_input = cifar10_input.read_cifar10(filename_queue)
read_image = tf.cast(read_input.uint8image, tf.float32)
return read_image
def evaluate():
with tf.Graph().as_default():
log_dir = '/home/kevin/tensorflow/CIFAR10/logs10000/'
test_dir = '/home/kevin/tensorflow/CIFAR10/data/cifar-10-batches-bin/'
n_test = 10000
# reading test data
images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
is_train=False,
batch_size= BATCH_SIZE,
shuffle=False)
logits = inference(images)
top_k_op = tf.nn.in_top_k(logits, labels, 1)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess = sess, coord = coord)
try:
num_iter = int(math.ceil(n_test / BATCH_SIZE))
true_count = 0
total_sample_count = num_iter * BATCH_SIZE
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count
print('precision = %.3f' % precision)
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
def inputs_origin(data_dir):
# only training set
filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in range(1, 6)]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Wrap the list of file names into the form of the queue in TensorFlow
filename_queue = tf.train.string_input_producer(filenames)
# The result of the returned read_input attribute uint8image is the image of the Tensor
read_input = cifar10_input.read_cifar10(filename_queue)
# Convert image to real form
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
return reshaped_image
def evaluate():
with tf.Graph().as_default() as g:
log_dir = '/home/caozhang/spyder_projects/cifar10_code/logs_no_one_hot/'
test_dir = '/home/caozhang/spyder_projects/cifar10_code/cifar-10-batches-bin/'
n_test = 10000
# reading test data
images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
is_train=False,
batch_size=BATCH_SIZE,
is_shuffle=False)
logits = inference(images)
top_k_op = tf.nn.in_top_k(logits, labels, 1)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
num_iter = int(math.ceil(n_test / BATCH_SIZE))
true_count = 0
total_sample_count = num_iter * BATCH_SIZE
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
# python2.*中的除法是取商，而python3.*中的除法是得到浮点型
precision = true_count / total_sample_count
print("precision: %.3f" % precision)
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
def evaluate_running():
with tf.Graph().as_default():
# reading test data
images, labels = cifar10_input.read_cifar10(data_path=test_data_path,
is_train=False,
batch_size=batch_size,
shuffle=False)
model = LeNet5(images, num_classes)
logits = model.logits
labels = tf.cast(labels, tf.int64)
top_k_op = tf.nn.in_top_k(logits, tf.argmax(labels, 1), 1)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
print("Reading checkpoints...")
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and ckpt.model_checkpoint_path:
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
saver.restore(sess, ckpt.model_checkpoint_path)
print('Loading success, global_step is %s' % global_step)
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
num_iter = int(math.ceil(n_test / batch_size))
true_count = 0
total_sample_count = num_iter * batch_size
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
precision = true_count / total_sample_count * 100.0
print('precision = %.2f%%' % precision)
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
def generate_cnn_inputs(bin_fnm):
"""Construct input for CIFAR evaluation using the Reader ops.
Args:
bin_fnm: filename that is used for binary input into the network
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
with tf.name_scope('input'):
# Create a queue that produces the filenames to read.
# filenames = ['C:/codes/stylar_hw/tensorflow_cifar10_tutorial/img_1.bin']
complete_filename = [os.path.join(FLAGS.image_dir, bin_fnm)]
filename_queue = tf.train.string_input_producer(complete_filename)
# Read examples from files in the filename queue.
read_input = read_cifar10(filename_queue)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# Image processing for evaluation.
# Crop the central [height, width] of the image.
resized_image = tf.image.resize_image_with_crop_or_pad(
reshaped_image, height, width)
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_standardization(resized_image)
# Set the shapes of tensors.
float_image.set_shape([height, width, 3])
read_input.label.set_shape([1])
images, label_batch = tf.train.batch([float_image, read_input.label],
batch_size=1,
num_threads=1,
capacity=1)
labels = tf.reshape(label_batch, [1])
if FLAGS.use_fp16:
images = tf.cast(images, tf.float16)
labels = tf.cast(labels, tf.float16)
return images, labels
def inputs_origin(data_dir):
# filenames一共5个，从data_batch_1.bin到data_batch_5.bin
# 读入的都是训练图像
filenames = [os.path.join(data_dir, 'data_batch_%d.bin' % i)
for i in xrange(1, 6)]
# 判断文件是否存在
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# 将文件名的list包装成TensorFlow中queue的形式
filename_queue = tf.train.string_input_producer(filenames)
# cifar10_input.read_cifar10是事先写好的从queue中读取文件的函数
# 返回的结果read_input的属性uint8image就是图像的Tensor
read_input = cifar10_input.read_cifar10(filename_queue)
# 将图片转换为实数形式
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
# 返回的reshaped_image是一张图片的tensor
# 我们应当这样理解reshaped_image：每次使用sess.run(reshaped_image)，就会取出一张图片
return reshaped_image
def train():
my_global_step = tf.Variable(0, name='global_step', trainable=False)
data_dir = 'data/'
log_dir = 'logs/'
images, labels = cifar10_input.read_cifar10(data_dir=data_dir,
is_train=True,
batch_size=BATCH_SIZE,
shuffle=True)
logits = inference(images)
loss = losses(logits, labels)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(loss)
saver = tf.train.Saver(tf.global_variables())
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners()
try:
for step in range(MAX_STEP):
if coord.should_stop():
break
_, loss_value = sess.run([train_op, loss])
if step % 200 == 0:
print('Step: %d, loss: %.4f' % (step, loss_value))
if step % 2000 == 0:
check_path = os.path.join(log_dir, 'model_ckpt')
saver.save(sess, check_path, global_step=step)
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
sess.close()
def evaluation():
with tf.Graph().as_default():
log_dir = 'logs/'
test_dir = 'data/'
n_test = 10000
images, labels = cifar10_input.read_cifar10(data_dir=test_dir,
is_train=False,
batch_size=BATCH_SIZE,
shuffle=False)
logits = inference(images)
top_k_op = tf.nn.in_top_k(logits, labels, 1)
saver = tf.train.Saver(tf.global_variables())
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
try:
num_iter = int(math.ceil(n_test / BATCH_SIZE))
true_count = 0
total_sample_count = num_iter * BATCH_SIZE
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count = np.sum(predictions)
step += 1
precision = true_count / total_sample_count
except Exception as e:
coord.request_stop(e)
finally:
coord.request_stop()
coord.join(threads)
def inputs_origin(data_dir):
#filenames一共5个，从data_batch1.bin到data_5.bin
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in range(1, 6)
]
#判断文件是否存在
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file:' + f)
#将文件名包装成TensorFlow中的queue形式
filenames_queue = tf.train.string_input_producer(filenames)
# cifar10_input.read_cifar10是事先写好的从queue中读取文件的函数
# 返回的结果read_input的属性uint8image就是图像的Tensor
read_input = cifar10_input.read_cifar10(filenames_queue)
#将图片转换为实数形式
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
# 返回的reshaped_image是一张图片的tensor
# 我们应当这样理解reshaped_image：每次使用sess.run(reshaped_image)，就会取出一张图片
return reshaped_image
class CIFAR10InputTest(tf.test.TestCase):
def _record(self, label, red, green, blue):
image_size = 32 * 32
record = bytes(
bytearray([label] + [red] * image_size + [green] * image_size +
[blue] * image_size))
expected = [[[red, green, blue]] * 32] * 32
return record, expected
def testSimple(self):
labels = [9, 3, 0]
records = [
self._record(labels[0], 0, 128, 255),
self._record(labels[1], 255, 0, 1),
self._record(labels[2], 254, 255, 0)
]
contents = b"".join([record for record, _ in records])
expected = [expected for _, expected in records]
filename = os.path.join(self.get_temp_dir(), "cifar")
open(filename, "wb").write(contents)
with self.test_session() as sess:
q = tf.FIFOQueue(99, [tf.string], shapes=())
q.enqueue([filename]).run()
q.close().run()
result = cifar10_input.read_cifar10(q)
for i in range(3):
key, label, uint8image = sess.run(
[result.key, result.label, result.uint8image])
self.assertEqual("%s:%d" % (filename, i), tf.compat.as_text(key))
self.assertEqual(labels[i], label)
self.assertAllEqual(expected[i], uint8image)
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run([result.key, result.uint8image])
#%%
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import os
import cifar10_input
os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'
os.environ['CUDA_VISIBLE_DEVICES'] = '7'
#%%
data_dir = '/data/minglin/data/cifar10_data/cifar-10-batches-bin/'
batch_size = 4
image_batch, label_batch = cifar10_input.read_cifar10(data_dir,
is_train=True,
batch_size=batch_size,
shuffle=True)
with tf.Session() as sess:
i = 0 # 控制只运行一个batch
coord = tf.train.Coordinator() # 开启协调器
threads = tf.train.start_queue_runners(coord=coord) # 开启队列
try:
while not coord.should_stop() and i < 1:
images, labels = sess.run([image_batch, label_batch])
# 显示label和image
for j in np.arange(batch_size):
print('labels: %d' % np.argmax(labels[j]))
plt.imshow(images[j, :, :, :])
plt.show()
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
if __name__ == '__main__':
with tf.Graph().as_default():
with tf.device('/cpu:0'):
train_batch, train_label_batch = cifar10_input.read_cifar10(
data_path=data_path,
is_train=True,
batch_size=batch_size,
shuffle=True)
val_batch, val_label_batch = cifar10_input.read_cifar10(
data_path=data_path,
is_train=False,
batch_size=batch_size,
shuffle=False)
xs = tf.placeholder(tf.float32, shape=[batch_size, img_h, img_w, 3])
ys = tf.placeholder(tf.int32, shape=[batch_size, num_classes])
keep_prob = tf.placeholder(tf.float32)
model = VGG16(xs, num_classes, keep_prob)
logits = model.logits
def testSimple(self):
labels = [9, 3, 0]
records = [self._record(labels[0], 0, 128, 255),
self._record(labels[1], 255, 0, 1),
self._record(labels[2], 254, 255, 0)]
contents = b"".join([record for record, _ in records])
expected = [expected for _, expected in records]
filename = os.path.join(self.get_temp_dir(), "cifar")
open(filename, "wb").write(contents)
with self.test_session() as sess:
q = tf.FIFOQueue(99, [tf.string], shapes=())
q.enqueue([filename]).run()
q.close().run()
result = cifar10_input.read_cifar10(q)
for i in range(3):
key, label, uint8image = sess.run([
result.key, result.label, result.uint8image])
self.assertEqual("%s:%d" % (filename, i), tf.compat.as_text(key))
self.assertEqual(labels[i], label)
self.assertAllEqual(expected[i], uint8image)
with self.assertRaises(tf.errors.OutOfRangeError):
sess.run([result.key, result.uint8image])
if __name__ == "__main__":
tf.test.main()
def train(vgg, timestamp=None):
'''
:param timestamp: Timestamp is used as a label to differentiate each experiment.
If None or there isn't any folder named the timestamp,
it will make new folder named current timestamp to save logs and models.
(e.g. logs/1506496800/train)
Otherwise(folder exists), it restore the model from the folder.
'''
log_dirs = './log/' + str(timestamp)
tmp_dirs = './tmp/' + str(timestamp)
if not os.path.exists(log_dirs):
os.makedirs(log_dirs + '/train')
os.makedirs(log_dirs + '/test')
os.makedirs(tmp_dirs)
for dir in [log_dirs, tmp_dirs]:
with open(os.path.join(dir, 'params.pkl'), 'wb'
) as f: # Save hyperparameters of vgg models to retrieve
pickle.dump(vgg.params, f)
with open(os.path.join(dir, "spec.txt"), "w") as f: # For logging
f.write(vgg.desc)
summary_op = tf.summary.merge_all()
train_summary_writer = tf.summary.FileWriter(log_dirs + '/train')
test_summary_writer = tf.summary.FileWriter(log_dirs + '/test')
with tf.Session() as sess:
train_batch_xs, train_batch_ys = cifar10_input.read_cifar10(
is_train=True, batch_size=BATCH_SIZE, shuffle=True)
test_batch_xs, test_batch_ys = cifar10_input.read_cifar10(
is_train=False, batch_size=BATCH_SIZE * 4, shuffle=False)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
vgg.load_weight_with_skip(sess, 'vgg16_weights.npz',
['fc6', 'fc7', 'fc8'])
for step in range(0, MAX_STEP + 1):
if coord.should_stop():
break
xs, ys, = sess.run([train_batch_xs,
train_batch_ys]) # Multi-thread
loss, acc, summaries, _ = vgg.update(sess, xs, ys, summary_op)
if step % 50 == 0:
print('Step: %d, loss: %.4f, accuracy: %.4f%%' %
(step, loss, acc * 100))
train_summary_writer.add_summary(summaries, step)
if step % 100 == 0:
xs, ys, = sess.run([test_batch_xs, test_batch_ys])
test_loss, test_acc, test_summaries = vgg.validate(
sess, xs, ys, summary_op)
test_result = 'Step: %d, loss: %.4f, accuracy: %.4f%%' % (
step, test_loss, test_acc * 100)
print('[Test]' + test_result)
if step % 5000 == 0:
with open(
os.path.join(
log_dirs,
"result_{}.txt".format(int(test_acc * 100))),
"w") as f:
f.write(test_result)
test_summary_writer.add_summary(test_summaries, step)
if step != 0:
if not os.path.exists('./tmp'):
os.makedirs('./tmp')
saver_path = saver.save(sess,
os.path.join(
'./tmp', str(timestamp),
'vgg.skpt'),
global_step=step)
print("Model saved in file: %s" % saver_path)
coord.request_stop()
coord.join(threads)
# author: Jason Howe
import os
import tensorflow as tf
from six.moves import xrange
import cifar10_input
data_dir = 'F:/cifar-10-batches-bin'
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in xrange(1, 6)
]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
result_original = cifar10_input.read_cifar10(filename_queue)
print(result_original.uint8image.get_shape)
print(filenames)
# with tf.Session() as sess:
# print(sess.run(result_original.label[0]))
import tensorflow as tf
from six.moves import xrange
import cifar10_input
import matplotlib.pyplot as plt
data_dir = os.path.join('/mnt/nas/ntu-rgbd/other_Datasets/cifar10_data',
'cifar-10-batches-bin')
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in xrange(1, 6)
]
for f in filenames:
if not tf.gfile.Exists(f):
raise ValueError('Failed to find file: ' + f)
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
read_input = cifar10_input.read_cifar10(filename_queue)
print(type(tf.Session().run(read_input.uint8image)))
'''
# Create a queue that produces the filenames to read.
filename_queue = tf.train.string_input_producer(filenames)
read_input = cifar10_input.read_cifar10(filename_queue)
from PIL import Image
import numpy as np
im = Image.open('images.jpeg')
im = (np.array(im))
r = im[:,:,0].flatten()
g = im[:,:,1].flatten()
def inputs(eval_data, data_dir, batch_size):
"""Construct input for CIFAR evaluation using the Reader ops.
Args:
eval_data: bool, indicating if one should use the train or eval data set.
data_dir: Path to the CIFAR-10 data directory.
batch_size: Number of images per batch.
Returns:
images: Images. 4D tensor of [batch_size, IMAGE_SIZE, IMAGE_SIZE, 3] size.
labels: Labels. 1D tensor of [batch_size] size.
"""
import os
num_examples_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN
filenames = []
if GET_TrainFile_By_BIN:
tempfilelist = os.listdir("/tmp/cifar10_data")
for item in tempfilelist:
if item.find(".bin") != -1:
filenames.append(
os.path.join("/tmp/cifar10_data", item).replace("\\", "/"))
filename_queue = tf.train.string_input_producer(
filenames, name="filename_queue_hcq", shuffle=True)
read_input = cifar10_input.read_cifar10(filename_queue)
else:
data_dir = trainPath
if len(os.listdir(data_dir)) != NUM_CLASSES:
raise Exception('图片分类总数与设置NUM_CLASSES参数不一致')
return
filenames, calist = labeltools.getfilelist(data_dir)
filename_queue = tf.train.string_input_producer(
filenames, name="filename_queue_hcq", shuffle=True)
read_input = read_from_imagenet(filename_queue, filenames)
reshaped_image = tf.cast(read_input.uint8image, tf.float32)
height = IMAGE_SIZE
width = IMAGE_SIZE
# Image processing for evaluation.
# Crop the central [height, width] of the image.
resized_image = tf.image.resize_image_with_crop_or_pad(
reshaped_image, width, height)
# Subtract off the mean and divide by the variance of the pixels.
float_image = tf.image.per_image_standardization(resized_image)
# Set the shapes of tensors.
float_image.set_shape([height, width, 3])
#read_input.label.set_shape([])
# Ensure that the random shuffling has good mixing properties.
min_fraction_of_examples_in_queue = 0.4
min_queue_examples = int(num_examples_per_epoch *
min_fraction_of_examples_in_queue)
# Generate a batch of images and labels by building up a queue of examples.
return _generate_image_and_label_batch(float_image,
read_input.label,
min_queue_examples,
batch_size,
shuffle=False)