def read_data(file_queue):
"""
Data is saved in binary files.
Each row has:
1st byte -> label
2nd-last byte -> 3D Volume [height, width, depth, channels]
Args:
file_queue -> a queue of file names saved as strings
Rtns:
An object with:
height -> volume height
width -> volume width
depth -> volume depth
nChan -> number of channels
key -> scalar tensor with file name and record number
label -> 1D int32 tensor with the associated label
img3_uint8 -> 4D uint8 tensor with image data
"""
class record_data(object):
pass
img3_obj = record_data()
# Dimensions of data
label_bytes = 1
img3_obj.height = CFG['height']
img3_obj.width = CFG['width']
img3_obj.depth = CFG['depth']
img3_obj.nChan = CFG['nChan']
# Size in memory
img3_bytes = img3_obj.height * img3_obj.width * img3_obj.depth * img3_obj.nChan
record_bytes = label_bytes + img3_bytes
# Read a record
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
img3_obj.key, value = reader.read(file_queue)
# Convert from a string to a vector of uint8 that is record_bytes long
record_bytes = tf.decode_raw(value, tf.uint8)
# First byte represent the label, which we convert from uint8 -> int32
img3_obj.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]),
tf.int32)
# Remaining bytes after the label represent the image, which we reshape from
# [depth * height * width] to [depth,height, width]
depth_major = tf.reshape(
tf.slice(record_bytes, [label_bytes], [img3_bytes]),
[img3_obj.depth, img3_obj.height, img3_obj.width, img3_obj.nChan])
img3_obj.img3_uint8 = tf.transpose(depth_major, [2, 1, 0, 3])
return img3_obj
def inputs_origin(filePath):
class CIFAR10Record(object):
pass
result = CIFAR10Record()
# Dimensions of the images in the CIFAR-10 dataset.
# See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
# input format.
label_bytes = 1 # 2 for CIFAR-100
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes
fileNames = [
os.path.join(filePath, 'data_batch_%d.bin' % _) for _ in range(1, 6)
] # in Python 3, rang() && xrange() == range()
for f in fileNames:
if not tf.gfile.Exists(f):
raise ValueError(
'Error at __Test02_exchangeCIFAR_Pic.py : Failed to find file: '
+ f)
# create a queue of pic list by tf.train.string_input_producer()
file_queue = tf.train.string_input_producer(fileNames)
# read file and the size of step is record_bytes
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(file_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
# Change pic into float32
reshape_image_data = tf.cast(result.uint8image, tf.float32)
# the type of pic of returned is a Tensor
# so whenever we sess.run(reshape_image) , we will take out a image
return reshape_image_data
def read_cifar10(filename_queue):
"""Reads and parses examples from CIFAR10 data files.
Recommendation: if you want N-way read parallelism, call this function
N times. This will give you N independent Readers reading different
files & positions within those files, which will give better mixing of
examples.
Args:
filename_queue: A queue of strings with the filenames to read from.
Returns:
An object representing a single example, with the following fields:
height: number of rows in the result (32)
width: number of columns in the result (32)
depth: number of color channels in the result (3)
key: a scalar string Tensor describing the filename & record number
for this example.
label: an int32 Tensor with the label in the range 0..9.
uint8image: a [height, width, depth] uint8 Tensor with the image data
"""
class CIFAR10Record(object):
pass
result = CIFAR10Record()
# Dimensions of the images in the CIFAR-10 dataset.
# See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
# input format.
label_bytes = 1 # 2 for CIFAR-100
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_f0(
file_pattern,
batch_size,
record_bytes=RECORD_BYTES,
capacity=2048,
min_after_dequeue=1536,
num_threads=8,
data_format='NCHW',
normalizer=None,
):
'''
Read only `sp` and `speaker`
Return:
`feature`: [b, c]
`speaker`: [b,]
'''
with tf.device('cpu'):
with tf.name_scope('InputSpectralFrame'):
files = tf.gfile.Glob(file_pattern)
filename_queue = tf.train.string_input_producer(files)
reader = tf.FixedLengthRecordReader(record_bytes)
_, value = reader.read(filename_queue)
value = tf.decode_raw(value, tf.float32)
value = tf.reshape(value, [FEAT_DIM,])
#feature = value[:SP_DIM] # NCHW format
feature = value[513+1:513*2+1]
if normalizer is not None:
feature = normalizer.forward_process(feature)
if data_format == 'NCHW':
feature = tf.reshape(feature, [1, 513, 1])
elif data_format == 'NHWC':
feature = tf.reshape(feature, [513, 1, 1])
else:
pass
speaker = tf.cast(value[-1], tf.int64)
#zk
# f0 = tf.cast(value[SP_DIM * 2], tf.int64)
return tf.train.shuffle_batch(
[feature, speaker],
batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue,
num_threads=num_threads,
# enqueue_many=True,
)
def read_nn(filename_queue):
"""Reads and parses examples from the data files.
Args:
filename_queue: A queue of strings with the filenames to read from.
Returns:
An object representing a single example, with the following fields:
height: number of rows in the result (32)
width: number of columns in the result (32)
depth: number of color channels in the result (3)
key: a scalar string Tensor describing the filename & record number
for this example.
label: an int32 Tensor with the label in the range 0..9.
uint8image: a [height, width, depth] uint8 Tensor with the image data
"""
class NNRecord(object):
pass
result = NNRecord()
# Dimensions of the images in the dataset.
label_bytes = 1
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes
# Read a record, getting filenames from the filename_queue.
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]),
tf.int32)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.slice(record_bytes, [label_bytes], [image_bytes]),
[result.depth, result.height, result.width])
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
print("here")
return result
def _get_raw_data_op(self, file_name_queue):
self._file_reader = tf.FixedLengthRecordReader(self._total_size)
(raw_key, raw_value) = self._file_reader.read(file_name_queue)
uint8_value = tf.decode_raw(raw_value, tf.uint8)
y = tf.squeeze(tf.slice(uint8_value, [0], [LABEL_SIZE]))
x = tf.slice(uint8_value, [LABEL_SIZE], [self._data_size])
return self._process_image(tf.cast(x,
tf.float32)), tf.cast(y, tf.int64)
def read_input(file_list):
reader = tf.FixedLengthRecordReader(record_bytes=3073)
key, value = reader.read(file_list)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.cast(tf.strided_slice(record_bytes, [0], [1]), tf.int32)
reshaped_bytes = tf.reshape(tf.strided_slice(record_bytes, [0], [3072]),
[3, 32, 32])
# [3,32,32] -> [32,32,3]
image_data = tf.transpose(reshaped_bytes, [1, 2, 0])
return image_data, label
def read_cifar10(data_dir, is_train, batch_size, shuffle):
img_width = 32
img_height = 32
img_depth = 3
label_bytes = 1
image_bytes = img_width * img_height * img_depth
with tf.name_scope('input'):
if is_train:
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % ii)
for ii in np.arange(1, 6)
]
else:
filenames = [os.path.join(data_dir, 'test_batch.bin')]
filenames_queue = tf.train.string_input_producer(filenames)
reader = tf.FixedLengthRecordReader(label_bytes + image_bytes)
key, value = reader.read(filenames_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.slice(record_bytes, [0], [label_bytes])
label = tf.cast(label, tf.int32)
img_raw = tf.slice(record_bytes, [label_bytes], [image_bytes])
img_raw = tf.reshape(img_raw, [img_depth, img_height, img_width])
img = tf.transpose(img_raw, (1, 2, 0)) #convert from D/H/W to H/W/D
img = tf.cast(img, tf.float32)
# data argumentation
# img = tf.random_crop(img,[24,24,3])
# img = tf.image.random_flip_left_right(img)
# img = tf.image.random_brightness(img,max_delta=63)
# img = tf.image.random_contrast(img,lower=0.2,upper=1.8)
# img = tf.image.per_image_standardization(img)
if shuffle:
img, label_batch = tf.train.shuffle_batch([img, label],
batch_size=batch_size,
num_threads=16,
capacity=2000,
min_after_dequeue=1500)
else:
img, label_batch = tf.train.batch([img, label],
batch_size=batch_size,
num_threads=16,
capacity=2000)
return img, tf.reshape(label_batch, [batch_size])
def read_shapenet(filename_queue):
"""Reads and parses examples from ShapeCat data files.
Recommendation: if you want N-way read parallelism, call this function
N times. This will give you N independent Readers reading different
files & positions within those files, which will give better mixing of
examples.
Args:
filename_queue: A queue of strings with the filenames to read from.
Returns:
An object representing a single example, with the following fields:
height: number of rows in the result (32)
width: number of columns in the result (32)
depth: number of color channels in the result (3)
key: a scalar string Tensor describing the filename & record number
for this example.
depth_images: a [width, height, number_of_depth_images] float32 Tensor with the image data
"""
class ShapenetCatRecord(object):
pass
result = ShapenetCatRecord()
# check glx_main.cc in depth_renderer-egl for input format.
label_bytes = 4 # int32_t for shapenet
result.n_images = 12 + 12 + 4 + 1 + 1 # See the ctor of RLDriver
result.height = config.DEFAULT_RES
result.width = config.DEFAULT_RES
result.depth_bytes = 4 # FP32 = 4 bytes
image_bytes = result.height * result.width * result.depth_bytes
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes * result.n_images
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, record_string = reader.read(filename_queue)
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.decode_raw(tf.substr(record_string, 0, label_bytes), tf.int32)
record_depth_image_string = tf.substr(record_string, label_bytes, image_bytes * result.n_images)
depth = tf.reshape(
tf.decode_raw(record_depth_image_string, tf.float32),
[result.n_images, result.height, result.width, 1])
result.image = depth
return result
def read_cifar10(filename_queue):
"""从二进制数据中获取一个样本的信息结构体(大小、数据、标签)
执行N次实现N路并行
Args:
filename_queue: 由filenames组成的字符串队列
Returns:
一个包含下诉变量的对象:
height: 行数
width: 列数
depth: 颜色通道数
key: 描述样本的名字&记录数的一个字符串scalar张量
label: 标识类别0..9.
uint8image: 一个带图片信息[height, width, depth] uint8 张量
"""
class CIFAR10Record(object):
pass # 保持句式结构,代表什么都不做
result = CIFAR10Record()
# 输入格式.
label_bytes = 1 # 2 for CIFAR-100 ?????
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# 每个记录都由定长字节图片和紧随其后的标签组成
record_bytes = label_bytes + image_bytes
# 读取一个记录,从filename_queue中获取文件名,CIFAR10没有header 或者footer,对应的字节数使用默认值0
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# 将一个字符串转换为一个uint8的张量
record_bytes = tf.decode_raw(value, tf.uint8)
# 获取代表标签的第一字节分片,并做uint8-->int32转换,strided_slice可以参看本人博客
# http://blog.csdn.net/banana1006034246/article/details/75092388
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# 获取图片信息,并reshape[depth * height * width]-->[depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# 使用tranpose将张量[depth, height, width]转置成[height, width, depth]
# 关于transpose函数参看本人博客http://blog.csdn.net/banana1006034246/article/details/75126815
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_cifar10(filename_queue):
"""Reads and parses examples from CIFAR10 data files.
Recommendation: if you want N-way read parallelism, call this function
N times. This will give you N independent Readers reading different
files & positions within those files, which will give better mixing of
examples.
Args:
filename_queue: A queue of strings with the filenames to read from.
Returns:
An object representing a single example, with the following fields:
height: number of rows in the result (32)
width: number of columns in the result (32)
depth: number of color channels in the result (3)
key: a scalar string Tensor describing the filename & record number
for this example.
label: an int32 Tensor with the label in the range 0..9.
uint8image: a [height, width, depth] uint8 Tensor with the image data
"""
class CIFAR10Record(object):
pass
result = CIFAR10Record()
label_bytes = 1
result.height, result.width, result.depth = 32, 32, 3
image_bytes = result.height * result.width * result.depth
record_bytes = label_bytes + image_bytes
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_binary(filename_queue, dimensions):
class BinaryRecord(object):
pass
result = BinaryRecord()
# Float type of elements
float_type = np.dtype('float32')
float_bytes = float_type.itemsize
# Calculate label dimensions
label_bytes = 1
# Calculate data dimensions
data_bytes = 29
# Calculate image dimensions
# result.height = 91
# result.width = 109
# result.depth = 91
result.height = dimensions[0]
result.width = dimensions[1]
result.depth = dimensions[2]
image_bytes = result.height * result.width * result.depth
# Every record has label followed by image
record_bytes = (label_bytes + data_bytes + image_bytes) * float_bytes
# Read a record - from filename in filename_queue
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of float32 that is of record_bytes long
record_bytes = tf.decode_raw(value, tf.float32)
# Extract label
result.label = tf.slice(record_bytes, [0], [label_bytes])
# Extract data
data_raw = tf.slice(record_bytes, [label_bytes], [data_bytes])
result.data = tf.reshape(data_raw, [data_bytes, 1])
# Extract image
# image_raw = tf.slice(record_bytes, [label_bytes + data_bytes], [result.height * result.width * result.depth])
image_raw = tf.slice(record_bytes, [data_bytes],
[result.height * result.width * result.depth])
# result.image = tf.reshape(image_raw, [result.height, result.width, result.depth, 1])
result.image = tf.reshape(image_raw,
[result.height, result.width, result.depth])
return result
def read_cifar10(filename_queue):
"""
Reads and parses examples from CIFAR10 data files.
Parammeters:
filename_queue: A queue of strings with the filenames to read from.
Returns:
An object representating a single example, with the following fields:
height: number of rows in the result (32)
width: numebr of columns in the result (32)
depth: number of color channels in the result (3)
key: a scalar string Tensor describing the filename & record number for this example.
label: an int32 Tensor with the label in the range 0...9.
uint8image: a [height, width, depth] uint8 Tensor with the image data.
"""
class CIFAR10Record(object):
pass
result = CIFAR10Record()
label_bytes = 1
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
record_bytes = label_bytes + image_bytes
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# with tf.Session() as sess:
# print('**********************')
# coord = tf.train.Coordinator()
# tf.train.start_queue_runners(sess, coord)
# print(sess.run(result.label))
# exit()
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def load(self, filename_queue):
reader = tf.FixedLengthRecordReader(record_bytes=self.record_bytes)
self.key, value = reader.read(filename_queue)
self.raw = tf.decode_raw(value, tf.uint8)
self.label = tf.cast(tf.strided_slice(self.raw, [0], [self.label_bytes]), tf.int32)
self.label.set_shape([1])
self.label = tf.one_hot(self.label, depth = 10, on_value = 1.0, off_value = 0.0, axis = -1)
depth_major = tf.reshape(
tf.strided_slice(self.raw,
[self.label_bytes],
[self.record_bytes]
),
[self.depth, self.height, self.width])
self.uint8image = tf.transpose(depth_major, [1, 2, 0])
def read_file(filename_queue):
label_bytes = 1
record_bytes = 32 * 32 * 3 + label_bytes #image data length + label data length
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, data = reader.read(filename_queue)
record_bytes = tf.decode_raw(data, tf.uint8)
label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
image = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + 32 * 32 * 3]),
[3, IMAGE_RESIZE, IMAGE_RESIZE])
image = tf.transpose(image, [1, 2, 0])
return image, label
def read(
file_pattern,
batch_size,
record_bytes=RECORD_BYTES,
capacity=256,
min_after_dequeue=128,
num_threads=8,
format='NHWC',
normalizer=None,
):
'''
Read only `sp` and `speaker`
Return:
`feature`: [b, c]
`speaker`: [b,]
'''
with tf.name_scope('InputSpectralFrame'):
files = tf.gfile.Glob(file_pattern)
filename_queue = tf.train.string_input_producer(files)
reader = tf.FixedLengthRecordReader(record_bytes)
_, value = reader.read(filename_queue)
print "value 1:", value.get_shape().as_list()
value = tf.decode_raw(value, tf.float32)
print "value 2:", value.get_shape().as_list()
value = tf.reshape(value, [
FEAT_DIM,
])
print "value 3:", value.get_shape().as_list()
feature = value[:SP_DIM] # NCHW format
print "feature shape:", feature.get_shape().as_list()
if normalizer is not None:
feature = normalizer.forward_process(feature)
if format == 'NCHW':
feature = tf.reshape(feature, [1, SP_DIM, 1])
elif format == 'NHWC':
feature = tf.reshape(feature, [SP_DIM, 1, 1])
else:
pass
speaker = tf.cast(value[-1], tf.int64)
print "shuffle batch"
return tf.train.shuffle_batch(
[feature, speaker],
batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue,
num_threads=num_threads,
# enqueue_many=True,
)
def read_cifar100(filename_queue,coarse_or_fine='fine'):
class CIFAR10Record(object):
pass
result = CIFAR10Record()
# Dimensions of the images in the CIFAR-10 dataset.
# See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
# input format.
#label_bytes = 2 # 2 for CIFAR-100
coarse_label_bytes=1
fine_label_bytes = 1
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = coarse_label_bytes + fine_label_bytes + image_bytes
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
coarse_label = tf.cast(tf.strided_slice(record_bytes, [0], [coarse_label_bytes]), tf.int32)
fine_label=tf.cast(tf.strided_slice(record_bytes,[coarse_label_bytes],
[coarse_label_bytes+fine_label_bytes]),tf.int32)
if coarse_or_fine=='fine':
result.label=fine_label#100个精细分类标签
else:
result.label=coarse_label #20个粗略分类
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [coarse_label_bytes+fine_label_bytes],
[coarse_label_bytes+fine_label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_raw_images(data_set):
dirs = './data/' + data_set + '/'
filename = list_binary_files(dirs)
filename_queue = tf.train.string_input_producer(filename)
if data_set is 'train':
image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth
record_bytes = image_bytes + 1
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32)
depth_major = tf.reshape(tf.slice(record_bytes, [1], [image_bytes]), [
FLAGS.depth, FLAGS.height, FLAGS.width])
uint8image = tf.transpose(depth_major, [1, 2, 0])
return label, uint8image
elif data_set is 'test':
image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth
record_bytes = image_bytes + 1
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
depth_major = tf.reshape(tf.slice(record_bytes, [0], [image_bytes]),
[FLAGS.depth, FLAGS.height, FLAGS.width])
uint8image = tf.transpose(depth_major, [1, 2, 0])
return uint8image
elif data_set is 'validation':
image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth
record_bytes = image_bytes + 1
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32)
depth_major = tf.reshape(tf.slice(record_bytes, [1], [image_bytes]), [
FLAGS.depth, FLAGS.height, FLAGS.width])
uint8image = tf.transpose(depth_major, [1, 2, 0])
return label, uint8image
def __read_cifar(filenames, shuffle=True, cifar100=False):
"""Reads and parses examples from CIFAR data files.
"""
# Dimensions of the images in the CIFAR-10 dataset.
# See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
# input format.
filename_queue = tf.train.string_input_producer(filenames,
shuffle=shuffle,
num_epochs=None)
label_bytes = 1 # 2 for CIFAR-100
if cifar100:
label_bytes = 2
height = 32
width = 32
depth = 3
image_bytes = height * width * depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
if cifar100:
#label = tf.cast(
# tf.strided_slice(record_bytes, [0], [1]), tf.int32) #20 calss
label = tf.cast(tf.strided_slice(record_bytes, [1], [label_bytes]),
tf.int32) #100 class
else:
label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]), [depth, height, width])
# Convert from [depth, height, width] to [height, width, depth].
image = tf.transpose(depth_major, [1, 2, 0])
return tf.cast(image, tf.float32), label
def read_cifar10(filename_queue):
# define a structure object which could contain different attributes
class CIFAR10Record(object):
pass
# result is the passing parameter in this function
result = CIFAR10Record()
# Dimensions of the images in the CIFAR-10 dataset.
label_bytes = 1
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
record_bytes = label_bytes + image_bytes
''' define the file reading rules: reader'''
# Read a record, getting file_names from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes,
name='data_reader')
''' start reading the file'''
# read the file name as well as the value in the file into a list
# format key: file_name, value: [length = record_bytes]
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long
# doesn't change the shape of vector but change the data type
# different from the function: tf.cast.
record_bytes = tf.decode_raw(value, tf.uint8)
# The first bytes represent the label, which we convert from uint8->int32.
# tf.strided_slice: choose data of index from 0 to label_bytes in the record_bytes
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32,
name='label_get')
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width],
name='image_get')
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_cifar10(is_train, batch_size, shuffle, data_dir='./cifar-10-batches-bin/'):
"""Read CIFAR10
Args:
data_dir: the directory of CIFAR10
is_train: boolen
batch_size:
shuffle:
Returns:
label: 1D tensor, tf.int32
image: 4D tensor, [batch_size, height, width, 3], tf.float32
"""
img_width = 32
img_height = 32
img_depth = 3
label_bytes = 1
image_bytes = img_width*img_height*img_depth
# Create queues for execution of queue runners
if is_train:
filenames = [os.path.join(data_dir, 'data_batch_%d.bin' %ii) for ii in np.arange(1, 6)]
else:
filenames = [os.path.join(data_dir, 'test_batch.bin')]
filename_queue = tf.train.string_input_producer(filenames)
reader = tf.FixedLengthRecordReader(label_bytes + image_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.slice(record_bytes, [0], [label_bytes])
label = tf.cast(label, tf.int32)
image_raw = tf.slice(record_bytes, [label_bytes], [image_bytes])
image_raw = tf.reshape(image_raw, [img_depth, img_height, img_width])
image = tf.transpose(image_raw, (1,2,0)) # C/H/W to H/W/C
image = tf.cast(image, tf.float32)
image = tf.image.per_image_standardization(image) # whitening the image
if shuffle:
images, label_batch = tf.train.shuffle_batch([image, label],
batch_size=batch_size, num_threads=64,
capacity=20000, min_after_dequeue=3000)
else:
images, label_batch = tf.train.batch([image, label],
batch_size=batch_size, num_threads=64, capacity=2000)
label_batch = tf.reshape(tf.cast(label_batch, dtype=tf.int32), [-1])
return images, label_batch
def read_record(image_file_queue, label_file_queue):
class MuctRecord(object):
pass
result = MuctRecord()
height = 64
width = 64
depth = 3
image_bytes = height * width * depth * 1
label_bytes = 2
image_reader = tf.FixedLengthRecordReader(record_bytes=image_bytes)
_, value = image_reader.read(image_file_queue)
result.image = tf.decode_raw(value, tf.uint8)
result.image = tf.reshape(result.image, [height, width, depth])
result.image = tf.cast(result.image, tf.float32)
label_reader = tf.FixedLengthRecordReader(record_bytes=label_bytes)
_, value = label_reader.read(label_file_queue)
result.label = tf.decode_raw(value, tf.int16)
result.label = tf.reshape(result.label, [1])
return result
def read_data_set(self, filename_queue):
record = Record()
reader = tf.FixedLengthRecordReader(record_bytes=self.record_bytes)
file_name, value = reader.read(filename_queue)
byte_record = tf.decode_raw(value, tf.uint8)
image_label = tf.cast(tf.slice(byte_record, [0], [LABEL_BYTES]),
tf.int32)
array_image = tf.strided_slice(byte_record, [LABEL_BYTES],
[self.record_bytes])
depth_major_image = tf.reshape(array_image,
[self.depth, self.height, self.width])
record.image = tf.transpose(depth_major_image, [1, 2, 0])
record.label = image_label[0]
return record
def read_raw_images(sess, data_set):
filename = ['./data/' + data_set + '_data.bin']
filename_queue = tf.train.string_input_producer(filename)
record_bytes = (FLAGS.raw_height) * (FLAGS.raw_width) * FLAGS.depth + 1
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
tf.train.start_queue_runners(sess=sess)
for i in range(0, 100):
result = sess.run(record_bytes)
print i, result[0]
image = result[1:len(result)]
def read_raw_images(data_set):
filename = ['./data/' + data_set + '_data.bin']
filename_queue = tf.train.string_input_producer(filename)
size = (FLAGS.img_h) * (FLAGS.img_w) * FLAGS.channel
record_bytes = size + 1
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32)
image = tf.reshape(tf.slice(record_bytes, [1], [size]),
[FLAGS.channel, FLAGS.img_h, FLAGS.img_w])
image_ = tf.transpose(image, [1, 2, 0])
return image_, label
def read_cifar10(data_dir, is_train, batch_size, shuffle):
img_width = 32
img_height = 32
img_depth = 3
label_bytes = 1
image_bytes = img_width * img_height * img_depth
with tf.name_scope('input'):
if is_train:
filenames = [
os.path.join(data_dir, 'data_batch_%d.bin' % i)
for i in np.arange(1, 6)
]
else:
filenames = os.path.join(data_dir, 'test_batch.bin')
filename_queue = tf.train.string_input_producer(filenames)
reader = tf.FixedLengthRecordReader(label_bytes + image_bytes)
key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
label = tf.slice(record_bytes, [0], [label_bytes])
label = tf.cast(label, tf.int32)
image_raw = tf.slice(record_bytes, [label_bytes], [image_bytes])
image_raw = tf.reshape(image_raw, [img_depth, img_height, img_depth])
image = tf.transpose(image_raw, (1, 2, 0))
image = tf.cast(image, tf.float32)
image = tf.image.per_image_standardization(image)
if shuffle:
image_batch, label_batch = tf.train.shuffle_batch(
[image, label],
batch_size=batch_size,
num_threads=16,
capacity=2000,
min_after_dequeue=1500)
else:
image_batch, label_batch = tf.train.batch([image, label],
batch_size=batch_size,
num_threads=16,
capacity=2000)
n_classes = 10
label_batch = tf.one_hot(label_batch, depth=n_classes)
return image_batch, tf.reshape(label_batch, [batch_size, n_classes])
def read_cifar10(filename_queue):
class CIFAR10Record(object):
pass
result = CIFAR10Record()
# Dimensions of the images in the CIFAR-10 dataset.
# See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the
# input format.
# 分类结果的长度，CIFAR-100长度为2
label_bytes = 1 # 2 for CIFAR-100
result.height = 32
result.width = 32
# 3位表示rgb颜色（0-255,0-255,0-255）
result.depth = 3
image_bytes = result.height * result.width * result.depth
# Every record consists of a label followed by the image, with a
# fixed number of bytes for each.
# 单个记录的总长度=分类结果长度+图片长度
record_bytes = label_bytes + image_bytes
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
# 读取
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_bytes = tf.decode_raw(value, tf.uint8)
# 第一位代表lable-图片的正确分类结果，从uint8转换为int32类型
# The first bytes represent the label, which we convert from uint8->int32.
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# 分类结果之后的数据代表图片，我们重新调整大小
# The remaining bytes after the label represent the image, which we reshape
# from [depth * height * width] to [depth, height, width].
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# 格式转换，从[颜色,高度,宽度]--》[高度,宽度,颜色]
# Convert from [depth, height, width] to [height, width, depth].
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_cifar10(filename_queue):
"""从CIFAR10数据文件中阅读并解析example。
推荐：如果你想要N路并行阅读，那么调用这个函数N次。这样会返回N个独立的Reader用来阅读那些文件里不同的文件和位置，这样会返回更好的混合example。
参数:
filename_queue: 文件名字符串队列。
返回:
一个object代表一个example，包括以下内容：
高: result的行数(32)
宽: result的列数(32)
深: result的色彩通道数(3)
key: 一个标量字符串描述这个example的文件名和record number。
标签: 一个带有标签（0..9）的int32 Tensor
uint8image: 一个带有图像数据的[height, width, depth] uint8 Tensor
"""
class CIFAR10Record(object):
pass
result = CIFAR10Record()
# CIFAR-10数据集中图像的维度。
label_bytes = 1
result.height = 32
result.width = 32
result.depth = 3
image_bytes = result.height * result.width * result.depth
# 每一个record包含一个标签和一个固定长度的用来描述图像的bytes。
record_bytes = label_bytes + image_bytes
# 阅读一个record，从filename_queue中获得filename。CIFAR-10格式没有header何footer，所以我们默认header——bytes和footer_bytes为0。
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
# 将一个字符串转换成一个uint8向量
record_bytes = tf.decode_raw(value, tf.uint8)
# 第一个bytes代表标签，我们将它转换成int32。
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
# 剩下的bytes代表图像，我们将它reshape成[depth, height, width]。
depth_major = tf.reshape(
tf.strided_slice(record_bytes, [label_bytes],
[label_bytes + image_bytes]),
[result.depth, result.height, result.width])
# 转换成[height, width, depth]
result.uint8image = tf.transpose(depth_major, [1, 2, 0])
return result
def read_cifar10_label(filename_queue):
class CIFAR10Record(object):
pass
result = CIFAR10Record()
label_bytes = 1 # 2 for CIFAR-100
record_bytes = label_bytes
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
record_bytes = tf.decode_raw(value, tf.uint8)
result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]),
tf.int32)
return result
def read_animals(filename_queue):
class ANIMAL10Record(object):
pass
result = IMAL10Record()
label_bytes = 1 # 2 for CIFAR-100
result.height = 140
result.width = 160
result.depth = 3
image_bytes = result.height * result.width * result.depth
record_bytes = label_bytes + image_bytes
reader = tf.FixedLengthRecordReader(record_bytes=record_bytes)
result.key, value = reader.read(filename_queue)
