def read_data(file_queue): """ Data is saved in binary files. Each row has: 1st byte -> label 2nd-last byte -> 3D Volume [height, width, depth, channels] Args: file_queue -> a queue of file names saved as strings Rtns: An object with: height -> volume height width -> volume width depth -> volume depth nChan -> number of channels key -> scalar tensor with file name and record number label -> 1D int32 tensor with the associated label img3_uint8 -> 4D uint8 tensor with image data """ class record_data(object): pass img3_obj = record_data() # Dimensions of data label_bytes = 1 img3_obj.height = CFG['height'] img3_obj.width = CFG['width'] img3_obj.depth = CFG['depth'] img3_obj.nChan = CFG['nChan'] # Size in memory img3_bytes = img3_obj.height * img3_obj.width * img3_obj.depth * img3_obj.nChan record_bytes = label_bytes + img3_bytes # Read a record reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) img3_obj.key, value = reader.read(file_queue) # Convert from a string to a vector of uint8 that is record_bytes long record_bytes = tf.decode_raw(value, tf.uint8) # First byte represent the label, which we convert from uint8 -> int32 img3_obj.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # Remaining bytes after the label represent the image, which we reshape from # [depth * height * width] to [depth,height, width] depth_major = tf.reshape( tf.slice(record_bytes, [label_bytes], [img3_bytes]), [img3_obj.depth, img3_obj.height, img3_obj.width, img3_obj.nChan]) img3_obj.img3_uint8 = tf.transpose(depth_major, [2, 1, 0, 3]) return img3_obj
def inputs_origin(filePath): class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes fileNames = [ os.path.join(filePath, 'data_batch_%d.bin' % _) for _ in range(1, 6) ] # in Python 3, rang() && xrange() == range() for f in fileNames: if not tf.gfile.Exists(f): raise ValueError( 'Error at __Test02_exchangeCIFAR_Pic.py : Failed to find file: ' + f) # create a queue of pic list by tf.train.string_input_producer() file_queue = tf.train.string_input_producer(fileNames) # read file and the size of step is record_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(file_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) # Change pic into float32 reshape_image_data = tf.cast(result.uint8image, tf.float32) # the type of pic of returned is a Tensor # so whenever we sess.run(reshape_image) , we will take out a image return reshape_image_data
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_f0( file_pattern, batch_size, record_bytes=RECORD_BYTES, capacity=2048, min_after_dequeue=1536, num_threads=8, data_format='NCHW', normalizer=None, ): ''' Read only `sp` and `speaker` Return: `feature`: [b, c] `speaker`: [b,] ''' with tf.device('cpu'): with tf.name_scope('InputSpectralFrame'): files = tf.gfile.Glob(file_pattern) filename_queue = tf.train.string_input_producer(files) reader = tf.FixedLengthRecordReader(record_bytes) _, value = reader.read(filename_queue) value = tf.decode_raw(value, tf.float32) value = tf.reshape(value, [FEAT_DIM,]) #feature = value[:SP_DIM] # NCHW format feature = value[513+1:513*2+1] if normalizer is not None: feature = normalizer.forward_process(feature) if data_format == 'NCHW': feature = tf.reshape(feature, [1, 513, 1]) elif data_format == 'NHWC': feature = tf.reshape(feature, [513, 1, 1]) else: pass speaker = tf.cast(value[-1], tf.int64) #zk # f0 = tf.cast(value[SP_DIM * 2], tf.int64) return tf.train.shuffle_batch( [feature, speaker], batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, num_threads=num_threads, # enqueue_many=True, )
def read_nn(filename_queue): """Reads and parses examples from the data files. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class NNRecord(object): pass result = NNRecord() # Dimensions of the images in the dataset. label_bytes = 1 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.slice(record_bytes, [label_bytes], [image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) print("here") return result
def _get_raw_data_op(self, file_name_queue): self._file_reader = tf.FixedLengthRecordReader(self._total_size) (raw_key, raw_value) = self._file_reader.read(file_name_queue) uint8_value = tf.decode_raw(raw_value, tf.uint8) y = tf.squeeze(tf.slice(uint8_value, [0], [LABEL_SIZE])) x = tf.slice(uint8_value, [LABEL_SIZE], [self._data_size]) return self._process_image(tf.cast(x, tf.float32)), tf.cast(y, tf.int64)
def read_input(file_list): reader = tf.FixedLengthRecordReader(record_bytes=3073) key, value = reader.read(file_list) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.strided_slice(record_bytes, [0], [1]), tf.int32) reshaped_bytes = tf.reshape(tf.strided_slice(record_bytes, [0], [3072]), [3, 32, 32]) # [3,32,32] -> [32,32,3] image_data = tf.transpose(reshaped_bytes, [1, 2, 0]) return image_data, label
def read_cifar10(data_dir, is_train, batch_size, shuffle): img_width = 32 img_height = 32 img_depth = 3 label_bytes = 1 image_bytes = img_width * img_height * img_depth with tf.name_scope('input'): if is_train: filenames = [ os.path.join(data_dir, 'data_batch_%d.bin' % ii) for ii in np.arange(1, 6) ] else: filenames = [os.path.join(data_dir, 'test_batch.bin')] filenames_queue = tf.train.string_input_producer(filenames) reader = tf.FixedLengthRecordReader(label_bytes + image_bytes) key, value = reader.read(filenames_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.slice(record_bytes, [0], [label_bytes]) label = tf.cast(label, tf.int32) img_raw = tf.slice(record_bytes, [label_bytes], [image_bytes]) img_raw = tf.reshape(img_raw, [img_depth, img_height, img_width]) img = tf.transpose(img_raw, (1, 2, 0)) #convert from D/H/W to H/W/D img = tf.cast(img, tf.float32) # data argumentation # img = tf.random_crop(img,[24,24,3]) # img = tf.image.random_flip_left_right(img) # img = tf.image.random_brightness(img,max_delta=63) # img = tf.image.random_contrast(img,lower=0.2,upper=1.8) # img = tf.image.per_image_standardization(img) if shuffle: img, label_batch = tf.train.shuffle_batch([img, label], batch_size=batch_size, num_threads=16, capacity=2000, min_after_dequeue=1500) else: img, label_batch = tf.train.batch([img, label], batch_size=batch_size, num_threads=16, capacity=2000) return img, tf.reshape(label_batch, [batch_size])
def read_shapenet(filename_queue): """Reads and parses examples from ShapeCat data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. depth_images: a [width, height, number_of_depth_images] float32 Tensor with the image data """ class ShapenetCatRecord(object): pass result = ShapenetCatRecord() # check glx_main.cc in depth_renderer-egl for input format. label_bytes = 4 # int32_t for shapenet result.n_images = 12 + 12 + 4 + 1 + 1 # See the ctor of RLDriver result.height = config.DEFAULT_RES result.width = config.DEFAULT_RES result.depth_bytes = 4 # FP32 = 4 bytes image_bytes = result.height * result.width * result.depth_bytes # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes * result.n_images reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, record_string = reader.read(filename_queue) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.decode_raw(tf.substr(record_string, 0, label_bytes), tf.int32) record_depth_image_string = tf.substr(record_string, label_bytes, image_bytes * result.n_images) depth = tf.reshape( tf.decode_raw(record_depth_image_string, tf.float32), [result.n_images, result.height, result.width, 1]) result.image = depth return result
def read_cifar10(filename_queue): """从二进制数据中获取一个样本的信息结构体(大小、数据、标签) 执行N次实现N路并行 Args: filename_queue: 由filenames组成的字符串队列 Returns: 一个包含下诉变量的对象: height: 行数 width: 列数 depth: 颜色通道数 key: 描述样本的名字&记录数的一个字符串scalar张量 label: 标识类别0..9. uint8image: 一个带图片信息[height, width, depth] uint8 张量 """ class CIFAR10Record(object): pass # 保持句式结构,代表什么都不做 result = CIFAR10Record() # 输入格式. label_bytes = 1 # 2 for CIFAR-100 ????? result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # 每个记录都由定长字节图片和紧随其后的标签组成 record_bytes = label_bytes + image_bytes # 读取一个记录,从filename_queue中获取文件名,CIFAR10没有header 或者footer,对应的字节数使用默认值0 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # 将一个字符串转换为一个uint8的张量 record_bytes = tf.decode_raw(value, tf.uint8) # 获取代表标签的第一字节分片,并做uint8-->int32转换,strided_slice可以参看本人博客 # http://blog.csdn.net/banana1006034246/article/details/75092388 result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # 获取图片信息,并reshape[depth * height * width]-->[depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # 使用tranpose将张量[depth, height, width]转置成[height, width, depth] # 关于transpose函数参看本人博客http://blog.csdn.net/banana1006034246/article/details/75126815 result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(filename_queue): """Reads and parses examples from CIFAR10 data files. Recommendation: if you want N-way read parallelism, call this function N times. This will give you N independent Readers reading different files & positions within those files, which will give better mixing of examples. Args: filename_queue: A queue of strings with the filenames to read from. Returns: An object representing a single example, with the following fields: height: number of rows in the result (32) width: number of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0..9. uint8image: a [height, width, depth] uint8 Tensor with the image data """ class CIFAR10Record(object): pass result = CIFAR10Record() label_bytes = 1 result.height, result.width, result.depth = 32, 32, 3 image_bytes = result.height * result.width * result.depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_binary(filename_queue, dimensions): class BinaryRecord(object): pass result = BinaryRecord() # Float type of elements float_type = np.dtype('float32') float_bytes = float_type.itemsize # Calculate label dimensions label_bytes = 1 # Calculate data dimensions data_bytes = 29 # Calculate image dimensions # result.height = 91 # result.width = 109 # result.depth = 91 result.height = dimensions[0] result.width = dimensions[1] result.depth = dimensions[2] image_bytes = result.height * result.width * result.depth # Every record has label followed by image record_bytes = (label_bytes + data_bytes + image_bytes) * float_bytes # Read a record - from filename in filename_queue reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of float32 that is of record_bytes long record_bytes = tf.decode_raw(value, tf.float32) # Extract label result.label = tf.slice(record_bytes, [0], [label_bytes]) # Extract data data_raw = tf.slice(record_bytes, [label_bytes], [data_bytes]) result.data = tf.reshape(data_raw, [data_bytes, 1]) # Extract image # image_raw = tf.slice(record_bytes, [label_bytes + data_bytes], [result.height * result.width * result.depth]) image_raw = tf.slice(record_bytes, [data_bytes], [result.height * result.width * result.depth]) # result.image = tf.reshape(image_raw, [result.height, result.width, result.depth, 1]) result.image = tf.reshape(image_raw, [result.height, result.width, result.depth]) return result
def read_cifar10(filename_queue): """ Reads and parses examples from CIFAR10 data files. Parammeters: filename_queue: A queue of strings with the filenames to read from. Returns: An object representating a single example, with the following fields: height: number of rows in the result (32) width: numebr of columns in the result (32) depth: number of color channels in the result (3) key: a scalar string Tensor describing the filename & record number for this example. label: an int32 Tensor with the label in the range 0...9. uint8image: a [height, width, depth] uint8 Tensor with the image data. """ class CIFAR10Record(object): pass result = CIFAR10Record() label_bytes = 1 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # with tf.Session() as sess: # print('**********************') # coord = tf.train.Coordinator() # tf.train.start_queue_runners(sess, coord) # print(sess.run(result.label)) # exit() depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def load(self, filename_queue): reader = tf.FixedLengthRecordReader(record_bytes=self.record_bytes) self.key, value = reader.read(filename_queue) self.raw = tf.decode_raw(value, tf.uint8) self.label = tf.cast(tf.strided_slice(self.raw, [0], [self.label_bytes]), tf.int32) self.label.set_shape([1]) self.label = tf.one_hot(self.label, depth = 10, on_value = 1.0, off_value = 0.0, axis = -1) depth_major = tf.reshape( tf.strided_slice(self.raw, [self.label_bytes], [self.record_bytes] ), [self.depth, self.height, self.width]) self.uint8image = tf.transpose(depth_major, [1, 2, 0])
def read_file(filename_queue): label_bytes = 1 record_bytes = 32 * 32 * 3 + label_bytes #image data length + label data length reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, data = reader.read(filename_queue) record_bytes = tf.decode_raw(data, tf.uint8) label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) image = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + 32 * 32 * 3]), [3, IMAGE_RESIZE, IMAGE_RESIZE]) image = tf.transpose(image, [1, 2, 0]) return image, label
def read( file_pattern, batch_size, record_bytes=RECORD_BYTES, capacity=256, min_after_dequeue=128, num_threads=8, format='NHWC', normalizer=None, ): ''' Read only `sp` and `speaker` Return: `feature`: [b, c] `speaker`: [b,] ''' with tf.name_scope('InputSpectralFrame'): files = tf.gfile.Glob(file_pattern) filename_queue = tf.train.string_input_producer(files) reader = tf.FixedLengthRecordReader(record_bytes) _, value = reader.read(filename_queue) print "value 1:", value.get_shape().as_list() value = tf.decode_raw(value, tf.float32) print "value 2:", value.get_shape().as_list() value = tf.reshape(value, [ FEAT_DIM, ]) print "value 3:", value.get_shape().as_list() feature = value[:SP_DIM] # NCHW format print "feature shape:", feature.get_shape().as_list() if normalizer is not None: feature = normalizer.forward_process(feature) if format == 'NCHW': feature = tf.reshape(feature, [1, SP_DIM, 1]) elif format == 'NHWC': feature = tf.reshape(feature, [SP_DIM, 1, 1]) else: pass speaker = tf.cast(value[-1], tf.int64) print "shuffle batch" return tf.train.shuffle_batch( [feature, speaker], batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, num_threads=num_threads, # enqueue_many=True, )
def read_cifar100(filename_queue,coarse_or_fine='fine'): class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. #label_bytes = 2 # 2 for CIFAR-100 coarse_label_bytes=1 fine_label_bytes = 1 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = coarse_label_bytes + fine_label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. coarse_label = tf.cast(tf.strided_slice(record_bytes, [0], [coarse_label_bytes]), tf.int32) fine_label=tf.cast(tf.strided_slice(record_bytes,[coarse_label_bytes], [coarse_label_bytes+fine_label_bytes]),tf.int32) if coarse_or_fine=='fine': result.label=fine_label#100个精细分类标签 else: result.label=coarse_label #20个粗略分类 # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [coarse_label_bytes+fine_label_bytes], [coarse_label_bytes+fine_label_bytes + image_bytes]), [result.depth, result.height, result.width]) # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_raw_images(data_set): dirs = './data/' + data_set + '/' filename = list_binary_files(dirs) filename_queue = tf.train.string_input_producer(filename) if data_set is 'train': image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth record_bytes = image_bytes + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) depth_major = tf.reshape(tf.slice(record_bytes, [1], [image_bytes]), [ FLAGS.depth, FLAGS.height, FLAGS.width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) return label, uint8image elif data_set is 'test': image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth record_bytes = image_bytes + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) depth_major = tf.reshape(tf.slice(record_bytes, [0], [image_bytes]), [FLAGS.depth, FLAGS.height, FLAGS.width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) return uint8image elif data_set is 'validation': image_bytes = FLAGS.height * FLAGS.width * FLAGS.depth record_bytes = image_bytes + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) depth_major = tf.reshape(tf.slice(record_bytes, [1], [image_bytes]), [ FLAGS.depth, FLAGS.height, FLAGS.width]) uint8image = tf.transpose(depth_major, [1, 2, 0]) return label, uint8image
def __read_cifar(filenames, shuffle=True, cifar100=False): """Reads and parses examples from CIFAR data files. """ # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. filename_queue = tf.train.string_input_producer(filenames, shuffle=shuffle, num_epochs=None) label_bytes = 1 # 2 for CIFAR-100 if cifar100: label_bytes = 2 height = 32 width = 32 depth = 3 image_bytes = height * width * depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. if cifar100: #label = tf.cast( # tf.strided_slice(record_bytes, [0], [1]), tf.int32) #20 calss label = tf.cast(tf.strided_slice(record_bytes, [1], [label_bytes]), tf.int32) #100 class else: label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [depth, height, width]) # Convert from [depth, height, width] to [height, width, depth]. image = tf.transpose(depth_major, [1, 2, 0]) return tf.cast(image, tf.float32), label
def read_cifar10(filename_queue): # define a structure object which could contain different attributes class CIFAR10Record(object): pass # result is the passing parameter in this function result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. label_bytes = 1 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. record_bytes = label_bytes + image_bytes ''' define the file reading rules: reader''' # Read a record, getting file_names from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.FixedLengthRecordReader(record_bytes=record_bytes, name='data_reader') ''' start reading the file''' # read the file name as well as the value in the file into a list # format key: file_name, value: [length = record_bytes] result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long # doesn't change the shape of vector but change the data type # different from the function: tf.cast. record_bytes = tf.decode_raw(value, tf.uint8) # The first bytes represent the label, which we convert from uint8->int32. # tf.strided_slice: choose data of index from 0 to label_bytes in the record_bytes result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32, name='label_get') # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape(tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width], name='image_get') # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(is_train, batch_size, shuffle, data_dir='./cifar-10-batches-bin/'): """Read CIFAR10 Args: data_dir: the directory of CIFAR10 is_train: boolen batch_size: shuffle: Returns: label: 1D tensor, tf.int32 image: 4D tensor, [batch_size, height, width, 3], tf.float32 """ img_width = 32 img_height = 32 img_depth = 3 label_bytes = 1 image_bytes = img_width*img_height*img_depth # Create queues for execution of queue runners if is_train: filenames = [os.path.join(data_dir, 'data_batch_%d.bin' %ii) for ii in np.arange(1, 6)] else: filenames = [os.path.join(data_dir, 'test_batch.bin')] filename_queue = tf.train.string_input_producer(filenames) reader = tf.FixedLengthRecordReader(label_bytes + image_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.slice(record_bytes, [0], [label_bytes]) label = tf.cast(label, tf.int32) image_raw = tf.slice(record_bytes, [label_bytes], [image_bytes]) image_raw = tf.reshape(image_raw, [img_depth, img_height, img_width]) image = tf.transpose(image_raw, (1,2,0)) # C/H/W to H/W/C image = tf.cast(image, tf.float32) image = tf.image.per_image_standardization(image) # whitening the image if shuffle: images, label_batch = tf.train.shuffle_batch([image, label], batch_size=batch_size, num_threads=64, capacity=20000, min_after_dequeue=3000) else: images, label_batch = tf.train.batch([image, label], batch_size=batch_size, num_threads=64, capacity=2000) label_batch = tf.reshape(tf.cast(label_batch, dtype=tf.int32), [-1]) return images, label_batch
def read_record(image_file_queue, label_file_queue): class MuctRecord(object): pass result = MuctRecord() height = 64 width = 64 depth = 3 image_bytes = height * width * depth * 1 label_bytes = 2 image_reader = tf.FixedLengthRecordReader(record_bytes=image_bytes) _, value = image_reader.read(image_file_queue) result.image = tf.decode_raw(value, tf.uint8) result.image = tf.reshape(result.image, [height, width, depth]) result.image = tf.cast(result.image, tf.float32) label_reader = tf.FixedLengthRecordReader(record_bytes=label_bytes) _, value = label_reader.read(label_file_queue) result.label = tf.decode_raw(value, tf.int16) result.label = tf.reshape(result.label, [1]) return result
def read_data_set(self, filename_queue): record = Record() reader = tf.FixedLengthRecordReader(record_bytes=self.record_bytes) file_name, value = reader.read(filename_queue) byte_record = tf.decode_raw(value, tf.uint8) image_label = tf.cast(tf.slice(byte_record, [0], [LABEL_BYTES]), tf.int32) array_image = tf.strided_slice(byte_record, [LABEL_BYTES], [self.record_bytes]) depth_major_image = tf.reshape(array_image, [self.depth, self.height, self.width]) record.image = tf.transpose(depth_major_image, [1, 2, 0]) record.label = image_label[0] return record
def read_raw_images(sess, data_set): filename = ['./data/' + data_set + '_data.bin'] filename_queue = tf.train.string_input_producer(filename) record_bytes = (FLAGS.raw_height) * (FLAGS.raw_width) * FLAGS.depth + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) tf.train.start_queue_runners(sess=sess) for i in range(0, 100): result = sess.run(record_bytes) print i, result[0] image = result[1:len(result)]
def read_raw_images(data_set): filename = ['./data/' + data_set + '_data.bin'] filename_queue = tf.train.string_input_producer(filename) size = (FLAGS.img_h) * (FLAGS.img_w) * FLAGS.channel record_bytes = size + 1 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.cast(tf.slice(record_bytes, [0], [1]), tf.int32) image = tf.reshape(tf.slice(record_bytes, [1], [size]), [FLAGS.channel, FLAGS.img_h, FLAGS.img_w]) image_ = tf.transpose(image, [1, 2, 0]) return image_, label
def read_cifar10(data_dir, is_train, batch_size, shuffle): img_width = 32 img_height = 32 img_depth = 3 label_bytes = 1 image_bytes = img_width * img_height * img_depth with tf.name_scope('input'): if is_train: filenames = [ os.path.join(data_dir, 'data_batch_%d.bin' % i) for i in np.arange(1, 6) ] else: filenames = os.path.join(data_dir, 'test_batch.bin') filename_queue = tf.train.string_input_producer(filenames) reader = tf.FixedLengthRecordReader(label_bytes + image_bytes) key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) label = tf.slice(record_bytes, [0], [label_bytes]) label = tf.cast(label, tf.int32) image_raw = tf.slice(record_bytes, [label_bytes], [image_bytes]) image_raw = tf.reshape(image_raw, [img_depth, img_height, img_depth]) image = tf.transpose(image_raw, (1, 2, 0)) image = tf.cast(image, tf.float32) image = tf.image.per_image_standardization(image) if shuffle: image_batch, label_batch = tf.train.shuffle_batch( [image, label], batch_size=batch_size, num_threads=16, capacity=2000, min_after_dequeue=1500) else: image_batch, label_batch = tf.train.batch([image, label], batch_size=batch_size, num_threads=16, capacity=2000) n_classes = 10 label_batch = tf.one_hot(label_batch, depth=n_classes) return image_batch, tf.reshape(label_batch, [batch_size, n_classes])
def read_cifar10(filename_queue): class CIFAR10Record(object): pass result = CIFAR10Record() # Dimensions of the images in the CIFAR-10 dataset. # See http://www.cs.toronto.edu/~kriz/cifar.html for a description of the # input format. # 分类结果的长度,CIFAR-100长度为2 label_bytes = 1 # 2 for CIFAR-100 result.height = 32 result.width = 32 # 3位表示rgb颜色(0-255,0-255,0-255) result.depth = 3 image_bytes = result.height * result.width * result.depth # Every record consists of a label followed by the image, with a # fixed number of bytes for each. # 单个记录的总长度=分类结果长度+图片长度 record_bytes = label_bytes + image_bytes # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. # 读取 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_bytes = tf.decode_raw(value, tf.uint8) # 第一位代表lable-图片的正确分类结果,从uint8转换为int32类型 # The first bytes represent the label, which we convert from uint8->int32. result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # 分类结果之后的数据代表图片,我们重新调整大小 # The remaining bytes after the label represent the image, which we reshape # from [depth * height * width] to [depth, height, width]. depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # 格式转换,从[颜色,高度,宽度]--》[高度,宽度,颜色] # Convert from [depth, height, width] to [height, width, depth]. result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10(filename_queue): """从CIFAR10数据文件中阅读并解析example。 推荐:如果你想要N路并行阅读,那么调用这个函数N次。这样会返回N个独立的Reader用来阅读那些文件里不同的文件和位置,这样会返回更好的混合example。 参数: filename_queue: 文件名字符串队列。 返回: 一个object代表一个example,包括以下内容: 高: result的行数(32) 宽: result的列数(32) 深: result的色彩通道数(3) key: 一个标量字符串描述这个example的文件名和record number。 标签: 一个带有标签(0..9)的int32 Tensor uint8image: 一个带有图像数据的[height, width, depth] uint8 Tensor """ class CIFAR10Record(object): pass result = CIFAR10Record() # CIFAR-10数据集中图像的维度。 label_bytes = 1 result.height = 32 result.width = 32 result.depth = 3 image_bytes = result.height * result.width * result.depth # 每一个record包含一个标签和一个固定长度的用来描述图像的bytes。 record_bytes = label_bytes + image_bytes # 阅读一个record,从filename_queue中获得filename。CIFAR-10格式没有header何footer,所以我们默认header——bytes和footer_bytes为0。 reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) # 将一个字符串转换成一个uint8向量 record_bytes = tf.decode_raw(value, tf.uint8) # 第一个bytes代表标签,我们将它转换成int32。 result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) # 剩下的bytes代表图像,我们将它reshape成[depth, height, width]。 depth_major = tf.reshape( tf.strided_slice(record_bytes, [label_bytes], [label_bytes + image_bytes]), [result.depth, result.height, result.width]) # 转换成[height, width, depth] result.uint8image = tf.transpose(depth_major, [1, 2, 0]) return result
def read_cifar10_label(filename_queue): class CIFAR10Record(object): pass result = CIFAR10Record() label_bytes = 1 # 2 for CIFAR-100 record_bytes = label_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue) record_bytes = tf.decode_raw(value, tf.uint8) result.label = tf.cast(tf.strided_slice(record_bytes, [0], [label_bytes]), tf.int32) return result
def read_animals(filename_queue): class ANIMAL10Record(object): pass result = IMAL10Record() label_bytes = 1 # 2 for CIFAR-100 result.height = 140 result.width = 160 result.depth = 3 image_bytes = result.height * result.width * result.depth record_bytes = label_bytes + image_bytes reader = tf.FixedLengthRecordReader(record_bytes=record_bytes) result.key, value = reader.read(filename_queue)