def batch_inputs(feature_map, data_files, height=2048, width=2448, batch_size=1, is_train=True, num_readers=1, num_preprocess_threads=4): # feature_map: 对应proto的数据映射。 # data_files: list类型,存放的是tfrecord的文件列表。 # batch_size: 一个批次batch的大小。 # is_train: DataProvider在train和test节点的表现形式有所不同,主要test时并不需要一个循环队列。 # num_reader: 每一个线程reader的个数。 # num_preprocess_threads: 处理数据的线程的个数。 with tf.name_scope('reader_defination'): # 创建文件队列,如果是训练,创建一个随机文件队列,如果是测试,创建一个顺序文件队列。 if is_train: filename_queue = tf.train.string_input_producer(data_files, shuffle=True, capacity=16) else: filename_queue = tf.train.string_input_producer(data_files, shuffle=False, capacity=1) # reader的个数至少为1。 num_readers = 1 if num_readers < 1 else num_readers if num_readers > 1: # 定义缓冲池的大小。 examples_per_shard = 1024 min_queue_examples = examples_per_shard * 16 if is_train: examples_queue = tf.RandomShuffleQueue(capacity=min_queue_examples + 3 * batch_size, min_after_dequeue=min_queue_examples, dtypes=[tf.string]) else: examples_queue = tf.FIFOQueue(capacity=examples_per_shard + 3 * batch_size, dtypes=[tf.string]) # 多个reader时对reader队列进行管理。 enqueue_ops = [] for _ in range(num_readers): reader = tf.TFRecordReader() _, value = reader.read(filename_queue) enqueue_ops.append(examples_queue.enqueue([value])) tf.train.queue_runner.add_queue_runner(tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops)) example_serialized = examples_queue.dequeue() else: reader = tf.TFRecordReader() _, example_serialized = reader.read(filename_queue) samples = [] for _ in range(num_preprocess_threads): features = tf.parse_single_example(example_serialized, feature_map) samples.append([image_processing(features['image/encoded'], height, width), features['image/format']]) batch_data = tf.train.batch_join(samples, batch_size=batch_size, capacity=2 * num_preprocess_threads * batch_size) data = tf.reshape(batch_data[0], [batch_size, -1]) label = tf.reshape(batch_data[1], [batch_size]) return (data, label)
def read_and_decode(filename_queue, label_type, shape): reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'image_raw': tf.FixedLenFeature([], tf.string), 'label_raw': tf.FixedLenFeature([], tf.string), }) image = tf.decode_raw(features['image_raw'], tf.uint8) image = tf.cast(image, tf.float32) image = (image - 127.5) * (1. / 128.0) image.set_shape([shape * shape * 3]) image = tf.reshape(image, [shape, shape, 3]) label = tf.decode_raw(features['label_raw'], tf.float32) if label_type == 'cls': image = tf.image.random_flip_left_right(image) image = tf.image.random_flip_up_down(image) label.set_shape([2]) elif label_type == 'bbx': label.set_shape([4]) elif label_type == 'pts': label.set_shape([10]) return image, label
def get_example(self, batch_size): """Get a single example from the tfrecord file. Args: batch_size: Int, minibatch size. Returns: tf.Example protobuf parsed from tfrecord. """ reader = tf.TFRecordReader() num_epochs = None if self.is_training else 1 capacity = batch_size path_queue = tf.train.input_producer( [self.record_path], num_epochs=num_epochs, shuffle=self.is_training, capacity=capacity) unused_key, serialized_example = reader.read(path_queue) features = { "note_str": tf.FixedLenFeature([], dtype=tf.string), "pitch": tf.FixedLenFeature([1], dtype=tf.int64), "velocity": tf.FixedLenFeature([1], dtype=tf.int64), "audio": tf.FixedLenFeature([64000], dtype=tf.float32), "qualities": tf.FixedLenFeature([10], dtype=tf.int64), "instrument_source": tf.FixedLenFeature([1], dtype=tf.int64), "instrument_family": tf.FixedLenFeature([1], dtype=tf.int64), } example = tf.parse_single_example(serialized_example, features) return example
def feed(self): """ Returns: images: 4D tensor [batch_size, image_width, image_height, image_depth] """ with tf.name_scope(self.name): filename_queue = tf.train.string_input_producer([self.tfrecords_file]) reader = tf.TFRecordReader() _, serialized_example = self.reader.read(filename_queue) features = tf.parse_single_example( serialized_example, features={ 'image/file_name': tf.FixedLenFeature([], tf.string), 'image/encoded_image': tf.FixedLenFeature([], tf.string), }) image_buffer = features['image/encoded_image'] image = tf.image.decode_jpeg(image_buffer, channels=3) image = self._preprocess(image) images = tf.train.shuffle_batch( [image], batch_size=self.batch_size, num_threads=self.num_threads, capacity=self.min_queue_examples + 3*self.batch_size, min_after_dequeue=self.min_queue_examples ) tf.summary.image('_input', images) return images
def input_queue(self) -> tf.Tensor: """ Creates the input queue. Returns ------- tf.Tensor A tensor containing batched instances read from the TFRecords file passed to this class """ filename_tensor = tf.constant( value=[str(file) for file in self._record_files], dtype=tf.string, name="filenames") filename_queue = tf.train.string_input_producer(filename_tensor, num_epochs=None) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) data = to_tensor(serialized_example, self._feature_shape) data_batches = tf.train.shuffle_batch([data], enqueue_many=False, batch_size=self._batch_size, num_threads=2, capacity=100 * self._batch_size, min_after_dequeue=50 * self._batch_size, allow_smaller_final_batch=True) # batches are batch major; also we assume that features are stored as [time, frequency] data_batches = tf.transpose(data_batches, perm=[1, 0, 2]) # [time, batch, frequency] return data_batches
def read_and_decode2stand(tfrecords_file, batch_size): # make an input queue from the tfrecord file filename_queue = tf.train.string_input_producer([tfrecords_file]) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) img_features = tf.parse_single_example( serialized_example, features={ 'label': tf.FixedLenFeature([], tf.int64), 'image_raw': tf.FixedLenFeature([], tf.string), }) image = tf.decode_raw(img_features['image_raw'], tf.uint8) image = tf.reshape(image, [H, W, Channels]) image = tf.cast(image, tf.float32) * (1.0 / 255) image = tf.image.per_image_standardization(image) # standardization # all the images of notMNIST are 200*150, you need to change the image size if you use other dataset. label = tf.cast(img_features['label'], tf.int32) image_batch, label_batch = tf.train.batch([image, label], batch_size=batch_size, num_threads=64, capacity=2000) # Change to ONE-HOT label_batch = tf.one_hot(label_batch, depth=n_classes) label_batch = tf.cast(label_batch, dtype=tf.int32) label_batch = tf.reshape(label_batch, [batch_size, n_classes]) print(label_batch) return image_batch, label_batch
def build_input(tfrecord_paths): """Builds the graph's input. Args: tfrecord_paths: List of paths to the input TFRecords Returns: serialized_example_tensor: The next serialized example. String scalar Tensor image_tensor: The decoded image of the example. Uint8 tensor, shape=[1, None, None,3] """ filename_queue = tf.train.string_input_producer(tfrecord_paths, shuffle=False, num_epochs=1) tf_record_reader = tf.TFRecordReader() _, serialized_example_tensor = tf_record_reader.read(filename_queue) features = tf.parse_single_example( serialized_example_tensor, features={ standard_fields.TfExampleFields.image_encoded: tf.FixedLenFeature([], tf.string), }) encoded_image = features[standard_fields.TfExampleFields.image_encoded] image_tensor = tf.image.decode_image(encoded_image, channels=3) image_tensor.set_shape([None, None, 3]) image_tensor = tf.expand_dims(image_tensor, 0) return serialized_example_tensor, image_tensor
def read_record(data_path, batch_size=1, size=512): feature = {'image': tf.FixedLenFeature(shape=(), dtype=tf.string), 'wall': tf.FixedLenFeature(shape=(), dtype=tf.string), 'close': tf.FixedLenFeature(shape=(), dtype=tf.string), 'room': tf.FixedLenFeature(shape=(), dtype=tf.string), 'close_wall': tf.FixedLenFeature(shape=(), dtype=tf.string)} # Create a list of filenames and pass it to a queue filename_queue = tf.train.string_input_producer([data_path], num_epochs=None, shuffle=False, capacity=batch_size*128) # Define a reader and read the next record reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) # Decode the record read by the reader features = tf.parse_single_example(serialized_example, features=feature) # Convert the image data from string back to the numbers image = tf.decode_raw(features['image'], tf.uint8) wall = tf.decode_raw(features['wall'], tf.uint8) close = tf.decode_raw(features['close'], tf.uint8) room = tf.decode_raw(features['room'], tf.uint8) close_wall = tf.decode_raw(features['close_wall'], tf.uint8) # Cast data image = tf.cast(image, dtype=tf.float32) wall = tf.cast(wall, dtype=tf.float32) close = tf.cast(close, dtype=tf.float32) # room = tf.cast(room, dtype=tf.float32) close_wall = tf.cast(close_wall, dtype=tf.float32) # Reshape image data into the original shape image = tf.reshape(image, [size, size, 3]) wall = tf.reshape(wall, [size, size, 1]) close = tf.reshape(close, [size, size, 1]) room = tf.reshape(room, [size, size]) close_wall = tf.reshape(close_wall, [size, size, 1]) # Any preprocessing here ... # normalize image = tf.divide(image, tf.constant(255.0)) wall = tf.divide(wall, tf.constant(255.0)) close = tf.divide(close, tf.constant(255.0)) close_wall = tf.divide(close_wall, tf.constant(255.0)) # Genereate one hot room label room_one_hot = tf.one_hot(room, 9, axis=-1) # Creates batches by randomly shuffling tensors images, walls, closes, rooms, close_walls = tf.train.shuffle_batch([image, wall, close, room_one_hot, close_wall], batch_size=batch_size, capacity=batch_size*128, num_threads=1, min_after_dequeue=batch_size*32) # images, walls = tf.train.shuffle_batch([image, wall], # batch_size=batch_size, capacity=batch_size*128, num_threads=1, min_after_dequeue=batch_size*32) return {'images': images, 'walls': walls, 'closes': closes, 'rooms': rooms, 'close_walls': close_walls}
def reader(self): """Return a reader for a single entry from the data set. See io_ops.py for details of Reader class. Returns: Reader object that reads the data set. """ return tf.TFRecordReader()
def train(self): #输入向量 inputs = tf.placeholder('float',[None,224,224,3]) #输出向量 outputs = tf.placeholder('float',[None,224,224,3]) #encoded是图片经过编码器的结果,decoded是图片又经过解码器的结果,decoded_encoded生成新图片经过编码器的结果 encoded,decoded,decoded_encoded = self.encoder_decoder(inputs) #内容损失 重建图像和原图(outputs,decoded) pixel_loss = tf.losses.mean_squared_error(decoded,outputs) #风格损失 原图经过编码器(encoded)和重建图像经过解码器(decoded_encoded) feature_loss = tf.losses.mean_squared_error(decoded_encoded,encoded) # loss = pixel_loss+ feature_loss loss=0.5*pixel_loss + 0.1*feature_loss opt= tf.train.AdamOptimizer(0.0001).minimize(loss) #训练集的位置 tfrecords_filename = self.tfrecord_path filename_queue = tf.train.string_input_producer([tfrecords_filename],num_epochs=100) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) feature2 = { 'image_raw': tf.FixedLenFeature([], tf.string)} features = tf.parse_single_example(serialized_example, features=feature2) image = tf.decode_raw(features['image_raw'], tf.uint8) image = tf.reshape(image,[224,224,3]) images = tf.train.shuffle_batch([image], batch_size=self.batch_size, capacity=30, min_after_dequeue=10) config = tf.ConfigProto() config.gpu_options.allow_growth = True with tf.Session(config = config)as sess : tf.global_variables_initializer().run() tf.local_variables_initializer().run() coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) saver = tf.train.Saver() for i in range (self.max_iterator): batch_x=sess.run(images) feed_dict = {inputs:batch_x, outputs : batch_x} _,p_loss,f_loss,reconstruct_imgs=sess.run([opt,pixel_loss,feature_loss,decoded],feed_dict=feed_dict) print('step %d | pixel_loss is %f | feature_loss is %f |'%(i,p_loss,f_loss)) if i % 5 ==0: result_img = np.clip(reconstruct_imgs[0],0,255).astype(np.uint8) imsave('result.jpg',result_img) saver.save(sess,self.checkpoint_path) coord.request_stop() coord.join(threads)
def _read_and_decode(filename_queue, image_pixel=96, distort=0): """Read a norb tf record file.""" reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example( serialized_example, # Defaults are not specified since both keys are required. features={ 'image_raw': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), 'height': tf.FixedLenFeature([], tf.int64), 'width': tf.FixedLenFeature([], tf.int64), 'depth': tf.FixedLenFeature([], tf.int64), 'meta': tf.FixedLenFeature([4], tf.int64), }) # Convert from a scalar string tensor (whose single string has # length image_pixels) to a uint8 tensor with shape # [image_pixels]. image = tf.decode_raw(features['image_raw'], tf.uint8) height = tf.cast(features['height'], tf.int32) depth = tf.cast(features['depth'], tf.int32) image = tf.reshape(image, tf.stack([depth, height, height])) image = tf.transpose(image, [1, 2, 0]) image = tf.cast(image, tf.float32) if image_pixel < 96: print('image resizing to {}'.format(image_pixel)) image = tf.image.resize_images(image, [image_pixel, image_pixel]) orig_images = image if image_pixel == 48: new_dim = 32 elif image_pixel == 32: new_dim = 22 if distort == 1: image = tf.image.random_brightness(image, max_delta=63) image = tf.image.random_contrast(image, lower=0.2, upper=1.8) image = tf.random_crop(image, tf.stack([new_dim, new_dim, depth])) # 0.26179938779 is 15 degress in radians image = tf.image.per_image_standardization(image) image_pixel = new_dim elif distort == 2: image = tf.image.resize_image_with_crop_or_pad(image, new_dim, new_dim) image = tf.image.per_image_standardization(image) image_pixel = new_dim else: image = image * (1.0 / 255.0) image = tf.div( tf.subtract(image, tf.reduce_min(image)), tf.subtract(tf.reduce_max(image), tf.reduce_min(image))) # Convert label from a scalar uint8 tensor to an int32 scalar. label = tf.cast(features['label'], tf.int32) return image, label, image_pixel, orig_images
def read_and_decode(filename_queue): reader = tf.TFRecordReader() key, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={'vol_raw': tf.FixedLenFeature([], tf.string)}) vol_str = tf.decode_raw(features['vol_raw'], tf.float32) volume = tf.reshape(vol_str, volume_shape) volume_batch = tf.train.shuffle_batch([volume], batch_size=batch_size, capacity=capacity, num_threads=num_threads, min_after_dequeue=min_after_dequeue) finalbatch = tf.expand_dims(volume_batch, -1) return finalbatch
def read_tfrecords(filename): filename_quene = tf.train.string_input_producer([filename]) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_quene) features = tf.parse_single_example( serialized_example, features={'data': tf.FixedLenFeature([32768], tf.int64)}) data = tf.reshape(features['data'], (32, 32, 32, 1)) train_data = tf.cast(data, tf.float32) label_data = tf.cast(data, tf.float32) return train_data, label_data
def read_bd_rm_record(data_path, batch_size=1, size=512): feature = {'image': tf.FixedLenFeature(shape=(), dtype=tf.string), 'boundary': tf.FixedLenFeature(shape=(), dtype=tf.string), 'room': tf.FixedLenFeature(shape=(), dtype=tf.string), 'door': tf.FixedLenFeature(shape=(), dtype=tf.string)} print("data_path=",data_path) # Create a list of filenames and pass it to a queue filename_queue = tf.train.string_input_producer([data_path], num_epochs=None, shuffle=False, capacity=batch_size*128) # filename_queue = tf.data.Dataset.from_tensor_slices([data_path]).shuffle(tf.shape([data_path], out_type=tf.int64)[0]).repeat(None) # Define a reader and read the next record reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) # Decode the record read by the reader features = tf.parse_single_example(serialized_example, features=feature) # Convert the image data from string back to the numbers image = tf.decode_raw(features['image'], tf.uint8) boundary = tf.decode_raw(features['boundary'], tf.uint8) room = tf.decode_raw(features['room'], tf.uint8) door = tf.decode_raw(features['door'], tf.uint8) # Cast data image = tf.cast(image, dtype=tf.float32) # Reshape image data into the original shape image = tf.reshape(image, [size, size, 3]) boundary = tf.reshape(boundary, [size, size]) room = tf.reshape(room, [size, size]) door = tf.reshape(door, [size, size]) # Any preprocessing here ... # normalize image = tf.divide(image, tf.constant(255.0)) # Genereate one hot room label label_boundary = tf.one_hot(boundary, 3, axis=-1) label_room = tf.one_hot(room, 9, axis=-1) # Creates batches by randomly shuffling tensors images, label_boundaries, label_rooms, label_doors = tf.train.shuffle_batch([image, label_boundary, label_room, door], batch_size=batch_size, capacity=batch_size*128, num_threads=1, min_after_dequeue=batch_size*32) # images, walls = tf.train.shuffle_batch([image, wall], # batch_size=batch_size, capacity=batch_size*128, num_threads=1, min_after_dequeue=batch_size*32) return {'images': images, 'label_boundaries': label_boundaries, 'label_rooms': label_rooms, 'label_doors': label_doors}
def _read_and_decode(filename_queue, image_pixel=28, distort=0): """Read tf records of MNIST images and labels.""" reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example( serialized_example, # Defaults are not specified since both keys are required. features={ 'image_raw': tf.FixedLenFeature([], tf.string), 'label': tf.FixedLenFeature([], tf.int64), 'height': tf.FixedLenFeature([], tf.int64), 'width': tf.FixedLenFeature([], tf.int64), 'depth': tf.FixedLenFeature([], tf.int64) }) # Convert from a scalar string tensor (whose single string has # length image_pixels) to a uint8 tensor with shape # [image_pixels]. image = tf.decode_raw(features['image_raw'], tf.uint8) image = tf.reshape(image, [image_pixel, image_pixel, 1]) print(image.get_shape()[0].value) image.set_shape([image_pixel, image_pixel, 1]) # OPTIONAL: Could reshape into a 28x28 image and apply distortions # here. Since we are not applying any distortions in this # example, and the next step expects the image to be flattened # into a vector, we don't bother. # Convert from [0, 255] -> [-0.5, 0.5] floats. image = tf.cast(image, tf.float32) * (1. / 255) if distort == 1: image = tf.reshape(image, [28, 28]) image = tf.random_crop(image, [24, 24]) # 0.26179938779 is 15 degress in radians # image = contrib_image.rotate(image, # random.uniform(-0.26179938779, 0.26179938779)) image = tf.reshape(image, [24, 24, 1]) elif distort == 2: image = tf.reshape(image, [28, 28]) image = tf.expand_dims(image, 2) image = tf.image.central_crop(image, central_fraction=24 / 28) image = tf.squeeze(image, 2) image = tf.reshape(image, [24, 24, 1]) # Convert label from a scalar uint8 tensor to an int32 scalar. label = tf.cast(features['label'], tf.int32) return image, label
def __init__(self, tfrecords_file, image_size=256, min_queue_examples=1000, batch_size=1, num_threads=8, name=''): """ Args: tfrecords_file: string, tfrecords file path min_queue_examples: integer, minimum number of samples to retain in the queue that provides of batches of examples batch_size: integer, number of images per batch num_threads: integer, number of preprocess threads """ self.tfrecords_file = tfrecords_file self.image_size = image_size self.min_queue_examples = min_queue_examples self.batch_size = batch_size self.num_threads = num_threads self.reader = tf.TFRecordReader() self.name = name
def read_tfRecord(tfRecord_path): filename_queue = tf.train.string_input_producer([tfRecord_path], shuffle=True) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'label': tf.FixedLenFeature([10], tf.int64), 'img_raw': tf.FixedLenFeature([], tf.string) }) img = tf.decode_raw(features['img_raw'], tf.uint8) img = tf.reshape(img, [32, 32, 3]) img = tf.cast(img, tf.float32) * (1.0 / 255) label = tf.cast(features['label'], tf.float32) return img, label
def read_and_decode(filename): filename_queue = tf.train.string_input_producer([filename]) reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'label': tf.FixedLenFeature([], tf.string), 'img_raw': tf.FixedLenFeature([], tf.string) }) image = tf.decode_raw(features['img_raw'], tf.uint8) image = tf.reshape(image, [32, 20, 1]) image = tf.cast(image, tf.float32) * (1. / 255) - 0.5 l = tf.decode_raw(features['label'], tf.float64) label = tf.reshape(tf.cast(l, tf.float32), [2]) return image, label
def read_and_decode(filename_queue): reader = tf.TFRecordReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example(serialized_example, features={ 'image_raw': tf.FixedLenFeature([], tf.string), 'label_raw': tf.FixedLenFeature([], tf.string), }) image = tf.decode_raw(features['image_raw'], tf.int16) image.set_shape([IMAGE_HEIGHT * IMAGE_WIDTH]) image = tf.cast(image, tf.float32) * (1. / 255) - 0.5 reshape_image = tf.reshape(image, [IMAGE_HEIGHT, IMAGE_WIDTH, 1]) label = tf.decode_raw(features['label_raw'], tf.uint8) label.set_shape([CHARS_NUM * CLASSES_NUM]) reshape_label = tf.reshape(label, [CHARS_NUM, CLASSES_NUM]) return tf.cast(reshape_image, tf.float32), tf.cast(reshape_label, tf.float32)
def read_smpl_data(filename_queue): """ Parses a smpl Example proto. It's contents are: 'pose' : 72-D float 'shape' : 10-D float """ with tf.name_scope(None, 'read_smpl_data', [filename_queue]): reader = tf.TFRecordReader() _, example_serialized = reader.read(filename_queue) feature_map = { 'pose': tf.FixedLenFeature((72, ), dtype=tf.float32), 'shape': tf.FixedLenFeature((10, ), dtype=tf.float32) } features = tf.parse_single_example(example_serialized, feature_map) pose = tf.cast(features['pose'], dtype=tf.float32) shape = tf.cast(features['shape'], dtype=tf.float32) return pose, shape
def load_patch_coordinates_from_filename_queue(filename_queue): """Loads coordinates and volume names from filename queue. Args: filename_queue: Tensorflow queue created from create_filename_queue() Returns: Tuple of coordinates (shape `[1, 3]`) and volume name (shape `[1]`) tensors. """ record_options = tf.python_io.TFRecordOptions( tf.python_io.TFRecordCompressionType.GZIP) keys, protos = tf.TFRecordReader( options=record_options).read(filename_queue) examples = tf.parse_single_example( protos, features=dict( center=tf.FixedLenFeature(shape=[1, 3], dtype=tf.int64), label_volume_name=tf.FixedLenFeature(shape=[1], dtype=tf.string), )) coord = examples['center'] volname = examples['label_volume_name'] return coord, volname
def _build(self): # Find split file from which we are going to read. split_path = os.path.join(self._dataset_dir, '{}.tfrecords'.format(self._split)) if not tf.gfile.Exists(split_path): raise InvalidDataDirectory( '"{}" does not exist.'.format(split_path)) # String input producer allows for a variable number of files to read # from. We just know we have a single file. filename_queue = tf.train.string_input_producer( [split_path], num_epochs=self._num_epochs, seed=self._seed) # Define reader to parse records. reader = tf.TFRecordReader() _, raw_record = reader.read(filename_queue) values, dtypes, names = self.read_record(raw_record) if self._random_shuffle: queue = tf.RandomShuffleQueue(capacity=100, min_after_dequeue=0, dtypes=dtypes, names=names, name='tfrecord_random_queue', seed=self._seed) else: queue = tf.FIFOQueue(capacity=100, dtypes=dtypes, names=names, name='tfrecord_fifo_queue') # Generate queueing ops for QueueRunner. enqueue_ops = [queue.enqueue(values)] * self._total_queue_ops self.queue_runner = tf.train.QueueRunner(queue, enqueue_ops) tf.train.add_queue_runner(self.queue_runner) return queue.dequeue()
def get_padded_batch(file_list, batch_size, input_size, label_shape=None, num_enqueuing_threads=4, shuffle=False): """Reads batches of SequenceExamples from TFRecords and pads them. Can deal with variable length SequenceExamples by padding each batch to the length of the longest sequence with zeros. Args: file_list: A list of paths to TFRecord files containing SequenceExamples. batch_size: The number of SequenceExamples to include in each batch. input_size: The size of each input vector. The returned batch of inputs will have a shape [batch_size, num_steps, input_size]. label_shape: Shape for labels. If not specified, will use []. num_enqueuing_threads: The number of threads to use for enqueuing SequenceExamples. shuffle: Whether to shuffle the batches. Returns: inputs: A tensor of shape [batch_size, num_steps, input_size] of floats32s. labels: A tensor of shape [batch_size, num_steps] of int64s. lengths: A tensor of shape [batch_size] of int32s. The lengths of each SequenceExample before padding. Raises: ValueError: If `shuffle` is True and `num_enqueuing_threads` is less than 2. """ file_queue = tf.train.string_input_producer(file_list) reader = tf.TFRecordReader() _, serialized_example = reader.read(file_queue) sequence_features = { 'inputs': tf.FixedLenSequenceFeature(shape=[input_size], dtype=tf.float32), 'labels': tf.FixedLenSequenceFeature(shape=label_shape or [], dtype=tf.int64)} _, sequence = tf.parse_single_sequence_example( serialized_example, sequence_features=sequence_features) length = tf.shape(sequence['inputs'])[0] input_tensors = [sequence['inputs'], sequence['labels'], length] if shuffle: if num_enqueuing_threads < 2: raise ValueError( '`num_enqueuing_threads` must be at least 2 when shuffling.') shuffle_threads = int(math.ceil(num_enqueuing_threads) / 2.) # Since there may be fewer records than SHUFFLE_MIN_AFTER_DEQUEUE, take the # minimum of that number and the number of records. min_after_dequeue = count_records( file_list, stop_at=SHUFFLE_MIN_AFTER_DEQUEUE) input_tensors = _shuffle_inputs( input_tensors, capacity=QUEUE_CAPACITY, min_after_dequeue=min_after_dequeue, num_threads=shuffle_threads) num_enqueuing_threads -= shuffle_threads tf.logging.info(input_tensors) return tf.train.batch( input_tensors, batch_size=batch_size, capacity=QUEUE_CAPACITY, num_threads=num_enqueuing_threads, dynamic_pad=True, allow_smaller_final_batch=False)
def load_preaggregated_data(self): # Return objects of this function X = None Y = None X_valid = None Y_valid = None # Load pre-aggregated training dataset tfrecord_file_list = os.listdir(self.preaggregated_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X = all_data Y = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this # Load pre-aggregated validation dataset tfrecord_file_list = os.listdir( self.preaggregated_validation_data_path) tfrecord_file_list = [ os.path.join(self.preaggregated_validation_data_path, k) for k in tfrecord_file_list ] print('Pre-aggregated file list = ' + str(tfrecord_file_list)) reader = tf.TFRecordReader() key, examples = reader.read( tf.train.string_input_producer( tfrecord_file_list, num_epochs=1)) # Only generate all data once name_to_features = { "input_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "input_mask": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), "segment_ids": tf.io.FixedLenFeature([self.max_seq_length], tf.int64), } parsed_example = tf.parse_single_example(examples, name_to_features) parsed_example_values = list(parsed_example.values()) # Reuse Keras Session sess = K.get_session() # Just read all data into array for now. # TODO: Implment generator to support very large dataset that is not fit into RAM all_data = [] sess.run(tf.initialize_local_variables()) tf.train.start_queue_runners(sess=sess) try: while True: data = sess.run(parsed_example_values) for i in range(len(data)): if len(all_data) <= i: all_data.append([]) all_data[i].append(data[i]) except tf.errors.OutOfRangeError: pass all_data = [np.array(a) for a in all_data] X_valid = all_data Y_valid = all_data[0] # Y is only 'input_ids' tensor K.clear_session() # sess object is not valid anymore after this #print(len(X_valid)) #print(len(Y_valid)) return (X, Y, X_valid, Y_valid)
import tensorflow.compat.v1 as tf import numpy as np # 读取tfrecord文件列表,这里只有一个tfrecord records_queue = tf.train.string_input_producer(['data.tfrecord'], num_epochs=2) # 创建一个TFRecord中的数据,每次只解析一个 reader = tf.TFRecordReader() _, serialized_example = reader.read(records_queue) features = tf.parse_single_example(serialized_example, features={ 'array_raw': tf.FixedLenFeature([], tf.string), 'height': tf.FixedLenFeature([], tf.int64), 'width': tf.FixedLenFeature([], tf.int64), 'depth': tf.FixedLenFeature([], tf.int64) }) # 解析出对应的值 array_raw = features['array_raw'] array = tf.decode_raw(array_raw, tf.float32) height = features['height'] width = features['width'] depth = features['depth'] # 创建会话 session = tf.Session() session.run(tf.global_variables_initializer()) session.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=session, coord=coord) # 循环5次解析文件流中的数据
def arbitrary_style_image_inputs(style_dataset_file, batch_size=None, image_size=None, center_crop=True, shuffle=True, augment_style_images=False, random_style_image_size=False, min_rand_image_size=128, max_rand_image_size=300): """Loads a batch of random style image given the path of tfrecord dataset. This method does not return pre-compute Gram matrices for the images like style_image_inputs. But it can provide data augmentation. If augment_style_images is equal to True, then style images will randomly modified (eg. changes in brightness, hue or saturation) for data augmentation. If random_style_image_size is set to True then all images in one batch will be resized to a random size. Args: style_dataset_file: str, path to the tfrecord dataset of style files. batch_size: int. If provided, batches style images. Defaults to None. image_size: int. The images will be resized bilinearly so that the smallest side has size image_size. Defaults to None. center_crop: bool. If True, center-crops to [image_size, image_size]. Defaults to False. shuffle: bool, whether to shuffle style files at random. Defaults to False. augment_style_images: bool. Wheather to augment style images or not. random_style_image_size: bool. If this value is True, then all the style images in one batch will be resized to a random size between min_rand_image_size and max_rand_image_size. min_rand_image_size: int. If random_style_image_size is True, this value specifies the minimum image size. max_rand_image_size: int. If random_style_image_size is True, this value specifies the maximum image size. Returns: 4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style image (with random changes for data augmentation if augment_style_image_size is set to true), and 0-D tensor for the style label, 4-D tensor of shape [1, ?, ?, 3] with values in [0, 1] for the style image without random changes for data augmentation. Raises: ValueError: if center cropping is requested but no image size is provided, or if batch size is specified but center-cropping or augment-style-images is not requested, or if both augment-style-images and center-cropping are requested. """ if center_crop and image_size is None: raise ValueError('center-cropping requires specifying the image size.') if center_crop and augment_style_images: raise ValueError( 'When augment_style_images is true images will be randomly cropped.' ) if batch_size is not None and not center_crop and not augment_style_images: raise ValueError( 'batching requires same image sizes (Set center-cropping or ' 'augment_style_images to true)') with tf.name_scope('style_image_processing'): # Force all input processing onto CPU in order to reserve the GPU for the # forward inference and back-propagation. with tf.device('/cpu:0'): filename_queue = tf.train.string_input_producer( [style_dataset_file], shuffle=False, capacity=1, name='filename_queue') if shuffle: examples_queue = tf.RandomShuffleQueue( capacity=64, min_after_dequeue=32, dtypes=[tf.string], name='random_examples_queue') else: examples_queue = tf.FIFOQueue(capacity=64, dtypes=[tf.string], name='fifo_examples_queue') reader = tf.TFRecordReader() _, value = reader.read(filename_queue) enqueue_ops = [examples_queue.enqueue([value])] tf.train.queue_runner.add_queue_runner( tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops)) example_serialized = examples_queue.dequeue() features = tf.parse_single_example( example_serialized, features={ 'label': tf.FixedLenFeature([], tf.int64), 'image_raw': tf.FixedLenFeature([], tf.string) }) image = tf.image.decode_jpeg(features['image_raw']) image.set_shape([None, None, 3]) label = features['label'] if image_size is not None: image_channels = int(image.shape[2]) if augment_style_images: image_orig = image image = tf.image.random_brightness(image, max_delta=0.8) image = tf.image.random_saturation(image, lower=0.5, upper=1.5) image = tf.image.random_hue(image, max_delta=0.2) image = tf.image.random_flip_left_right(image) image = tf.image.random_flip_up_down(image) random_larger_image_size = tf.random_uniform( [], minval=image_size + 2, maxval=image_size + 200, dtype=tf.int32) image = _aspect_preserving_resize( image, random_larger_image_size) image = tf.random_crop( image, size=[image_size, image_size, image_channels]) image.set_shape([image_size, image_size, image_channels]) image_orig = _aspect_preserving_resize( image_orig, image_size + 2) image_orig = _central_crop([image_orig], image_size, image_size)[0] image_orig.set_shape([image_size, image_size, 3]) elif center_crop: image = _aspect_preserving_resize(image, image_size + 2) image = _central_crop([image], image_size, image_size)[0] image.set_shape([image_size, image_size, image_channels]) image_orig = image else: image = _aspect_preserving_resize(image, image_size) image_orig = image image = tf.to_float(image) / 255.0 image_orig = tf.to_float(image_orig) / 255.0 if batch_size is None: image = tf.expand_dims(image, 0) else: [image, image_orig, label] = tf.train.batch([image, image_orig, label], batch_size=batch_size) if random_style_image_size: # Selects a random size for the style images and resizes all the images # in the batch to that size. image = _aspect_preserving_resize( image, tf.random_uniform([], minval=min_rand_image_size, maxval=max_rand_image_size, dtype=tf.int32)) return image, label, image_orig
def style_image_inputs(style_dataset_file, batch_size=None, image_size=None, square_crop=False, shuffle=True): """Loads a batch of random style image given the path of tfrecord dataset. Args: style_dataset_file: str, path to the tfrecord dataset of style files. The dataset is produced via the create_style_dataset.py script and is made of Example protobufs with the following features: * 'image_raw': byte encoding of the JPEG string of the style image. * 'label': integer identifier of the style image in [0, N - 1], where N is the number of examples in the dataset. * 'vgg_16/<LAYER_NAME>': Gram matrix at layer <LAYER_NAME> of the VGG-16 network (<LAYER_NAME> in {conv,pool}{1,2,3,4,5}) for the style image. batch_size: int. If provided, batches style images. Defaults to None. image_size: int. The images will be resized bilinearly so that the smallest side has size image_size. Defaults to None. square_crop: bool. If True, square-crops to [image_size, image_size]. Defaults to False. shuffle: bool, whether to shuffle style files at random. Defaults to True. Returns: If batch_size is defined, a 4-D tensor of shape [batch_size, ?, ?, 3] with values in [0, 1] for the style image, and 1-D tensor for the style label. Raises: ValueError: if center cropping is requested but no image size is provided, or if batch size is specified but center-cropping is not requested. """ vgg_layers = [ 'vgg_16/conv1', 'vgg_16/pool1', 'vgg_16/conv2', 'vgg_16/pool2', 'vgg_16/conv3', 'vgg_16/pool3', 'vgg_16/conv4', 'vgg_16/pool4', 'vgg_16/conv5', 'vgg_16/pool5' ] if square_crop and image_size is None: raise ValueError('center-cropping requires specifying the image size.') if batch_size is not None and not square_crop: raise ValueError('batching requires center-cropping.') with tf.name_scope('style_image_processing'): filename_queue = tf.train.string_input_producer([style_dataset_file], shuffle=False, capacity=1, name='filename_queue') if shuffle: examples_queue = tf.RandomShuffleQueue( capacity=64, min_after_dequeue=32, dtypes=[tf.string], name='random_examples_queue') else: examples_queue = tf.FIFOQueue(capacity=64, dtypes=[tf.string], name='fifo_examples_queue') reader = tf.TFRecordReader() _, value = reader.read(filename_queue) enqueue_ops = [examples_queue.enqueue([value])] tf.train.queue_runner.add_queue_runner( tf.train.queue_runner.QueueRunner(examples_queue, enqueue_ops)) example_serialized = examples_queue.dequeue() features = tf.parse_single_example( example_serialized, features={ 'label': tf.FixedLenFeature([], tf.int64), 'image_raw': tf.FixedLenFeature([], tf.string), 'vgg_16/conv1': tf.FixedLenFeature([64, 64], tf.float32), 'vgg_16/pool1': tf.FixedLenFeature([64, 64], tf.float32), 'vgg_16/conv2': tf.FixedLenFeature([128, 128], tf.float32), 'vgg_16/pool2': tf.FixedLenFeature([128, 128], tf.float32), 'vgg_16/conv3': tf.FixedLenFeature([256, 256], tf.float32), 'vgg_16/pool3': tf.FixedLenFeature([256, 256], tf.float32), 'vgg_16/conv4': tf.FixedLenFeature([512, 512], tf.float32), 'vgg_16/pool4': tf.FixedLenFeature([512, 512], tf.float32), 'vgg_16/conv5': tf.FixedLenFeature([512, 512], tf.float32), 'vgg_16/pool5': tf.FixedLenFeature([512, 512], tf.float32) }) image = tf.image.decode_jpeg(features['image_raw']) label = features['label'] gram_matrices = [features[vgg_layer] for vgg_layer in vgg_layers] image.set_shape([None, None, 3]) if image_size: if square_crop: image = _aspect_preserving_resize(image, image_size + 2) image = _central_crop([image], image_size, image_size)[0] image.set_shape([image_size, image_size, 3]) else: image = _aspect_preserving_resize(image, image_size) image = tf.to_float(image) / 255.0 if batch_size is None: image = tf.expand_dims(image, 0) else: image_label_gram_matrices = tf.train.batch([image, label] + gram_matrices, batch_size=batch_size) image, label = image_label_gram_matrices[:2] gram_matrices = image_label_gram_matrices[2:] gram_matrices = dict( (vgg_layer, gram_matrix) for vgg_layer, gram_matrix in zip(vgg_layers, gram_matrices)) return image, label, gram_matrices
def reader(self): return tf.TFRecordReader()