def _count_matrix_input(self, filenames, submatrix_rows, submatrix_cols): """Creates ops that read submatrix shards from disk.""" random.shuffle(filenames) filename_queue = tf.train.string_input_producer(filenames) reader = tf.WholeFileReader() _, serialized_example = reader.read(filename_queue) features = tf.parse_single_example( serialized_example, features={ 'global_row': tf.FixedLenFeature([submatrix_rows], dtype=tf.int64), 'global_col': tf.FixedLenFeature([submatrix_cols], dtype=tf.int64), 'sparse_local_row': tf.VarLenFeature(dtype=tf.int64), 'sparse_local_col': tf.VarLenFeature(dtype=tf.int64), 'sparse_value': tf.VarLenFeature(dtype=tf.float32) }) global_row = features['global_row'] global_col = features['global_col'] sparse_local_row = features['sparse_local_row'].values sparse_local_col = features['sparse_local_col'].values sparse_count = features['sparse_value'].values sparse_indices = tf.concat( axis=1, values=[tf.expand_dims(sparse_local_row, 1), tf.expand_dims(sparse_local_col, 1)]) count = tf.sparse_to_dense(sparse_indices, [submatrix_rows, submatrix_cols], sparse_count) return global_row, global_col, count
def load_train_batch(self): """Load a batch of training instances. """ seed = random.randint(0, 2**31 - 1) # Load the list of training files into queues file_list = self.format_file_list(self.dataset_dir, 'train') image_paths_queue = tf.train.string_input_producer( file_list['image_file_list'], seed=seed, shuffle=True) cam_paths_queue = tf.train.string_input_producer( file_list['cam_file_list'], seed=seed, shuffle=True) self.steps_per_epoch = int( len(file_list['image_file_list']) // self.batch_size) # Load images img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) image_seq = tf.image.decode_jpeg(image_contents) tgt_image, src_image_stack = \ self.unpack_image_sequence( image_seq, self.img_height, self.img_width, self.num_source) # Load camera intrinsics cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for i in range(9): rec_def.append([1.]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) # Form training batches src_image_stack, tgt_image, intrinsics = \ tf.train.batch([src_image_stack, tgt_image, intrinsics], batch_size=self.batch_size) # Data augmentation image_all = tf.concat([tgt_image, src_image_stack], axis=3) image_all, intrinsics = self.data_augmentation(image_all, intrinsics, self.img_height, self.img_width) tgt_image = image_all[:, :, :, :3] src_image_stack = image_all[:, :, :, 3:] intrinsics = self.get_multi_scale_intrinsics(intrinsics, self.num_scales) return tgt_image, src_image_stack, intrinsics
def load_images(input_dir, input_name=''): input_paths = glob.glob(os.path.join(input_dir, "*.jpg")) decode = tf.image.decode_jpeg if len(input_paths) == 0: input_paths = glob.glob(os.path.join(input_dir, "*.png")) decode = tf.image.decode_png if len(input_paths) == 0: raise Exception("%s contains no images (jpg/png)" % input_dir) else: def get_name(path): name, _ = os.path.splitext(os.path.basename(path)) return name # if the image names are numbers, sort by the value rather than asciibetically # having sorted inputs means that the outputs are sorted in test mode if all(get_name(path).isdigit() for path in input_paths): input_paths = sorted(input_paths, key=lambda path: int(get_name(path))) else: input_paths = sorted(input_paths) with tf.name_scope("load_%simages" % input_name): path_queue = tf.train.string_input_producer( input_paths, shuffle=a.mode == "train") reader = tf.WholeFileReader() paths, contents = reader.read(path_queue) raw_input = decode(contents) raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32) assertion = tf.assert_equal( tf.shape(raw_input)[2], 3, message="image does not have 3 channels") with tf.control_dependencies([assertion]): raw_input = tf.identity(raw_input) raw_input.set_shape([None, None, 3]) return len(input_paths), paths, raw_input
def _preproc_image_batch(self, batch_size, num_threads=1): # Read image file from disk and decode JPEG reader = tf.WholeFileReader() image_filename, image_raw = reader.read(self._filename_queue) image_data = tf.image.decode_jpeg(image_raw, channels=3) # Image preprocessing image_data = tf.image.convert_image_dtype(image_data, dtype=tf.float32) image_data = tf.expand_dims(image_data, 0) image_data = tf.image.resize_bilinear(image_data, [image_size, image_size], align_corners=False) image_data = tf.squeeze(image_data) image_data = image_data / 255.0 # Read a batch of preprocessing images from queue image_batch = tf.train.batch([image_data, image_filename], batch_size, num_threads=num_threads, allow_smaller_final_batch=True) return image_batch
def read_data(self): """Provides images and camera intrinsics.""" with tf.name_scope('data_loading'): with tf.name_scope('enqueue_paths'): seed = random.randint(0, 2**31 - 1) self.file_lists = self.compile_file_list(self.data_dir, self.input_file) image_paths_queue = tf.train.string_input_producer( self.file_lists['image_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None) ) seg_paths_queue = tf.train.string_input_producer( self.file_lists['segment_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) cam_paths_queue = tf.train.string_input_producer( self.file_lists['cam_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) seg_reader = tf.WholeFileReader() _, seg_contents = seg_reader.read(seg_paths_queue) if self.file_extension == 'jpg': image_seq = tf.image.decode_jpeg(image_contents) seg_seq = tf.image.decode_jpeg(seg_contents, channels=3) elif self.file_extension == 'png': image_seq = tf.image.decode_png(image_contents, channels=3) seg_seq = tf.image.decode_png(seg_contents, channels=3) with tf.name_scope('load_intrinsics'): cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for _ in range(9): rec_def.append([1.0]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) with tf.name_scope('convert_image'): image_seq = self.preprocess_image(image_seq) # Converts to float. if self.random_color: with tf.name_scope('image_augmentation'): image_seq = self.augment_image_colorspace(image_seq) image_stack = self.unpack_images(image_seq) seg_stack = self.unpack_images(seg_seq) if self.flipping_mode != FLIP_NONE: random_flipping = (self.flipping_mode == FLIP_RANDOM) with tf.name_scope('image_augmentation_flip'): image_stack, seg_stack, intrinsics = self.augment_images_flip( image_stack, seg_stack, intrinsics, randomized=random_flipping) if self.random_scale_crop: with tf.name_scope('image_augmentation_scale_crop'): image_stack, seg_stack, intrinsics = self.augment_images_scale_crop( image_stack, seg_stack, intrinsics, self.img_height, self.img_width) with tf.name_scope('multi_scale_intrinsics'): intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics, self.num_scales) intrinsic_mat.set_shape([self.num_scales, 3, 3]) intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat) intrinsic_mat_inv.set_shape([self.num_scales, 3, 3]) if self.imagenet_norm: im_mean = tf.tile( tf.constant(IMAGENET_MEAN), multiples=[self.seq_length]) im_sd = tf.tile( tf.constant(IMAGENET_SD), multiples=[self.seq_length]) image_stack_norm = (image_stack - im_mean) / im_sd else: image_stack_norm = image_stack with tf.name_scope('batching'): if self.shuffle: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.shuffle_batch( [image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv], batch_size=self.batch_size, num_threads=self.threads, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size, min_after_dequeue=self.queue_size) else: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.batch( [image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv], batch_size=self.batch_size, num_threads=1, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size) return (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv)
def make_data_tensor(self, train=True): if train: folders = self.metatrain_character_folders # number of tasks, not number of meta-iterations. (divide by metabatch size to measure) num_total_batches = 200000 if FLAGS.task_type == "ne": print("Inside ne train") folders = self.metatrain_character_folders[:50] # 10 classification tasks num_total_batches = 5000 else: folders = self.metaval_character_folders num_total_batches = 600 if FLAGS.task_type == "ne": print("inside ne val") if FLAGS.test_set: folders = self.metaval_character_folders[:15] else: folders = self.metaval_character_folders[:15] # 3 classification tasks num_total_batches = 60 # print("folders", folders) # make list of files print('Generating filenames') if FLAGS.task_setting == 'ne' and train: # all_filenames = [] # random.shuffle(folders) # for i in range(len(folders)/self.num_classes): # #sampled_character_folders = random.sample(folders, self.num_classes) # #random.shuffle(sampled_character_folders) # sampled_character_folders = folders[i*self.num_classes:(i+1)*self.num_classes] # labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # # make sure the above isn't randomized order # labels = [li[0] for li in labels_and_images] # filenames = [li[1] for li in labels_and_images] # all_filenames.extend(filenames) all_filenames = [] print("len folders", len(folders)) random.shuffle(folders) task_folders_new = [] for i in range(int(len(folders)/self.num_classes)): # sampled_character_folders = random.sample(folders, self.num_classes) # random.shuffle(sampled_character_folders) sampled_character_folders = folders[i*self.num_classes:(i+1)*self.num_classes] task_folders_temp = itertools.permutations(sampled_character_folders) task_folders_new.extend(task_folders_temp) # print("task_folders_new", task_folders_new) print("len of task_folders_new", len(task_folders_new)) random.shuffle(task_folders_new) for i in range(len(task_folders_new)): sampled_character_folders = task_folders_new[i] # print("scf", sampled_character_folders) labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) else: all_filenames = [] for _ in range(num_total_batches): sampled_character_folders = random.sample(folders, self.num_classes) random.shuffle(sampled_character_folders) labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) # make queue for tensorflow to read from filename_queue = tf.train.string_input_producer(tf.convert_to_tensor(all_filenames), shuffle=False) print('Generating image processing ops') image_reader = tf.WholeFileReader() _, image_file = image_reader.read(filename_queue) if FLAGS.datasource == 'miniimagenet': image = tf.image.decode_jpeg(image_file, channels=3) image.set_shape((self.img_size[0],self.img_size[1],3)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 else: image = tf.image.decode_png(image_file) image.set_shape((self.img_size[0],self.img_size[1],1)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 image = 1.0 - image # invert num_preprocess_threads = 1 # TODO - enable this to be set to >1 min_queue_examples = 256 examples_per_batch = self.num_classes * self.num_samples_per_class batch_image_size = self.batch_size * examples_per_batch print('Batching images') print("batch_image_size", batch_image_size) images = tf.train.batch( [image], batch_size = batch_image_size, num_threads=num_preprocess_threads, capacity=min_queue_examples + 3 * batch_image_size, ) print("len images", images.shape) all_image_batches, all_label_batches = [], [] print('Manipulating image data to be right shape') for i in range(self.batch_size): image_batch = images[i*examples_per_batch:(i+1)*examples_per_batch] if FLAGS.datasource == 'omniglot': # omniglot augments the dataset by rotating digits to create new classes # get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes) rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]), self.num_classes) # print("labels", labels) label_batch = tf.convert_to_tensor(labels) new_list, new_label_list = [], [] # shuffles the data within a batch, class labels remain fixed for k in range(self.num_samples_per_class): class_idxs = tf.range(0, self.num_classes) class_idxs = tf.random_shuffle(class_idxs) true_idxs = class_idxs * self.num_samples_per_class + k new_list.append(tf.gather(image_batch,true_idxs)) if FLAGS.datasource == 'omniglot': # and FLAGS.train: new_list[-1] = tf.stack([tf.reshape(tf.image.rot90( tf.reshape(new_list[-1][ind], [self.img_size[0], self.img_size[1],1]), k=tf.cast(rotations[0, class_idxs[ind]], tf.int32)), (self.dim_input,)) for ind in range(self.num_classes)]) new_label_list.append(tf.gather(label_batch, true_idxs)) new_list = tf.concat(new_list, 0) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input] new_label_list = tf.concat(new_label_list, 0) all_image_batches.append(new_list) all_label_batches.append(new_label_list) all_image_batches = tf.stack(all_image_batches) all_label_batches = tf.stack(all_label_batches) print("all_image_batches", all_image_batches) all_label_batches = tf.one_hot(all_label_batches, self.num_classes) return all_image_batches, all_label_batches
def make_data_tensor(self, train=True): if train: folders = self.metatrain_character_folders # number of tasks, not number of meta-iterations. (divide by metabatch size to measure) if FLAGS.expt_number == '14a': folders = folders[:(5 * self.num_classes)] elif FLAGS.expt_number == '14b': folders = folders[:(10 * self.num_classes)] elif FLAGS.expt_number == '14c': folders = folders[:(25 * self.num_classes)] elif FLAGS.expt_number == '14d': folders = folders[:(50 * self.num_classes)] elif FLAGS.expt_number == '14e': folders = folders[:(75 * self.num_classes)] elif FLAGS.expt_number == '14f': folders = folders[:(100 * self.num_classes)] elif FLAGS.expt_number == '14g': folders = folders[:(150 * self.num_classes)] elif FLAGS.expt_number == '14h': folders = folders[:(200 * self.num_classes)] if FLAGS.expt_number == '6' or FLAGS.expt_number == '8': print("Inside expt number 6/8") num_total_batches = 26400 elif FLAGS.expt_number == '6a' or FLAGS.expt_number == '8a': print("Inside expt number 6a/8a") num_total_batches = 1100 else: num_total_batches = 200000 else: folders = self.metaval_character_folders num_total_batches = 600 # make list of files print('Generating filenames') if FLAGS.expt_number == '2' and train: print('Inside expt number 2') all_filenames = [] """ go over the folders once, group the adjacent 5 classes together as one task. Non-exclusive """ for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif (FLAGS.expt_number == '9a' or FLAGS.expt_number == '9b' or FLAGS.expt_number == '9c' or FLAGS.expt_number == '11a1' or FLAGS.expt_number == '11a2' or FLAGS.expt_number == '11a3' or ('14' in FLAGS.expt_number)) and train: print('Inside expt number 9a/b/c/11a1/11a2/11a3/14s') all_filenames = [] """ go over the folders multiple times, group the adjacent 5 classes together as one task. Non-exclusive """ if FLAGS.expt_number == '9a' or FLAGS.expt_number == '11a1' or FLAGS.expt_number == '11a2' or FLAGS.expt_number == '11a3' or ( '14' in FLAGS.expt_number): total_num_tasks = 200000 elif FLAGS.expt_number == '9b': total_num_tasks = 26400 elif FLAGS.expt_number == '9c': total_num_tasks = 1100 for _ in range( int(total_num_tasks / (len(folders) / self.num_classes))): #9b #for _ in range(int(num_total_batches/(len(folders)/self.num_classes))): #9a for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] #random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif (FLAGS.expt_number == '7a' or FLAGS.expt_number == '7b' or FLAGS.expt_number == '7c' or FLAGS.expt_number == '11b1' or FLAGS.expt_number == '11b2' or FLAGS.expt_number == '11b3') and train: print('Inside expt number 7a/b/c/11b1/11b2/11b3') all_filenames = [] """ go over the folders multiple times, group the adjacent 5 classes together as one task. Non-exclusive """ if FLAGS.expt_number == '7a' or FLAGS.expt_number == '11b1' or FLAGS.expt_number == '11b2' or FLAGS.expt_number == '11b3': total_num_tasks = 200000 elif FLAGS.expt_number == '7b': total_num_tasks = 26400 elif FLAGS.expt_number == '7c': total_num_tasks = 1100 for _ in range( int(total_num_tasks / (len(folders) / self.num_classes))): #7b #for _ in range(int(num_total_batches/(len(folders)/self.num_classes))): #7a for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif FLAGS.expt_number == '3' and train: print('Inside expt number 3') all_filenames = [] """ removed shuffling of classes in the init function. classes from the same alphabet together now. go over the folders once, group the adjacent 5 classes together as one task. Non-exclusive """ for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] #random.shuffle(sampled_character_folders) #print("Task ", task_count, ": ", sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif (FLAGS.expt_number == '4' or FLAGS.expt_number == '5' or FLAGS.expt_number == '8c') and train: print('Inside expt number 4/5/8c') all_filenames = [] """ go over the folders once, group the adjacent 5 classes together as one task. get all permutations of that task. Shuffle these tasks """ task_folders_new = [] for task_count in range(int(len(folders) / self.num_classes)): sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] task_folders_temp = permutations(sampled_character_folders) task_folders_new.extend(task_folders_temp) print('total number of tasks: ', len(task_folders_new)) random.shuffle(task_folders_new) for task_count in range(len(task_folders_new)): #sampled_character_folders = random.sample(folders, self.num_classes) #sampled_character_folders = folders[task_count*self.num_classes: (task_count+1)*self.num_classes] sampled_character_folders = task_folders_new[task_count] #random.shuffle(sampled_character_folders) #print("Task ", task_count, ": ", sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif FLAGS.expt_number == '4a' and train: print('Inside expt number 4a') all_filenames = [] """ go over all tasks group the same 5 classes together, but shuffle them everytime. Do it till you reach 1100 tasks """ i = 0 while i < 1100: for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] random.shuffle(sampled_character_folders) #print("Task ", task_count, ": ", sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) i += 1 if i >= 1100: break elif (FLAGS.expt_number == '12a' or FLAGS.expt_number == '12b' or FLAGS.expt_number == '12c' or FLAGS.expt_number == '12d' or FLAGS.expt_number == '12e' or FLAGS.expt_number == '12f') and train: print('Inside expt number 12s') all_filenames = [] """ make tasks first. Then go over the tasks mulitiple times to collect data """ if FLAGS.expt_number == '12a': n_tasks = 220 elif FLAGS.expt_number == '12b': n_tasks = 1100 elif FLAGS.expt_number == '12c': n_tasks = 5500 elif FLAGS.expt_number == '12d': n_tasks = 26400 elif FLAGS.expt_number == '12e': n_tasks = 27500 elif FLAGS.expt_number == '12f': n_tasks = 137500 task_folders_new = [] for task_count in range(n_tasks): sampled_character_folders = random.sample( folders, self.num_classes) #task_folders_temp = permutations(sampled_character_folders) task_folders_new.append(sampled_character_folders) print('total number of tasks: ', len(task_folders_new)) #random.shuffle(task_folders_new) for task_count in range(num_total_batches): sampled_character_folders = task_folders_new[task_count % n_tasks] labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) else: all_filenames = [] print('Inside expt number 1/6/6a/8/8a/11c1/11c2/11c3') for _ in range(num_total_batches): sampled_character_folders = random.sample( folders, self.num_classes) random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False, train=train) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) # make queue for tensorflow to read from filename_queue = tf.train.string_input_producer( tf.convert_to_tensor(all_filenames), shuffle=False) print('Generating image processing ops') image_reader = tf.WholeFileReader() _, image_file = image_reader.read(filename_queue) if FLAGS.datasource == 'miniimagenet': image = tf.image.decode_jpeg(image_file, channels=3) image.set_shape((self.img_size[0], self.img_size[1], 3)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 else: image = tf.image.decode_png(image_file) image.set_shape((self.img_size[0], self.img_size[1], 1)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 image = 1.0 - image # invert num_preprocess_threads = 1 # TODO - enable this to be set to >1 min_queue_examples = 256 examples_per_batch = self.num_classes * self.num_samples_per_class batch_image_size = self.batch_size * examples_per_batch print('Batching images') images = tf.train.batch( [image], batch_size=batch_image_size, num_threads=num_preprocess_threads, capacity=min_queue_examples + 3 * batch_image_size, ) all_image_batches, all_label_batches = [], [] print('Manipulating image data to be right shape') for i in range(self.batch_size): image_batch = images[i * examples_per_batch:(i + 1) * examples_per_batch] if FLAGS.datasource == 'omniglot': # omniglot augments the dataset by rotating digits to create new classes # get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes) rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]), self.num_classes) label_batch = tf.convert_to_tensor(labels) new_list, new_label_list = [], [] for k in range(self.num_samples_per_class): class_idxs = tf.range(0, self.num_classes) class_idxs = tf.random_shuffle(class_idxs) true_idxs = class_idxs * self.num_samples_per_class + k new_list.append(tf.gather(image_batch, true_idxs)) if FLAGS.datasource == 'omniglot': # and FLAGS.train: new_list[-1] = tf.stack([ tf.reshape( tf.image.rot90(tf.reshape( new_list[-1][ind], [self.img_size[0], self.img_size[1], 1]), k=tf.cast( rotations[0, class_idxs[ind]], tf.int32)), (self.dim_input, )) for ind in range(self.num_classes) ]) new_label_list.append(tf.gather(label_batch, true_idxs)) new_list = tf.concat( new_list, 0 ) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input] new_label_list = tf.concat(new_label_list, 0) all_image_batches.append(new_list) all_label_batches.append(new_label_list) all_image_batches = tf.stack(all_image_batches) all_label_batches = tf.stack(all_label_batches) all_label_batches = tf.one_hot(all_label_batches, self.num_classes) return all_image_batches, all_label_batches
def load_examples(): if a.input_dir is None or not os.path.exists(a.input_dir): raise Exception("input_dir does not exist") input_paths = glob.glob(os.path.join(a.input_dir, "*.jpg")) decode = tf.image.decode_jpeg if len(input_paths) == 0: input_paths = glob.glob(os.path.join(a.input_dir, "*.png")) decode = tf.image.decode_png if len(input_paths) == 0: raise Exception("input_dir contains no image files") def get_name(path): name, _ = os.path.splitext(os.path.basename(path)) return name # if the image names are numbers, sort by the value rather than asciibetically # having sorted inputs means that the outputs are sorted in test mode if all(get_name(path).isdigit() for path in input_paths): input_paths = sorted(input_paths, key=lambda path: int(get_name(path))) else: input_paths = sorted(input_paths) with tf.name_scope("load_images"): path_queue = tf.train.string_input_producer(input_paths, shuffle=a.mode == "train") reader = tf.WholeFileReader() paths, contents = reader.read(path_queue) raw_input = decode(contents) raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32) assertion = tf.assert_equal(tf.shape(raw_input)[2], 3, message="image does not have 3 channels") with tf.control_dependencies([assertion]): raw_input = tf.identity(raw_input) raw_input.set_shape([None, None, 3]) if a.lab_colorization: # load color and brightness from image, no B image exists here lab = rgb_to_lab(raw_input) L_chan, a_chan, b_chan = preprocess_lab(lab) a_images = tf.expand_dims(L_chan, axis=2) b_images = tf.stack([a_chan, b_chan], axis=2) else: # break apart image pair and move to range [-1, 1] width = tf.shape(raw_input)[1] # [height, width, channels] a_images = preprocess(raw_input[:, :width // 2, :]) b_images = preprocess(raw_input[:, width // 2:, :]) if a.which_direction == "AtoB": inputs, targets = [a_images, b_images] elif a.which_direction == "BtoA": inputs, targets = [b_images, a_images] else: raise Exception("invalid direction") # synchronize seed for image operations so that we do the same operations to both # input and output images seed = random.randint(0, 2**31 - 1) def transform(image): r = image if a.flip: r = tf.image.random_flip_left_right(r, seed=seed) # area produces a nice downscaling, but does nearest neighbor for upscaling # assume we're going to be doing downscaling here r = tf.image.resize_images(r, [a.scale_size, a.scale_size], method=tf.image.ResizeMethod.AREA) offset = tf.cast(tf.floor( tf.random_uniform([2], 0, a.scale_size - CROP_SIZE + 1, seed=seed)), dtype=tf.int32) if a.scale_size > CROP_SIZE: r = tf.image.crop_to_bounding_box(r, offset[0], offset[1], CROP_SIZE, CROP_SIZE) elif a.scale_size < CROP_SIZE: raise Exception("scale size cannot be less than crop size") return r with tf.name_scope("input_images"): input_images = transform(inputs) with tf.name_scope("target_images"): target_images = transform(targets) paths_batch, inputs_batch, targets_batch = tf.train.batch( [paths, input_images, target_images], batch_size=a.batch_size) steps_per_epoch = int(math.ceil(len(input_paths) / a.batch_size)) return Examples( paths=paths_batch, inputs=inputs_batch, targets=targets_batch, count=len(input_paths), steps_per_epoch=steps_per_epoch, )
def make_data_tensor(self, train=True, ne=False, ne_task_folders=None): ne_tasks_n = 1 if train or ne: folders = self.metatrain_character_folders # number of tasks, not number of meta-iterations. (divide by metabatch size to measure) num_total_batches = 200000 else: folders = self.metaval_character_folders num_total_batches = 600 task_folders = [] # make list of files print('Generating filenames') all_filenames = [] if (not train) and ne: print('Inside test ne filename generation') for outer_task_count in range( int(200000 / (len(folders) / self.num_classes))): for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] #random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif train and ne: print('Inside train ne filename generation') """ Shuffle the tasks """ task_folders = [] for outer_task_count in range( int(200000 / (len(folders) / self.num_classes))): for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] task_folders.append(sampled_character_folders) random.shuffle(task_folders) for task_count in range(len(task_folders)): #random.shuffle(sampled_character_folders) sampled_character_folders = task_folders[task_count] labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif train and (not ne) and (FLAGS.expt_number == '7' or FLAGS.expt_number == '8'): #no overlap between b1 and b2 for task_count in range(num_total_batches): ne_folders = [] for i in range(task_count * ne_tasks_n, (task_count + 1) * ne_tasks_n): ne_folders.extend(ne_task_folders[i % len(ne_task_folders)]) #print('ne_folders', ne_folders) folders_temp = set(folders) - set(ne_folders) sampled_character_folders = random.sample( folders_temp, self.num_classes) #print('folders', folders) #print('sampled_character_folders', sampled_character_folders) #sample from set(folders) - set(ne_folders) random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) else: if (FLAGS.expt_number == '2' or FLAGS.expt_number == '5') and train: for task_count in range(int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] #random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) elif (FLAGS.expt_number == '3' or FLAGS.expt_number == '6') and train: for _ in range( int(num_total_batches / (len(folders) / self.num_classes))): for task_count in range( int(len(folders) / self.num_classes)): #sampled_character_folders = random.sample(folders, self.num_classes) sampled_character_folders = folders[task_count * self.num_classes: (task_count + 1) * self.num_classes] #random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) else: print('In expt 1/4') for _ in range(num_total_batches): sampled_character_folders = random.sample( folders, self.num_classes) #random.shuffle(sampled_character_folders) labels_and_images = get_images( sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False) # make sure the above isn't randomized order labels = [li[0] for li in labels_and_images] filenames = [li[1] for li in labels_and_images] all_filenames.extend(filenames) if (not train) and ne: batch_size = self.batch_size * int(len(folders) / self.num_classes) print('ne batch size: ', batch_size) elif train and ne: batch_size = self.batch_size * ne_tasks_n else: batch_size = self.batch_size # make queue for tensorflow to read from filename_queue = tf.train.string_input_producer( tf.convert_to_tensor(all_filenames), shuffle=False) print('Generating image processing ops') image_reader = tf.WholeFileReader() _, image_file = image_reader.read(filename_queue) if FLAGS.datasource == 'miniimagenet': image = tf.image.decode_jpeg(image_file, channels=3) image.set_shape((self.img_size[0], self.img_size[1], 3)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 else: image = tf.image.decode_png(image_file) image.set_shape((self.img_size[0], self.img_size[1], 1)) image = tf.reshape(image, [self.dim_input]) image = tf.cast(image, tf.float32) / 255.0 image = 1.0 - image # invert num_preprocess_threads = 1 # TODO - enable this to be set to >1 min_queue_examples = 256 examples_per_batch = self.num_classes * self.num_samples_per_class batch_image_size = batch_size * examples_per_batch print('Batching images') images = tf.train.batch( [image], batch_size=batch_image_size, num_threads=num_preprocess_threads, capacity=min_queue_examples + 3 * batch_image_size, ) all_image_batches, all_label_batches = [], [] print('Manipulating image data to be right shape') for i in range(batch_size): image_batch = images[ i * examples_per_batch:(i + 1) * examples_per_batch] #examples from the same task if FLAGS.datasource == 'omniglot': # omniglot augments the dataset by rotating digits to create new classes # get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes) rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]), self.num_classes) label_batch = tf.convert_to_tensor(labels) new_list, new_label_list = [], [] for k in range(self.num_samples_per_class): class_idxs = tf.range(0, self.num_classes) class_idxs = tf.random_shuffle(class_idxs) true_idxs = class_idxs * self.num_samples_per_class + k new_list.append(tf.gather(image_batch, true_idxs)) if FLAGS.datasource == 'omniglot': # and FLAGS.train: new_list[-1] = tf.stack([ tf.reshape( tf.image.rot90(tf.reshape( new_list[-1][ind], [self.img_size[0], self.img_size[1], 1]), k=tf.cast( rotations[0, class_idxs[ind]], tf.int32)), (self.dim_input, )) for ind in range(self.num_classes) ]) new_label_list.append(tf.gather(label_batch, true_idxs)) new_list = tf.concat( new_list, 0 ) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input] new_label_list = tf.concat(new_label_list, 0) all_image_batches.append(new_list) all_label_batches.append(new_label_list) all_image_batches = tf.stack(all_image_batches) all_label_batches = tf.stack(all_label_batches) all_label_batches = tf.one_hot(all_label_batches, self.num_classes) print("label_batches", all_label_batches) return all_image_batches, all_label_batches, task_folders
def load_examples(): if a.input_dir is None or not os.path.exists(a.input_dir): raise Exception("input_dir does not exist") input_paths = glob.glob(os.path.join(a.input_dir, "*", "*.jpg")) decode = tf.image.decode_jpeg if len(input_paths) == 0: input_paths = glob.glob(os.path.join(a.input_dir, "*", "*.png")) decode = tf.image.decode_png print("len = ", len(input_paths)) if len(input_paths) == 0: raise Exception("input_dir contains no image files") def get_name(path): name, _ = os.path.splitext(os.path.basename(path)) return name if all(get_name(path).isdigit() for path in input_paths): input_paths = sorted(input_paths, key=lambda path: int(get_name(path))) else: # if the image names are numbers, sort by the value rather than asciibetically # having sorted inputs means that the outputs are sorted in test mode input_paths = sorted(input_paths) with tf.name_scope("load_images"): path_queue = tf.train.string_input_producer(input_paths, shuffle=a.mode == "train") reader = tf.WholeFileReader() paths, contents = reader.read(path_queue) raw_input = decode(contents) raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32) assertion = tf.assert_equal(tf.shape(raw_input)[2], 1, message="image does not have 1 channels") with tf.control_dependencies([assertion]): raw_input = tf.identity(raw_input) raw_input.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH * 2, 1]) # break apart image pair and move to range [-1, 1] width = tf.shape(raw_input)[1] # [height, width, channels] a_images = preprocess(raw_input[:, :width // 2, :]) b_images = preprocess(raw_input[:, width // 2:, :]) if a.which_direction == "AtoB": inputs, targets = [a_images, b_images] elif a.which_direction == "BtoA": inputs, targets = [b_images, a_images] else: raise Exception("invalid direction") # synchronize seed for image operations so that we do the same operations to both # input and output images seed = random.randint(0, 2**31 - 1) def transform(image): r = image if a.flip: r = tf.image.random_flip_left_right(r, seed=seed) if a.mode == "train": # crop image h = r.get_shape().as_list()[0] w = r.get_shape().as_list()[1] h_offset = tf.cast(tf.floor( tf.random_uniform([1], 0, h - CROP_SIZE + 1, seed=seed)), dtype=tf.int32) w_offset = tf.cast(tf.floor( tf.random_uniform([1], 0, w - CROP_SIZE + 1, seed=seed)), dtype=tf.int32) r = tf.image.crop_to_bounding_box(r, h_offset[0], w_offset[0], CROP_SIZE, CROP_SIZE) return r with tf.name_scope("input_images"): input_images = transform(inputs) with tf.name_scope("target_images"): target_images = transform(targets) paths_batch, inputs_batch, targets_batch = tf.train.batch( [paths, input_images, target_images], batch_size=a.batch_size) steps_per_epoch = int(math.ceil(len(input_paths) / a.batch_size)) print(inputs_batch.get_shape().as_list()) return Examples( paths=paths_batch, inputs=inputs_batch, targets=targets_batch, count=len(input_paths), steps_per_epoch=steps_per_epoch, )
def read_data(self): """Provides images and camera intrinsics.""" with tf.name_scope("data_loading"): with tf.name_scope("enqueue_paths"): seed = random.randint(0, 2**31 - 1) self.file_lists = self.compile_file_list( self.data_dir, self.input_file) image_paths_queue = tf.train.string_input_producer( self.file_lists["image_file_list"], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None), ) seg_paths_queue = tf.train.string_input_producer( self.file_lists["segment_file_list"], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None), ) img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) seg_reader = tf.WholeFileReader() _, seg_contents = seg_reader.read(seg_paths_queue) if self.file_extension == "jpg": image_seq = tf.image.decode_jpeg(image_contents) seg_seq = tf.image.decode_jpeg(seg_contents, channels=3) elif self.file_extension == "png": image_seq = tf.image.decode_png(image_contents, channels=3) seg_seq = tf.image.decode_png(seg_contents, channels=3) with tf.name_scope("load_intrinsics"): intrinsics = tf.random.uniform(shape=(3, 3)) with tf.name_scope("convert_image"): image_seq = self.preprocess_image( image_seq) # Converts to float. if self.random_color: with tf.name_scope("image_augmentation"): image_seq = self.augment_image_colorspace(image_seq) image_stack = self.unpack_images(image_seq) seg_stack = self.unpack_images(seg_seq) if self.flipping_mode != FLIP_NONE: random_flipping = self.flipping_mode == FLIP_RANDOM with tf.name_scope("image_augmentation_flip"): image_stack, seg_stack, intrinsics = self.augment_images_flip( image_stack, seg_stack, intrinsics, randomized=random_flipping) if self.random_scale_crop: with tf.name_scope("image_augmentation_scale_crop"): image_stack, seg_stack, intrinsics = self.augment_images_scale_crop( image_stack, seg_stack, intrinsics, self.img_height, self.img_width, ) with tf.name_scope("multi_scale_intrinsics"): intrinsic_mat = self.get_multi_scale_intrinsics( intrinsics, self.num_scales) intrinsic_mat.set_shape([self.num_scales, 3, 3]) intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat) intrinsic_mat_inv.set_shape([self.num_scales, 3, 3]) if self.imagenet_norm: im_mean = tf.tile(tf.constant(IMAGENET_MEAN), multiples=[self.seq_length]) im_sd = tf.tile(tf.constant(IMAGENET_SD), multiples=[self.seq_length]) image_stack_norm = (image_stack - im_mean) / im_sd else: image_stack_norm = image_stack with tf.name_scope("batching"): if self.shuffle: ( image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv, ) = tf.train.shuffle_batch( [ image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv, ], batch_size=self.batch_size, num_threads=self.threads, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size, min_after_dequeue=self.queue_size, ) else: ( image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv, ) = tf.train.batch( [ image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv, ], batch_size=self.batch_size, num_threads=1, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size, ) return ( image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv, )