def _count_matrix_input(self, filenames, submatrix_rows, submatrix_cols):
"""Creates ops that read submatrix shards from disk."""
random.shuffle(filenames)
filename_queue = tf.train.string_input_producer(filenames)
reader = tf.WholeFileReader()
_, serialized_example = reader.read(filename_queue)
features = tf.parse_single_example(
serialized_example,
features={
'global_row': tf.FixedLenFeature([submatrix_rows], dtype=tf.int64),
'global_col': tf.FixedLenFeature([submatrix_cols], dtype=tf.int64),
'sparse_local_row': tf.VarLenFeature(dtype=tf.int64),
'sparse_local_col': tf.VarLenFeature(dtype=tf.int64),
'sparse_value': tf.VarLenFeature(dtype=tf.float32)
})
global_row = features['global_row']
global_col = features['global_col']
sparse_local_row = features['sparse_local_row'].values
sparse_local_col = features['sparse_local_col'].values
sparse_count = features['sparse_value'].values
sparse_indices = tf.concat(
axis=1, values=[tf.expand_dims(sparse_local_row, 1),
tf.expand_dims(sparse_local_col, 1)])
count = tf.sparse_to_dense(sparse_indices, [submatrix_rows, submatrix_cols],
sparse_count)
return global_row, global_col, count
def load_train_batch(self):
"""Load a batch of training instances.
"""
seed = random.randint(0, 2**31 - 1)
# Load the list of training files into queues
file_list = self.format_file_list(self.dataset_dir, 'train')
image_paths_queue = tf.train.string_input_producer(
file_list['image_file_list'], seed=seed, shuffle=True)
cam_paths_queue = tf.train.string_input_producer(
file_list['cam_file_list'], seed=seed, shuffle=True)
self.steps_per_epoch = int(
len(file_list['image_file_list']) // self.batch_size)
# Load images
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
image_seq = tf.image.decode_jpeg(image_contents)
tgt_image, src_image_stack = \
self.unpack_image_sequence(
image_seq, self.img_height, self.img_width, self.num_source)
# Load camera intrinsics
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for i in range(9):
rec_def.append([1.])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
# Form training batches
src_image_stack, tgt_image, intrinsics = \
tf.train.batch([src_image_stack, tgt_image, intrinsics],
batch_size=self.batch_size)
# Data augmentation
image_all = tf.concat([tgt_image, src_image_stack], axis=3)
image_all, intrinsics = self.data_augmentation(image_all, intrinsics,
self.img_height,
self.img_width)
tgt_image = image_all[:, :, :, :3]
src_image_stack = image_all[:, :, :, 3:]
intrinsics = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
return tgt_image, src_image_stack, intrinsics
def load_images(input_dir, input_name=''):
input_paths = glob.glob(os.path.join(input_dir, "*.jpg"))
decode = tf.image.decode_jpeg
if len(input_paths) == 0:
input_paths = glob.glob(os.path.join(input_dir, "*.png"))
decode = tf.image.decode_png
if len(input_paths) == 0:
raise Exception("%s contains no images (jpg/png)" % input_dir)
else:
def get_name(path):
name, _ = os.path.splitext(os.path.basename(path))
return name
# if the image names are numbers, sort by the value rather than asciibetically
# having sorted inputs means that the outputs are sorted in test mode
if all(get_name(path).isdigit() for path in input_paths):
input_paths = sorted(input_paths,
key=lambda path: int(get_name(path)))
else:
input_paths = sorted(input_paths)
with tf.name_scope("load_%simages" % input_name):
path_queue = tf.train.string_input_producer(
input_paths, shuffle=a.mode == "train")
reader = tf.WholeFileReader()
paths, contents = reader.read(path_queue)
raw_input = decode(contents)
raw_input = tf.image.convert_image_dtype(raw_input,
dtype=tf.float32)
assertion = tf.assert_equal(
tf.shape(raw_input)[2],
3,
message="image does not have 3 channels")
with tf.control_dependencies([assertion]):
raw_input = tf.identity(raw_input)
raw_input.set_shape([None, None, 3])
return len(input_paths), paths, raw_input
def _preproc_image_batch(self, batch_size, num_threads=1):
# Read image file from disk and decode JPEG
reader = tf.WholeFileReader()
image_filename, image_raw = reader.read(self._filename_queue)
image_data = tf.image.decode_jpeg(image_raw, channels=3)
# Image preprocessing
image_data = tf.image.convert_image_dtype(image_data, dtype=tf.float32)
image_data = tf.expand_dims(image_data, 0)
image_data = tf.image.resize_bilinear(image_data,
[image_size, image_size],
align_corners=False)
image_data = tf.squeeze(image_data)
image_data = image_data / 255.0
# Read a batch of preprocessing images from queue
image_batch = tf.train.batch([image_data, image_filename],
batch_size,
num_threads=num_threads,
allow_smaller_final_batch=True)
return image_batch
def read_data(self):
"""Provides images and camera intrinsics."""
with tf.name_scope('data_loading'):
with tf.name_scope('enqueue_paths'):
seed = random.randint(0, 2**31 - 1)
self.file_lists = self.compile_file_list(self.data_dir, self.input_file)
image_paths_queue = tf.train.string_input_producer(
self.file_lists['image_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None)
)
seg_paths_queue = tf.train.string_input_producer(
self.file_lists['segment_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
cam_paths_queue = tf.train.string_input_producer(
self.file_lists['cam_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
seg_reader = tf.WholeFileReader()
_, seg_contents = seg_reader.read(seg_paths_queue)
if self.file_extension == 'jpg':
image_seq = tf.image.decode_jpeg(image_contents)
seg_seq = tf.image.decode_jpeg(seg_contents, channels=3)
elif self.file_extension == 'png':
image_seq = tf.image.decode_png(image_contents, channels=3)
seg_seq = tf.image.decode_png(seg_contents, channels=3)
with tf.name_scope('load_intrinsics'):
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for _ in range(9):
rec_def.append([1.0])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
with tf.name_scope('convert_image'):
image_seq = self.preprocess_image(image_seq) # Converts to float.
if self.random_color:
with tf.name_scope('image_augmentation'):
image_seq = self.augment_image_colorspace(image_seq)
image_stack = self.unpack_images(image_seq)
seg_stack = self.unpack_images(seg_seq)
if self.flipping_mode != FLIP_NONE:
random_flipping = (self.flipping_mode == FLIP_RANDOM)
with tf.name_scope('image_augmentation_flip'):
image_stack, seg_stack, intrinsics = self.augment_images_flip(
image_stack, seg_stack, intrinsics,
randomized=random_flipping)
if self.random_scale_crop:
with tf.name_scope('image_augmentation_scale_crop'):
image_stack, seg_stack, intrinsics = self.augment_images_scale_crop(
image_stack, seg_stack, intrinsics, self.img_height,
self.img_width)
with tf.name_scope('multi_scale_intrinsics'):
intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
intrinsic_mat.set_shape([self.num_scales, 3, 3])
intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat)
intrinsic_mat_inv.set_shape([self.num_scales, 3, 3])
if self.imagenet_norm:
im_mean = tf.tile(
tf.constant(IMAGENET_MEAN), multiples=[self.seq_length])
im_sd = tf.tile(
tf.constant(IMAGENET_SD), multiples=[self.seq_length])
image_stack_norm = (image_stack - im_mean) / im_sd
else:
image_stack_norm = image_stack
with tf.name_scope('batching'):
if self.shuffle:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.shuffle_batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=self.threads,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size,
min_after_dequeue=self.queue_size)
else:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=1,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size)
return (image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv)
def make_data_tensor(self, train=True):
if train:
folders = self.metatrain_character_folders
# number of tasks, not number of meta-iterations. (divide by metabatch size to measure)
num_total_batches = 200000
if FLAGS.task_type == "ne":
print("Inside ne train")
folders = self.metatrain_character_folders[:50] # 10 classification tasks
num_total_batches = 5000
else:
folders = self.metaval_character_folders
num_total_batches = 600
if FLAGS.task_type == "ne":
print("inside ne val")
if FLAGS.test_set:
folders = self.metaval_character_folders[:15]
else:
folders = self.metaval_character_folders[:15] # 3 classification tasks
num_total_batches = 60
# print("folders", folders)
# make list of files
print('Generating filenames')
if FLAGS.task_setting == 'ne' and train:
# all_filenames = []
# random.shuffle(folders)
# for i in range(len(folders)/self.num_classes):
# #sampled_character_folders = random.sample(folders, self.num_classes)
# #random.shuffle(sampled_character_folders)
# sampled_character_folders = folders[i*self.num_classes:(i+1)*self.num_classes]
# labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False)
# # make sure the above isn't randomized order
# labels = [li[0] for li in labels_and_images]
# filenames = [li[1] for li in labels_and_images]
# all_filenames.extend(filenames)
all_filenames = []
print("len folders", len(folders))
random.shuffle(folders)
task_folders_new = []
for i in range(int(len(folders)/self.num_classes)):
# sampled_character_folders = random.sample(folders, self.num_classes)
# random.shuffle(sampled_character_folders)
sampled_character_folders = folders[i*self.num_classes:(i+1)*self.num_classes]
task_folders_temp = itertools.permutations(sampled_character_folders)
task_folders_new.extend(task_folders_temp)
# print("task_folders_new", task_folders_new)
print("len of task_folders_new", len(task_folders_new))
random.shuffle(task_folders_new)
for i in range(len(task_folders_new)):
sampled_character_folders = task_folders_new[i]
# print("scf", sampled_character_folders)
labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
else:
all_filenames = []
for _ in range(num_total_batches):
sampled_character_folders = random.sample(folders, self.num_classes)
random.shuffle(sampled_character_folders)
labels_and_images = get_images(sampled_character_folders, range(self.num_classes), nb_samples=self.num_samples_per_class, shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
# make queue for tensorflow to read from
filename_queue = tf.train.string_input_producer(tf.convert_to_tensor(all_filenames), shuffle=False)
print('Generating image processing ops')
image_reader = tf.WholeFileReader()
_, image_file = image_reader.read(filename_queue)
if FLAGS.datasource == 'miniimagenet':
image = tf.image.decode_jpeg(image_file, channels=3)
image.set_shape((self.img_size[0],self.img_size[1],3))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
else:
image = tf.image.decode_png(image_file)
image.set_shape((self.img_size[0],self.img_size[1],1))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
image = 1.0 - image # invert
num_preprocess_threads = 1 # TODO - enable this to be set to >1
min_queue_examples = 256
examples_per_batch = self.num_classes * self.num_samples_per_class
batch_image_size = self.batch_size * examples_per_batch
print('Batching images')
print("batch_image_size", batch_image_size)
images = tf.train.batch(
[image],
batch_size = batch_image_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 3 * batch_image_size,
)
print("len images", images.shape)
all_image_batches, all_label_batches = [], []
print('Manipulating image data to be right shape')
for i in range(self.batch_size):
image_batch = images[i*examples_per_batch:(i+1)*examples_per_batch]
if FLAGS.datasource == 'omniglot':
# omniglot augments the dataset by rotating digits to create new classes
# get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes)
rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]), self.num_classes)
# print("labels", labels)
label_batch = tf.convert_to_tensor(labels)
new_list, new_label_list = [], []
# shuffles the data within a batch, class labels remain fixed
for k in range(self.num_samples_per_class):
class_idxs = tf.range(0, self.num_classes)
class_idxs = tf.random_shuffle(class_idxs)
true_idxs = class_idxs * self.num_samples_per_class + k
new_list.append(tf.gather(image_batch,true_idxs))
if FLAGS.datasource == 'omniglot': # and FLAGS.train:
new_list[-1] = tf.stack([tf.reshape(tf.image.rot90(
tf.reshape(new_list[-1][ind], [self.img_size[0], self.img_size[1],1]),
k=tf.cast(rotations[0, class_idxs[ind]], tf.int32)), (self.dim_input,))
for ind in range(self.num_classes)])
new_label_list.append(tf.gather(label_batch, true_idxs))
new_list = tf.concat(new_list, 0) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input]
new_label_list = tf.concat(new_label_list, 0)
all_image_batches.append(new_list)
all_label_batches.append(new_label_list)
all_image_batches = tf.stack(all_image_batches)
all_label_batches = tf.stack(all_label_batches)
print("all_image_batches", all_image_batches)
all_label_batches = tf.one_hot(all_label_batches, self.num_classes)
return all_image_batches, all_label_batches
def make_data_tensor(self, train=True):
if train:
folders = self.metatrain_character_folders
# number of tasks, not number of meta-iterations. (divide by metabatch size to measure)
if FLAGS.expt_number == '14a':
folders = folders[:(5 * self.num_classes)]
elif FLAGS.expt_number == '14b':
folders = folders[:(10 * self.num_classes)]
elif FLAGS.expt_number == '14c':
folders = folders[:(25 * self.num_classes)]
elif FLAGS.expt_number == '14d':
folders = folders[:(50 * self.num_classes)]
elif FLAGS.expt_number == '14e':
folders = folders[:(75 * self.num_classes)]
elif FLAGS.expt_number == '14f':
folders = folders[:(100 * self.num_classes)]
elif FLAGS.expt_number == '14g':
folders = folders[:(150 * self.num_classes)]
elif FLAGS.expt_number == '14h':
folders = folders[:(200 * self.num_classes)]
if FLAGS.expt_number == '6' or FLAGS.expt_number == '8':
print("Inside expt number 6/8")
num_total_batches = 26400
elif FLAGS.expt_number == '6a' or FLAGS.expt_number == '8a':
print("Inside expt number 6a/8a")
num_total_batches = 1100
else:
num_total_batches = 200000
else:
folders = self.metaval_character_folders
num_total_batches = 600
# make list of files
print('Generating filenames')
if FLAGS.expt_number == '2' and train:
print('Inside expt number 2')
all_filenames = []
"""
go over the folders once, group the adjacent 5 classes together as one task. Non-exclusive
"""
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif (FLAGS.expt_number == '9a' or FLAGS.expt_number == '9b'
or FLAGS.expt_number == '9c' or FLAGS.expt_number == '11a1'
or FLAGS.expt_number == '11a2' or FLAGS.expt_number == '11a3' or
('14' in FLAGS.expt_number)) and train:
print('Inside expt number 9a/b/c/11a1/11a2/11a3/14s')
all_filenames = []
"""
go over the folders multiple times, group the adjacent 5 classes together as one task. Non-exclusive
"""
if FLAGS.expt_number == '9a' or FLAGS.expt_number == '11a1' or FLAGS.expt_number == '11a2' or FLAGS.expt_number == '11a3' or (
'14' in FLAGS.expt_number):
total_num_tasks = 200000
elif FLAGS.expt_number == '9b':
total_num_tasks = 26400
elif FLAGS.expt_number == '9c':
total_num_tasks = 1100
for _ in range(
int(total_num_tasks /
(len(folders) / self.num_classes))): #9b
#for _ in range(int(num_total_batches/(len(folders)/self.num_classes))): #9a
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
#random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif (FLAGS.expt_number == '7a' or FLAGS.expt_number == '7b'
or FLAGS.expt_number == '7c' or FLAGS.expt_number == '11b1'
or FLAGS.expt_number == '11b2'
or FLAGS.expt_number == '11b3') and train:
print('Inside expt number 7a/b/c/11b1/11b2/11b3')
all_filenames = []
"""
go over the folders multiple times, group the adjacent 5 classes together as one task. Non-exclusive
"""
if FLAGS.expt_number == '7a' or FLAGS.expt_number == '11b1' or FLAGS.expt_number == '11b2' or FLAGS.expt_number == '11b3':
total_num_tasks = 200000
elif FLAGS.expt_number == '7b':
total_num_tasks = 26400
elif FLAGS.expt_number == '7c':
total_num_tasks = 1100
for _ in range(
int(total_num_tasks /
(len(folders) / self.num_classes))): #7b
#for _ in range(int(num_total_batches/(len(folders)/self.num_classes))): #7a
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif FLAGS.expt_number == '3' and train:
print('Inside expt number 3')
all_filenames = []
"""
removed shuffling of classes in the init function. classes from the same alphabet together now.
go over the folders once, group the adjacent 5 classes together as one task. Non-exclusive
"""
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
#random.shuffle(sampled_character_folders)
#print("Task ", task_count, ": ", sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif (FLAGS.expt_number == '4' or FLAGS.expt_number == '5'
or FLAGS.expt_number == '8c') and train:
print('Inside expt number 4/5/8c')
all_filenames = []
"""
go over the folders once, group the adjacent 5 classes together as one task.
get all permutations of that task. Shuffle these tasks
"""
task_folders_new = []
for task_count in range(int(len(folders) / self.num_classes)):
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
task_folders_temp = permutations(sampled_character_folders)
task_folders_new.extend(task_folders_temp)
print('total number of tasks: ', len(task_folders_new))
random.shuffle(task_folders_new)
for task_count in range(len(task_folders_new)):
#sampled_character_folders = random.sample(folders, self.num_classes)
#sampled_character_folders = folders[task_count*self.num_classes: (task_count+1)*self.num_classes]
sampled_character_folders = task_folders_new[task_count]
#random.shuffle(sampled_character_folders)
#print("Task ", task_count, ": ", sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif FLAGS.expt_number == '4a' and train:
print('Inside expt number 4a')
all_filenames = []
"""
go over all tasks group the same 5 classes together, but shuffle them everytime. Do it till you reach 1100 tasks
"""
i = 0
while i < 1100:
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
random.shuffle(sampled_character_folders)
#print("Task ", task_count, ": ", sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
i += 1
if i >= 1100:
break
elif (FLAGS.expt_number == '12a' or FLAGS.expt_number == '12b'
or FLAGS.expt_number == '12c' or FLAGS.expt_number == '12d'
or FLAGS.expt_number == '12e'
or FLAGS.expt_number == '12f') and train:
print('Inside expt number 12s')
all_filenames = []
"""
make tasks first.
Then go over the tasks mulitiple times to collect data
"""
if FLAGS.expt_number == '12a':
n_tasks = 220
elif FLAGS.expt_number == '12b':
n_tasks = 1100
elif FLAGS.expt_number == '12c':
n_tasks = 5500
elif FLAGS.expt_number == '12d':
n_tasks = 26400
elif FLAGS.expt_number == '12e':
n_tasks = 27500
elif FLAGS.expt_number == '12f':
n_tasks = 137500
task_folders_new = []
for task_count in range(n_tasks):
sampled_character_folders = random.sample(
folders, self.num_classes)
#task_folders_temp = permutations(sampled_character_folders)
task_folders_new.append(sampled_character_folders)
print('total number of tasks: ', len(task_folders_new))
#random.shuffle(task_folders_new)
for task_count in range(num_total_batches):
sampled_character_folders = task_folders_new[task_count %
n_tasks]
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
else:
all_filenames = []
print('Inside expt number 1/6/6a/8/8a/11c1/11c2/11c3')
for _ in range(num_total_batches):
sampled_character_folders = random.sample(
folders, self.num_classes)
random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False,
train=train)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
# make queue for tensorflow to read from
filename_queue = tf.train.string_input_producer(
tf.convert_to_tensor(all_filenames), shuffle=False)
print('Generating image processing ops')
image_reader = tf.WholeFileReader()
_, image_file = image_reader.read(filename_queue)
if FLAGS.datasource == 'miniimagenet':
image = tf.image.decode_jpeg(image_file, channels=3)
image.set_shape((self.img_size[0], self.img_size[1], 3))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
else:
image = tf.image.decode_png(image_file)
image.set_shape((self.img_size[0], self.img_size[1], 1))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
image = 1.0 - image # invert
num_preprocess_threads = 1 # TODO - enable this to be set to >1
min_queue_examples = 256
examples_per_batch = self.num_classes * self.num_samples_per_class
batch_image_size = self.batch_size * examples_per_batch
print('Batching images')
images = tf.train.batch(
[image],
batch_size=batch_image_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 3 * batch_image_size,
)
all_image_batches, all_label_batches = [], []
print('Manipulating image data to be right shape')
for i in range(self.batch_size):
image_batch = images[i * examples_per_batch:(i + 1) *
examples_per_batch]
if FLAGS.datasource == 'omniglot':
# omniglot augments the dataset by rotating digits to create new classes
# get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes)
rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]),
self.num_classes)
label_batch = tf.convert_to_tensor(labels)
new_list, new_label_list = [], []
for k in range(self.num_samples_per_class):
class_idxs = tf.range(0, self.num_classes)
class_idxs = tf.random_shuffle(class_idxs)
true_idxs = class_idxs * self.num_samples_per_class + k
new_list.append(tf.gather(image_batch, true_idxs))
if FLAGS.datasource == 'omniglot': # and FLAGS.train:
new_list[-1] = tf.stack([
tf.reshape(
tf.image.rot90(tf.reshape(
new_list[-1][ind],
[self.img_size[0], self.img_size[1], 1]),
k=tf.cast(
rotations[0, class_idxs[ind]],
tf.int32)), (self.dim_input, ))
for ind in range(self.num_classes)
])
new_label_list.append(tf.gather(label_batch, true_idxs))
new_list = tf.concat(
new_list, 0
) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input]
new_label_list = tf.concat(new_label_list, 0)
all_image_batches.append(new_list)
all_label_batches.append(new_label_list)
all_image_batches = tf.stack(all_image_batches)
all_label_batches = tf.stack(all_label_batches)
all_label_batches = tf.one_hot(all_label_batches, self.num_classes)
return all_image_batches, all_label_batches
def load_examples():
if a.input_dir is None or not os.path.exists(a.input_dir):
raise Exception("input_dir does not exist")
input_paths = glob.glob(os.path.join(a.input_dir, "*.jpg"))
decode = tf.image.decode_jpeg
if len(input_paths) == 0:
input_paths = glob.glob(os.path.join(a.input_dir, "*.png"))
decode = tf.image.decode_png
if len(input_paths) == 0:
raise Exception("input_dir contains no image files")
def get_name(path):
name, _ = os.path.splitext(os.path.basename(path))
return name
# if the image names are numbers, sort by the value rather than asciibetically
# having sorted inputs means that the outputs are sorted in test mode
if all(get_name(path).isdigit() for path in input_paths):
input_paths = sorted(input_paths, key=lambda path: int(get_name(path)))
else:
input_paths = sorted(input_paths)
with tf.name_scope("load_images"):
path_queue = tf.train.string_input_producer(input_paths,
shuffle=a.mode == "train")
reader = tf.WholeFileReader()
paths, contents = reader.read(path_queue)
raw_input = decode(contents)
raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32)
assertion = tf.assert_equal(tf.shape(raw_input)[2],
3,
message="image does not have 3 channels")
with tf.control_dependencies([assertion]):
raw_input = tf.identity(raw_input)
raw_input.set_shape([None, None, 3])
if a.lab_colorization:
# load color and brightness from image, no B image exists here
lab = rgb_to_lab(raw_input)
L_chan, a_chan, b_chan = preprocess_lab(lab)
a_images = tf.expand_dims(L_chan, axis=2)
b_images = tf.stack([a_chan, b_chan], axis=2)
else:
# break apart image pair and move to range [-1, 1]
width = tf.shape(raw_input)[1] # [height, width, channels]
a_images = preprocess(raw_input[:, :width // 2, :])
b_images = preprocess(raw_input[:, width // 2:, :])
if a.which_direction == "AtoB":
inputs, targets = [a_images, b_images]
elif a.which_direction == "BtoA":
inputs, targets = [b_images, a_images]
else:
raise Exception("invalid direction")
# synchronize seed for image operations so that we do the same operations to both
# input and output images
seed = random.randint(0, 2**31 - 1)
def transform(image):
r = image
if a.flip:
r = tf.image.random_flip_left_right(r, seed=seed)
# area produces a nice downscaling, but does nearest neighbor for upscaling
# assume we're going to be doing downscaling here
r = tf.image.resize_images(r, [a.scale_size, a.scale_size],
method=tf.image.ResizeMethod.AREA)
offset = tf.cast(tf.floor(
tf.random_uniform([2], 0, a.scale_size - CROP_SIZE + 1,
seed=seed)),
dtype=tf.int32)
if a.scale_size > CROP_SIZE:
r = tf.image.crop_to_bounding_box(r, offset[0], offset[1],
CROP_SIZE, CROP_SIZE)
elif a.scale_size < CROP_SIZE:
raise Exception("scale size cannot be less than crop size")
return r
with tf.name_scope("input_images"):
input_images = transform(inputs)
with tf.name_scope("target_images"):
target_images = transform(targets)
paths_batch, inputs_batch, targets_batch = tf.train.batch(
[paths, input_images, target_images], batch_size=a.batch_size)
steps_per_epoch = int(math.ceil(len(input_paths) / a.batch_size))
return Examples(
paths=paths_batch,
inputs=inputs_batch,
targets=targets_batch,
count=len(input_paths),
steps_per_epoch=steps_per_epoch,
)
def make_data_tensor(self, train=True, ne=False, ne_task_folders=None):
ne_tasks_n = 1
if train or ne:
folders = self.metatrain_character_folders
# number of tasks, not number of meta-iterations. (divide by metabatch size to measure)
num_total_batches = 200000
else:
folders = self.metaval_character_folders
num_total_batches = 600
task_folders = []
# make list of files
print('Generating filenames')
all_filenames = []
if (not train) and ne:
print('Inside test ne filename generation')
for outer_task_count in range(
int(200000 / (len(folders) / self.num_classes))):
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
#random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif train and ne:
print('Inside train ne filename generation')
"""
Shuffle the tasks
"""
task_folders = []
for outer_task_count in range(
int(200000 / (len(folders) / self.num_classes))):
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
task_folders.append(sampled_character_folders)
random.shuffle(task_folders)
for task_count in range(len(task_folders)):
#random.shuffle(sampled_character_folders)
sampled_character_folders = task_folders[task_count]
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif train and (not ne) and (FLAGS.expt_number == '7'
or FLAGS.expt_number == '8'):
#no overlap between b1 and b2
for task_count in range(num_total_batches):
ne_folders = []
for i in range(task_count * ne_tasks_n,
(task_count + 1) * ne_tasks_n):
ne_folders.extend(ne_task_folders[i %
len(ne_task_folders)])
#print('ne_folders', ne_folders)
folders_temp = set(folders) - set(ne_folders)
sampled_character_folders = random.sample(
folders_temp, self.num_classes)
#print('folders', folders)
#print('sampled_character_folders', sampled_character_folders) #sample from set(folders) - set(ne_folders)
random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
else:
if (FLAGS.expt_number == '2'
or FLAGS.expt_number == '5') and train:
for task_count in range(int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
#random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
elif (FLAGS.expt_number == '3'
or FLAGS.expt_number == '6') and train:
for _ in range(
int(num_total_batches /
(len(folders) / self.num_classes))):
for task_count in range(
int(len(folders) / self.num_classes)):
#sampled_character_folders = random.sample(folders, self.num_classes)
sampled_character_folders = folders[task_count *
self.num_classes:
(task_count + 1) *
self.num_classes]
#random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
else:
print('In expt 1/4')
for _ in range(num_total_batches):
sampled_character_folders = random.sample(
folders, self.num_classes)
#random.shuffle(sampled_character_folders)
labels_and_images = get_images(
sampled_character_folders,
range(self.num_classes),
nb_samples=self.num_samples_per_class,
shuffle=False)
# make sure the above isn't randomized order
labels = [li[0] for li in labels_and_images]
filenames = [li[1] for li in labels_and_images]
all_filenames.extend(filenames)
if (not train) and ne:
batch_size = self.batch_size * int(len(folders) / self.num_classes)
print('ne batch size: ', batch_size)
elif train and ne:
batch_size = self.batch_size * ne_tasks_n
else:
batch_size = self.batch_size
# make queue for tensorflow to read from
filename_queue = tf.train.string_input_producer(
tf.convert_to_tensor(all_filenames), shuffle=False)
print('Generating image processing ops')
image_reader = tf.WholeFileReader()
_, image_file = image_reader.read(filename_queue)
if FLAGS.datasource == 'miniimagenet':
image = tf.image.decode_jpeg(image_file, channels=3)
image.set_shape((self.img_size[0], self.img_size[1], 3))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
else:
image = tf.image.decode_png(image_file)
image.set_shape((self.img_size[0], self.img_size[1], 1))
image = tf.reshape(image, [self.dim_input])
image = tf.cast(image, tf.float32) / 255.0
image = 1.0 - image # invert
num_preprocess_threads = 1 # TODO - enable this to be set to >1
min_queue_examples = 256
examples_per_batch = self.num_classes * self.num_samples_per_class
batch_image_size = batch_size * examples_per_batch
print('Batching images')
images = tf.train.batch(
[image],
batch_size=batch_image_size,
num_threads=num_preprocess_threads,
capacity=min_queue_examples + 3 * batch_image_size,
)
all_image_batches, all_label_batches = [], []
print('Manipulating image data to be right shape')
for i in range(batch_size):
image_batch = images[
i * examples_per_batch:(i + 1) *
examples_per_batch] #examples from the same task
if FLAGS.datasource == 'omniglot':
# omniglot augments the dataset by rotating digits to create new classes
# get rotation per class (e.g. 0,1,2,0,0 if there are 5 classes)
rotations = tf.multinomial(tf.log([[1., 1., 1., 1.]]),
self.num_classes)
label_batch = tf.convert_to_tensor(labels)
new_list, new_label_list = [], []
for k in range(self.num_samples_per_class):
class_idxs = tf.range(0, self.num_classes)
class_idxs = tf.random_shuffle(class_idxs)
true_idxs = class_idxs * self.num_samples_per_class + k
new_list.append(tf.gather(image_batch, true_idxs))
if FLAGS.datasource == 'omniglot': # and FLAGS.train:
new_list[-1] = tf.stack([
tf.reshape(
tf.image.rot90(tf.reshape(
new_list[-1][ind],
[self.img_size[0], self.img_size[1], 1]),
k=tf.cast(
rotations[0, class_idxs[ind]],
tf.int32)), (self.dim_input, ))
for ind in range(self.num_classes)
])
new_label_list.append(tf.gather(label_batch, true_idxs))
new_list = tf.concat(
new_list, 0
) # has shape [self.num_classes*self.num_samples_per_class, self.dim_input]
new_label_list = tf.concat(new_label_list, 0)
all_image_batches.append(new_list)
all_label_batches.append(new_label_list)
all_image_batches = tf.stack(all_image_batches)
all_label_batches = tf.stack(all_label_batches)
all_label_batches = tf.one_hot(all_label_batches, self.num_classes)
print("label_batches", all_label_batches)
return all_image_batches, all_label_batches, task_folders
def load_examples():
if a.input_dir is None or not os.path.exists(a.input_dir):
raise Exception("input_dir does not exist")
input_paths = glob.glob(os.path.join(a.input_dir, "*", "*.jpg"))
decode = tf.image.decode_jpeg
if len(input_paths) == 0:
input_paths = glob.glob(os.path.join(a.input_dir, "*", "*.png"))
decode = tf.image.decode_png
print("len = ", len(input_paths))
if len(input_paths) == 0:
raise Exception("input_dir contains no image files")
def get_name(path):
name, _ = os.path.splitext(os.path.basename(path))
return name
if all(get_name(path).isdigit() for path in input_paths):
input_paths = sorted(input_paths, key=lambda path: int(get_name(path)))
else:
# if the image names are numbers, sort by the value rather than asciibetically
# having sorted inputs means that the outputs are sorted in test mode
input_paths = sorted(input_paths)
with tf.name_scope("load_images"):
path_queue = tf.train.string_input_producer(input_paths,
shuffle=a.mode == "train")
reader = tf.WholeFileReader()
paths, contents = reader.read(path_queue)
raw_input = decode(contents)
raw_input = tf.image.convert_image_dtype(raw_input, dtype=tf.float32)
assertion = tf.assert_equal(tf.shape(raw_input)[2],
1,
message="image does not have 1 channels")
with tf.control_dependencies([assertion]):
raw_input = tf.identity(raw_input)
raw_input.set_shape([IMAGE_HEIGHT, IMAGE_WIDTH * 2, 1])
# break apart image pair and move to range [-1, 1]
width = tf.shape(raw_input)[1] # [height, width, channels]
a_images = preprocess(raw_input[:, :width // 2, :])
b_images = preprocess(raw_input[:, width // 2:, :])
if a.which_direction == "AtoB":
inputs, targets = [a_images, b_images]
elif a.which_direction == "BtoA":
inputs, targets = [b_images, a_images]
else:
raise Exception("invalid direction")
# synchronize seed for image operations so that we do the same operations to both
# input and output images
seed = random.randint(0, 2**31 - 1)
def transform(image):
r = image
if a.flip:
r = tf.image.random_flip_left_right(r, seed=seed)
if a.mode == "train":
# crop image
h = r.get_shape().as_list()[0]
w = r.get_shape().as_list()[1]
h_offset = tf.cast(tf.floor(
tf.random_uniform([1], 0, h - CROP_SIZE + 1, seed=seed)),
dtype=tf.int32)
w_offset = tf.cast(tf.floor(
tf.random_uniform([1], 0, w - CROP_SIZE + 1, seed=seed)),
dtype=tf.int32)
r = tf.image.crop_to_bounding_box(r, h_offset[0], w_offset[0],
CROP_SIZE, CROP_SIZE)
return r
with tf.name_scope("input_images"):
input_images = transform(inputs)
with tf.name_scope("target_images"):
target_images = transform(targets)
paths_batch, inputs_batch, targets_batch = tf.train.batch(
[paths, input_images, target_images], batch_size=a.batch_size)
steps_per_epoch = int(math.ceil(len(input_paths) / a.batch_size))
print(inputs_batch.get_shape().as_list())
return Examples(
paths=paths_batch,
inputs=inputs_batch,
targets=targets_batch,
count=len(input_paths),
steps_per_epoch=steps_per_epoch,
)
def read_data(self):
"""Provides images and camera intrinsics."""
with tf.name_scope("data_loading"):
with tf.name_scope("enqueue_paths"):
seed = random.randint(0, 2**31 - 1)
self.file_lists = self.compile_file_list(
self.data_dir, self.input_file)
image_paths_queue = tf.train.string_input_producer(
self.file_lists["image_file_list"],
seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None),
)
seg_paths_queue = tf.train.string_input_producer(
self.file_lists["segment_file_list"],
seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None),
)
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
seg_reader = tf.WholeFileReader()
_, seg_contents = seg_reader.read(seg_paths_queue)
if self.file_extension == "jpg":
image_seq = tf.image.decode_jpeg(image_contents)
seg_seq = tf.image.decode_jpeg(seg_contents, channels=3)
elif self.file_extension == "png":
image_seq = tf.image.decode_png(image_contents, channels=3)
seg_seq = tf.image.decode_png(seg_contents, channels=3)
with tf.name_scope("load_intrinsics"):
intrinsics = tf.random.uniform(shape=(3, 3))
with tf.name_scope("convert_image"):
image_seq = self.preprocess_image(
image_seq) # Converts to float.
if self.random_color:
with tf.name_scope("image_augmentation"):
image_seq = self.augment_image_colorspace(image_seq)
image_stack = self.unpack_images(image_seq)
seg_stack = self.unpack_images(seg_seq)
if self.flipping_mode != FLIP_NONE:
random_flipping = self.flipping_mode == FLIP_RANDOM
with tf.name_scope("image_augmentation_flip"):
image_stack, seg_stack, intrinsics = self.augment_images_flip(
image_stack,
seg_stack,
intrinsics,
randomized=random_flipping)
if self.random_scale_crop:
with tf.name_scope("image_augmentation_scale_crop"):
image_stack, seg_stack, intrinsics = self.augment_images_scale_crop(
image_stack,
seg_stack,
intrinsics,
self.img_height,
self.img_width,
)
with tf.name_scope("multi_scale_intrinsics"):
intrinsic_mat = self.get_multi_scale_intrinsics(
intrinsics, self.num_scales)
intrinsic_mat.set_shape([self.num_scales, 3, 3])
intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat)
intrinsic_mat_inv.set_shape([self.num_scales, 3, 3])
if self.imagenet_norm:
im_mean = tf.tile(tf.constant(IMAGENET_MEAN),
multiples=[self.seq_length])
im_sd = tf.tile(tf.constant(IMAGENET_SD),
multiples=[self.seq_length])
image_stack_norm = (image_stack - im_mean) / im_sd
else:
image_stack_norm = image_stack
with tf.name_scope("batching"):
if self.shuffle:
(
image_stack,
image_stack_norm,
seg_stack,
intrinsic_mat,
intrinsic_mat_inv,
) = tf.train.shuffle_batch(
[
image_stack,
image_stack_norm,
seg_stack,
intrinsic_mat,
intrinsic_mat_inv,
],
batch_size=self.batch_size,
num_threads=self.threads,
capacity=self.queue_size +
QUEUE_BUFFER * self.batch_size,
min_after_dequeue=self.queue_size,
)
else:
(
image_stack,
image_stack_norm,
seg_stack,
intrinsic_mat,
intrinsic_mat_inv,
) = tf.train.batch(
[
image_stack,
image_stack_norm,
seg_stack,
intrinsic_mat,
intrinsic_mat_inv,
],
batch_size=self.batch_size,
num_threads=1,
capacity=self.queue_size +
QUEUE_BUFFER * self.batch_size,
)
return (
image_stack,
image_stack_norm,
seg_stack,
intrinsic_mat,
intrinsic_mat_inv,
)