def make_sample_grid_and_save(self,
ops,
batch_size=10,
random=12,
interpolation=16,
height=16,
save_to_disk=True):
# Gather images
pool_size = random * height + 2 * height
current_size = 0
with tf.Graph().as_default():
data_in = self.test_data.make_one_shot_iterator().get_next()
with tf.Session() as sess_new:
images = []
labels = []
while current_size < pool_size:
data_dict = sess_new.run(data_in)
images.append(data_dict["x"])
labels.append(data_dict["label"].argmax(1))
current_size += images[-1].shape[0]
images = np.concatenate(images, axis=0)[:pool_size]
labels = np.concatenate(labels, axis=0)[:pool_size]
def batched_op(op, op_input, array):
return np.concatenate(
[
self.tf_sess.run(op, feed_dict={
op_input: array[x:x + batch_size]})
for x in range(0, array.shape[0], batch_size)
],
axis=0)
if random:
random_x = images[:random * height]
random_l = labels[:random * height]
random_y = batched_op(ops.ae, ops.x, random_x)
if ops.gap_weight is not None:
encode_map = batched_op(ops.encode, ops.x, random_x)
gap_weight = self.tf_sess.run(ops.gap_weight) # 2x10
encode_map, origin = utils.convert_to_heatmap(encode_map, gap_weight,
random_x.copy()[:,:,:, ::-1], random_l)
fig_size = 32
big_images = np.zeros(shape=[height * fig_size, random * fig_size, 3], dtype=np.uint8)
big_encode = np.zeros(shape=[height * fig_size, random * fig_size, 3], dtype=np.uint8)
big_encode_map = np.zeros(shape=[height * fig_size, random * 2 * fig_size, 3], dtype=np.uint8)
for i in range(encode_map.shape[0]):
col = i % random
row = (i - col) // random
big_encode_map[row * fig_size: (row + 1) * fig_size,
col * fig_size * 2: (col + 1) * fig_size * 2] = \
np.concatenate((origin[i], encode_map[i]), axis=1)
big_images[row * fig_size: (row + 1) * fig_size,
col * fig_size: (col + 1) * fig_size] = origin[i]
big_encode[row * fig_size: (row + 1) * fig_size,
col * fig_size: (col + 1) * fig_size] = encode_map[i]
encode_map = big_encode_map
cv2.imwrite(os.path.join(self.image_dir, "heatmap%d.png" % (self.cur_nimg >> 10)),
encode_map)
cv2.imwrite(os.path.join(self.image_dir, "heatmap_origin%d.png" % (self.cur_nimg >> 10)),
big_images)
cv2.imwrite(os.path.join(self.image_dir, "heatmap_map%d.png" % (self.cur_nimg >> 10)),
big_encode)
randoms = np.concatenate([random_x, random_y], axis=2)
image_random = utils.images_to_grid(
randoms.reshape((height, random) + randoms.shape[1:]))
else:
image_random = None
# Interpolations
interpolation_x = images[-2 * height:]
latent_x = batched_op(ops.encode, ops.x, interpolation_x)
latents = []
for x in range(interpolation):
latents.append((latent_x[:height] * (interpolation - x - 1) +
latent_x[height:] * x) / float(interpolation - 1))
latents = np.concatenate(latents, axis=0)
interpolation_y = batched_op(ops.decode, ops.h, latents)
interpolation_y = interpolation_y.reshape(
(interpolation, height) + interpolation_y.shape[1:])
interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
image_interpolation = utils.images_to_grid(interpolation_y)
# Interpolations
latents_slerp = []
dots = np.sum(latent_x[:height] * latent_x[height:],
tuple(range(1, len(latent_x.shape))),
keepdims=True)
norms = np.sum(latent_x * latent_x,
tuple(range(1, len(latent_x.shape))),
keepdims=True)
cosine_dist = dots / np.sqrt(norms[:height] * norms[height:])
omega = np.arccos(cosine_dist)
for x in range(interpolation):
t = x / float(interpolation - 1)
latents_slerp.append(
np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
np.sin(t * omega) / np.sin(omega) * latent_x[height:])
latents_slerp = np.concatenate(latents_slerp, axis=0)
interpolation_y_slerp = batched_op(ops.decode, ops.h, latents_slerp)
interpolation_y_slerp = interpolation_y_slerp.reshape(
(interpolation, height) + interpolation_y_slerp.shape[1:])
interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
image_interpolation_slerp = utils.images_to_grid(interpolation_y_slerp)
## generate synthetic images
random_latents = np.random.standard_normal(latents.shape)
samples_y = batched_op(ops.decode, ops.h, random_latents)
samples_y = samples_y.reshape((interpolation, height) + samples_y.shape[1:])
samples_y = samples_y.transpose(1, 0, 2, 3, 4)
image_samples = utils.images_to_grid(samples_y)
if random:
image = np.concatenate(
[image_random, image_interpolation, image_interpolation_slerp,
image_samples], axis=1)
else:
image = np.concatenate(
[image_interpolation, image_interpolation_slerp,
image_samples], axis=1)
if save_to_disk:
utils.save_images(utils.to_png(image), self.image_dir,
self.cur_nimg)
return (image_random, image_interpolation, image_interpolation_slerp,
image_samples)
def make_sample_grid_and_save(self,
ops,
batch_size=10,
random=4,
interpolation=16,
height=16,
save_to_disk=True):
# Gather images
pool_size = random * height + 2 * height
current_size = 0
with tf.Graph().as_default():
data_in = self.test_data.make_one_shot_iterator().get_next()
with tf.Session() as sess_new:
images = []
while current_size < pool_size:
images.append(sess_new.run(data_in)['x'])
current_size += images[-1].shape[0]
images = np.concatenate(images, axis=0)[:pool_size]
def batched_op(op, op_input, array):
return np.concatenate([
self.tf_sess.run(op,
feed_dict={op_input: array[x:x + batch_size]})
for x in range(0, array.shape[0], batch_size)
],
axis=0)
def batched_op_list(op, op_input, array):
return np.concatenate([
self.tf_sess.run(op,
feed_dict={
op_input[0]: array[0][x:x + batch_size],
op_input[1]: array[1][x:x + batch_size],
op_input[2]: array[2][x:x + batch_size]
})
for x in range(0, array[0].shape[0], batch_size)
],
axis=0)
# Random reconstructions
if random:
random_x = images[:random * height]
random_y = batched_op(ops.ae, ops.x, random_x)
randoms = np.concatenate([random_x, random_y], axis=2)
image_random = utils.images_to_grid(
randoms.reshape((height, random) + randoms.shape[1:]))
else:
image_random = None
# Linear interpolations
interpolation_x = images[-2 * height:]
latent_x = batched_op(ops.encode, ops.x, interpolation_x)
latents = []
for x in range(interpolation):
latents.append((latent_x[:height] *
(interpolation - x - 1) + latent_x[height:] * x) /
float(interpolation - 1))
latents = np.concatenate(latents, axis=0)
interpolation_y = batched_op(ops.decode, ops.h, latents)
interpolation_y = interpolation_y.reshape((interpolation, height) +
interpolation_y.shape[1:])
interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
image_interpolation = utils.images_to_grid(interpolation_y)
# Free interpolation
interpolation_x = images[-2 * height:]
latent_x = batched_op(ops.encode, ops.x, interpolation_x)
latents = []
for x in range(interpolation):
alpha = x / float(interpolation - 1) * np.ones(
shape=[height, 1, 1, 1], dtype=np.float32)
interp_latent = batched_op_list(
ops.my_encode_mix, [ops.h, ops.h_b, ops.alpha_h],
[latent_x[height:], latent_x[:height], alpha])
latents.append(interp_latent)
latents = np.concatenate(latents, axis=0)
interpolation_y = batched_op(ops.decode, ops.h, latents)
interpolation_y = interpolation_y.reshape((interpolation, height) +
interpolation_y.shape[1:])
interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
oppreserve_interpolation2 = utils.images_to_grid(interpolation_y)
# generate synthetic images
random_latents = np.random.standard_normal(latents.shape)
samples_y = batched_op(ops.decode, ops.h, random_latents)
samples_y = samples_y.reshape((interpolation, height) +
samples_y.shape[1:])
samples_y = samples_y.transpose(1, 0, 2, 3, 4)
image_samples = utils.images_to_grid(samples_y)
if random:
image = np.concatenate([
image_random, image_interpolation, oppreserve_interpolation2,
image_samples
],
axis=1)
else:
image = np.concatenate([
image_interpolation, oppreserve_interpolation2, image_samples
],
axis=1)
if save_to_disk:
utils.save_images(utils.to_png(image), self.image_dir,
self.cur_nimg)
return (image_random, image_interpolation, oppreserve_interpolation2,
image_samples)
def make_sample_grid_and_save(self,
ops,
batch_size=16,
random=4,
interpolation=16,
height=16,
save_to_disk=True):
# Gather images
pool_size = random * height + 2 * height
current_size = 0
with tf.Graph().as_default():
data_in = self.test_data.make_one_shot_iterator().get_next()
with tf.Session() as sess_new:
images = []
while current_size < pool_size:
images.append(sess_new.run(data_in)['x'])
current_size += images[-1].shape[0]
images = np.concatenate(images, axis=0)[:pool_size]
def batched_op(op, op_input, array):
return np.concatenate(
[
self.tf_sess.run(op, feed_dict={
op_input: array[x:x + batch_size]})
for x in range(0, array.shape[0], batch_size)
],
axis=0)
# Random reconstructions
if random:
random_x = images[:random * height]
random_y = batched_op(ops.ae, ops.x, random_x)
randoms = np.concatenate([random_x, random_y], axis=2)
image_random = utils.images_to_grid(
randoms.reshape((height, random) + randoms.shape[1:]))
else:
image_random = None
# Interpolations
interpolation_x = images[-2 * height:]
latent_x = batched_op(ops.encode, ops.x, interpolation_x)
latents = []
for x in range(interpolation):
latents.append((latent_x[:height] * (interpolation - x - 1) +
latent_x[height:] * x) / float(interpolation - 1))
latents = np.concatenate(latents, axis=0)
interpolation_y = batched_op(ops.decode, ops.h, latents)
interpolation_y = interpolation_y.reshape(
(interpolation, height) + interpolation_y.shape[1:])
interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
image_interpolation = utils.images_to_grid(interpolation_y)
latents_slerp = []
dots = np.sum(latent_x[:height] * latent_x[height:],
tuple(range(1, len(latent_x.shape))),
keepdims=True)
norms = np.sum(latent_x * latent_x,
tuple(range(1, len(latent_x.shape))),
keepdims=True)
cosine_dist = dots / np.sqrt(norms[:height] * norms[height:])
omega = np.arccos(cosine_dist)
for x in range(interpolation):
t = x / float(interpolation - 1)
latents_slerp.append(
np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
np.sin(t * omega) / np.sin(omega) * latent_x[height:])
latents_slerp = np.concatenate(latents_slerp, axis=0)
interpolation_y_slerp = batched_op(ops.decode, ops.h, latents_slerp)
interpolation_y_slerp = interpolation_y_slerp.reshape(
(interpolation, height) + interpolation_y_slerp.shape[1:])
interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
image_interpolation_slerp = utils.images_to_grid(interpolation_y_slerp)
random_latents = np.random.standard_normal(latents.shape)
samples_y = batched_op(ops.decode, ops.h, random_latents)
samples_y = samples_y.reshape(
(interpolation, height) + samples_y.shape[1:])
samples_y = samples_y.transpose(1, 0, 2, 3, 4)
image_samples = utils.images_to_grid(samples_y)
if random:
image = np.concatenate(
[image_random, image_interpolation, image_interpolation_slerp,
image_samples], axis=1)
else:
image = np.concatenate(
[image_interpolation, image_interpolation_slerp,
image_samples], axis=1)
if save_to_disk:
utils.save_images(utils.to_png(image), self.image_dir,
self.cur_nimg)
return (image_random, image_interpolation, image_interpolation_slerp,
image_samples)
def make_sample_grid_and_save(self,
ops,
batch_size=16,
random=4,
interpolation=16,
height=16,
save_to_disk=True):
"""
:param ops: AEops class, including train_op
:param batch_size:
:param random: number of reconstructed images = random * height
:param interpolation: number of interpolation, namely the row number of compositive image
:param height: number of hight
:param save_to_disk:
:return: recon, inter, slerp, samples
"""
# Gather images
pool_size = random * height + 2 * height # 96
current_size = 0
with tf.Graph().as_default():
data_in = self.test_data.make_one_shot_iterator().get_next()
with tf.Session() as sess_new:
images = []
while current_size < pool_size:
images.append(sess_new.run(data_in)['x'])
current_size += images[-1].shape[0]
images = np.concatenate(images,
axis=0)[:pool_size] # [96, 32, 32, 1]
def batched_op(op, op_input, array):
return np.concatenate([
self.tf_sess.run(op,
feed_dict={op_input: array[x:x + batch_size]})
for x in range(0, array.shape[0], batch_size)
],
axis=0)
# 1. Random reconstructions
if random: # not zero
random_x = images[:random * height] # [64, 32, 32, 1]
random_y = batched_op(ops.ae, ops.x, random_x)
randoms = np.concatenate(
[random_x, random_y],
axis=2) # ae output: [64, 32, 32, 1] => [64, 32, 64, 1]
image_random = utils.images_to_grid( # [16, 4, 32, 64, 1] => [512, 256, 1]
randoms.reshape((height, random) + randoms.shape[1:]))
else:
image_random = None
# 2. Interpolations
interpolation_x = images[-2 * height:] # [32, 32, 32, 1]
latent_x = batched_op(ops.encode, ops.x,
interpolation_x) # [32, 4, 4, 16]
latents = []
for x in range(interpolation):
latents.append((latent_x[:height] *
(interpolation - x - 1) + latent_x[height:] * x) /
float(interpolation - 1))
latents = np.concatenate(latents, axis=0) # [256, 4, 4, 16]
interpolation_y = batched_op(ops.decode, ops.h,
latents) # [256, 32, 32, 1]
interpolation_y = interpolation_y.reshape( # [16, 16, 32, 32, 1]
(interpolation, height) + interpolation_y.shape[1:])
interpolation_y = interpolation_y.transpose(1, 0, 2, 3, 4)
image_interpolation = utils.images_to_grid(
interpolation_y) # [512, 512, 1]
# 3. Interpolation by slerp
latents_slerp = []
dots = np.sum(latent_x[:height] * latent_x[height:],
tuple(range(1, len(latent_x.shape))),
keepdims=True) # [16, 1, 1, 1]
norms = np.sum(latent_x * latent_x,
tuple(range(1, len(latent_x.shape))),
keepdims=True) # [32, 1, 1, 1]
cosine_dist = dots / np.sqrt(
norms[:height] * norms[height:]) # [16, 1, 1, 1]
omega = np.arccos(cosine_dist)
for x in range(interpolation):
t = x / float(interpolation - 1)
latents_slerp.append(
np.sin((1 - t) * omega) / np.sin(omega) * latent_x[:height] +
np.sin(t * omega) / np.sin(omega) * latent_x[height:])
latents_slerp = np.concatenate(
latents_slerp, axis=0) # 16 of[16, 4, 4, 16] => [256, 4, 4, 16]
interpolation_y_slerp = batched_op(ops.decode, ops.h,
latents_slerp) # [256, 32, 32, 1]
interpolation_y_slerp = interpolation_y_slerp.reshape(
(interpolation, height) +
interpolation_y_slerp.shape[1:]) # [16, 16, 32, 32, 1]
interpolation_y_slerp = interpolation_y_slerp.transpose(1, 0, 2, 3, 4)
image_interpolation_slerp = utils.images_to_grid(
interpolation_y_slerp) # [512, 512, 1]
# 4. get decoder by random normal dist of hidden h
random_latents = np.random.standard_normal(
latents.shape) # [256, 4, 4, 16]
samples_y = batched_op(ops.decode, ops.h, random_latents)
samples_y = samples_y.reshape((interpolation, height) +
samples_y.shape[1:])
samples_y = samples_y.transpose(1, 0, 2, 3, 4)
image_samples = utils.images_to_grid(samples_y) # [512, 512, 1]
if random: # [512, 256+512+512+512, 1]
image = np.concatenate([
image_random, image_interpolation, image_interpolation_slerp,
image_samples
],
axis=1)
else:
image = np.concatenate([
image_interpolation, image_interpolation_slerp, image_samples
],
axis=1)
if save_to_disk:
utils.save_images(utils.to_png(image), self.image_dir,
self.cur_nimg)
return image_random, image_interpolation, image_interpolation_slerp, image_samples