def sample(self, data, mgen):
if self.num_channels == 3:
data = np.cast[np.float32]((data - 127.5) / 127.5)
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update({model.x: ds[i] for i, model in enumerate(self.models)})
feed_dict.update({model.x_bar: ds[i] for i, model in enumerate(self.models)})
masks_np = [mgen.gen(self.batch_size//self.nr_gpu) for i in range(self.nr_gpu)]
feed_dict.update({model.masks: masks_np[i] for i, model in enumerate(self.models)})
feed_dict.update({model.input_masks: masks_np[i] for i, model in enumerate(self.models)})
ret = self.sess.run([model.z_mu for model in self.models]+[model.z_log_sigma_sq for model in self.models], feed_dict=feed_dict)
z_mu = np.concatenate(ret[:len(ret)//2], axis=0)
z_log_sigma_sq = np.concatenate(ret[len(ret)//2:], axis=0)
z_sigma = np.sqrt(np.exp(z_log_sigma_sq))
z = np.random.normal(loc=z_mu, scale=z_sigma)
z = np.split(z, self.nr_gpu)
feed_dict.update({model.z_ph:z[i] for i, model in enumerate(self.models)})
feed_dict.update({model.use_z_ph: True for model in self.models})
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i], num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
x_gen = [ds[i].copy() for i in range(self.nr_gpu)]
for yi in range(self.img_size):
for xi in range(self.img_size):
if np.min(np.array([masks_np[i][:, yi, xi] for i in range(self.nr_gpu)])) > 0:
continue
feed_dict.update({model.x_bar:x_gen[i] for i, model in enumerate(self.models)})
x_hats = self.sess.run([model.x_hat for model in self.models], feed_dict=feed_dict)
for i in range(self.nr_gpu):
bmask = broadcast_masks_np(masks_np[i][:, yi, xi] , num_channels=self.num_channels)
x_gen[i][:, yi, xi, :] = x_hats[i][:, yi, xi, :] * (1.-bmask) + x_gen[i][:, yi, xi, :] * bmask
gen_data = np.concatenate(x_gen, axis=0)
return ori_data, masked_data, gen_data
def _examine_posterior(self, image, mgen, masks_np=None):
data = np.array([image for i in range(self.batch_size)])
if self.num_channels == 3:
data = np.cast[np.float32]((data - 127.5) / 127.5)
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update(
{model.x: ds[i]
for i, model in enumerate(self.models)})
feed_dict.update(
{model.x_bar: ds[i]
for i, model in enumerate(self.models)})
if masks_np is None:
masks_np = [
mgen.gen(self.batch_size // self.nr_gpu)
for i in range(self.nr_gpu)
]
if self.phase == 'pvae':
feed_dict.update({
model.masks: np.zeros_like(masks_np[i])
for i, model in enumerate(self.models)
})
elif self.phase == 'ce':
feed_dict.update({
model.masks: masks_np[i]
for i, model in enumerate(self.models)
})
feed_dict.update({
model.input_masks: masks_np[i]
for i, model in enumerate(self.models)
})
ret = self.sess.run([model.z_mu for model in self.models] +
[model.z_log_sigma_sq for model in self.models],
feed_dict=feed_dict)
z_mu = np.concatenate(ret[:len(ret) // 2], axis=0)
z_log_sigma_sq = np.concatenate(ret[len(ret) // 2:], axis=0)
z_sigma = np.sqrt(np.exp(z_log_sigma_sq))
z = np.random.normal(loc=z_mu, scale=z_sigma)
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i],
num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
return z_mu, z_sigma, ori_data, masked_data, masks_np
def sample(self, data, mgen, same_inputs=False, use_mask_at=None):
if self.num_channels == 3:
data = np.cast[np.float32]((data - 127.5) / 127.5)
if same_inputs:
for i in range(data.shape[0]):
data[i] = data[3]
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update(
{model.x: ds[i]
for i, model in enumerate(self.models)})
feed_dict.update(
{model.x_bar: ds[i]
for i, model in enumerate(self.models)})
if use_mask_at is not None:
masks_np = np.load(use_mask_at)['masks']
masks_np = np.split(masks_np, self.nr_gpu)
else:
masks_np = [
mgen.gen(self.batch_size // self.nr_gpu)
for i in range(self.nr_gpu)
]
np.savez(mgen.name + "_" + self.data_set,
masks=np.concatenate(masks_np))
if same_inputs:
for g in range(self.nr_gpu):
for i in range(self.batch_size // self.nr_gpu):
masks_np[g][i] = masks_np[0][0]
feed_dict.update(
{model.masks: masks_np[i]
for i, model in enumerate(self.models)})
#
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i],
num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
x_gen = [ds[i].copy() for i in range(self.nr_gpu)]
for yi in range(self.img_size):
for xi in range(self.img_size):
if np.min(
np.array([
masks_np[i][:, yi, xi] for i in range(self.nr_gpu)
])) > 0:
continue
feed_dict.update({
model.x_bar: x_gen[i]
for i, model in enumerate(self.models)
})
x_hats = self.sess.run([model.x_hat for model in self.models],
feed_dict=feed_dict)
for i in range(self.nr_gpu):
bmask = broadcast_masks_np(masks_np[i][:, yi, xi],
num_channels=self.num_channels)
x_gen[i][:, yi, xi, :] = x_hats[i][:, yi, xi, :] * (
1. - bmask) + x_gen[i][:, yi, xi, :] * bmask
gen_data = np.concatenate(x_gen, axis=0)
if self.num_channels == 1:
masks_np = np.concatenate(masks_np, axis=0)
masks_np = broadcast_masks_np(masks_np,
num_channels=self.num_channels)
masked_data += (1 - masks_np) * 0.5
return ori_data, masked_data, gen_data
def latent_traversal(self,
context,
traversal_range=[-6, 6],
num_traversal_step=13,
mgen=None):
self.num_traversal_step = num_traversal_step
if self.num_channels == 3:
image = np.cast[np.float32]((context - 127.5) / 127.5)
else:
image = context
num_instances = num_traversal_step * self.z_dim
assert num_instances <= self.batch_size, "cannot feed all the instances into GPUs"
data = np.stack([image.copy() for i in range(self.batch_size)], axis=0)
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update(
{model.x: ds[i]
for i, model in enumerate(self.models)})
masks_np = [
mgen.gen(self.batch_size // self.nr_gpu)
for i in range(self.nr_gpu)
]
if self.phase == 'pvae':
feed_dict.update({
model.masks: np.zeros_like(masks_np[i])
for i, model in enumerate(self.models)
})
elif self.phase == 'ce':
feed_dict.update({
model.masks: masks_np[i]
for i, model in enumerate(self.models)
})
feed_dict.update({
model.input_masks: masks_np[i]
for i, model in enumerate(self.models)
})
ret = self.sess.run([model.z_mu for model in self.models] +
[model.z_log_sigma_sq for model in self.models],
feed_dict=feed_dict)
z_mu = np.concatenate(ret[:len(ret) // 2], axis=0)
z_log_sigma_sq = np.concatenate(ret[len(ret) // 2:], axis=0)
z_sigma = np.sqrt(np.exp(z_log_sigma_sq))
z = z_mu #np.random.normal(loc=z_mu, scale=z_sigma)
for i in range(z.shape[0]):
z[i] = z[0].copy()
for i in range(self.z_dim):
z[i * num_traversal_step:(i + 1) * num_traversal_step,
i] = np.linspace(start=traversal_range[0],
stop=traversal_range[1],
num=num_traversal_step)
z = np.split(z, self.nr_gpu)
feed_dict.update(
{model.z_ph: z[i]
for i, model in enumerate(self.models)})
feed_dict.update({model.use_z_ph: True for model in self.models})
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i],
num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
x_gen = [ds[i].copy() for i in range(self.nr_gpu)]
for yi in range(self.img_size):
for xi in range(self.img_size):
if np.min(
np.array([
masks_np[i][:, yi, xi] for i in range(self.nr_gpu)
])) > 0:
continue
feed_dict.update({
model.x_bar: x_gen[i]
for i, model in enumerate(self.models)
})
x_hats = self.sess.run([model.x_hat for model in self.models],
feed_dict=feed_dict)
for i in range(self.nr_gpu):
bmask = broadcast_masks_np(masks_np[i][:, yi, xi],
num_channels=self.num_channels)
x_gen[i][:, yi, xi, :] = x_hats[i][:, yi, xi, :] * (
1. - bmask) + x_gen[i][:, yi, xi, :] * bmask
gen_data = np.concatenate(x_gen, axis=0)
return ori_data[:
num_instances], masked_data[:
num_instances], gen_data[:
num_instances]
def controlled_sample(self, data, mgen):
if self.num_channels == 3:
data = np.cast[np.float32]((data - 127.5) / 127.5)
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update(
{model.x: ds[i]
for i, model in enumerate(self.models)})
feed_dict.update(
{model.x_bar: ds[i]
for i, model in enumerate(self.models)})
masks_np = [
mgen.gen(self.batch_size // self.nr_gpu)
for i in range(self.nr_gpu)
]
if self.phase == 'pvae':
feed_dict.update({
model.masks: np.zeros_like(masks_np[i])
for i, model in enumerate(self.models)
})
elif self.phase == 'ce':
feed_dict.update({
model.masks: masks_np[i]
for i, model in enumerate(self.models)
})
feed_dict.update({
model.input_masks: masks_np[i]
for i, model in enumerate(self.models)
})
ret = self.sess.run([model.z_mu for model in self.models] +
[model.z_log_sigma_sq for model in self.models],
feed_dict=feed_dict)
z_mu = np.concatenate(ret[:len(ret) // 2], axis=0)
z_log_sigma_sq = np.concatenate(ret[len(ret) // 2:], axis=0)
z_sigma = np.sqrt(np.exp(z_log_sigma_sq))
z = np.random.normal(loc=z_mu, scale=z_sigma)
# for i in range(z.shape[0]):
# z[i] = z[0]
# z += np.random.normal(loc=np.zeros_like(z_mu), scale=np.ones_like(z_sigma))
for i in range(8):
for j in range(4):
z[i * 4 + j] = z[i * 4]
z = np.split(z, self.nr_gpu)
feed_dict.update(
{model.z_ph: z[i]
for i, model in enumerate(self.models)})
feed_dict.update({model.use_z_ph: True for model in self.models})
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i],
num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
x_gen = [ds[i].copy() for i in range(self.nr_gpu)]
for yi in range(self.img_size):
for xi in range(self.img_size):
feed_dict.update({
model.x_bar: x_gen[i]
for i, model in enumerate(self.models)
})
x_hats = self.sess.run([model.x_hat for model in self.models],
feed_dict=feed_dict)
for i in range(self.nr_gpu):
x_gen[i][:, yi, xi, :] = x_hats[i][:, yi, xi, :]
gen_data = np.concatenate(x_gen, axis=0)
return ori_data, masked_data, gen_data
def sample(self, data, mgen, same_inputs=False, use_mask_at=None):
if self.num_channels == 3:
data = np.cast[np.float32]((data - 127.5) / 127.5)
if same_inputs:
for i in range(data.shape[0]):
data[i] = data[3]
ori_data = data.copy()
ds = np.split(data.copy(), self.nr_gpu)
feed_dict = {}
feed_dict.update({model.is_training: False for model in self.models})
feed_dict.update({model.dropout_p: 0.0 for model in self.models})
feed_dict.update(
{model.x: ds[i]
for i, model in enumerate(self.models)})
feed_dict.update(
{model.x_bar: ds[i]
for i, model in enumerate(self.models)})
if use_mask_at is not None:
masks_np = np.load(use_mask_at)['masks']
masks_np = np.split(masks_np, self.nr_gpu)
else:
masks_np = [
mgen.gen(self.batch_size // self.nr_gpu)
for i in range(self.nr_gpu)
]
np.savez(mgen.name + "_" + self.data_set,
masks=np.concatenate(masks_np))
# masks_np = [mgen.gen(self.batch_size//self.nr_gpu) for i in range(self.nr_gpu)]
if self.phase == 'pvae':
feed_dict.update({
model.masks: np.zeros_like(masks_np[i])
for i, model in enumerate(self.models)
})
elif self.phase == 'ce':
feed_dict.update({
model.masks: masks_np[i]
for i, model in enumerate(self.models)
})
feed_dict.update({
model.input_masks: masks_np[i]
for i, model in enumerate(self.models)
})
ret = self.sess.run([model.z_mu for model in self.models] +
[model.z_log_sigma_sq for model in self.models],
feed_dict=feed_dict)
z_mu = np.concatenate(ret[:len(ret) // 2], axis=0)
z_log_sigma_sq = np.concatenate(ret[len(ret) // 2:], axis=0)
z_sigma = np.sqrt(np.exp(z_log_sigma_sq))
z = np.random.normal(loc=z_mu, scale=z_sigma)
z = np.split(z, self.nr_gpu)
feed_dict.update(
{model.z_ph: z[i]
for i, model in enumerate(self.models)})
feed_dict.update({model.use_z_ph: True for model in self.models})
for i in range(self.nr_gpu):
ds[i] *= broadcast_masks_np(masks_np[i],
num_channels=self.num_channels)
masked_data = np.concatenate(ds, axis=0)
x_gen = [ds[i].copy() for i in range(self.nr_gpu)]
for yi in range(self.img_size):
for xi in range(self.img_size):
if np.min(
np.array([
masks_np[i][:, yi, xi] for i in range(self.nr_gpu)
])) > 0:
continue
feed_dict.update({
model.x_bar: x_gen[i]
for i, model in enumerate(self.models)
})
x_hats = self.sess.run([model.x_hat for model in self.models],
feed_dict=feed_dict)
for i in range(self.nr_gpu):
bmask = broadcast_masks_np(masks_np[i][:, yi, xi],
num_channels=self.num_channels)
x_gen[i][:, yi, xi, :] = x_hats[i][:, yi, xi, :] * (
1. - bmask) + x_gen[i][:, yi, xi, :] * bmask
gen_data = np.concatenate(x_gen, axis=0)
if self.num_channels == 1:
masks_np = np.concatenate(masks_np, axis=0)
masks_np = broadcast_masks_np(masks_np,
num_channels=self.num_channels)
masked_data += (1 - masks_np) * 0.5
return ori_data, masked_data, gen_data
with tf.Session(config=config) as sess:
sess.run(initializer)
ckpt_file = args.save_dir + '/params_' + args.data_set + '.ckpt'
print('restoring parameters from', ckpt_file)
saver.restore(sess, ckpt_file)
sample_mgen = get_generator('bottom quarter', args.img_size)
fill_region = sample_mgen.gen(1)[0]
# sample_mgen = get_generator('transparent', args.img_size)
# fill_region = get_generator('full', args.img_size).gen(1)[0]
data = next(test_data)
from blocks.helpers import broadcast_masks_np
data = data.astype(np.float32) * broadcast_masks_np(fill_region, 3)
test_data.reset()
# vdata = np.cast[np.float32]((data - 127.5) / 127.5)
# visualize_samples(vdata, "/data/ziz/jxu/gpu-results/show_original.png", layout=[8,8])
img = []
for i in [7]: #[5,7,8]: #[5, 7, 8, 18, 27, 44, 74, 77]:
sample_x = latent_traversal(sess,
data[i],
traversal_range=[-6, 6],
num_traversal_step=13,
fill_region=fill_region,
mgen=sample_mgen)
view = visualize_samples(sample_x,
None,