def montage_summaries(self, x, y, g, y_hat, args):
n_examples = 64
n = math.floor(math.sqrt(n_examples))
with tf.variable_scope('montage_preprocess'):
y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True)
y_hat = tf.reshape(y_hat, tf.shape(y))
g = tf.reshape(g, tf.shape(y))
y = y / 10.0
y_hat = y_hat / 10.0
y_bar = y_bar / 10.0
g = g / 10.0
# if args.model_version == 'baseline':
# g = g / 10.0
# else:
# g = (g - tf.reduce_min(g)) / (tf.reduce_max(g) - tf.reduce_min(g))
with arg_scope([hem.montage],
num_examples=n_examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='x')
hem.montage(y, name='y')
hem.montage(tf.ones_like(y) * y_bar, name='y_bar')
hem.montage(g, name='g', colorize=False)
# hem.montage(tf.nn.sigmoid(g), name='g_sigmoid')
hem.montage(y_hat, name='y_hat')
def summarize_layers(scope, layers, montage=False):
with tf.variable_scope(scope):
# activations
for l in layers:
layer_name = hem.tensor_name(l)
layer_name = '/'.join(layer_name.split('/')[0:2])
l = tf.transpose(l, [0, 2, 3, 1])
tf.summary.histogram(layer_name, l)
tf.summary.scalar(layer_name + '/sparsity', tf.nn.zero_fraction(l))
tf.summary.scalar(layer_name + '/mean', tf.reduce_mean(l))
if montage:
hem.montage(tf.transpose(l[0], [2, 0, 1]), name=layer_name + '/montage')
def summaries(x, d, avg_grads, args):
with tf.variable_scope('examples'):
n = int(sqrt(args.examples))
hem.montage(hem.rescale(x[0:args.examples], (-1, 1), (0, 1)),
n,
n,
name='inputs')
hem.montage(hem.rescale(d[0:args.examples], (-1, 1), (0, 1)),
n,
n,
name='outputs')
hem.summarize_activations()
hem.summarize_losses()
hem.summarize_weights_biases()
hem.summarize_gradients(avg_grads)
def montage_summaries(x, y, m, args):
n = math.floor(math.sqrt(args.examples))
with tf.variable_scope('model'):
with arg_scope([hem.montage],
num_examples=args.examples,
height=n,
width=n,
colorize=True):
hem.montage(y, name='real_depths')
hem.montage(x, name='real_images')
hem.montage(tf.expand_dims(tf.expand_dims(tf.reduce_mean(y, axis=[2, 3]), -1), -1), name='real_average_depths')
hem.montage(tf.expand_dims(tf.expand_dims(m, -1), -1), name='predicted_average_depths')
def montage_summaries(self, x, y, g, y_hat, args):
n_examples = 64
n = math.floor(math.sqrt(n_examples))
with tf.variable_scope('montage_preprocess'):
y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True)
y_hat = tf.reshape(y_hat, tf.shape(y)) # reattach shape info
g = tf.reshape(g, tf.shape(y)) # reattach shape info
y = y / 10.0
y_hat = y_hat / 10.0
y_bar = y_bar / 10.0
g = g / 10.0
with arg_scope([hem.montage],
num_examples=n_examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='x')
hem.montage(y, name='y')
hem.montage(tf.ones_like(y) * y_bar, name='y_bar')
hem.montage(g, name='g')
hem.montage(y_hat, name='y_hat')
def summaries(x, y, x_hat, y_hat, x_grads, y_grads, args):
n = math.floor(math.sqrt(args.examples))
with arg_scope([hem.montage], height=n, width=n):
x = hem.rescale(x, (-1, 1), (0, 1))
y = hem.rescale(y, (-1, 1), (0, 1))
x_hat = hem.rescale(x_hat, (-1, 1), (0, 1))
y_hat = hem.rescale(y_hat, (-1, 1), (0, 1))
hem.montage(x[0:args.examples], name='x')
hem.montage(y[0:args.examples], name='y', colorize=True)
hem.montage(x_hat[0:args.examples], name='x_hat')
hem.montage(y_hat[0:args.examples], name='y_hat', colorize=True)
hem.summarize_gradients(x_grads)
hem.summarize_gradients(y_grads)
hem.summarize_losses()
hem.summarize_activations(False)
hem.summarize_weights_biases()
def _summaries(g, x, args):
"""Add specialized summaries to the graph.
This adds summaries that:
- Track variability in generator samples given random noise vectors.
- Track how much of the noise vector is used.
- Generate examples from the real and learned distributions.
Args:
g: Tensor, the generator's output. i.e., the fake images.
x: Tensor, the real (input) images.
args: Argparse structure.
Returns:
None
"""
# 1. generate multiple samples using a single image
with tf.variable_scope(tf.get_variable_scope()):
gpu_id = 0
with tf.device(hem.variables_on_cpu(gpu_id)):
with tf.name_scope('tower_{}'.format(gpu_id)) as scope:
# print('input shape', x.shape)
with tf.variable_scope('generator'):
# using first image in batch, form new one with just this image
x_repeated = tf.stack([x[0]] * args.batch_size)
x_rgb, x_depth = _split(x_repeated)
# then create a new path for the generator using just this dataset
import copy
args_copy = copy.copy(args)
args_copy.batch_norm = False
d = generator(x_rgb, args_copy, reuse=True)
# scale to [0, 1]
sampler_rgb, sampler_depth = _split(d)
sampler_rgb = hem.rescale(sampler_rgb, (-1, 1), (0, 1))
sampler_depth = hem.rescale(sampler_depth, (-1, 1), (0, 1))
with tf.variable_scope('sampler'):
hem.montage(sampler_rgb[0:args.examples], 8, 8, name='images')
hem.montage(sampler_depth[0:args.examples], 8, 8, name='depths')
# and track the variance of the depth predictions
mean, var = tf.nn.moments(sampler_depth, axes=[0])
hem.scalars(('predicted_depth_mean', tf.reduce_mean(mean)),
('predicted_depth_var', tf.reduce_mean(var)))
# and the images (temporary, for calibration/sanity check)
x_rgb, x_depth = _split(x)
mean, var = tf.nn.moments(x_depth, axes=[0])
hem.scalars(('real_depth_mean', tf.reduce_mean(mean)),
('real_depth_var', tf.reduce_mean(var)))
sampler_rgb = tf.transpose(sampler_rgb, [0, 2, 3, 1])
sampler_depth = tf.transpose(sampler_depth, [0, 2, 3, 1])
# mean, var = tf.nn.moments(tf.image.rgb_to_grayscale(sampler_rgb), axes=[0])
axis = [0, -1]
mean, var = tf.nn.moments(tf.image.rgb_to_grayscale(sampler_rgb),
axes=[0])
hem.scalars(('image_mean', tf.reduce_mean(mean)),
('image_var', tf.reduce_mean(var)))
mean, var = tf.nn.moments(sampler_depth, axes=[0])
hem.scalars(('depth_mean', tf.reduce_mean(mean)),
('depth_var', tf.reduce_mean(var)))
# 2. generate summaries for real and fake images
with tf.variable_scope('examples'):
hem.histograms(('fake', g), ('real', x))
# rescale, split, and colorize
x = hem.rescale(x[0:args.examples], (-1, 1), (0, 1))
g = hem.rescale(g[0:args.examples], (-1, 1), (0, 1))
hem.histograms(('fake_rescaled', g), ('real_rescaled', x))
x_rgb, x_depth = _split(x)
g_rgb, g_depth = _split(g)
# note: these are rescaled to (0, 1)
hem.histograms(('real_depth', x_depth), ('fake_depth', g_depth),
('real_rgb', x_rgb), ('fake_rgb', g_rgb))
# add montages
# TODO shouldn't be fixed to 8, but to ceil(sqrt(args.examples))
hem.montage(x_rgb, 8, 8, name='real/images')
hem.montage(x_depth, 8, 8, name='real/depths')
hem.montage(g_rgb, 8, 8, name='fake/images')
hem.montage(g_depth, 8, 8, name='fake/depths')
# 3. add additional summaries for weights and biases in e_c1 (the initial noise layer)
# TODO don't iterate through list, but grab directly by full name
with tf.variable_scope('noise'):
for l in tf.get_collection('weights'):
if 'e_c1' in hem.tensor_name(l):
print(l, hem.tensor_name(l))
x_rgb, x_noise = _split(l)
hem.histograms(('rgb/weights', x_rgb),
('noise/weights', x_noise))
hem.scalars(
('rgb/weights/sparsity', tf.nn.zero_fraction(x_rgb)),
('noise/weights/sparsity', tf.nn.zero_fraction(x_noise)),
('noise/weights/mean', tf.reduce_mean(x_noise)),
('rgb/weights/mean', tf.reduce_mean(x_rgb)))
break
def montage_summaries(x, y, g,
x_sample, y_sample, g_sampler,
x_noise, g_noise,
x_shuffled, y_shuffled, g_shuffle,
args):
n = math.floor(math.sqrt(args.examples))
if args.g_arch in ['E2']:
x, x_loc, y_loc, mean_vec = tf.split(x, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_noise, x_loc_noise, y_loc_noise, mean_vec_noise = tf.split(x_noise, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_shuffled, x_loc_shuffled, y_loc_shuffled, mean_vec_shuffled = tf.split(x_shuffled, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_sample, x_loc_sample, y_loc_sample, mean_vec_sample = tf.split(x_sample, num_or_size_splits=[3, 1, 1, 1], axis=1)
with tf.variable_scope('model'):
hem.montage(x_loc, num_examples=args.examples, height=n, width=n, name='x_loc', colorize=True)
hem.montage(y_loc, num_examples=args.examples, height=n, width=n, name='y_loc', colorize=True)
hem.montage(mean_vec, num_examples=args.examples, height=n, width=n, name='mean_vec', colorize=True)
with tf.variable_scope('montage_preprocess'):
g = tf.reshape(g, tf.shape(y)) # reattach
g = hem.rescale(g, (-1, 1), (0, 1))
y = hem.rescale(y, (-1, 1), (0, 1))
g_sampler = tf.reshape(g_sampler, tf.shape(y)) # reattach
g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1))
y_sample = hem.rescale(y_sample, (-1, 1), (0, 1))
# add image montages
with arg_scope([hem.montage],
num_examples=args.examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='images')
hem.montage(y, name='real_depths')
hem.montage(g, name='fake_depths')
with tf.variable_scope('sampler'):
hem.montage(x_sample, name='images')
hem.montage(y_sample, name='real_depths')
hem.montage(g_sampler, name='fake_depths')
with tf.variable_scope('shuffled'):
hem.montage(x_shuffled, name='images')
hem.montage(g_shuffle, name='fake_depths')
with tf.variable_scope('noise'):
hem.montage(x_noise, name='images')
hem.montage(g_noise, name='fake_depths')
def montage_summaries(x_rgb, x_depth, g, args, x_repeated_rgb, x_repeated_depth, g_sampler, d_real, d_fake):
"""Adds montage summaries to the graph for the images, generator, and sampler.
Args:
x_rgb: Tensor, the real RGB image input batch.
x_depth: Tensor, the real depth input batch.
g: Tensor, the generator's output.
args: Argparse struct.
x_repeated_rgb: Tensor, the real RGB image inputs to the sampler path.
x_repeated_depth: Tensor, the real RGB depth inputs to the sampler path.
g_sampler: Tensor, the sampler generator outputs.
Returns:
None.
"""
# example image montages
n = math.floor(math.sqrt(args.examples))
# rescale
with tf.variable_scope('montage_preprocess'):
g = tf.reshape(g, tf.shape(x_depth)) # re-attach lost shape info
g = hem.rescale(g, (-1, 1), (0, 1))
x_depth = hem.rescale(x_depth, (-1, 1), (0, 1))
g_sampler = tf.reshape(g_sampler, tf.shape(x_depth))
g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1))
x_repeated_depth = hem.rescale(x_repeated_depth, (-1, 1), (0, 1))
# add image montages
with arg_scope([hem.montage],
height=n,
width=n):
# example batches
hem.montage(x_rgb[0:args.examples], name='real/images')
hem.montage(x_depth[0:args.examples], name='real/depths', colorize=True)
hem.montage(x_rgb[0:args.examples], name='fake/images')
hem.montage(g[0:args.examples], name='fake/depths', colorize=True)
# sampler
hem.montage(x_repeated_rgb[0:args.examples], name='sampler/images')
hem.montage(x_repeated_depth[0:args.examples], name='sampler/depths', colorize=True)
hem.montage(g_sampler[0:args.examples], name='sampler/fake', colorize=True)
# discriminator patches
hem.montage(d_fake[0:args.examples], name='discriminator_patches/fake')
hem.montage(d_real[0:args.examples], name='discriminator_patches/real')
