def montage_summaries(self, x, y, g, y_hat, args): n_examples = 64 n = math.floor(math.sqrt(n_examples)) with tf.variable_scope('montage_preprocess'): y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True) y_hat = tf.reshape(y_hat, tf.shape(y)) g = tf.reshape(g, tf.shape(y)) y = y / 10.0 y_hat = y_hat / 10.0 y_bar = y_bar / 10.0 g = g / 10.0 # if args.model_version == 'baseline': # g = g / 10.0 # else: # g = (g - tf.reduce_min(g)) / (tf.reduce_max(g) - tf.reduce_min(g)) with arg_scope([hem.montage], num_examples=n_examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='x') hem.montage(y, name='y') hem.montage(tf.ones_like(y) * y_bar, name='y_bar') hem.montage(g, name='g', colorize=False) # hem.montage(tf.nn.sigmoid(g), name='g_sigmoid') hem.montage(y_hat, name='y_hat')
def summarize_layers(scope, layers, montage=False): with tf.variable_scope(scope): # activations for l in layers: layer_name = hem.tensor_name(l) layer_name = '/'.join(layer_name.split('/')[0:2]) l = tf.transpose(l, [0, 2, 3, 1]) tf.summary.histogram(layer_name, l) tf.summary.scalar(layer_name + '/sparsity', tf.nn.zero_fraction(l)) tf.summary.scalar(layer_name + '/mean', tf.reduce_mean(l)) if montage: hem.montage(tf.transpose(l[0], [2, 0, 1]), name=layer_name + '/montage')
def summaries(x, d, avg_grads, args): with tf.variable_scope('examples'): n = int(sqrt(args.examples)) hem.montage(hem.rescale(x[0:args.examples], (-1, 1), (0, 1)), n, n, name='inputs') hem.montage(hem.rescale(d[0:args.examples], (-1, 1), (0, 1)), n, n, name='outputs') hem.summarize_activations() hem.summarize_losses() hem.summarize_weights_biases() hem.summarize_gradients(avg_grads)
def montage_summaries(x, y, m, args): n = math.floor(math.sqrt(args.examples)) with tf.variable_scope('model'): with arg_scope([hem.montage], num_examples=args.examples, height=n, width=n, colorize=True): hem.montage(y, name='real_depths') hem.montage(x, name='real_images') hem.montage(tf.expand_dims(tf.expand_dims(tf.reduce_mean(y, axis=[2, 3]), -1), -1), name='real_average_depths') hem.montage(tf.expand_dims(tf.expand_dims(m, -1), -1), name='predicted_average_depths')
def montage_summaries(self, x, y, g, y_hat, args): n_examples = 64 n = math.floor(math.sqrt(n_examples)) with tf.variable_scope('montage_preprocess'): y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True) y_hat = tf.reshape(y_hat, tf.shape(y)) # reattach shape info g = tf.reshape(g, tf.shape(y)) # reattach shape info y = y / 10.0 y_hat = y_hat / 10.0 y_bar = y_bar / 10.0 g = g / 10.0 with arg_scope([hem.montage], num_examples=n_examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='x') hem.montage(y, name='y') hem.montage(tf.ones_like(y) * y_bar, name='y_bar') hem.montage(g, name='g') hem.montage(y_hat, name='y_hat')
def summaries(x, y, x_hat, y_hat, x_grads, y_grads, args): n = math.floor(math.sqrt(args.examples)) with arg_scope([hem.montage], height=n, width=n): x = hem.rescale(x, (-1, 1), (0, 1)) y = hem.rescale(y, (-1, 1), (0, 1)) x_hat = hem.rescale(x_hat, (-1, 1), (0, 1)) y_hat = hem.rescale(y_hat, (-1, 1), (0, 1)) hem.montage(x[0:args.examples], name='x') hem.montage(y[0:args.examples], name='y', colorize=True) hem.montage(x_hat[0:args.examples], name='x_hat') hem.montage(y_hat[0:args.examples], name='y_hat', colorize=True) hem.summarize_gradients(x_grads) hem.summarize_gradients(y_grads) hem.summarize_losses() hem.summarize_activations(False) hem.summarize_weights_biases()
def _summaries(g, x, args): """Add specialized summaries to the graph. This adds summaries that: - Track variability in generator samples given random noise vectors. - Track how much of the noise vector is used. - Generate examples from the real and learned distributions. Args: g: Tensor, the generator's output. i.e., the fake images. x: Tensor, the real (input) images. args: Argparse structure. Returns: None """ # 1. generate multiple samples using a single image with tf.variable_scope(tf.get_variable_scope()): gpu_id = 0 with tf.device(hem.variables_on_cpu(gpu_id)): with tf.name_scope('tower_{}'.format(gpu_id)) as scope: # print('input shape', x.shape) with tf.variable_scope('generator'): # using first image in batch, form new one with just this image x_repeated = tf.stack([x[0]] * args.batch_size) x_rgb, x_depth = _split(x_repeated) # then create a new path for the generator using just this dataset import copy args_copy = copy.copy(args) args_copy.batch_norm = False d = generator(x_rgb, args_copy, reuse=True) # scale to [0, 1] sampler_rgb, sampler_depth = _split(d) sampler_rgb = hem.rescale(sampler_rgb, (-1, 1), (0, 1)) sampler_depth = hem.rescale(sampler_depth, (-1, 1), (0, 1)) with tf.variable_scope('sampler'): hem.montage(sampler_rgb[0:args.examples], 8, 8, name='images') hem.montage(sampler_depth[0:args.examples], 8, 8, name='depths') # and track the variance of the depth predictions mean, var = tf.nn.moments(sampler_depth, axes=[0]) hem.scalars(('predicted_depth_mean', tf.reduce_mean(mean)), ('predicted_depth_var', tf.reduce_mean(var))) # and the images (temporary, for calibration/sanity check) x_rgb, x_depth = _split(x) mean, var = tf.nn.moments(x_depth, axes=[0]) hem.scalars(('real_depth_mean', tf.reduce_mean(mean)), ('real_depth_var', tf.reduce_mean(var))) sampler_rgb = tf.transpose(sampler_rgb, [0, 2, 3, 1]) sampler_depth = tf.transpose(sampler_depth, [0, 2, 3, 1]) # mean, var = tf.nn.moments(tf.image.rgb_to_grayscale(sampler_rgb), axes=[0]) axis = [0, -1] mean, var = tf.nn.moments(tf.image.rgb_to_grayscale(sampler_rgb), axes=[0]) hem.scalars(('image_mean', tf.reduce_mean(mean)), ('image_var', tf.reduce_mean(var))) mean, var = tf.nn.moments(sampler_depth, axes=[0]) hem.scalars(('depth_mean', tf.reduce_mean(mean)), ('depth_var', tf.reduce_mean(var))) # 2. generate summaries for real and fake images with tf.variable_scope('examples'): hem.histograms(('fake', g), ('real', x)) # rescale, split, and colorize x = hem.rescale(x[0:args.examples], (-1, 1), (0, 1)) g = hem.rescale(g[0:args.examples], (-1, 1), (0, 1)) hem.histograms(('fake_rescaled', g), ('real_rescaled', x)) x_rgb, x_depth = _split(x) g_rgb, g_depth = _split(g) # note: these are rescaled to (0, 1) hem.histograms(('real_depth', x_depth), ('fake_depth', g_depth), ('real_rgb', x_rgb), ('fake_rgb', g_rgb)) # add montages # TODO shouldn't be fixed to 8, but to ceil(sqrt(args.examples)) hem.montage(x_rgb, 8, 8, name='real/images') hem.montage(x_depth, 8, 8, name='real/depths') hem.montage(g_rgb, 8, 8, name='fake/images') hem.montage(g_depth, 8, 8, name='fake/depths') # 3. add additional summaries for weights and biases in e_c1 (the initial noise layer) # TODO don't iterate through list, but grab directly by full name with tf.variable_scope('noise'): for l in tf.get_collection('weights'): if 'e_c1' in hem.tensor_name(l): print(l, hem.tensor_name(l)) x_rgb, x_noise = _split(l) hem.histograms(('rgb/weights', x_rgb), ('noise/weights', x_noise)) hem.scalars( ('rgb/weights/sparsity', tf.nn.zero_fraction(x_rgb)), ('noise/weights/sparsity', tf.nn.zero_fraction(x_noise)), ('noise/weights/mean', tf.reduce_mean(x_noise)), ('rgb/weights/mean', tf.reduce_mean(x_rgb))) break
def montage_summaries(x, y, g, x_sample, y_sample, g_sampler, x_noise, g_noise, x_shuffled, y_shuffled, g_shuffle, args): n = math.floor(math.sqrt(args.examples)) if args.g_arch in ['E2']: x, x_loc, y_loc, mean_vec = tf.split(x, num_or_size_splits=[3, 1, 1, 1], axis=1) x_noise, x_loc_noise, y_loc_noise, mean_vec_noise = tf.split(x_noise, num_or_size_splits=[3, 1, 1, 1], axis=1) x_shuffled, x_loc_shuffled, y_loc_shuffled, mean_vec_shuffled = tf.split(x_shuffled, num_or_size_splits=[3, 1, 1, 1], axis=1) x_sample, x_loc_sample, y_loc_sample, mean_vec_sample = tf.split(x_sample, num_or_size_splits=[3, 1, 1, 1], axis=1) with tf.variable_scope('model'): hem.montage(x_loc, num_examples=args.examples, height=n, width=n, name='x_loc', colorize=True) hem.montage(y_loc, num_examples=args.examples, height=n, width=n, name='y_loc', colorize=True) hem.montage(mean_vec, num_examples=args.examples, height=n, width=n, name='mean_vec', colorize=True) with tf.variable_scope('montage_preprocess'): g = tf.reshape(g, tf.shape(y)) # reattach g = hem.rescale(g, (-1, 1), (0, 1)) y = hem.rescale(y, (-1, 1), (0, 1)) g_sampler = tf.reshape(g_sampler, tf.shape(y)) # reattach g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1)) y_sample = hem.rescale(y_sample, (-1, 1), (0, 1)) # add image montages with arg_scope([hem.montage], num_examples=args.examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='images') hem.montage(y, name='real_depths') hem.montage(g, name='fake_depths') with tf.variable_scope('sampler'): hem.montage(x_sample, name='images') hem.montage(y_sample, name='real_depths') hem.montage(g_sampler, name='fake_depths') with tf.variable_scope('shuffled'): hem.montage(x_shuffled, name='images') hem.montage(g_shuffle, name='fake_depths') with tf.variable_scope('noise'): hem.montage(x_noise, name='images') hem.montage(g_noise, name='fake_depths')
def montage_summaries(x_rgb, x_depth, g, args, x_repeated_rgb, x_repeated_depth, g_sampler, d_real, d_fake): """Adds montage summaries to the graph for the images, generator, and sampler. Args: x_rgb: Tensor, the real RGB image input batch. x_depth: Tensor, the real depth input batch. g: Tensor, the generator's output. args: Argparse struct. x_repeated_rgb: Tensor, the real RGB image inputs to the sampler path. x_repeated_depth: Tensor, the real RGB depth inputs to the sampler path. g_sampler: Tensor, the sampler generator outputs. Returns: None. """ # example image montages n = math.floor(math.sqrt(args.examples)) # rescale with tf.variable_scope('montage_preprocess'): g = tf.reshape(g, tf.shape(x_depth)) # re-attach lost shape info g = hem.rescale(g, (-1, 1), (0, 1)) x_depth = hem.rescale(x_depth, (-1, 1), (0, 1)) g_sampler = tf.reshape(g_sampler, tf.shape(x_depth)) g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1)) x_repeated_depth = hem.rescale(x_repeated_depth, (-1, 1), (0, 1)) # add image montages with arg_scope([hem.montage], height=n, width=n): # example batches hem.montage(x_rgb[0:args.examples], name='real/images') hem.montage(x_depth[0:args.examples], name='real/depths', colorize=True) hem.montage(x_rgb[0:args.examples], name='fake/images') hem.montage(g[0:args.examples], name='fake/depths', colorize=True) # sampler hem.montage(x_repeated_rgb[0:args.examples], name='sampler/images') hem.montage(x_repeated_depth[0:args.examples], name='sampler/depths', colorize=True) hem.montage(g_sampler[0:args.examples], name='sampler/fake', colorize=True) # discriminator patches hem.montage(d_fake[0:args.examples], name='discriminator_patches/fake') hem.montage(d_real[0:args.examples], name='discriminator_patches/real')