def g_mean_provided2(self, x, y_bar, args, reuse=False): with tf.variable_scope('encoder', reuse=reuse), \ arg_scope([hem.conv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=tf.contrib.layers.xavier_initializer, activation=tf.nn.relu): # 65x65x3 x = tf.concat([x, tf.ones((args.batch_size, 1, 65, 65))], axis=1) e1 = hem.conv2d(x, 4, 64, name='e1') # 31x31x64 # e1 = tf.concat([e1, tf.ones((args.batch_size, 1, 31, 31)) * y_bar], axis=1) e2 = hem.conv2d(e1, 64, 128, name='e2') # 14x14x128 e3 = hem.conv2d(e2, 128, 256, name='e3') # 5x5x256 e4 = hem.conv2d(e3, 256, 512, name='e4') # 1x1x512 with tf.variable_scope('decoder', reuse=reuse), \ arg_scope([hem.deconv2d, hem.conv2d], reuse=reuse, filter_size=5, stride=2, init=tf.contrib.layers.xavier_initializer, padding='VALID', activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x512 y_hat = hem.deconv2d(e4, 512, 256, output_shape=(args.batch_size, 256, 5, 5), name='d1') # 5x5x256 y_hat = tf.concat([y_hat, e3], axis=1) # 5x5x512 y_hat = hem.deconv2d(y_hat, 512, 128, output_shape=(args.batch_size, 128, 14, 14), name='d2') # 14x14x128 y_hat = tf.concat([y_hat, e2], axis=1) # 14x14x256 y_hat = hem.deconv2d(y_hat, 256, 64, output_shape=(args.batch_size, 64, 31, 31), name='d3') # 31x31x64 y_hat = tf.concat([y_hat, e1], axis=1) # 31x31x128 y_hat = hem.conv2d(y_hat, 128, 1, stride=1, filter_size=1, padding='SAME', activation=None, name='d4') # 31x31x1 y_hat = hem.crop_to_bounding_box(y_hat, 0, 0, 29, 29) # 29x29x1 # y_hat = tf.maximum(y_hat, tf.zeros_like(y_hat)) return y_hat
def discriminatorE2(x, y, args, reuse=False): with arg_scope([hem.conv2d], reuse=reuse, activation=lambda x: hem.lrelu(x, leak=0.2), init=tf.contrib.layers.xavier_initializer, padding='SAME', filter_size=5, stride=2): # x = 64x64x3, y = 32x32x1 with tf.variable_scope('rgb_path'): h1 = hem.conv2d(x, 6, 64, name='hx1') # 32x32x64 h1 = hem.conv2d(h1, 64, 128, name='hx2') # 16x16x128 h1 = hem.conv2d(h1, 128, 256, name='hx3') # 8x8x256 h1 = hem.conv2d(h1, 256, 512, name='hx4') # 4x4x512 h1 = hem.conv2d(h1, 512, 1024, padding='VALID', filter_size=4, name='hx5') # 1x1x1024 with tf.variable_scope('depth_path'): h2 = hem.conv2d(y, 1, 128, name='hy1') # 16x16x128 h2 = hem.conv2d(h2, 128, 256, name='hy2') # 8x8x256 h2 = hem.conv2d(h2, 256, 512, name='hy3') # 4x4x512 h2 = hem.conv2d(h2, 512, 1024, filter_size=4, padding='VALID', name='hy4') # 1x1x1024 with tf.variable_scope('combined_path'): h = tf.concat([h1, h2], axis=1) # 1x1x2048 h = hem.conv2d(h, 2048, 1024, stride=1, filter_size=1, padding='SAME', name='h1') # 1x1x1024 h = hem.conv2d(h, 1024, 512, stride=1, filter_size=1, padding='SAME', name='h2') # 1x1x512 h = hem.conv2d(h, 512, 256, stride=1, filter_size=1, padding='SAME', name='h3') # 1x1x256 h = hem.conv2d(h, 256, 128, stride=1, filter_size=1, padding='SAME', name='h4') # 1x1x1x128 h = hem.conv2d(h, 128, 64, stride=1, filter_size=1, padding='SAME', name='h5') # 1x1x1x64 h = hem.conv2d(h, 64, 1, stride=1, filter_size=1, padding='SAME', name='h6', activation=None) # 1x1x1x64 # output, logits return tf.nn.sigmoid(h), h
def montage_summaries(self, x, y, g, y_hat, args): n_examples = 64 n = math.floor(math.sqrt(n_examples)) with tf.variable_scope('montage_preprocess'): y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True) y_hat = tf.reshape(y_hat, tf.shape(y)) g = tf.reshape(g, tf.shape(y)) y = y / 10.0 y_hat = y_hat / 10.0 y_bar = y_bar / 10.0 g = g / 10.0 # if args.model_version == 'baseline': # g = g / 10.0 # else: # g = (g - tf.reduce_min(g)) / (tf.reduce_max(g) - tf.reduce_min(g)) with arg_scope([hem.montage], num_examples=n_examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='x') hem.montage(y, name='y') hem.montage(tf.ones_like(y) * y_bar, name='y_bar') hem.montage(g, name='g', colorize=False) # hem.montage(tf.nn.sigmoid(g), name='g_sigmoid') hem.montage(y_hat, name='y_hat')
def discriminator(x, y, args, reuse=False): """Adds (PatchGAN) discriminator nodes to the graph, given RGB and D inputs. Args: x_rgb: Tensor, the RGB component of the real or fake images. x_depth: Tensor, the depth component of the real or fake images. args: Argparse struct. reuse: Bool, whether to reuse variables. Returns: Tensor, contains 70x70 patches with a probability assigned to each. """ with arg_scope([hem.conv2d], reuse=reuse, use_batch_norm=args.batch_norm_disc, activation=lambda x: hem.lrelu(x, leak=0.2), init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02), filter_size=4, stride=2): x_y = tf.concat([x, y], axis=1) h = hem.conv2d(x_y, 4, 64, name='m1', use_batch_norm=False) h = hem.conv2d(h, 64, 128, name='m2') h = hem.conv2d(h, 128, 256, name='m3') h = hem.conv2d(h, 256, 512, name='m4') h = hem.conv2d(h, 512, 1, name='m5', activation=None) # , activation=tf.nn.sigmoid) # hem.montage(tf.nn.sigmoid(h), ) # layer outout, logits return tf.nn.sigmoid(h), h
def d_mean_provided2(self, x, y, y_bar, args, reuse=False): with arg_scope([hem.conv2d], reuse=reuse, activation=lambda x: hem.lrelu(x, leak=0.2), init=tf.contrib.layers.xavier_initializer, padding='VALID', filter_size=5, stride=2): # x = 65x65x3, y = 29x29x1 with tf.variable_scope('rgb_path'): x = tf.concat([x, tf.ones((args.batch_size, 1, 65, 65)) * y_bar], axis=1) h1 = hem.conv2d(x, 4, 64, name='hx1') # 31x31x64 h1 = hem.conv2d(h1, 64, 128, name='hx2') # 14x14x128 h1 = hem.conv2d(h1, 128, 256, name='hx3') # 5x5x256 h1 = hem.conv2d(h1, 256, 512, name='hx4') # 1x1x512 with tf.variable_scope('depth_path'): h2 = tf.concat([y, tf.ones_like(y) * y_bar], axis=1, name='hy0') h2 = hem.conv2d(h2, 2, 128, name='hy1') # 14x14x128 h2 = hem.conv2d(h2, 128, 256, name='hy2') # 5x5x256 h2 = hem.conv2d(h2, 256, 512, name='hy3') # 1x1x512 with tf.variable_scope('combined_path'): h = tf.concat([h1, h2], axis=1) # 1x1x1024 h = hem.conv2d(h, 1024, 1024, stride=1, filter_size=1, padding='SAME', name='h1') # 1x1x768 h = hem.conv2d(h, 1024, 512, stride=1, filter_size=1, padding='SAME', name='h2') # 1x1x384 h = hem.conv2d(h, 512, 1, stride=1, filter_size=1, padding='SAME', name='h3', activation=None) # 1x1x1 # output, logits return tf.nn.sigmoid(h), h
def generator(batch_size, latent_size, args, reuse=False): """Adds generator nodes to the graph. From noise, applies deconv2d until image is scaled up to match the dataset. """ # final_activation = tf.tanh if args.model in ['wgan', 'iwgan'] else tf.nn.sigmoid output_dim = 64 * 64 * 3 with arg_scope([dense, deconv2d], reuse=reuse, use_batch_norm=True, activation=tf.nn.relu): z = tf.random_normal([batch_size, latent_size]) y = dense(z, latent_size, 4 * 4 * 4 * latent_size, name='fc1') y = tf.reshape(y, [-1, 4, 4, 4 * latent_size]) y = deconv2d(y, 4 * latent_size, 2 * latent_size, 5, 2, name='dc1') y = deconv2d(y, 2 * latent_size, latent_size, 5, 2, name='dc2') y = deconv2d(y, latent_size, int(latent_size / 2), 5, 2, name='dc3') y = deconv2d(y, int(latent_size / 2), 3, 5, 2, name='dc4', activation=tf.tanh, use_batch_norm=False) y = tf.reshape(y, [-1, output_dim]) return y
def E2(x, args, reuse=False): with arg_scope([hem.conv2d], reuse=reuse, stride=2, padding='SAME', filter_size=5, init=tf.contrib.layers.xavier_initializer, activation=tf.nn.relu): # 427 x 561 x 3 l1 = hem.conv2d(x, 3, 64, name='l1') # 214 x 281 x 64 # print('l1', l1) l2 = hem.conv2d(l1, 64, 128, name='l2') # 107, 141 x 128 # print('l2', l2) l3 = hem.conv2d(l2, 128, 256, name='l3') # 54 x 71 x 256 # print('l3', l3) l4 = hem.conv2d(l3, 256, 512, name='l4') # 27 x 36 x 512 # print('l4', l4) l5 = hem.conv2d(l4, 512, 1024, name='l5') # 14 x 18 x 1024 # print('l5', l5) l6 = hem.conv2d(l5, 1024, 2048, name='l6') # 4 x 5 x 2048 # print('l6', l6) f = hem.flatten(l6) # 4096 # print('f', f) l7 = hem.dense(f, 4096, 2048, name='l7') # 2048 # print('l7', l7) l8 = hem.dense(l7, 2048, 1, activation=tf.nn.sigmoid, name='l8') # 1 # print('l8', l8) return l8
def discriminator(x, args, reuse=False): """Adds discriminator nodes to the graph. From the input image, successively applies convolutions with striding to scale down layer sizes until we get to a single output value, representing the discriminator's estimate of fake vs real. The single final output acts similar to a sigmoid activation function. Args: x: Tensor, input. args: Argparse structure. reuse: Boolean, whether to reuse variables. Returns: Final output of discriminator pipeline. """ use_bn = False if args.model == 'iwgan' else True final_activation = None if args.model in ['wgan', 'iwgan' ] else tf.nn.sigmoid with arg_scope([hem.conv2d], use_batch_norm=use_bn, activation=hem.lrelu, reuse=reuse): x = tf.reshape(x, [-1, 3, 64, 64]) x = hem.conv2d(x, 3, args.latent_size, 5, 2, name='c1', use_batch_norm=False) x = hem.conv2d(x, args.latent_size, args.latent_size * 2, 5, 2, name='c2') x = hem.conv2d(x, args.latent_size * 2, args.latent_size * 4, 5, 2, name='c3') x = tf.reshape(x, [-1, 4 * 4 * 4 * args.latent_size]) x = hem.dense(x, 4 * 4 * 4 * args.latent_size, 1, use_batch_norm=False, activation=final_activation, name='fc2', reuse=reuse) x = tf.reshape(x, [-1]) return x
def _decoder(x, args, reuse=False): """ Adds decoder nodes to the graph. Args: x: Tensor, the latent variable tensor from the encoder. args: Argparse structure. reuse: Boolean, whether to reuse pinned variables (in multi-GPU environment). Returns: Tensor, a single-channel image representing the predicted depth map. """ e_layers = tf.get_collection('conv_layers') with arg_scope([hem.dense, hem.conv2d, hem.deconv2d], reuse=reuse, use_batch_norm=args.batch_norm, dropout=args.dropout, activation=tf.nn.relu): skip_axis = 1 skip_m = 2 if args.skip_layers else 1 # TODO add better support for skip layers in layer ops # input = 4x4x32 x = hem.conv2d(x, 32, 96, 1, name='d_c1') # output = 4x4x96 + e_c5 x = tf.concat( (x, e_layers[4]), axis=skip_axis) if args.skip_layers else x x = hem.conv2d(x, 96 * skip_m, 256, 1, name='d_c2') # output = 4x4x256 + e_c4 x = tf.concat( (x, e_layers[3]), axis=skip_axis) if args.skip_layers else x x = hem.deconv2d(x, 256 * skip_m, 256, 5, 2, name='d_dc1') # output = 8x8x256 + e_c3 x = tf.concat( (x, e_layers[2]), axis=skip_axis) if args.skip_layers else x x = hem.deconv2d(x, 256 * skip_m, 128, 5, 2, name='d_dc2') # output = 16x16x128 + e_c2 x = tf.concat( (x, e_layers[1]), axis=skip_axis) if args.skip_layers else x x = hem.deconv2d(x, 128 * skip_m, 64, 5, 2, name='d_dc3') # output = 32x32x64 + e_c1 x = tf.concat( (x, e_layers[0]), axis=skip_axis) if args.skip_layers else x x = hem.deconv2d(x, 64 * skip_m, 1, 5, 2, name='d_dc4', activation=tf.nn.tanh, dropout=0, use_batch_norm=False) # output = 64x64x1 return x
def predictor(y, reuse=False): # given the depth map y, generate a rgb scene x with tf.variable_scope('predictor, reuse=reuse'),\ arg_scope([hem.conv2d, hem.deconv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02), activation=lambda x: hem.lrelu(x, leak=0.2)): y = hem.conv2d(y, 1, 3, stride=1, activation=tf.tanh) return y
def latent(x, latent_size, reuse=False): """Add latant nodes to the graph. Args: x: Tensor, output from encoder. latent_size: Integer, size of latent vector. reuse: Boolean, whether to reuse variables. """ with arg_scope([dense], reuse = reuse): x = flatten(x) x = dense(x, 32*4*4, latent_size, name='d1') return x
def montage_summaries(x, y, m, args): n = math.floor(math.sqrt(args.examples)) with tf.variable_scope('model'): with arg_scope([hem.montage], num_examples=args.examples, height=n, width=n, colorize=True): hem.montage(y, name='real_depths') hem.montage(x, name='real_images') hem.montage(tf.expand_dims(tf.expand_dims(tf.reduce_mean(y, axis=[2, 3]), -1), -1), name='real_average_depths') hem.montage(tf.expand_dims(tf.expand_dims(m, -1), -1), name='predicted_average_depths')
def montage_summaries(x_rgb, x_depth, g, args, x_repeated_rgb, x_repeated_depth, g_sampler): """Adds montage summaries to the graph for the images, generator, and sampler. Args: x_rgb: Tensor, the real RGB image input batch. x_depth: Tensor, the real depth input batch. g: Tensor, the generator's output. args: Argparse struct. x_repeated_rgb: Tensor, the real RGB image inputs to the sampler path. x_repeated_depth: Tensor, the real RGB depth inputs to the sampler path. g_sampler: Tensor, the sampler generator outputs. Returns: None. """ # print('x shape', x_rgb.shape) # print('y shape', x_depth.shape) # print('y_hat shape', g.shape) # print('x_repeated shape', x_repeated_depth.shape) # print('y_hat_repeated shape', g_sampler.shape) # example image montages n = math.floor(math.sqrt(args.examples)) # rescale with tf.variable_scope('montage_preprocess'): g = tf.reshape(g, tf.shape(x_depth)) # re-attach lost shape info g = hem.rescale(g, (-1, 1), (0, 1)) x_depth = hem.rescale(x_depth, (-1, 1), (0, 1)) g_sampler = tf.reshape(g_sampler, tf.shape(x_depth)) g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1)) x_repeated_depth = hem.rescale(x_repeated_depth, (-1, 1), (0, 1)) # x_depth = tf.reshape(x_depth, (-1, 1, 65, 65)) # add image montages with arg_scope([hem.montage], height=n, width=n): # example batches hem.montage(x_rgb[0:args.examples], name='real/images') hem.montage(x_depth[0:args.examples], name='real/depths', colorize=True) hem.montage(x_rgb[0:args.examples], name='fake/images') hem.montage(g[0:args.examples], name='fake/depths', colorize=True) # sampler hem.montage(x_repeated_rgb[0:args.examples], name='sampler/images') hem.montage(x_repeated_depth[0:args.examples], name='sampler/depths', colorize=True) hem.montage(g_sampler[0:args.examples], name='sampler/fake', colorize=True)
def latent(x, args, reuse=False): """Add latant nodes to the graph. Args: x: Tensor, output from encoder. args: Argparse struct, containing: latent_size, integer describing size of the latent vector reuse: Boolean, whether to reuse variables. """ with arg_scope([hem.dense], reuse=reuse): x = hem.flatten(x) x = hem.dense(x, 32 * 4 * 4, args.latent_size, name='d1') return x
def generatorE2(x, args, reuse=False): with tf.variable_scope('encoder', reuse=reuse), \ arg_scope([hem.conv2d], reuse=reuse, stride=2, padding='SAME', filter_size=5, init=tf.contrib.layers.xavier_initializer, activation=tf.nn.relu): # 64x64x3 noise = tf.random_uniform([args.batch_size, 1, 64, 64], minval=-1.0, maxval=1.0) x = tf.concat([x, noise], axis=1) # 64x64x4 e1 = hem.conv2d(x, 7, 64, name='e1') # 32x32x64 e2 = hem.conv2d(e1, 64, 128, name='e2') # 16x16x128 e3 = hem.conv2d(e2, 128, 256, name='e3') # 8x8x256 e4 = hem.conv2d(e3, 256, 512, name='e4') # 4x4x512 e5 = hem.conv2d(e4, 512, 1024, filter_size=4, padding='VALID', name='e5') # 1x1x1024 with tf.variable_scope('decoder', reuse=reuse), \ arg_scope([hem.deconv2d, hem.conv2d], reuse=reuse, stride=2, init=tf.contrib.layers.xavier_initializer, padding='SAME', filter_size=5, activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x1024 y = hem.deconv2d(e5, 1024, 512, output_shape=(args.batch_size, 512, 4, 4), filter_size=4, padding='VALID', name='d1') # 4x4x512 y = tf.concat([y, e4], axis=1) # 4x4x1024 y = hem.deconv2d(y, 1024, 256, output_shape=(args.batch_size, 256, 8, 8), name='d2') # 8x8x256 y = tf.concat([y, e3], axis=1) # 8x8x512 y = hem.deconv2d(y, 512, 128, output_shape=(args.batch_size, 128, 16, 16), name='d3') # 16x16x128 y = tf.concat([y, e2], axis=1) # 16x16x256 y = hem.deconv2d(y, 256, 64, output_shape=(args.batch_size, 64, 32, 32), name='d4') # 32x32x64 y = tf.concat([y, e1], axis=1) # 32x32x128 y = hem.conv2d(y, 128, 1, stride=1, filter_size=1, activation=tf.nn.tanh, name='d5') # 31x31x1 return y
def encoder(x, reuse=False): with arg_scope([hem.conv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=hem.xavier_initializer, use_batch_norm=True, activation=lambda x: hem.lrelu(x, leak=0.2)): x_rep = hem.conv2d(x, 3, 6, name='e1', use_batch_norm=False) x_rep = hem.conv2d(x_rep, 6, 12, name='e2') x_rep = hem.conv2d(x_rep, 12, 24, name='e3') x_rep = hem.conv2d(x_rep, 24, 48, name='e4') x_rep = hem.conv2d(x_rep, 48, 192, name='e5') x_rep = hem.conv2d(x_rep, 192, 384, name='e6') # 5x5x192 return x_rep
def summaries(x, y, x_hat, y_hat, x_grads, y_grads, args): n = math.floor(math.sqrt(args.examples)) with arg_scope([hem.montage], height=n, width=n): x = hem.rescale(x, (-1, 1), (0, 1)) y = hem.rescale(y, (-1, 1), (0, 1)) x_hat = hem.rescale(x_hat, (-1, 1), (0, 1)) y_hat = hem.rescale(y_hat, (-1, 1), (0, 1)) hem.montage(x[0:args.examples], name='x') hem.montage(y[0:args.examples], name='y', colorize=True) hem.montage(x_hat[0:args.examples], name='x_hat') hem.montage(y_hat[0:args.examples], name='y_hat', colorize=True) hem.summarize_gradients(x_grads) hem.summarize_gradients(y_grads) hem.summarize_losses() hem.summarize_activations(False) hem.summarize_weights_biases()
def latent(x, batch_size, latent_size, reuse=False): """Adds latent nodes for sampling and reparamaterization. Args: x: Tensor, input images. batch_size: Integer, batch size. latent_size: Integer, size of latent vector. reuse: Boolean, whether to reuse variables. """ with arg_scope([dense], reuse=reuse): flat = flatten(x) z_mean = dense(flat, 32 * 4 * 4, latent_size, name='d1') z_stddev = dense(flat, 32 * 4 * 4, latent_size, name='d2') samples = tf.random_normal([batch_size, latent_size], 0, 1) z = (z_mean + (z_stddev * samples)) return (samples, z, z_mean, z_stddev)
def encoder(x, args, reuse=False): """Adds encoder nodes to the graph. Args: x: Tensor, input images. args: Argparse struct reuse: Boolean, whether to reuse variables. """ with arg_scope([hem.conv2d], reuse=reuse, activation=hem.lrelu): x = hem.conv2d(x, 3, 64, 5, 2, name='c1') x = hem.conv2d(x, 64, 128, 5, 2, name='c2') x = hem.conv2d(x, 128, 256, 5, 2, name='c3') x = hem.conv2d(x, 256, 256, 5, 2, name='c4') x = hem.conv2d(x, 256, 96, 1, name='c5') x = hem.conv2d(x, 96, 32, 1, name='c6') return x
def discriminator(y, reuse=False): # given a depth map y, determine whether it is real or fake with tf.variable_scope('discriminator', reuse=reuse),\ arg_scope([hem.conv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02), activation=lambda x: hem.lrelu(x, leak=0.2)): y = hem.conv2d(y, 1, 64, name='d1') y = hem.conv2d(y, 64, 128, name='d2') y = hem.conv2d(y, 128, 256, name='d3') y = hem.conv2d(y, 256, 512, name='d4') y = hem.conv2d(y, 512, 256, name='d5') y = hem.conv2d(y, 256, 1, name='d6', activation=tf.sigmoid) return y
def montage_summaries(x, y, g, x_sample, y_sample, g_sampler, x_noise, g_noise, x_shuffled, y_shuffled, g_shuffle, args): n = math.floor(math.sqrt(args.examples)) if args.g_arch in ['E2']: x, x_loc, y_loc, mean_vec = tf.split(x, num_or_size_splits=[3, 1, 1, 1], axis=1) x_noise, x_loc_noise, y_loc_noise, mean_vec_noise = tf.split(x_noise, num_or_size_splits=[3, 1, 1, 1], axis=1) x_shuffled, x_loc_shuffled, y_loc_shuffled, mean_vec_shuffled = tf.split(x_shuffled, num_or_size_splits=[3, 1, 1, 1], axis=1) x_sample, x_loc_sample, y_loc_sample, mean_vec_sample = tf.split(x_sample, num_or_size_splits=[3, 1, 1, 1], axis=1) with tf.variable_scope('model'): hem.montage(x_loc, num_examples=args.examples, height=n, width=n, name='x_loc', colorize=True) hem.montage(y_loc, num_examples=args.examples, height=n, width=n, name='y_loc', colorize=True) hem.montage(mean_vec, num_examples=args.examples, height=n, width=n, name='mean_vec', colorize=True) with tf.variable_scope('montage_preprocess'): g = tf.reshape(g, tf.shape(y)) # reattach g = hem.rescale(g, (-1, 1), (0, 1)) y = hem.rescale(y, (-1, 1), (0, 1)) g_sampler = tf.reshape(g_sampler, tf.shape(y)) # reattach g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1)) y_sample = hem.rescale(y_sample, (-1, 1), (0, 1)) # add image montages with arg_scope([hem.montage], num_examples=args.examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='images') hem.montage(y, name='real_depths') hem.montage(g, name='fake_depths') with tf.variable_scope('sampler'): hem.montage(x_sample, name='images') hem.montage(y_sample, name='real_depths') hem.montage(g_sampler, name='fake_depths') with tf.variable_scope('shuffled'): hem.montage(x_shuffled, name='images') hem.montage(g_shuffle, name='fake_depths') with tf.variable_scope('noise'): hem.montage(x_noise, name='images') hem.montage(g_noise, name='fake_depths')
def encoder(x, args, reuse=False): """Adds encoder nodes to the graph. Args: x: Tensor, input images. reuse: Boolean, whether to reuse variables. """ with arg_scope([hem.conv2d], reuse = reuse, activation = hem.lrelu, use_batch_norm = True): x = hem.conv2d(x, 3, 64, 5, 2, name='c1') x = hem.conv2d(x, 64, 128, 5, 2, name='c2') x = hem.conv2d(x, 128, 256, 5, 2, name='c3') x = hem.conv2d(x, 256, 256, 5, 2, name='c4') x = hem.conv2d(x, 256, 96, 1, name='c5') x = hem.conv2d(x, 96, 32, 1, name='c6') #, activation=tf.nn.sigmoid) return x
def decoder(x_rep, args, channel_output=3, reuse=False): with arg_scope([hem.conv2d, hem.deconv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=hem.xavier_initializer, use_batch_norm=True, activation=lambda x: hem.lrelu(x, leak=0.2)): x_hat = hem.deconv2d(x_rep, 384, 192, output_shape=(args.batch_size, 192, 5, 5), name='d1') x_hat = hem.deconv2d(x_hat, 192, 48, output_shape=(args.batch_size, 48, 13, 13), name='d2') x_hat = hem.deconv2d(x_hat, 48, 24, output_shape=(args.batch_size, 24, 29, 29), name='d3') x_hat = hem.deconv2d(x_hat, 24, 12, output_shape=(args.batch_size, 12, 61, 61), name='d4') x_hat = hem.deconv2d(x_hat, 12, 6, output_shape=(args.batch_size, 6, 126, 126), name='d5') x_hat = hem.deconv2d(x_hat, 6, channel_output, activation=tf.tanh, output_shape=(args.batch_size, channel_output, 256, 256), name='d6') return x_hat
def decoder(x, args, reuse=False): """Adds decoder nodes to the graph. Args: x: Tensor, encoded image representation. args: Argparse struct reuse: Boolean, whether to reuse variables. """ with arg_scope([hem.dense, hem.conv2d, hem.deconv2d], reuse=reuse, activation=tf.nn.relu): x = hem.dense(x, args.latent_size, 32 * 4 * 4, name='d1') x = tf.reshape(x, [-1, 32, 4, 4]) # un-flatten x = hem.conv2d(x, 32, 96, 1, name='c1') x = hem.conv2d(x, 96, 256, 1, name='c2') x = hem.deconv2d(x, 256, 256, 5, 2, name='dc1') x = hem.deconv2d(x, 256, 128, 5, 2, name='dc2') x = hem.deconv2d(x, 128, 64, 5, 2, name='dc3') x = hem.deconv2d(x, 64, 3, 5, 2, name='dc4', activation=tf.nn.tanh) return x
def decoder(x, latent_size, reuse=False): """Adds decoder nodes to the graph. Args: x: Tensor, encoded image representation. latent_size: Integer, size of latent vector. reuse: Boolean, whether to reuse variables. """ with arg_scope([dense, conv2d, deconv2d], reuse = reuse, activation = tf.nn.relu): x = dense(x, latent_size, 32*4*4, name='d1') x = tf.reshape(x, [-1, 4, 4, 32]) # un-flatten x = conv2d(x, 32, 96, 1, name='c1') x = conv2d(x, 96, 256, 1, name='c2') x = deconv2d(x, 256, 256, 5, 2, name='dc1') x = deconv2d(x, 256, 128, 5, 2, name='dc2') x = deconv2d(x, 128, 64, 5, 2, name='dc3') x = deconv2d(x, 64, 3, 5, 2, name='dc4', activation=tf.nn.tanh) return x
def generator(z, x, reuse=False): # given the noise z (256x256x1) and rgb map x (256x256x3), generate a depth map y (256x256x1) with tf.variable_scope('generator', reuse=reuse),\ arg_scope([hem.conv2d, hem.deconv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02), activation=lambda x: hem.lrelu(x, leak=0.2)): y = tf.concat([x, z], axis=1) y = hem.conv2d(y, 4, 64, name='g1') y = hem.conv2d(y, 64, 128, name='g2') y = hem.conv2d(y, 128, 256, name='g3') y = hem.conv2d(y, 256, 512, name='g4') y = hem.deconv2d(y, 512, 256, name='g5') y = hem.deconv2d(y, 256, 128, name='g6') y = hem.deconv2d(y, 128, 64, name='g7') y = hem.deconv2d(y, 64, 1, name='g8', activation=tf.tanh) return y
def montage_summaries(self, x, y, g, y_hat, args): n_examples = 64 n = math.floor(math.sqrt(n_examples)) with tf.variable_scope('montage_preprocess'): y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True) y_hat = tf.reshape(y_hat, tf.shape(y)) # reattach shape info g = tf.reshape(g, tf.shape(y)) # reattach shape info y = y / 10.0 y_hat = y_hat / 10.0 y_bar = y_bar / 10.0 g = g / 10.0 with arg_scope([hem.montage], num_examples=n_examples, height=n, width=n, colorize=True): with tf.variable_scope('model'): hem.montage(x, name='x') hem.montage(y, name='y') hem.montage(tf.ones_like(y) * y_bar, name='y_bar') hem.montage(g, name='g') hem.montage(y_hat, name='y_hat')
def discriminator(x, args, reuse=False): """Adds discriminator nodes to the graph. From the input image, successively applies convolutions with striding to scale down layer sizes until we get to a single output value, representing the discriminator's estimate of fake vs real. Args: x: Tensor, the input (image) tensor. This should be a 4D Tensor with 3 RGB channels† and 1 depth channel. args: Argparse struct. reuse: Boolean, whether to reuse the variables in this scope. Returns: Tensor, the discriminator's estimation of whether the inputs are fake or real. """ with arg_scope([hem.conv2d], activation=hem.lrelu, reuse=reuse): x = hem.conv2d(x, 4, 64, 5, 2, name='disc_c1') x = hem.conv2d(x, 64, 128, 5, 2, name='disc_c2') x = hem.conv2d(x, 128, 256, 5, 2, name='disc_c3') x = hem.conv2d(x, 256, 512, 1, name='disc_c4') x = hem.conv2d(x, 512, 1, 1, name='disc_c5', activation=tf.nn.sigmoid) return x
def _encoder(x, args, reuse=False): """Adds encoder nodes to the graph. Args: x: Tensor, the input tensor/images. args: Arparse struct. reuse: Boolean, whether to reuse variables. Returns: Tensor, the encoded latent variables for the given input. """ with arg_scope([hem.conv2d], reuse=reuse, use_batch_norm=args.batch_norm, dropout=args.dropout, activation=hem.lrelu): noise = tf.random_uniform((args.batch_size, 64, 64, 1), minval=-1.0, maxval=1.0) noise = tf.transpose(noise, [0, 3, 1, 2]) x = tf.concat([x, noise], axis=1) # input = 64x646x4 x = hem.conv2d(x, 4, 64, 5, 2, name='e_c1', dropout=0, use_batch_norm=False) # output = 32x32x64 x = hem.conv2d(x, 64, 128, 5, 2, name='e_c2') # output = 16x16x128 x = hem.conv2d(x, 128, 256, 5, 2, name='e_c3') # output = 8x8x256 x = hem.conv2d(x, 256, 256, 5, 2, name='e_c4') # output = 4x4x256 x = hem.conv2d(x, 256, 96, 1, name='e_c5') # output = 4x4x96 x = hem.conv2d(x, 96, 32, 1, name='e_c6') # output = 4x4x32 return x
def g_baseline(self, x, args, reuse=False): with tf.variable_scope('encoder', reuse=reuse), \ arg_scope([hem.conv2d], reuse=reuse, filter_size=5, stride=2, padding='VALID', use_batch_norm=args.e_bn, init=tf.contrib.layers.xavier_initializer, activation=tf.nn.relu): # 65x65x3 if args.noise_layer == 'x': noise = tf.random_uniform([args.batch_size, 1, 65, 65], minval=0, maxval=1) e1 = hem.conv2d(tf.concat([x, noise], axis=1), 4, 64, name='e1') # 31x31x64 else: e1 = hem.conv2d(x, 3, 64, name='e1') if args.noise_layer == 'e1': noise = tf.random_uniform([args.batch_size, 1, 31, 31], minval=0, maxval=1) e2 = hem.conv2d(tf.concat([e1, noise], axis=1), 65, 128, name='e2') # 14x14x128 else: e2 = hem.conv2d(e1, 64, 128, name='e2') # 14x14x128 if args.noise_layer == 'e2': noise = tf.random_uniform([args.batch_size, 1, 14, 14], minval=0, maxval=1) e3 = hem.conv2d(tf.concat([e2, noise], axis=1), 129, 256, name='e3') # 5x5x256 else: e3 = hem.conv2d(e2, 128, 256, name='e3') # 5x5x256 if args.noise_layer == 'e3': noise = tf.random_uniform([args.batch_size, 1, 5, 5], minval=0, maxval=1) e4 = hem.conv2d(tf.concat([e3, noise], axis=1), 257, 512, name='e4') # 1x1x512 else: e4 = hem.conv2d(e3, 256, 512, name='e4') # 1x1x512 with tf.variable_scope('decoder', reuse=reuse), \ arg_scope([hem.deconv2d, hem.conv2d], reuse=reuse, filter_size=5, stride=2, init=tf.contrib.layers.xavier_initializer, padding='VALID', activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x512 # TODO: noise could be of size 512, instead of 1 if args.noise_layer == 'e4': noise = tf.random_uniform([args.batch_size, 1, 1, 1], minval=0, maxval=1) y_hat = hem.deconv2d(tf.concat([e4, noise], axis=1), 513, 256, output_shape=(args.batch_size, 256, 5, 5), name='d1') # 5x5x256 elif args.noise_layer == 'e4-512': noise = tf.random_uniform([args.batch_size, 512, 1, 1], minval=0, maxval=1) y_hat = hem.deconv2d(tf.concat([e4, noise], axis=1), 1024, 256, output_shape=(args.batch_size, 256, 5, 5), name='d1') # 5x5x256 else: y_hat = hem.deconv2d(e4, 512, 256, output_shape=(args.batch_size, 256, 5, 5), name='d1') # 5x5x256 y_hat = tf.concat([y_hat, e3], axis=1) # 5x5x512 if args.noise_layer == 'd2': noise = tf.random_uniform([args.batch_size, 1, 5, 5], minval=0, maxval=1) y_hat = hem.deconv2d(tf.concat([y_hat, noise], axis=1), 513, 128, output_shape=(args.batch_size, 128, 14, 14), name='d2') # 14x14x128z else: y_hat = hem.deconv2d(y_hat, 512, 128, output_shape=(args.batch_size, 128, 14, 14), name='d2') # 14x14x128z y_hat = tf.concat([y_hat, e2], axis=1) # 14x14x256 if args.noise_layer == 'd3': noise = tf.random_uniform([args.batch_size, 1, 14, 14], minval=0, maxval=1) y_hat = hem.deconv2d(tf.concat([y_hat, noise], axis=1), 257, 64, output_shape=(args.batch_size, 64, 31, 31), name='d3') # 31x31x64 else: y_hat = hem.deconv2d(y_hat, 256, 64, output_shape=(args.batch_size, 64, 31, 31), name='d3') # 31x31x64 y_hat = tf.concat([y_hat, e1], axis=1) # 31x31x128 if args.noise_layer == 'd4': noise = tf.random_uniform([args.batch_size, 1, 31, 31], minval=0, maxval=1) y_hat = hem.conv2d(tf.concat([y_hat, noise], axis=1), 129, 1, stride=1, filter_size=1, padding='SAME', activation=None, name='d4') # 31x31x1 else: y_hat = hem.conv2d(y_hat, 128, 1, stride=1, filter_size=1, padding='SAME', activation=None, name='d4') # 31x31x1 y_hat = hem.crop_to_bounding_box(y_hat, 0, 0, 29, 29) # 29x29x1 #y_hat = tf.maximum(y_hat, tf.zeros_like(y_hat)) return y_hat