def g_mean_provided2(self, x, y_bar, args, reuse=False):
with tf.variable_scope('encoder', reuse=reuse), \
arg_scope([hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=tf.contrib.layers.xavier_initializer,
activation=tf.nn.relu): # 65x65x3
x = tf.concat([x, tf.ones((args.batch_size, 1, 65, 65))], axis=1)
e1 = hem.conv2d(x, 4, 64, name='e1') # 31x31x64
# e1 = tf.concat([e1, tf.ones((args.batch_size, 1, 31, 31)) * y_bar], axis=1)
e2 = hem.conv2d(e1, 64, 128, name='e2') # 14x14x128
e3 = hem.conv2d(e2, 128, 256, name='e3') # 5x5x256
e4 = hem.conv2d(e3, 256, 512, name='e4') # 1x1x512
with tf.variable_scope('decoder', reuse=reuse), \
arg_scope([hem.deconv2d, hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
init=tf.contrib.layers.xavier_initializer,
padding='VALID',
activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x512
y_hat = hem.deconv2d(e4, 512, 256, output_shape=(args.batch_size, 256, 5, 5), name='d1') # 5x5x256
y_hat = tf.concat([y_hat, e3], axis=1) # 5x5x512
y_hat = hem.deconv2d(y_hat, 512, 128, output_shape=(args.batch_size, 128, 14, 14), name='d2') # 14x14x128
y_hat = tf.concat([y_hat, e2], axis=1) # 14x14x256
y_hat = hem.deconv2d(y_hat, 256, 64, output_shape=(args.batch_size, 64, 31, 31), name='d3') # 31x31x64
y_hat = tf.concat([y_hat, e1], axis=1) # 31x31x128
y_hat = hem.conv2d(y_hat, 128, 1, stride=1, filter_size=1, padding='SAME', activation=None,
name='d4') # 31x31x1
y_hat = hem.crop_to_bounding_box(y_hat, 0, 0, 29, 29) # 29x29x1
# y_hat = tf.maximum(y_hat, tf.zeros_like(y_hat))
return y_hat
def discriminatorE2(x, y, args, reuse=False):
with arg_scope([hem.conv2d],
reuse=reuse,
activation=lambda x: hem.lrelu(x, leak=0.2),
init=tf.contrib.layers.xavier_initializer,
padding='SAME',
filter_size=5,
stride=2): # x = 64x64x3, y = 32x32x1
with tf.variable_scope('rgb_path'):
h1 = hem.conv2d(x, 6, 64, name='hx1') # 32x32x64
h1 = hem.conv2d(h1, 64, 128, name='hx2') # 16x16x128
h1 = hem.conv2d(h1, 128, 256, name='hx3') # 8x8x256
h1 = hem.conv2d(h1, 256, 512, name='hx4') # 4x4x512
h1 = hem.conv2d(h1, 512, 1024, padding='VALID', filter_size=4, name='hx5') # 1x1x1024
with tf.variable_scope('depth_path'):
h2 = hem.conv2d(y, 1, 128, name='hy1') # 16x16x128
h2 = hem.conv2d(h2, 128, 256, name='hy2') # 8x8x256
h2 = hem.conv2d(h2, 256, 512, name='hy3') # 4x4x512
h2 = hem.conv2d(h2, 512, 1024, filter_size=4, padding='VALID', name='hy4') # 1x1x1024
with tf.variable_scope('combined_path'):
h = tf.concat([h1, h2], axis=1) # 1x1x2048
h = hem.conv2d(h, 2048, 1024, stride=1, filter_size=1, padding='SAME', name='h1') # 1x1x1024
h = hem.conv2d(h, 1024, 512, stride=1, filter_size=1, padding='SAME', name='h2') # 1x1x512
h = hem.conv2d(h, 512, 256, stride=1, filter_size=1, padding='SAME', name='h3') # 1x1x256
h = hem.conv2d(h, 256, 128, stride=1, filter_size=1, padding='SAME', name='h4') # 1x1x1x128
h = hem.conv2d(h, 128, 64, stride=1, filter_size=1, padding='SAME', name='h5') # 1x1x1x64
h = hem.conv2d(h, 64, 1, stride=1, filter_size=1, padding='SAME', name='h6',
activation=None) # 1x1x1x64
# output, logits
return tf.nn.sigmoid(h), h
def montage_summaries(self, x, y, g, y_hat, args):
n_examples = 64
n = math.floor(math.sqrt(n_examples))
with tf.variable_scope('montage_preprocess'):
y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True)
y_hat = tf.reshape(y_hat, tf.shape(y))
g = tf.reshape(g, tf.shape(y))
y = y / 10.0
y_hat = y_hat / 10.0
y_bar = y_bar / 10.0
g = g / 10.0
# if args.model_version == 'baseline':
# g = g / 10.0
# else:
# g = (g - tf.reduce_min(g)) / (tf.reduce_max(g) - tf.reduce_min(g))
with arg_scope([hem.montage],
num_examples=n_examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='x')
hem.montage(y, name='y')
hem.montage(tf.ones_like(y) * y_bar, name='y_bar')
hem.montage(g, name='g', colorize=False)
# hem.montage(tf.nn.sigmoid(g), name='g_sigmoid')
hem.montage(y_hat, name='y_hat')
def discriminator(x, y, args, reuse=False):
"""Adds (PatchGAN) discriminator nodes to the graph, given RGB and D inputs.
Args:
x_rgb: Tensor, the RGB component of the real or fake images.
x_depth: Tensor, the depth component of the real or fake images.
args: Argparse struct.
reuse: Bool, whether to reuse variables.
Returns:
Tensor, contains 70x70 patches with a probability assigned to each.
"""
with arg_scope([hem.conv2d],
reuse=reuse,
use_batch_norm=args.batch_norm_disc,
activation=lambda x: hem.lrelu(x, leak=0.2),
init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02),
filter_size=4,
stride=2):
x_y = tf.concat([x, y], axis=1)
h = hem.conv2d(x_y, 4, 64, name='m1', use_batch_norm=False)
h = hem.conv2d(h, 64, 128, name='m2')
h = hem.conv2d(h, 128, 256, name='m3')
h = hem.conv2d(h, 256, 512, name='m4')
h = hem.conv2d(h, 512, 1, name='m5', activation=None) # , activation=tf.nn.sigmoid)
# hem.montage(tf.nn.sigmoid(h), )
# layer outout, logits
return tf.nn.sigmoid(h), h
def d_mean_provided2(self, x, y, y_bar, args, reuse=False):
with arg_scope([hem.conv2d],
reuse=reuse,
activation=lambda x: hem.lrelu(x, leak=0.2),
init=tf.contrib.layers.xavier_initializer,
padding='VALID',
filter_size=5,
stride=2): # x = 65x65x3, y = 29x29x1
with tf.variable_scope('rgb_path'):
x = tf.concat([x, tf.ones((args.batch_size, 1, 65, 65)) * y_bar], axis=1)
h1 = hem.conv2d(x, 4, 64, name='hx1') # 31x31x64
h1 = hem.conv2d(h1, 64, 128, name='hx2') # 14x14x128
h1 = hem.conv2d(h1, 128, 256, name='hx3') # 5x5x256
h1 = hem.conv2d(h1, 256, 512, name='hx4') # 1x1x512
with tf.variable_scope('depth_path'):
h2 = tf.concat([y, tf.ones_like(y) * y_bar], axis=1, name='hy0')
h2 = hem.conv2d(h2, 2, 128, name='hy1') # 14x14x128
h2 = hem.conv2d(h2, 128, 256, name='hy2') # 5x5x256
h2 = hem.conv2d(h2, 256, 512, name='hy3') # 1x1x512
with tf.variable_scope('combined_path'):
h = tf.concat([h1, h2], axis=1) # 1x1x1024
h = hem.conv2d(h, 1024, 1024, stride=1, filter_size=1, padding='SAME', name='h1') # 1x1x768
h = hem.conv2d(h, 1024, 512, stride=1, filter_size=1, padding='SAME', name='h2') # 1x1x384
h = hem.conv2d(h, 512, 1, stride=1, filter_size=1, padding='SAME', name='h3', activation=None) # 1x1x1
# output, logits
return tf.nn.sigmoid(h), h
def generator(batch_size, latent_size, args, reuse=False):
"""Adds generator nodes to the graph.
From noise, applies deconv2d until image is scaled up to match
the dataset.
"""
# final_activation = tf.tanh if args.model in ['wgan', 'iwgan'] else tf.nn.sigmoid
output_dim = 64 * 64 * 3
with arg_scope([dense, deconv2d],
reuse=reuse,
use_batch_norm=True,
activation=tf.nn.relu):
z = tf.random_normal([batch_size, latent_size])
y = dense(z, latent_size, 4 * 4 * 4 * latent_size, name='fc1')
y = tf.reshape(y, [-1, 4, 4, 4 * latent_size])
y = deconv2d(y, 4 * latent_size, 2 * latent_size, 5, 2, name='dc1')
y = deconv2d(y, 2 * latent_size, latent_size, 5, 2, name='dc2')
y = deconv2d(y, latent_size, int(latent_size / 2), 5, 2, name='dc3')
y = deconv2d(y,
int(latent_size / 2),
3,
5,
2,
name='dc4',
activation=tf.tanh,
use_batch_norm=False)
y = tf.reshape(y, [-1, output_dim])
return y
def E2(x, args, reuse=False):
with arg_scope([hem.conv2d],
reuse=reuse,
stride=2,
padding='SAME',
filter_size=5,
init=tf.contrib.layers.xavier_initializer,
activation=tf.nn.relu): # 427 x 561 x 3
l1 = hem.conv2d(x, 3, 64, name='l1') # 214 x 281 x 64
# print('l1', l1)
l2 = hem.conv2d(l1, 64, 128, name='l2') # 107, 141 x 128
# print('l2', l2)
l3 = hem.conv2d(l2, 128, 256, name='l3') # 54 x 71 x 256
# print('l3', l3)
l4 = hem.conv2d(l3, 256, 512, name='l4') # 27 x 36 x 512
# print('l4', l4)
l5 = hem.conv2d(l4, 512, 1024, name='l5') # 14 x 18 x 1024
# print('l5', l5)
l6 = hem.conv2d(l5, 1024, 2048, name='l6') # 4 x 5 x 2048
# print('l6', l6)
f = hem.flatten(l6) # 4096
# print('f', f)
l7 = hem.dense(f, 4096, 2048, name='l7') # 2048
# print('l7', l7)
l8 = hem.dense(l7, 2048, 1, activation=tf.nn.sigmoid, name='l8') # 1
# print('l8', l8)
return l8
def discriminator(x, args, reuse=False):
"""Adds discriminator nodes to the graph.
From the input image, successively applies convolutions with
striding to scale down layer sizes until we get to a single
output value, representing the discriminator's estimate of fake
vs real. The single final output acts similar to a sigmoid
activation function.
Args:
x: Tensor, input.
args: Argparse structure.
reuse: Boolean, whether to reuse variables.
Returns:
Final output of discriminator pipeline.
"""
use_bn = False if args.model == 'iwgan' else True
final_activation = None if args.model in ['wgan', 'iwgan'
] else tf.nn.sigmoid
with arg_scope([hem.conv2d],
use_batch_norm=use_bn,
activation=hem.lrelu,
reuse=reuse):
x = tf.reshape(x, [-1, 3, 64, 64])
x = hem.conv2d(x,
3,
args.latent_size,
5,
2,
name='c1',
use_batch_norm=False)
x = hem.conv2d(x,
args.latent_size,
args.latent_size * 2,
5,
2,
name='c2')
x = hem.conv2d(x,
args.latent_size * 2,
args.latent_size * 4,
5,
2,
name='c3')
x = tf.reshape(x, [-1, 4 * 4 * 4 * args.latent_size])
x = hem.dense(x,
4 * 4 * 4 * args.latent_size,
1,
use_batch_norm=False,
activation=final_activation,
name='fc2',
reuse=reuse)
x = tf.reshape(x, [-1])
return x
def _decoder(x, args, reuse=False):
"""
Adds decoder nodes to the graph.
Args:
x: Tensor, the latent variable tensor from the encoder.
args: Argparse structure.
reuse: Boolean, whether to reuse pinned variables (in multi-GPU environment).
Returns:
Tensor, a single-channel image representing the predicted depth map.
"""
e_layers = tf.get_collection('conv_layers')
with arg_scope([hem.dense, hem.conv2d, hem.deconv2d],
reuse=reuse,
use_batch_norm=args.batch_norm,
dropout=args.dropout,
activation=tf.nn.relu):
skip_axis = 1
skip_m = 2 if args.skip_layers else 1
# TODO add better support for skip layers in layer ops
# input = 4x4x32
x = hem.conv2d(x, 32, 96, 1, name='d_c1') # output = 4x4x96 + e_c5
x = tf.concat(
(x, e_layers[4]), axis=skip_axis) if args.skip_layers else x
x = hem.conv2d(x, 96 * skip_m, 256, 1,
name='d_c2') # output = 4x4x256 + e_c4
x = tf.concat(
(x, e_layers[3]), axis=skip_axis) if args.skip_layers else x
x = hem.deconv2d(x, 256 * skip_m, 256, 5, 2,
name='d_dc1') # output = 8x8x256 + e_c3
x = tf.concat(
(x, e_layers[2]), axis=skip_axis) if args.skip_layers else x
x = hem.deconv2d(x, 256 * skip_m, 128, 5, 2,
name='d_dc2') # output = 16x16x128 + e_c2
x = tf.concat(
(x, e_layers[1]), axis=skip_axis) if args.skip_layers else x
x = hem.deconv2d(x, 128 * skip_m, 64, 5, 2,
name='d_dc3') # output = 32x32x64 + e_c1
x = tf.concat(
(x, e_layers[0]), axis=skip_axis) if args.skip_layers else x
x = hem.deconv2d(x,
64 * skip_m,
1,
5,
2,
name='d_dc4',
activation=tf.nn.tanh,
dropout=0,
use_batch_norm=False) # output = 64x64x1
return x
def predictor(y, reuse=False):
# given the depth map y, generate a rgb scene x
with tf.variable_scope('predictor, reuse=reuse'),\
arg_scope([hem.conv2d, hem.deconv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02),
activation=lambda x: hem.lrelu(x, leak=0.2)):
y = hem.conv2d(y, 1, 3, stride=1, activation=tf.tanh)
return y
def latent(x, latent_size, reuse=False):
"""Add latant nodes to the graph.
Args:
x: Tensor, output from encoder.
latent_size: Integer, size of latent vector.
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([dense], reuse = reuse):
x = flatten(x)
x = dense(x, 32*4*4, latent_size, name='d1')
return x
def montage_summaries(x, y, m, args):
n = math.floor(math.sqrt(args.examples))
with tf.variable_scope('model'):
with arg_scope([hem.montage],
num_examples=args.examples,
height=n,
width=n,
colorize=True):
hem.montage(y, name='real_depths')
hem.montage(x, name='real_images')
hem.montage(tf.expand_dims(tf.expand_dims(tf.reduce_mean(y, axis=[2, 3]), -1), -1), name='real_average_depths')
hem.montage(tf.expand_dims(tf.expand_dims(m, -1), -1), name='predicted_average_depths')
def montage_summaries(x_rgb, x_depth, g, args, x_repeated_rgb,
x_repeated_depth, g_sampler):
"""Adds montage summaries to the graph for the images, generator, and sampler.
Args:
x_rgb: Tensor, the real RGB image input batch.
x_depth: Tensor, the real depth input batch.
g: Tensor, the generator's output.
args: Argparse struct.
x_repeated_rgb: Tensor, the real RGB image inputs to the sampler path.
x_repeated_depth: Tensor, the real RGB depth inputs to the sampler path.
g_sampler: Tensor, the sampler generator outputs.
Returns:
None.
"""
# print('x shape', x_rgb.shape)
# print('y shape', x_depth.shape)
# print('y_hat shape', g.shape)
# print('x_repeated shape', x_repeated_depth.shape)
# print('y_hat_repeated shape', g_sampler.shape)
# example image montages
n = math.floor(math.sqrt(args.examples))
# rescale
with tf.variable_scope('montage_preprocess'):
g = tf.reshape(g, tf.shape(x_depth)) # re-attach lost shape info
g = hem.rescale(g, (-1, 1), (0, 1))
x_depth = hem.rescale(x_depth, (-1, 1), (0, 1))
g_sampler = tf.reshape(g_sampler, tf.shape(x_depth))
g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1))
x_repeated_depth = hem.rescale(x_repeated_depth, (-1, 1), (0, 1))
# x_depth = tf.reshape(x_depth, (-1, 1, 65, 65))
# add image montages
with arg_scope([hem.montage], height=n, width=n):
# example batches
hem.montage(x_rgb[0:args.examples], name='real/images')
hem.montage(x_depth[0:args.examples],
name='real/depths',
colorize=True)
hem.montage(x_rgb[0:args.examples], name='fake/images')
hem.montage(g[0:args.examples], name='fake/depths', colorize=True)
# sampler
hem.montage(x_repeated_rgb[0:args.examples], name='sampler/images')
hem.montage(x_repeated_depth[0:args.examples],
name='sampler/depths',
colorize=True)
hem.montage(g_sampler[0:args.examples],
name='sampler/fake',
colorize=True)
def latent(x, args, reuse=False):
"""Add latant nodes to the graph.
Args:
x: Tensor, output from encoder.
args: Argparse struct, containing:
latent_size, integer describing size of the latent vector
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([hem.dense], reuse=reuse):
x = hem.flatten(x)
x = hem.dense(x, 32 * 4 * 4, args.latent_size, name='d1')
return x
def generatorE2(x, args, reuse=False):
with tf.variable_scope('encoder', reuse=reuse), \
arg_scope([hem.conv2d],
reuse=reuse,
stride=2,
padding='SAME',
filter_size=5,
init=tf.contrib.layers.xavier_initializer,
activation=tf.nn.relu): # 64x64x3
noise = tf.random_uniform([args.batch_size, 1, 64, 64], minval=-1.0, maxval=1.0)
x = tf.concat([x, noise], axis=1) # 64x64x4
e1 = hem.conv2d(x, 7, 64, name='e1') # 32x32x64
e2 = hem.conv2d(e1, 64, 128, name='e2') # 16x16x128
e3 = hem.conv2d(e2, 128, 256, name='e3') # 8x8x256
e4 = hem.conv2d(e3, 256, 512, name='e4') # 4x4x512
e5 = hem.conv2d(e4, 512, 1024, filter_size=4, padding='VALID', name='e5') # 1x1x1024
with tf.variable_scope('decoder', reuse=reuse), \
arg_scope([hem.deconv2d, hem.conv2d],
reuse=reuse,
stride=2,
init=tf.contrib.layers.xavier_initializer,
padding='SAME',
filter_size=5,
activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x1024
y = hem.deconv2d(e5, 1024, 512, output_shape=(args.batch_size, 512, 4, 4), filter_size=4, padding='VALID',
name='d1') # 4x4x512
y = tf.concat([y, e4], axis=1) # 4x4x1024
y = hem.deconv2d(y, 1024, 256, output_shape=(args.batch_size, 256, 8, 8), name='d2') # 8x8x256
y = tf.concat([y, e3], axis=1) # 8x8x512
y = hem.deconv2d(y, 512, 128, output_shape=(args.batch_size, 128, 16, 16), name='d3') # 16x16x128
y = tf.concat([y, e2], axis=1) # 16x16x256
y = hem.deconv2d(y, 256, 64, output_shape=(args.batch_size, 64, 32, 32), name='d4') # 32x32x64
y = tf.concat([y, e1], axis=1) # 32x32x128
y = hem.conv2d(y, 128, 1, stride=1, filter_size=1, activation=tf.nn.tanh, name='d5') # 31x31x1
return y
def encoder(x, reuse=False):
with arg_scope([hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=hem.xavier_initializer,
use_batch_norm=True,
activation=lambda x: hem.lrelu(x, leak=0.2)):
x_rep = hem.conv2d(x, 3, 6, name='e1', use_batch_norm=False)
x_rep = hem.conv2d(x_rep, 6, 12, name='e2')
x_rep = hem.conv2d(x_rep, 12, 24, name='e3')
x_rep = hem.conv2d(x_rep, 24, 48, name='e4')
x_rep = hem.conv2d(x_rep, 48, 192, name='e5')
x_rep = hem.conv2d(x_rep, 192, 384, name='e6') # 5x5x192
return x_rep
def summaries(x, y, x_hat, y_hat, x_grads, y_grads, args):
n = math.floor(math.sqrt(args.examples))
with arg_scope([hem.montage], height=n, width=n):
x = hem.rescale(x, (-1, 1), (0, 1))
y = hem.rescale(y, (-1, 1), (0, 1))
x_hat = hem.rescale(x_hat, (-1, 1), (0, 1))
y_hat = hem.rescale(y_hat, (-1, 1), (0, 1))
hem.montage(x[0:args.examples], name='x')
hem.montage(y[0:args.examples], name='y', colorize=True)
hem.montage(x_hat[0:args.examples], name='x_hat')
hem.montage(y_hat[0:args.examples], name='y_hat', colorize=True)
hem.summarize_gradients(x_grads)
hem.summarize_gradients(y_grads)
hem.summarize_losses()
hem.summarize_activations(False)
hem.summarize_weights_biases()
def latent(x, batch_size, latent_size, reuse=False):
"""Adds latent nodes for sampling and reparamaterization.
Args:
x: Tensor, input images.
batch_size: Integer, batch size.
latent_size: Integer, size of latent vector.
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([dense], reuse=reuse):
flat = flatten(x)
z_mean = dense(flat, 32 * 4 * 4, latent_size, name='d1')
z_stddev = dense(flat, 32 * 4 * 4, latent_size, name='d2')
samples = tf.random_normal([batch_size, latent_size], 0, 1)
z = (z_mean + (z_stddev * samples))
return (samples, z, z_mean, z_stddev)
def encoder(x, args, reuse=False):
"""Adds encoder nodes to the graph.
Args:
x: Tensor, input images.
args: Argparse struct
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([hem.conv2d], reuse=reuse, activation=hem.lrelu):
x = hem.conv2d(x, 3, 64, 5, 2, name='c1')
x = hem.conv2d(x, 64, 128, 5, 2, name='c2')
x = hem.conv2d(x, 128, 256, 5, 2, name='c3')
x = hem.conv2d(x, 256, 256, 5, 2, name='c4')
x = hem.conv2d(x, 256, 96, 1, name='c5')
x = hem.conv2d(x, 96, 32, 1, name='c6')
return x
def discriminator(y, reuse=False):
# given a depth map y, determine whether it is real or fake
with tf.variable_scope('discriminator', reuse=reuse),\
arg_scope([hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02),
activation=lambda x: hem.lrelu(x, leak=0.2)):
y = hem.conv2d(y, 1, 64, name='d1')
y = hem.conv2d(y, 64, 128, name='d2')
y = hem.conv2d(y, 128, 256, name='d3')
y = hem.conv2d(y, 256, 512, name='d4')
y = hem.conv2d(y, 512, 256, name='d5')
y = hem.conv2d(y, 256, 1, name='d6', activation=tf.sigmoid)
return y
def montage_summaries(x, y, g,
x_sample, y_sample, g_sampler,
x_noise, g_noise,
x_shuffled, y_shuffled, g_shuffle,
args):
n = math.floor(math.sqrt(args.examples))
if args.g_arch in ['E2']:
x, x_loc, y_loc, mean_vec = tf.split(x, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_noise, x_loc_noise, y_loc_noise, mean_vec_noise = tf.split(x_noise, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_shuffled, x_loc_shuffled, y_loc_shuffled, mean_vec_shuffled = tf.split(x_shuffled, num_or_size_splits=[3, 1, 1, 1], axis=1)
x_sample, x_loc_sample, y_loc_sample, mean_vec_sample = tf.split(x_sample, num_or_size_splits=[3, 1, 1, 1], axis=1)
with tf.variable_scope('model'):
hem.montage(x_loc, num_examples=args.examples, height=n, width=n, name='x_loc', colorize=True)
hem.montage(y_loc, num_examples=args.examples, height=n, width=n, name='y_loc', colorize=True)
hem.montage(mean_vec, num_examples=args.examples, height=n, width=n, name='mean_vec', colorize=True)
with tf.variable_scope('montage_preprocess'):
g = tf.reshape(g, tf.shape(y)) # reattach
g = hem.rescale(g, (-1, 1), (0, 1))
y = hem.rescale(y, (-1, 1), (0, 1))
g_sampler = tf.reshape(g_sampler, tf.shape(y)) # reattach
g_sampler = hem.rescale(g_sampler, (-1, 1), (0, 1))
y_sample = hem.rescale(y_sample, (-1, 1), (0, 1))
# add image montages
with arg_scope([hem.montage],
num_examples=args.examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='images')
hem.montage(y, name='real_depths')
hem.montage(g, name='fake_depths')
with tf.variable_scope('sampler'):
hem.montage(x_sample, name='images')
hem.montage(y_sample, name='real_depths')
hem.montage(g_sampler, name='fake_depths')
with tf.variable_scope('shuffled'):
hem.montage(x_shuffled, name='images')
hem.montage(g_shuffle, name='fake_depths')
with tf.variable_scope('noise'):
hem.montage(x_noise, name='images')
hem.montage(g_noise, name='fake_depths')
def encoder(x, args, reuse=False):
"""Adds encoder nodes to the graph.
Args:
x: Tensor, input images.
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([hem.conv2d],
reuse = reuse,
activation = hem.lrelu,
use_batch_norm = True):
x = hem.conv2d(x, 3, 64, 5, 2, name='c1')
x = hem.conv2d(x, 64, 128, 5, 2, name='c2')
x = hem.conv2d(x, 128, 256, 5, 2, name='c3')
x = hem.conv2d(x, 256, 256, 5, 2, name='c4')
x = hem.conv2d(x, 256, 96, 1, name='c5')
x = hem.conv2d(x, 96, 32, 1, name='c6') #, activation=tf.nn.sigmoid)
return x
def decoder(x_rep, args, channel_output=3, reuse=False):
with arg_scope([hem.conv2d, hem.deconv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=hem.xavier_initializer,
use_batch_norm=True,
activation=lambda x: hem.lrelu(x, leak=0.2)):
x_hat = hem.deconv2d(x_rep,
384,
192,
output_shape=(args.batch_size, 192, 5, 5),
name='d1')
x_hat = hem.deconv2d(x_hat,
192,
48,
output_shape=(args.batch_size, 48, 13, 13),
name='d2')
x_hat = hem.deconv2d(x_hat,
48,
24,
output_shape=(args.batch_size, 24, 29, 29),
name='d3')
x_hat = hem.deconv2d(x_hat,
24,
12,
output_shape=(args.batch_size, 12, 61, 61),
name='d4')
x_hat = hem.deconv2d(x_hat,
12,
6,
output_shape=(args.batch_size, 6, 126, 126),
name='d5')
x_hat = hem.deconv2d(x_hat,
6,
channel_output,
activation=tf.tanh,
output_shape=(args.batch_size, channel_output,
256, 256),
name='d6')
return x_hat
def decoder(x, args, reuse=False):
"""Adds decoder nodes to the graph.
Args:
x: Tensor, encoded image representation.
args: Argparse struct
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([hem.dense, hem.conv2d, hem.deconv2d],
reuse=reuse,
activation=tf.nn.relu):
x = hem.dense(x, args.latent_size, 32 * 4 * 4, name='d1')
x = tf.reshape(x, [-1, 32, 4, 4]) # un-flatten
x = hem.conv2d(x, 32, 96, 1, name='c1')
x = hem.conv2d(x, 96, 256, 1, name='c2')
x = hem.deconv2d(x, 256, 256, 5, 2, name='dc1')
x = hem.deconv2d(x, 256, 128, 5, 2, name='dc2')
x = hem.deconv2d(x, 128, 64, 5, 2, name='dc3')
x = hem.deconv2d(x, 64, 3, 5, 2, name='dc4', activation=tf.nn.tanh)
return x
def decoder(x, latent_size, reuse=False):
"""Adds decoder nodes to the graph.
Args:
x: Tensor, encoded image representation.
latent_size: Integer, size of latent vector.
reuse: Boolean, whether to reuse variables.
"""
with arg_scope([dense, conv2d, deconv2d],
reuse = reuse,
activation = tf.nn.relu):
x = dense(x, latent_size, 32*4*4, name='d1')
x = tf.reshape(x, [-1, 4, 4, 32]) # un-flatten
x = conv2d(x, 32, 96, 1, name='c1')
x = conv2d(x, 96, 256, 1, name='c2')
x = deconv2d(x, 256, 256, 5, 2, name='dc1')
x = deconv2d(x, 256, 128, 5, 2, name='dc2')
x = deconv2d(x, 128, 64, 5, 2, name='dc3')
x = deconv2d(x, 64, 3, 5, 2, name='dc4', activation=tf.nn.tanh)
return x
def generator(z, x, reuse=False):
# given the noise z (256x256x1) and rgb map x (256x256x3), generate a depth map y (256x256x1)
with tf.variable_scope('generator', reuse=reuse),\
arg_scope([hem.conv2d, hem.deconv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
init=lambda: tf.random_normal_initializer(mean=0, stddev=0.02),
activation=lambda x: hem.lrelu(x, leak=0.2)):
y = tf.concat([x, z], axis=1)
y = hem.conv2d(y, 4, 64, name='g1')
y = hem.conv2d(y, 64, 128, name='g2')
y = hem.conv2d(y, 128, 256, name='g3')
y = hem.conv2d(y, 256, 512, name='g4')
y = hem.deconv2d(y, 512, 256, name='g5')
y = hem.deconv2d(y, 256, 128, name='g6')
y = hem.deconv2d(y, 128, 64, name='g7')
y = hem.deconv2d(y, 64, 1, name='g8', activation=tf.tanh)
return y
def montage_summaries(self, x, y, g, y_hat, args):
n_examples = 64
n = math.floor(math.sqrt(n_examples))
with tf.variable_scope('montage_preprocess'):
y_bar = tf.reduce_mean(y, axis=[2, 3], keep_dims=True)
y_hat = tf.reshape(y_hat, tf.shape(y)) # reattach shape info
g = tf.reshape(g, tf.shape(y)) # reattach shape info
y = y / 10.0
y_hat = y_hat / 10.0
y_bar = y_bar / 10.0
g = g / 10.0
with arg_scope([hem.montage],
num_examples=n_examples,
height=n,
width=n,
colorize=True):
with tf.variable_scope('model'):
hem.montage(x, name='x')
hem.montage(y, name='y')
hem.montage(tf.ones_like(y) * y_bar, name='y_bar')
hem.montage(g, name='g')
hem.montage(y_hat, name='y_hat')
def discriminator(x, args, reuse=False):
"""Adds discriminator nodes to the graph.
From the input image, successively applies convolutions with
striding to scale down layer sizes until we get to a single
output value, representing the discriminator's estimate of fake
vs real.
Args:
x: Tensor, the input (image) tensor. This should be a 4D Tensor with 3 RGB channels† and 1 depth channel.
args: Argparse struct.
reuse: Boolean, whether to reuse the variables in this scope.
Returns:
Tensor, the discriminator's estimation of whether the inputs are fake or real.
"""
with arg_scope([hem.conv2d], activation=hem.lrelu, reuse=reuse):
x = hem.conv2d(x, 4, 64, 5, 2, name='disc_c1')
x = hem.conv2d(x, 64, 128, 5, 2, name='disc_c2')
x = hem.conv2d(x, 128, 256, 5, 2, name='disc_c3')
x = hem.conv2d(x, 256, 512, 1, name='disc_c4')
x = hem.conv2d(x, 512, 1, 1, name='disc_c5', activation=tf.nn.sigmoid)
return x
def _encoder(x, args, reuse=False):
"""Adds encoder nodes to the graph.
Args:
x: Tensor, the input tensor/images.
args: Arparse struct.
reuse: Boolean, whether to reuse variables.
Returns:
Tensor, the encoded latent variables for the given input.
"""
with arg_scope([hem.conv2d],
reuse=reuse,
use_batch_norm=args.batch_norm,
dropout=args.dropout,
activation=hem.lrelu):
noise = tf.random_uniform((args.batch_size, 64, 64, 1),
minval=-1.0,
maxval=1.0)
noise = tf.transpose(noise, [0, 3, 1, 2])
x = tf.concat([x, noise], axis=1)
# input = 64x646x4
x = hem.conv2d(x,
4,
64,
5,
2,
name='e_c1',
dropout=0,
use_batch_norm=False) # output = 32x32x64
x = hem.conv2d(x, 64, 128, 5, 2, name='e_c2') # output = 16x16x128
x = hem.conv2d(x, 128, 256, 5, 2, name='e_c3') # output = 8x8x256
x = hem.conv2d(x, 256, 256, 5, 2, name='e_c4') # output = 4x4x256
x = hem.conv2d(x, 256, 96, 1, name='e_c5') # output = 4x4x96
x = hem.conv2d(x, 96, 32, 1, name='e_c6') # output = 4x4x32
return x
def g_baseline(self, x, args, reuse=False):
with tf.variable_scope('encoder', reuse=reuse), \
arg_scope([hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
padding='VALID',
use_batch_norm=args.e_bn,
init=tf.contrib.layers.xavier_initializer,
activation=tf.nn.relu): # 65x65x3
if args.noise_layer == 'x':
noise = tf.random_uniform([args.batch_size, 1, 65, 65],
minval=0,
maxval=1)
e1 = hem.conv2d(tf.concat([x, noise], axis=1),
4,
64,
name='e1') # 31x31x64
else:
e1 = hem.conv2d(x, 3, 64, name='e1')
if args.noise_layer == 'e1':
noise = tf.random_uniform([args.batch_size, 1, 31, 31],
minval=0,
maxval=1)
e2 = hem.conv2d(tf.concat([e1, noise], axis=1),
65,
128,
name='e2') # 14x14x128
else:
e2 = hem.conv2d(e1, 64, 128, name='e2') # 14x14x128
if args.noise_layer == 'e2':
noise = tf.random_uniform([args.batch_size, 1, 14, 14],
minval=0,
maxval=1)
e3 = hem.conv2d(tf.concat([e2, noise], axis=1),
129,
256,
name='e3') # 5x5x256
else:
e3 = hem.conv2d(e2, 128, 256, name='e3') # 5x5x256
if args.noise_layer == 'e3':
noise = tf.random_uniform([args.batch_size, 1, 5, 5],
minval=0,
maxval=1)
e4 = hem.conv2d(tf.concat([e3, noise], axis=1),
257,
512,
name='e4') # 1x1x512
else:
e4 = hem.conv2d(e3, 256, 512, name='e4') # 1x1x512
with tf.variable_scope('decoder', reuse=reuse), \
arg_scope([hem.deconv2d, hem.conv2d],
reuse=reuse,
filter_size=5,
stride=2,
init=tf.contrib.layers.xavier_initializer,
padding='VALID',
activation=lambda x: hem.lrelu(x, leak=0.2)): # 1x1x512
# TODO: noise could be of size 512, instead of 1
if args.noise_layer == 'e4':
noise = tf.random_uniform([args.batch_size, 1, 1, 1],
minval=0,
maxval=1)
y_hat = hem.deconv2d(tf.concat([e4, noise], axis=1),
513,
256,
output_shape=(args.batch_size, 256, 5, 5),
name='d1') # 5x5x256
elif args.noise_layer == 'e4-512':
noise = tf.random_uniform([args.batch_size, 512, 1, 1],
minval=0,
maxval=1)
y_hat = hem.deconv2d(tf.concat([e4, noise], axis=1),
1024,
256,
output_shape=(args.batch_size, 256, 5, 5),
name='d1') # 5x5x256
else:
y_hat = hem.deconv2d(e4,
512,
256,
output_shape=(args.batch_size, 256, 5, 5),
name='d1') # 5x5x256
y_hat = tf.concat([y_hat, e3], axis=1) # 5x5x512
if args.noise_layer == 'd2':
noise = tf.random_uniform([args.batch_size, 1, 5, 5],
minval=0,
maxval=1)
y_hat = hem.deconv2d(tf.concat([y_hat, noise], axis=1),
513,
128,
output_shape=(args.batch_size, 128, 14,
14),
name='d2') # 14x14x128z
else:
y_hat = hem.deconv2d(y_hat,
512,
128,
output_shape=(args.batch_size, 128, 14,
14),
name='d2') # 14x14x128z
y_hat = tf.concat([y_hat, e2], axis=1) # 14x14x256
if args.noise_layer == 'd3':
noise = tf.random_uniform([args.batch_size, 1, 14, 14],
minval=0,
maxval=1)
y_hat = hem.deconv2d(tf.concat([y_hat, noise], axis=1),
257,
64,
output_shape=(args.batch_size, 64, 31,
31),
name='d3') # 31x31x64
else:
y_hat = hem.deconv2d(y_hat,
256,
64,
output_shape=(args.batch_size, 64, 31,
31),
name='d3') # 31x31x64
y_hat = tf.concat([y_hat, e1], axis=1) # 31x31x128
if args.noise_layer == 'd4':
noise = tf.random_uniform([args.batch_size, 1, 31, 31],
minval=0,
maxval=1)
y_hat = hem.conv2d(tf.concat([y_hat, noise], axis=1),
129,
1,
stride=1,
filter_size=1,
padding='SAME',
activation=None,
name='d4') # 31x31x1
else:
y_hat = hem.conv2d(y_hat,
128,
1,
stride=1,
filter_size=1,
padding='SAME',
activation=None,
name='d4') # 31x31x1
y_hat = hem.crop_to_bounding_box(y_hat, 0, 0, 29, 29) # 29x29x1
#y_hat = tf.maximum(y_hat, tf.zeros_like(y_hat))
return y_hat
