def discriminator(x, label):
with tf.variable_scope('Discriminator'):
if layers > 1:
with tf.variable_scope('rgb_layer_{}'.format(layers - 2)):
d0 = pool(x)
d0 = leaky_relu(conv2d(d0, self.channels[layers - 1], 1))
with tf.variable_scope('rgb_layer_{}'.format(layers - 1)):
d1 = leaky_relu(conv2d(x, self.channels[layers], 1))
for i in reversed(range(layers)):
with tf.variable_scope('layer_{}'.format(i)):
if i == 0:
d1 = minibatch_stddev(d1)
with tf.variable_scope('1'):
d1 = leaky_relu(conv2d(d1, self.channels[i]))
with tf.variable_scope('2'):
if i == 0:
d1 = leaky_relu(conv2d(d1, self.channels[0], 2, 2))
else:
d1 = leaky_relu(conv2d(d1, self.channels[i]))
if i > 0:
d1 = pool(d1)
if i == layers - 1 and layers > 1:
d1 = self._reparameterize(d0, d1)
with tf.variable_scope('dense'):
d = tf.concat([tf.layers.flatten(d1), tf.layers.flatten(label)], axis=1)
d = dense_layer(d, 1)
return d
def generator(z, label):
with tf.variable_scope('Generator'):
z = z[:self.batch_size[layers]]
with tf.variable_scope('latent_vector'):
g1 = tf.expand_dims(tf.expand_dims(z, axis=-1), axis=-1)
g1 = tf.concat([g1, label], axis=1)
for i in range(layers):
with tf.variable_scope('layer_{}'.format(i)):
if i > 0:
g1 = resize(g1)
if i == layers - 1 and layers > 1:
g0 = g1
with tf.variable_scope('1'):
if i == 0:
g1 = pixelwise_norm(leaky_relu(conv2d_transpose(
g1, [tf.shape(g1)[0], self.channels[0], 2, 8]
)))
else:
g1 = pixelwise_norm(leaky_relu(conv2d(g1, self.channels[i])))
with tf.variable_scope('2'):
g1 = pixelwise_norm(leaky_relu(conv2d(g1, self.channels[i])))
with tf.variable_scope('rgb_layer_{}'.format(layers - 1)):
g1 = conv2d(g1, self.channel_num, 1, weight_norm=False)
if layers > 1:
with tf.variable_scope('rgb_layer_{}'.format(layers - 2)):
g0 = conv2d(g0, self.channel_num, 1, weight_norm=False)
g = self._reparameterize(g0, g1)
else:
g = g1
return tf.tanh(g)
def up_sample4(ipt, name='up_sample4', reuse=False, is_training=True):
with tf.variable_scope(name):
c3s1k256 = ops.conv2d(ipt,
256,
3,
1,
norm=None,
activation=None,
reuse=reuse,
is_training=is_training,
name='c3s1k256_1')
d_to_s_1 = tf.depth_to_space(c3s1k256, 2)
relu1 = ops.leaky_relu(d_to_s_1)
c3s1k256 = ops.conv2d(relu1,
256,
3,
1,
norm=None,
activation=None,
reuse=reuse,
is_training=is_training,
name='c3s1k256_2')
d_to_s_2 = tf.depth_to_space(c3s1k256, 2)
relu2 = ops.leaky_relu(d_to_s_2)
return ops.conv2d(relu2,
3,
3,
1,
1,
norm=None,
activation=None,
name='c3s1k3',
reuse=reuse,
is_training=is_training)
def dblock(name, inputs, num_filters, data_format):
with tf.variable_scope(name):
x = ops.conv2d('conv', inputs, num_filters[0], 3, data_format)
x = ops.leaky_relu(x)
x = ops.conv2d_down('conv_down', x, num_filters[1], 3, data_format)
x = ops.leaky_relu(x)
return x
def gblock(name, inputs, filters, data_format):
with tf.variable_scope(name):
x = ops.conv2d_up('conv_up', inputs, filters, 3, data_format)
x = ops.leaky_relu(x)
x = ops.pixel_norm(x, data_format)
x = ops.conv2d('conv', x, filters, 3, data_format)
x = ops.leaky_relu(x)
x = ops.pixel_norm(x, data_format)
return x
def generator(x,
last_layer_resolution,
cfg,
is_training=True,
scope='Generator'):
def rname(resolution):
return str(resolution) + 'x' + str(resolution)
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
with tf.variable_scope("4x4"):
fn4 = cfg.resolution_to_filt_num[4]
x = ops.pixel_norm(x, cfg.data_format)
x = ops.dense('dense', x, 4 * 4 * fn4, cfg.data_format)
if cfg.data_format == 'NHWC':
x = tf.reshape(x, [-1, 4, 4, fn4])
else:
x = tf.reshape(x, [-1, fn4, 4, 4])
x = ops.leaky_relu(x)
x = ops.pixel_norm(x, cfg.data_format)
x = ops.conv2d('conv', x, fn4, 3, cfg.data_format)
x = ops.leaky_relu(x)
x = ops.pixel_norm(x, cfg.data_format)
resolution = 8
prev_x = None
while resolution <= last_layer_resolution:
filt_num = cfg.resolution_to_filt_num[resolution]
prev_x = x
x = gblock(rname(resolution), x, filt_num, cfg.data_format)
resolution *= 2
resolution = resolution // 2
if resolution > cfg.starting_resolution:
t = tf.get_variable(
rname(resolution) + '_t',
shape=[],
collections=[tf.GraphKeys.GLOBAL_VARIABLES, "lerp"],
dtype=tf.float32,
initializer=tf.zeros_initializer(),
trainable=False)
x1 = ops.to_rgb('to_rgb_' + rname(resolution // 2), prev_x,
cfg.data_format)
x1 = ops.upscale2d(x1, cfg.data_format)
x2 = ops.to_rgb('to_rgb_' + rname(resolution), x, cfg.data_format)
x = ops.lerp_clip(x1, x2, t)
else:
x = ops.to_rgb('to_rgb_' + rname(resolution), x, cfg.data_format)
x_shape = utils.int_shape(x)
assert (resolution == x_shape[1 if cfg.data_format == 'NHWC' else 3])
assert (resolution == x_shape[2])
return x
def discriminator(images, labels, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse:
scope.reuse_variables()
# conv1
conv1 = ops.conv_2d(images, 64, scope="conv1")
# leakly ReLu
h1 = ops.leaky_relu(conv1)
# conv2
conv2 = ops.conv_2d(h1, 128, scope="conv2")
# batch norm
norm2 = ops.batch_norm(conv2, scope="batch_norm2", is_training=True)
# leaky ReLU
h2 = ops.leaky_relu(norm2)
# conv3
conv3 = ops.conv_2d(h2, 256, scope="conv3")
# batch norm
norm3 = ops.batch_norm(conv3, scope="batch_norm3", is_training=True)
# leaky ReLU
h3 = ops.leaky_relu(norm3)
# conv4
conv4 = ops.conv_2d(h3, 512, scope="conv4")
# batch norm
norm4 = ops.batch_norm(conv4, scope="batch_norm4", is_training=True)
# leaky ReLU
h4 = ops.leaky_relu(norm4)
# reshape
h4_reshape = tf.reshape(h4, [FLAGS.batch_size, -1])
# source logits
source_logits = ops.fc(h4_reshape, 1, scope="source_logits")
# class logits
class_logits = ops.fc(
h4_reshape, FLAGS.n_classes, scope="class_logits")
return source_logits, class_logits
def generator(self, inputs, reuse=False, is_training=True):
####################################################
# Define the structure of generator, you may use the
# generator structure of DCGAN. ops.py defines some
# layers that you may use.
####################################################
with tf.variable_scope('generator', reuse=reuse):
size = self.image_size # 64
s2 = self.get_required_conv_out_size(size, 2) # 32
s4 = self.get_required_conv_out_size(s2, 2) # 16
s8 = self.get_required_conv_out_size(s4, 2) # 8
s16 = self.get_required_conv_out_size(s8, 2) # 4
img_ch = 3
# project z and reshape
fc1 = ops.linear(inputs, self.image_size * 8 * s16**2,
'fc') # [11,8192]
fc1_reshaped = tf.reshape(fc1, [-1, s16, s16, self.image_size * 8])
h0 = ops.leaky_relu(fc1_reshaped) #[11,4,4,512]
conv1 = ops.convt2d(h0,
[self.batch_size, s8, s8, self.image_size * 4],
name='g_h1')
h1 = ops.leaky_relu(conv1) # [11,8,8,256]
conv2 = ops.convt2d(h1,
[self.batch_size, s4, s4, self.image_size * 2],
name='g_h2')
h2 = ops.leaky_relu(conv2) # [11,16,16,128]
conv3 = ops.convt2d(h2,
[self.batch_size, s2, s2, self.image_size * 1],
name='g_h3')
h3 = ops.leaky_relu(conv3) # [11,32,32,64]
h4 = ops.convt2d(h3, [self.batch_size, size, size, img_ch],
name='g_h4') # [11,64,64,3]
return h4
def discriminator(image, reuse=False):
with tf.variable_scope('discriminator', reuse=reuse):
conv_1 = ops.conv_2d(image, 64, scope='conv_1')
relu_1 = ops.leaky_relu(conv_1)
conv_2 = ops.conv_2d(relu_1, 128, scope='conv_2')
conv_2_norm = ops.batch_norm(conv_2, True, scope="batch_norm_2")
relu_2 = ops.leaky_relu(conv_2_norm)
conv_3 = ops.conv_2d(relu_2, 256, scope='conv_3')
conv_3_norm = ops.batch_norm(conv_3, True, scope="batch_norm_3")
relu_3 = ops.leaky_relu(conv_3_norm)
conv_4 = ops.conv_2d(relu_3, 512, scope='conv_4')
conv_4_norm = ops.batch_norm(conv_4, True, scope="batch_norm_4")
relu_4 = ops.leaky_relu(conv_4_norm)
relu_4_flat = tf.reshape(relu_4, [flags.batch_size, -1])
source_logits = ops.full_connect(relu_4_flat, 1, scope='source_logits')
class_logits = ops.full_connect(relu_4_flat, 1, scope='class_logits')
return source_logits, class_logits
def __call__(self, inputs, train_phase):
with tf.variable_scope(self.name, reuse=tf.AUTO_REUSE):
# inputs = tf.random_crop(inputs, [-1, 70, 70, 3])
inputs = conv("conv1_1", inputs, 64, 3, 2)
inputs = leaky_relu(inputs, 0.2)
# inputs = conv("conv1_2", inputs, 64, 3, is_SN=True)
# inputs = leaky_relu(inputs, 0.2)
inputs = conv("conv2_1", inputs, 128, 3, 2)
inputs = batchnorm(inputs, train_phase, "BN1")
inputs = leaky_relu(inputs, 0.2)
# inputs = conv("conv2_2", inputs, 128, 3, is_SN=True)
# inputs = leaky_relu(inputs, 0.2)
inputs = conv("conv3_1", inputs, 256, 3, 2)
inputs = batchnorm(inputs, train_phase, "BN2")
inputs = leaky_relu(inputs, 0.2)
# inputs = conv("conv3_2", inputs, 256, 3, is_SN=True)
# inputs = leaky_relu(inputs, 0.2)
inputs = conv("conv4_1", inputs, 512, 3, 2)
inputs = batchnorm(inputs, train_phase, "BN3")
inputs = leaky_relu(inputs, 0.2)
# inputs = fully_connected("fc", inputs, 512, is_SN=True)
output = fully_connected("output", inputs, 1)
return output
def discriminator(x, resolution, cfg, is_training=True, scope='Discriminator'):
assert (cfg.data_format == 'NCHW' or cfg.data_format == 'NHWC')
def rname(resolution):
return str(resolution) + 'x' + str(resolution)
def fmap(resolution):
return cfg.resolution_to_filt_num[resolution]
x_shape = utils.int_shape(x)
assert (resolution == x_shape[1 if cfg.data_format == 'NHWC' else 3])
assert (resolution == x_shape[2])
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
if resolution > cfg.starting_resolution:
x1 = ops.downscale2d(x, cfg.data_format)
x1 = ops.from_rgb('from_rgb_' + rname(resolution // 2), x1,
fmap(resolution // 2), cfg.data_format)
x2 = ops.from_rgb('from_rgb_' + rname(resolution), x,
fmap(resolution // 2), cfg.data_format)
t = tf.get_variable(
rname(resolution) + '_t',
shape=[],
dtype=tf.float32,
collections=[tf.GraphKeys.GLOBAL_VARIABLES, "lerp"],
initializer=tf.zeros_initializer(),
trainable=False)
num_filters = [fmap(resolution), fmap(resolution // 2)]
x2 = dblock(rname(resolution), x2, num_filters, cfg.data_format)
x = ops.lerp_clip(x1, x2, t)
resolution = resolution // 2
else:
x = ops.from_rgb('from_rgb_' + rname(resolution), x,
fmap(resolution), cfg.data_format)
while resolution >= 4:
if resolution == 4:
x = ops.minibatch_stddev_layer(x, cfg.data_format)
num_filters = [fmap(resolution), fmap(resolution // 2)]
x = dblock(rname(resolution), x, num_filters, cfg.data_format)
resolution = resolution // 2
x = ops.dense('2x2', x, fmap(resolution), cfg.data_format)
x = ops.leaky_relu(x)
x = ops.dense('output', x, 1, cfg.data_format)
return x
def encoder(x, scope="spade_encoder"):
"""Encoder that outputs global N(mu, sig) parameters.
Args:
x: [B, H, W, 4] an RGBD image (usually the initial image) which is used to
sample noise from a distirbution to feed into the refinement
network. Range [0, 1].
scope: (str) variable scope
Returns:
(mu, logvar) are [B, 256] tensors of parameters defining a normal
distribution to sample from.
"""
x = 2 * x - 1
num_channel = 16
with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE):
x = ops.sn_conv(x, num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_0")
x = ops.instance_norm(x, scope="inst_norm_0")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 2 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_1")
x = ops.instance_norm(x, scope="inst_norm_1")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 4 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_2")
x = ops.instance_norm(x, scope="inst_norm_2")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_3")
x = ops.instance_norm(x, scope="inst_norm_3")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_4")
x = ops.instance_norm(x, scope="inst_norm_4")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_5")
x = ops.instance_norm(x, scope="inst_norm_5")
x = ops.leaky_relu(x, 0.2)
mu = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING,
scope="linear_mu")
logvar = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING,
scope="linear_logvar")
return mu, logvar
def spade_resblock(tensor,
condition,
channel_out,
use_spectral_norm=False,
scope="spade_resblock"):
"""A SPADE resblock.
Args:
tensor: [B, H, W, C] image to be generated
condition: [B, H, W, D] conditioning image to compute affine
normalization parameters.
channel_out: (int) The number of channels of the output tensor
use_spectral_norm: (bool) If true, use spectral normalization in conv layers
scope: (str) The variable scope
Returns:
The output of a spade residual block
"""
channel_in = tensor.get_shape().as_list()[-1]
channel_middle = min(channel_in, channel_out)
with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE):
x = spade(tensor,
condition,
use_spectral_norm=use_spectral_norm,
scope="spade_0")
x = ops.leaky_relu(x, 0.2)
# This one always uses spectral norm.
x = ops.sn_conv(x,
channel_middle,
kernel_size=3,
use_spectral_norm=True,
scope="conv_0")
x = spade(x,
condition,
use_spectral_norm=use_spectral_norm,
scope="spade_1")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x,
channel_out,
kernel_size=3,
use_spectral_norm=True,
scope="conv_1")
if channel_in != channel_out:
x_in = spade(tensor,
condition,
use_spectral_norm=use_spectral_norm,
scope="shortcut_spade")
x_in = ops.sn_conv(x_in,
channel_out,
kernel_size=1,
stride=1,
use_bias=False,
use_spectral_norm=True,
scope="shortcut_conv")
else:
x_in = tensor
out = x_in + x
return out
def discriminator(images, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse:
scope.reuse_variables()
# conv1
conv1 = ops.conv_2d(images, 64, scope="conv1")
# leakly ReLu
h1 = ops.leaky_relu(conv1)
# conv2
conv2 = ops.conv_2d(h1, 128, scope="conv2")
# batch norm
norm2 = ops.batch_norm(conv2,
scope="batch_norm2",
is_training=FLAGS.is_train)
# leaky ReLU
h2 = ops.leaky_relu(norm2)
# conv3
conv3 = ops.conv_2d(h2, 256, scope="conv3")
# batch norm
norm3 = ops.batch_norm(conv3,
scope="batch_norm3",
is_training=FLAGS.is_train)
# leaky ReLU
h3 = ops.leaky_relu(norm3)
# conv4
conv4 = ops.conv_2d(h3, 512, scope="conv4")
# batch norm
norm4 = ops.batch_norm(conv4,
scope="batch_norm4",
is_training=FLAGS.is_train)
# leaky ReLU
h4 = ops.leaky_relu(norm4)
conv5 = ops.conv_2d(h4, 1024, scope="conv5")
conv5 = tf.nn.dropout(conv5, 0.5, name='conv_5_drop_out')
norm5 = ops.batch_norm(conv5,
scope="batch_norm5",
is_training=FLAGS.is_train)
h5 = ops.leaky_relu(norm5)
# reshape
h5_reshape = tf.reshape(h5, [FLAGS.batch_size, -1])
# source logits
source_logits = ops.fc(h5_reshape, 1, scope="source_logits")
# class logits
class_logits = ops.fc(h5_reshape,
FLAGS.num_classes,
scope="class_logits",
decay=4e-3)
return source_logits, class_logits
def refinement_network(rgbd, mask, z, scope="spade_generator"):
"""Refines rgbd, mask based on noise z.
H, W should be divisible by 2 ** num_up_layers
Args:
rgbd: [B, H, W, 4] the rendered view to be refined
mask: [B, H, W, 1] binary mask of unknown regions. 1 where known and 0 where
unknown
z: [B, D] a noise vector to be used as noise for the generator
scope: (str) variable scope
Returns:
[B, H, W, 4] refined rgbd image.
"""
img = 2 * rgbd - 1
img = tf.concat([img, mask], axis=-1)
num_channel = 32
num_up_layers = 5
out_channels = 4 # For RGBD
batch_size, im_height, im_width, unused_c = rgbd.get_shape().as_list()
init_h = im_height // (2**num_up_layers)
init_w = im_width // (2**num_up_layers)
with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE):
x = ops.fully_connected(z, 16 * num_channel * init_h * init_w,
"fc_expand_z")
x = tf.reshape(x, [batch_size, init_h, init_w, 16 * num_channel])
x = spade.spade_resblock(
x,
img,
16 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="head")
x = ops.double_size(x)
x = spade.spade_resblock(
x,
img,
16 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="middle_0")
x = spade.spade_resblock(
x,
img,
16 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="middle_1")
x = ops.double_size(x)
x = spade.spade_resblock(
x,
img,
8 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="up_0")
x = ops.double_size(x)
x = spade.spade_resblock(
x,
img,
4 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="up_1")
x = ops.double_size(x)
x = spade.spade_resblock(
x,
img,
2 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="up_2")
x = ops.double_size(x)
x = spade.spade_resblock(
x,
img,
1 * num_channel,
use_spectral_norm=config.USE_SPECTRAL_NORMALIZATION,
scope="up_3")
x = ops.leaky_relu(x, 0.2)
# Pre-trained checkpoint uses default conv scoping.
x = ops.sn_conv(x, out_channels, kernel_size=3)
x = tf.tanh(x)
return 0.5 * (x + 1)
