def __call__(self, feature, reuse=False):
with tf.variable_scope(self.name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
h0 = lrelu(conv2d(feature, 64, name='d_h0_conv'))
# h0 is (16 x 16 x 64)
h1 = lrelu(
instance_norm(
conv2d(h0, self.hidden_dim * 2, name='d_h1_conv'),
'd_bn1'))
# h1 is (8 x 8 x self.hidden_dim*2)
h2 = lrelu(
instance_norm(
conv2d(h1, self.hidden_dim * 4, name='d_h2_conv'),
'd_bn2'))
# h2 is (4 x 4 x self.hidden_dim*4)
h3 = lrelu(
instance_norm(
conv2d(h2, self.hidden_dim * 8, s=1, name='d_h3_conv'),
'd_bn3'))
# h3 is (4 x 4 x self.hidden_dim*8)
h4 = tf.nn.sigmoid(conv2d(h3, 1, s=1, name='d_h3_pred'))
# h4 is (4 x 4 x 1)
return tf.identity(h4, name=self.name + '_output')
def discriminate(self, x_var, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(conv2d(x_var, output_dim=64, name='dis_conv1'))
conv2 = lrelu(
instance_norm(conv2d(conv1, output_dim=128, name='dis_conv2'),
scope='dis_bn1'))
conv3 = lrelu(
instance_norm(conv2d(conv2, output_dim=256, name='dis_conv3'),
scope='dis_bn2'))
conv4 = conv2d(conv3, output_dim=512, name='dis_conv4')
middle_conv = conv4
conv4 = lrelu(instance_norm(conv4, scope='dis_bn3'))
conv5 = lrelu(
instance_norm(conv2d(conv4, output_dim=1024, name='dis_conv5'),
scope='dis_bn4'))
conv6 = conv2d(conv5,
output_dim=2,
k_w=4,
k_h=4,
d_h=1,
d_w=1,
padding='VALID',
name='dis_conv6')
return conv6, middle_conv
def encode_decode(self, x_var, x_exemplar, reuse=False):
with tf.variable_scope("encode_decode") as scope:
if reuse == True:
scope.reuse_variables()
x_var = tf.concat([x_var, x_exemplar], axis=3)
conv1 = tf.nn.relu(
instance_norm(conv2d(x_var, output_dim=64, k_w=7, k_h=7, d_w=1, d_h=1, name='e_c1'), scope='e_in1'))
conv2 = tf.nn.relu(
instance_norm(conv2d(conv1, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c2'), scope='e_in2'))
conv3 = tf.nn.relu(
instance_norm(conv2d(conv2, output_dim=256, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c3'), scope='e_in3'))
r1 = Residual(conv3, residual_name='re_1')
r2 = Residual(r1, residual_name='re_2')
r3 = Residual(r2, residual_name='re_3')
r4 = Residual(r3, residual_name='re_4')
r5 = Residual(r4, residual_name='re_5')
r6 = Residual(r5, residual_name='re_6')
g_deconv1 = tf.nn.relu(instance_norm(de_conv(r6, output_shape=[self.batch_size,
self.output_size/2, self.output_size/2, 128], name='gen_deconv1'), scope="gen_in"))
# for 1
g_deconv_1_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1,
output_shape=[self.batch_size, self.output_size, self.output_size, 32], name='g_deconv_1_1'), scope='gen_in_1_1'))
g_deconv_1_1_x = tf.concat([g_deconv_1_1, x_var], axis=3)
x_tilde1 = conv2d(g_deconv_1_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_1_2')
return tf.nn.tanh(x_tilde1)
def Kdiscriminator(self, x, reuse=False, name='Kd_'):
with tf.variable_scope(name_or_scope=name, reuse=reuse):
x = Conv(name='conv1',
x=x,
filter_size=4,
in_filters=self.channel,
out_filters=self.n_feats,
strides=2,
padding="SAME")
x = instance_norm(name='inst_norm1', x=x, dim=self.n_feats)
x = tf.nn.leaky_relu(x)
prev = 1
n = 1
for i in range(self.Kdiscrim_blocks):
prev = n
n = min(2**(i + 1), 8)
x = Conv(name='conv%02d' % i,
x=x,
filter_size=4,
in_filters=self.n_feats * prev,
out_filters=self.n_feats * n,
strides=2,
padding="SAME")
x = instance_norm(name='instance_norm%02d' % i,
x=x,
dim=self.n_feats * n)
x = tf.nn.leaky_relu(x)
prev = n
n = min(2**self.Kdiscrim_blocks, 8)
x = Conv(name='conv_d1',
x=x,
filter_size=4,
in_filters=self.n_feats * prev,
out_filters=self.n_feats * n,
strides=1,
padding="SAME")
x = instance_norm(name='instance_norm_d1',
x=x,
dim=self.n_feats * n)
x = tf.nn.leaky_relu(x)
x = Conv(name='conv_d2',
x=x,
filter_size=4,
in_filters=self.n_feats * n,
out_filters=1,
strides=1,
padding="SAME")
x = tf.nn.sigmoid(x)
return x
def residule_block(x, dim, ks=3, s=1, name='res'):
p = int((ks - 1) / 2)
y = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT") #CONSTANT
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c1'),
name + '_in1')
y = tf.pad(tf.nn.relu(y), [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT")
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c2'),
name + '_in2')
return y + x
def encode(self, x, reuse=False):
with tf.variable_scope("encode") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=32,
k_w=7,
k_h=7,
d_w=1,
d_h=1,
name='e_c1'),
scope='e_in1'))
conv2 = tf.nn.relu(
instance_norm(conv2d(conv1,
output_dim=64,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
name='e_c2'),
scope='e_in2'))
conv3 = tf.nn.relu(
instance_norm(conv2d(conv2,
output_dim=128,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
name='e_c3'),
scope='e_in3'))
conv4 = tf.nn.relu(
instance_norm(conv2d(conv3,
output_dim=128,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
name='e_c4'),
scope='e_in4'))
bottleneck = tf.reshape(conv4, [self.batch_size, -1])
content = fully_connect(bottleneck,
output_size=128,
scope='e_ful1')
return content
def encode_decode_2(self, x, reuse=False):
with tf.variable_scope("encode_decode_2") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(
instance_norm(
conv2d(x,
output_dim=64,
k_w=5,
k_h=5,
d_w=1,
d_h=1,
name='e_c1'),
scope='e_in1',
))
conv2 = lrelu(
instance_norm(conv2d(conv1, output_dim=128, name='e_c2'),
scope='e_in2'))
conv3 = lrelu(
instance_norm(conv2d(conv2, output_dim=256, name='e_c3'),
scope='e_in3'))
# for x_{1}
de_conv1 = lrelu(
instance_norm(
de_conv(conv3,
output_shape=[self.batch_size, 64, 64, 128],
name='e_d1',
k_h=3,
k_w=3),
scope='e_in4',
))
de_conv2 = lrelu(
instance_norm(
de_conv(de_conv1,
output_shape=[self.batch_size, 128, 128, 64],
name='e_d2',
k_w=3,
k_h=3),
scope='e_in5',
))
x_tilde1 = conv2d(de_conv2,
output_dim=3,
d_h=1,
d_w=1,
name='e_c4')
return x_tilde1
def __call__(self, input, reuse=False):
with tf.variable_scope(self.name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
act = tf.nn.relu
_, h, w, c = input.shape.as_list()
# Justin Johnson's model from https://github.com/jcjohnson/fast-neural-style/
# The network with 9 blocks consists of: c7s1-32, d64, d128, R128, R128, R128,
# R128, R128, R128, R128, R128, R128, u64, u32, c7s1-3
# input shape == (32 x 32 x 128)
c1 = act(
instance_norm(conv2d(input, c, 7, 1, name='g_e1_c'),
'g_e1_bn'))
# c1 shape == (32 x 32 x 128)
c2 = act(
instance_norm(conv2d(c1, c * 2, 3, 2, name='g_e2_c'),
'g_e2_bn'))
# c2 shape == (16 x 16 x 256)
c3 = act(
instance_norm(conv2d(c2, c * 4, 3, 2, name='g_e3_c'),
'g_e3_bn'))
# c3 shape == (8 x 8 x 512)
# define G network with 9 resnet blocks
r1 = residule_block(c3, c * 4, name='g_r1')
r2 = residule_block(r1, c * 4, name='g_r2')
r3 = residule_block(r2, c * 4, name='g_r3')
r4 = residule_block(r3, c * 4, name='g_r4')
r5 = residule_block(r4, c * 4, name='g_r5')
r6 = residule_block(r5, c * 4, name='g_r6')
r7 = residule_block(r6, c * 4, name='g_r7')
r8 = residule_block(r7, c * 4, name='g_r8')
r9 = residule_block(r8, c * 4, name='g_r9')
d1 = act(
instance_norm(deconv2d(r9, c * 2, 3, 2, name='g_d1_dc'),
'g_d1_bn'))
# d1 shape == (16 x 16 x 256)
d2 = act(
instance_norm(deconv2d(d1, c, 3, 2, name='g_d2_dc'),
'g_d2_bn'))
# d1 shape == (32 x 32 x 128)
output = tf.nn.tanh(deconv2d(d2, c, 7, 1, name='g_output_dc'))
# output shape == (32, 32, 128)
return tf.identity(output, name=self.name + '_output')
def encode_decode_1(self, x, reuse=False):
with tf.variable_scope("encode_decode_1") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(instance_norm(conv2d(x, output_dim=64, k_w=5, k_h=5, d_w=1, d_h=1, name='e_c1'), scope='e_in1'))
conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='e_c2'), scope='e_in2'))
conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='e_c3'), scope='e_in3'))
# for x_{1}
de_conv1 = lrelu(instance_norm(de_conv(conv3, output_shape=[self.batch_size, 64, 64, 128]
, name='e_d1', k_h=3, k_w=3), scope='e_in4'))
de_conv2 = lrelu(instance_norm(de_conv(de_conv1, output_shape=[self.batch_size, 128, 128, 64]
, name='e_d2', k_w=3, k_h=3), scope='e_in5'))
x_tilde1 = conv2d(de_conv2, output_dim=3, d_h=1, d_w=1, name='e_c4')
return x_tilde1
def discriminate(self, x_var, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(conv2d(x_var, output_dim=64, name='dis_conv1'))
conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='dis_conv2'), scope='dis_bn1'))
conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='dis_conv3'), scope='dis_bn2'))
conv4 = conv2d(conv3, output_dim=512, name='dis_conv4')
middle_conv = conv4
conv4 = lrelu(instance_norm(conv4, scope='dis_bn3'))
conv5 = lrelu(instance_norm(conv2d(conv4, output_dim=1024, name='dis_conv5'), scope='dis_bn4'))
conv6 = conv2d(conv5, output_dim=2, k_w=4, k_h=4, d_h=1, d_w=1, padding='VALID', name='dis_conv6')
return conv6, middle_conv
def generator(self, x, c, reuse=False):
print("Generator ...........")
with tf.variable_scope('generator') as scope:
if (reuse):
scope.reuse_variables()
c = tf.reshape(c, [-1, 1, 1, self.feature_length])
c = tf.tile(c, [1, 64, 64, 1])
inputs = tf.concat([x, c], axis=3)
# print(inputs)
# Iinitial concat conv layer
# pad 3 => https://arxiv.org/pdf/1711.09020.pdf
outputs = tf.pad(inputs, [[0, 0], [3, 3], [3, 3], [0, 0]])
outputs = ops.conv2d(outputs, 64, 7, 1, 'VALID', scope="g_init")
# Downsize twice
outputs = ops.conv2d(outputs, 128, 4, 2, scope="g_down_sample_1")
outputs = ops.conv2d(outputs, 256, 4, 2, scope="g_down_sample_2")
# Resblock CONV-(N256, K3x3, S1, P1), IN, ReLU
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_1")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_2")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_3")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_4")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_5")
outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_6")
# Upsampling twice
outputs = ops.instance_norm(
ops.relu(
ops.deconv2d(outputs, 128, 4, 2, scope='g_upsampling_1')),
'g_in_1')
outputs = ops.instance_norm(
ops.relu(
ops.deconv2d(outputs, 64, 4, 2, scope='g_upsampling_2')),
'g_in_2')
# pad 3 => https://arxiv.org/pdf/1711.09020.pdf
outputs = tf.pad(outputs, [[0, 0], [3, 3], [3, 3], [0, 0]])
outputs = ops.tanh(
ops.conv2d(outputs, 3, 7, 1, 'VALID', scope='g_out'))
return outputs
def residule_block(inputs, filters, kernel_size, stride, scope):
with tf.variable_scope(scope):
outputs = ops.conv2d(inputs,
filters,
kernel_size,
stride,
scope='conv_1')
outputs = ops.instance_norm(outputs, 'in_1')
outputs = ops.relu(outputs)
outputs = ops.conv2d(outputs,
filters,
kernel_size,
stride,
scope='conv_2')
outputs = ops.instance_norm(outputs, 'in_2')
return outputs + inputs
def spade(x,
condition,
num_hidden=128,
use_spectral_norm=False,
scope="spade"):
"""Spatially Adaptive Instance Norm implementation.
Given x, applies a normalization that is conditioned on condition.
Args:
x: [B, H, W, C] A tensor to apply normalization
condition: [B, H', W', C'] A tensor to condition the normalization
parameters
num_hidden: (int) The number of intermediate channels to create the SPADE
layer with
use_spectral_norm: (bool) If true, creates convolutions with spectral
normalization applied to its weights
scope: (str) The variable scope
Returns:
A tensor that has been normalized by parameters estimated by cond.
"""
channel = x.shape[-1]
with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE):
x_normed = ops.instance_norm(x)
# Produce affine parameters from conditioning image.
# First resize.
height, width = x.get_shape().as_list()[1:3]
condition = diff_resize_area(condition, [height, width])
condition = ops.sn_conv(condition,
num_hidden,
kernel_size=3,
use_spectral_norm=use_spectral_norm,
scope="conv_cond")
condition = tf.nn.relu(condition)
gamma = ops.sn_conv(condition,
channel,
kernel_size=3,
use_spectral_norm=use_spectral_norm,
scope="gamma",
pad_type="CONSTANT")
beta = ops.sn_conv(condition,
channel,
kernel_size=3,
use_spectral_norm=use_spectral_norm,
scope="beta",
pad_type="CONSTANT")
out = x_normed * (1 + gamma) + beta
return out
def encode_decode(self, input_x, img_mask, guided_fp_left, guided_fp_right, use_sp=False, reuse=False):
with tf.variable_scope("ed") as scope:
if reuse == True:
scope.reuse_variables()
#encode
x = tf.concat([input_x, img_mask], axis=3)
for i in range(6):
c_dim = np.minimum(16 * np.power(2, i), 256)
if i == 0:
x = tf.nn.relu(
instance_norm(conv2d(x, output_dim=c_dim, k_w=7, k_h=7, d_w=1, d_h=1, use_sp=use_sp, name='e_c{}'.format(i))
, scope='e_in_{}'.format(i)))
else:
x = tf.nn.relu(
instance_norm(conv2d(x, output_dim=c_dim, k_w=4, k_h=4, d_w=2, d_h=2, use_sp=use_sp, name='e_c{}'.format(i))
, scope='e_in_{}'.format(i)))
bottleneck = tf.reshape(x, shape=[self.batch_size, -1])
bottleneck = fully_connect(bottleneck, output_size=256, use_sp=use_sp, scope='e_ful1')
bottleneck = tf.concat([bottleneck, guided_fp_left, guided_fp_right], axis=1)
de_x = tf.nn.relu(fully_connect(bottleneck, output_size=256*8*8, use_sp=use_sp, scope='d_ful1'))
de_x = tf.reshape(de_x, shape=[self.batch_size, 8, 8, 256])
#de_x = tf.tile(de_x, (1, 8, 8, 1), name='tile')
#decode
for i in range(5):
c_dim = np.maximum(256 / np.power(2, i), 16)
output_dim = 16 * np.power(2, i)
print de_x
de_x = tf.nn.relu(instance_norm(de_conv(de_x, output_shape=[self.batch_size, output_dim, output_dim, c_dim], use_sp=use_sp,
name='g_deconv_{}'.format(i)), scope='g_in_{}'.format(i)))
#de_x = tf.concat([de_x, input_x], axis=3)
x_tilde1 = conv2d(de_x, output_dim=3, k_w=7, k_h=7, d_h=1, d_w=1, use_sp=use_sp, name='g_conv1')
return tf.nn.tanh(x_tilde1)
def encoder(x, scope="spade_encoder"):
"""Encoder that outputs global N(mu, sig) parameters.
Args:
x: [B, H, W, 4] an RGBD image (usually the initial image) which is used to
sample noise from a distirbution to feed into the refinement
network. Range [0, 1].
scope: (str) variable scope
Returns:
(mu, logvar) are [B, 256] tensors of parameters defining a normal
distribution to sample from.
"""
x = 2 * x - 1
num_channel = 16
with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE):
x = ops.sn_conv(x, num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_0")
x = ops.instance_norm(x, scope="inst_norm_0")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 2 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_1")
x = ops.instance_norm(x, scope="inst_norm_1")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 4 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_2")
x = ops.instance_norm(x, scope="inst_norm_2")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_3")
x = ops.instance_norm(x, scope="inst_norm_3")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_4")
x = ops.instance_norm(x, scope="inst_norm_4")
x = ops.leaky_relu(x, 0.2)
x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2,
use_bias=True, use_spectral_norm=True, scope="conv_5")
x = ops.instance_norm(x, scope="inst_norm_5")
x = ops.leaky_relu(x, 0.2)
mu = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING,
scope="linear_mu")
logvar = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING,
scope="linear_logvar")
return mu, logvar
def encode_decode2(self, x, img_mask, pg=1, is_trans=False, alpha_trans=0.01, reuse=False):
with tf.variable_scope("ed") as scope:
if reuse == True:
scope.reuse_variables()
x = tf.concat([x, img_mask], axis=3)
if is_trans:
x_trans = downscale2d(x)
#fromrgb
x_trans = tf.nn.relu(instance_norm(conv2d(x_trans, output_dim=self.get_nf(pg - 2), k_w=1, k_h=1, d_h=1, d_w=1,
name='gen_rgb_e_{}'.format(x_trans.shape[1])), scope='gen_rgb_e_in_{}'.format(x_trans.shape[1])))
#fromrgb
x = tf.nn.relu(instance_norm(conv2d(x, output_dim=self.get_nf(pg - 1), k_w=1, k_h=1, d_h=1, d_w=1,
name='gen_rgb_e_{}'.format(x.shape[1])), scope='gen_rgb_e_in_{}'.format(x.shape[1])))
for i in range(pg - 1):
print "encode", x.shape
x = tf.nn.relu(instance_norm(conv2d(x, output_dim=self.get_nf(pg - 2 - i), d_h=1, d_w=1,
name='gen_conv_e_{}'.format(x.shape[1])), scope='gen_conv_e_in_{}'.format(x.shape[1])))
x = downscale2d(x)
if i == 0 and is_trans:
x = alpha_trans * x + (1 - alpha_trans) * x_trans
up_x = tf.nn.relu(
instance_norm(dilated_conv2d(x, output_dim=512, k_w=3, k_h=3, rate=4, name='gen_conv_dilated'),
scope='gen_conv_in'))
up_x = tf.nn.relu(instance_norm(conv2d(up_x, output_dim=self.get_nf(1), d_w=1, d_h=1, name='gen_conv_d'),
scope='gen_conv_d_in_{}'.format(x.shape[1])))
for i in range(pg - 1):
print "decode", up_x.shape
if i == pg - 2 and is_trans:
#torgb
up_x_trans = conv2d(up_x, output_dim=self.channel, k_w=1, k_h=1, d_w=1, d_h=1,
name='gen_rgb_d_{}'.format(up_x.shape[1]))
up_x_trans = upscale(up_x_trans, 2)
up_x = upscale(up_x, 2)
up_x = tf.nn.relu(instance_norm(conv2d(up_x, output_dim=self.get_nf(i + 1), d_w=1, d_h=1,
name='gen_conv_d_{}'.format(up_x.shape[1])), scope='gen_conv_d_in_{}'.format(up_x.shape[1])))
#torgb
up_x = conv2d(up_x, output_dim=self.channel, k_w=1, k_h=1, d_w=1, d_h=1,
name='gen_rgb_d_{}'.format(up_x.shape[1]))
if pg == 1: up_x = up_x
else:
if is_trans: up_x = (1 - alpha_trans) * up_x_trans + alpha_trans * up_x
else:
up_x = up_x
return up_x
def patch_discriminator(rgbd_sequence, scope="spade_discriminator"):
"""Creates a patch discriminator to process RGBD values.
Args:
rgbd_sequence: [B, H, W, 4] A batch of RGBD images.
scope: (str) variable scope
Returns:
(list of features, logits)
"""
num_channel = 64
num_layers = 4
features = []
with tf.variable_scope(scope, reuse=tf.AUTO_REUSE):
x = ops.sn_conv(rgbd_sequence,
num_channel,
kernel_size=4,
stride=2,
sn=False)
channel = num_channel
for i in range(1, num_layers):
stride = 1 if i == num_layers - 1 else 2
channel = min(channel * 2, 512)
x = ops.sn_conv(x,
channel,
kernel_size=4,
stride=stride,
sn=True,
scope="conv_{}".format(i))
x = ops.instance_norm(x, scope="inst_norm_{}".format(i))
x = tf.nn.lrelu(x, 0.2)
features.append(x)
logit = ops.sn_conv(x,
1,
kernel_size=4,
stride=1,
sn=False,
scope="D_logit")
return features, logit
def generator(self,
input_x,
img_mask,
guided_fp_left,
guided_fp_right,
use_sp=False,
reuse=False):
with tf.variable_scope("generator") as scope:
if reuse == True:
scope.reuse_variables()
x = tf.concat([input_x, img_mask], axis=3)
u_fp_list = []
for i in range(6):
c_dim = np.minimum(16 * np.power(2, i), 256)
if i == 0:
x = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=c_dim,
k_w=7,
k_h=7,
d_w=1,
d_h=1,
use_sp=use_sp,
name='conv_{}'.format(i)),
scope='conv_IN_{}'.format(i)))
else:
x = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=c_dim,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
use_sp=use_sp,
name='conv_{}'.format(i)),
scope='conv_IN_{}'.format(i)))
if i < 5:
u_fp_list.append(x)
bottleneck = tf.reshape(x, shape=[self.batch_size, -1])
bottleneck = fully_connect(bottleneck,
output_size=256,
use_sp=use_sp,
scope='FC1')
bottleneck = tf.concat(
[bottleneck, guided_fp_left, guided_fp_right], axis=1)
de_x = tf.nn.relu(
fully_connect(bottleneck,
output_size=256 * 8 * 8,
use_sp=use_sp,
scope='FC2'))
de_x = tf.reshape(de_x, shape=[self.batch_size, 8, 8, 256])
for i in range(5):
c_dim = np.maximum(256 / np.power(2, i), 16)
output_dim = 16 * np.power(2, i)
de_x = tf.nn.relu(
instance_norm(de_conv(de_x,
output_shape=[
self.batch_size, output_dim,
output_dim, c_dim
],
use_sp=use_sp,
name='deconv_{}'.format(i)),
scope='deconv_IN_{}'.format(i)))
if i < 4:
de_x = tf.concat(
[de_x, u_fp_list[len(u_fp_list) - (i + 1)]], axis=3)
recon_img1 = conv2d(de_x,
output_dim=3,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
use_sp=use_sp,
name='output_conv')
return tf.nn.tanh(recon_img1)
def generator(images, options, reuse=False, name='gen'):
# reuse or not
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
def residule_block(x, dim, ks=3, s=1, name='res'):
p = int((ks - 1) / 2)
y = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT") #CONSTANT
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c1'),
name + '_in1')
y = tf.pad(tf.nn.relu(y), [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT")
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c2'),
name + '_in2')
return y + x
# down sampling
c0 = tf.pad(images, [[0, 0], [3, 3], [3, 3], [0, 0]], "CONSTANT")
c1 = relu(
instance_norm(
conv2d(c0,
options.nf,
ks=7,
s=1,
padding='VALID',
name='gen_ds_conv1'), 'in1_1'))
c2 = relu(
instance_norm(
conv2d(c1, 2 * options.nf, ks=4, s=2, name='gen_ds_conv2'),
'in1_2'))
c3 = relu(
instance_norm(
conv2d(c2, 4 * options.nf, ks=4, s=2, name='gen_ds_conv3'),
'in1_3'))
# bottleneck
r1 = residule_block(c3, options.nf * 4, name='g_r1')
r2 = residule_block(r1, options.nf * 4, name='g_r2')
r3 = residule_block(r2, options.nf * 4, name='g_r3')
r4 = residule_block(r3, options.nf * 4, name='g_r4')
r5 = residule_block(r4, options.nf * 4, name='g_r5')
r6 = residule_block(r5, options.nf * 4, name='g_r6')
# up sampling
d1 = relu(
instance_norm(
deconv2d(r6, options.nf * 2, 4, 2, name='g_us_dconv1'),
'g_d1_in'))
d2 = relu(
instance_norm(deconv2d(d1, options.nf, 4, 2, name='g_us_dconv2'),
'g_d2_in'))
d2 = tf.pad(d2, [[0, 0], [3, 3], [3, 3], [0, 0]], "CONSTANT")
pred = tf.nn.tanh(conv2d(d2, 3, 7, 1, padding='VALID',
name='g_pred_c'))
return pred
def generator(images, options, reuse=False, name='gen'):
# reuse or not
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
# down sampling
x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1'))
x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3'))
# bottleneck
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6'))
# up sampling
x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1'))
x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2'))
x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3'))
return x
def encode_decode(self, x, reuse=False):
with tf.variable_scope("encode_decode") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=64,
k_w=7,
k_h=7,
d_w=1,
d_h=1,
name='e_c1'),
scope='e_in1'))
conv2 = tf.nn.relu(
instance_norm(conv2d(conv1,
output_dim=128,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
name='e_c2'),
scope='e_in2'))
conv3 = tf.nn.relu(
instance_norm(conv2d(conv2,
output_dim=256,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
name='e_c3'),
scope='e_in3'))
r1 = Residual(conv3, residual_name='re_1')
r2 = Residual(r1, residual_name='re_2')
r3 = Residual(r2, residual_name='re_3')
r4 = Residual(r3, residual_name='re_4')
r5 = Residual(r4, residual_name='re_5')
r6 = Residual(r5, residual_name='re_6')
g_deconv1 = tf.nn.relu(
instance_norm(de_conv(r6,
output_shape=[
self.batch_size,
self.output_size / 2,
self.output_size / 2, 128
],
name='gen_deconv1'),
scope="gen_in"))
# for 1
g_deconv_1_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='g_deconv_1_1'),
scope='gen_in_1_1'))
g_deconv_1_1_x = tf.concat([g_deconv_1_1, x], axis=3)
x_tilde1 = conv2d(g_deconv_1_1_x,
output_dim=self.channel,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
name='gen_conv_1_2')
# for 2
g_deconv_2_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='g_deconv_2_1'),
scope='gen_in_2_1'))
g_deconv_2_1_x = tf.concat([g_deconv_2_1, x], axis=3)
x_tilde2 = conv2d(g_deconv_2_1_x,
output_dim=self.channel,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
name='gen_conv_2_2')
# for 3
g_deconv_3_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='gen_deconv3_1'),
scope='gen_in_3_1'))
g_deconv_3_1_x = tf.concat([g_deconv_3_1, x], axis=3)
g_deconv_3_2 = conv2d(g_deconv_3_1_x,
output_dim=32,
k_w=3,
k_h=3,
d_h=1,
d_w=1,
name='gen_conv_3_2')
x_tilde3 = conv2d(g_deconv_3_2,
output_dim=3,
k_h=3,
k_w=3,
d_h=1,
d_w=1,
name='gen_conv_3_3')
# for 4
g_deconv_4_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='gen_deconv4_1'),
scope='gen_in_4_1'))
g_deconv_4_1_x = tf.concat([g_deconv_4_1, x], axis=3)
g_deconv_4_2 = conv2d(g_deconv_4_1_x,
output_dim=32,
k_w=3,
k_h=3,
d_h=1,
d_w=1,
name='gen_conv_4_2')
x_tilde4 = conv2d(g_deconv_4_2,
output_dim=3,
k_h=3,
k_w=3,
d_h=1,
d_w=1,
name='gen_conv_4_3')
# for 5
g_deconv_5_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='gen_deconv5_1'),
scope='gen_in_5_1'))
g_deconv_5_1_x = tf.concat([g_deconv_5_1, x], axis=3)
g_deconv_5_2 = conv2d(g_deconv_5_1_x,
output_dim=32,
k_w=3,
k_h=3,
d_h=1,
d_w=1,
name='gen_conv_5_2')
x_tilde5 = conv2d(g_deconv_5_2,
output_dim=3,
k_h=3,
k_w=3,
d_h=1,
d_w=1,
name='gen_conv_5_3')
# for 6
g_deconv_6_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='gen_deconv6_1'),
scope='gen_in_6_1'))
g_deconv_6_1_x = tf.concat([g_deconv_6_1, x], axis=3)
g_deconv_6_2 = conv2d(g_deconv_6_1_x,
output_dim=32,
k_w=3,
k_h=3,
d_h=1,
d_w=1,
name='gen_conv_6_2')
x_tilde6 = conv2d(g_deconv_6_2,
output_dim=3,
k_h=3,
k_w=3,
d_h=1,
d_w=1,
name='gen_conv_6_3')
# for 7
g_deconv_7_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='g_deconv_7_1'),
scope='gen_in_7_1'))
g_deconv_7_1_x = tf.concat([g_deconv_7_1, x], axis=3)
x_tilde7 = conv2d(g_deconv_7_1_x,
output_dim=self.channel,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
name='gen_conv_7_2')
# for 8
g_deconv_8_1 = tf.nn.relu(
instance_norm(de_conv(g_deconv1,
output_shape=[
self.batch_size, self.output_size,
self.output_size, 64
],
name='g_deconv_8_1'),
scope='gen_in_8_1'))
g_deconv_8_1_x = tf.concat([g_deconv_8_1, x], axis=3)
x_tilde8 = conv2d(g_deconv_8_1_x,
output_dim=self.channel,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
name='gen_conv_8_2')
return tf.nn.tanh(x_tilde1), tf.nn.tanh(x_tilde2), tf.nn.tanh(x_tilde3), \
tf.nn.tanh(x_tilde4), tf.nn.tanh(x_tilde5), tf.nn.tanh(x_tilde6), tf.nn.tanh(x_tilde7), tf.nn.tanh(x_tilde8)
def generator(images, options, reuse=False, name='gen'):
# down sampling
x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1'))
x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3'))
# bottleneck
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5'))
x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6'))
# up sampling
x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1'))
x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2'))
x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3'))
return x
def generator_multiunet(image,
gf_dim,
reuse=False,
name="generator",
output_c_dim=-1,
istraining=True):
if istraining:
dropout_rate = 0.5
else:
dropout_rate = 1.0
with tf.variable_scope(name):
# image is 256 x 256 x input_c_dim
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
# image is (256 x 256 x input_c_dim)
e1 = instance_norm(conv2d(image, gf_dim, name='g_e1_conv'), 'g_bn_e1')
# e1 is (128 x 128 x self.gf_dim)
e2 = instance_norm(conv2d(lrelu(e1), gf_dim * 2, name='g_e2_conv'),
'g_bn_e2')
# e2 is (64 x 64 x self.gf_dim*2)
e3 = instance_norm(conv2d(lrelu(e2), gf_dim * 4, name='g_e3_conv'),
'g_bn_e3')
# e3 is (32 x 32 x self.gf_dim*4)
e4 = instance_norm(conv2d(lrelu(e3), gf_dim * 8, name='g_e4_conv'),
'g_bn_e4')
# e4 is (16 x 16 x self.gf_dim*8)
e5 = instance_norm(conv2d(lrelu(e4), gf_dim * 8, name='g_e5_conv'),
'g_bn_e5')
# e5 is (8 x 8 x self.gf_dim*8)
e6 = instance_norm(conv2d(lrelu(e5), gf_dim * 8, name='g_e6_conv'),
'g_bn_e6')
# e6 is (4 x 4 x self.gf_dim*8)
e7 = instance_norm(
conv2d(lrelu(e6),
gf_dim * 8,
ks=3,
s=1,
padding='VALID',
name='g_e7_conv'), 'g_bn_e7')
# e7 is (2 x 2 x self.gf_dim*8)
e8 = instance_norm(
conv2d(lrelu(e7),
gf_dim * 16,
ks=2,
s=1,
padding='VALID',
name='g_e8_conv'), 'g_bn_e8')
# e8 is (1 x 1 x self.gf_dim*8)
d1 = deconv2d(tf.nn.relu(e8),
gf_dim * 8,
ks=2,
s=1,
padding='VALID',
name='g_d1')
d1 = tf.nn.dropout(d1, dropout_rate)
d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3)
# d1 is (2 x 2 x self.gf_dim*8*2)
d2 = deconv2d(tf.nn.relu(d1),
gf_dim * 8,
ks=3,
s=1,
padding='VALID',
name='g_d2')
d2 = tf.nn.dropout(d2, dropout_rate)
d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3)
# d2 is (4 x 4 x self.gf_dim*8*2)
d3 = deconv2d(tf.nn.relu(d2), gf_dim * 8, name='g_d3')
d3 = tf.nn.dropout(d3, dropout_rate)
d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3)
# d3 is (8 x 8 x self.gf_dim*8*2)
d4 = deconv2d(tf.nn.relu(d3), gf_dim * 8, name='g_d4')
d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3)
# d4 is (16 x 16 x self.gf_dim*8*2)
d5 = deconv2d(tf.nn.relu(d4), gf_dim * 4, name='g_d5')
d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3)
# d5 is (32 x 32 x self.gf_dim*4*2)
d6 = deconv2d(tf.nn.relu(d5), gf_dim * 2, name='g_d6')
d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3)
# d6 is (64 x 64 x self.gf_dim*2*2)
d7 = deconv2d(tf.nn.relu(d6), gf_dim, name='g_d7')
d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3)
# d7 is (128 x 128 x self.gf_dim*1*2)
d6_pre = deconv2d(tf.nn.relu(d5), output_c_dim, name='g_d6_pre')
# d6_pre is (64 x 64 x output_c_dim)
d7_pre = deconv2d(tf.nn.relu(d6), output_c_dim, name='g_d7_pre')
# d7_pre is (128 x 128 x output_c_dim)
d8_pre = deconv2d(tf.nn.relu(d7), output_c_dim, name='g_d8_pre')
# d8_pre is (256 x 256 x output_c_dim)
return tf.nn.tanh(d8_pre), tf.nn.tanh(d7_pre), tf.nn.tanh(d6_pre)
def build(self, input_, reuse=False, name="generator"):
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
e1 = instance_norm(
conv2d(input_, self.args.gf_dim, name='g_e1_conv'))
e2 = instance_norm(
conv2d(lrelu(e1), self.args.gf_dim * 2, name='g_e2_conv'),
'g_bn_e2')
e3 = instance_norm(
conv2d(lrelu(e2), self.args.gf_dim * 4, name='g_e3_conv'),
'g_bn_e3')
e4 = instance_norm(
conv2d(lrelu(e3), self.args.gf_dim * 8, name='g_e4_conv'),
'g_bn_e4')
e5 = instance_norm(
conv2d(lrelu(e4), self.args.gf_dim * 8, name='g_e5_conv'),
'g_bn_e5')
e6 = instance_norm(
conv2d(lrelu(e5), self.args.gf_dim * 8, name='g_e6_conv'),
'g_bn_e6')
e7 = instance_norm(
conv2d(lrelu(e6), self.args.gf_dim * 8, name='g_e7_conv'),
'g_bn_e7')
z_mean = instance_norm(
conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e8_conv'),
'mean')
z_logvar = instance_norm(
conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e9_conv'),
'logvar')
eps = tf.random_normal(shape=tf.shape(z_mean))
decoder_input = tf.add(z_mean,
tf.exp(z_logvar / 2) * eps,
name='decoder_input')
d1 = deconv2d(tf.nn.relu(decoder_input),
self.args.gf_dim * 8,
name='g_d1')
d1 = tf.nn.dropout(d1, self.dropout_rate)
d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3)
d2 = deconv2d(tf.nn.relu(d1), self.args.gf_dim * 8, name='g_d2')
d2 = tf.nn.dropout(d2, self.dropout_rate)
d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3)
d3 = deconv2d(tf.nn.relu(d2), self.args.gf_dim * 8, name='g_d3')
d3 = tf.nn.dropout(d3, self.dropout_rate)
d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3)
d4 = deconv2d(tf.nn.relu(d3), self.args.gf_dim * 8, name='g_d4')
d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3)
d5 = deconv2d(tf.nn.relu(d4), self.args.gf_dim * 4, name='g_d5')
d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3)
d6 = deconv2d(tf.nn.relu(d5), self.args.gf_dim * 2, name='g_d6')
d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3)
d7 = deconv2d(tf.nn.relu(d6), self.args.gf_dim, name='g_d7')
d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3)
d8 = deconv2d(tf.nn.relu(d7), self.args.out_dim, name='g_d8')
return z_mean, z_logvar, tf.nn.tanh(d8)
