def generator_pix2pix(self, image, reuse=False):
output_size = self.patch_size
s = math.ceil(output_size/16.0)*16
s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16)
# gf_dim = 16 # Dimension of gen filters in first conv layer.
with tf.variable_scope("generator") as scope:
# image is 128 x 128 x (input_c_dim + output_c_dim)
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse == False
# do we need here???
#image = image / 255.0
# liuas 2018.5.9
# trick: using lrelu instead of relu
ngf = 16 # number of generator filters in first conv layer
# encoder_1: [batch, 16, 16, 3] => [batch, 8, 8, ngf]
conv1 = conv2d(image, ngf, k_h=4, k_w=4, name='adv_g_enc1')
conv2 = layer_norm(conv2d(lrelu(conv1, 0.2), ngf*2, k_h=4, k_w=4, name='adv_g_enc2'), name='adv_g_enc2ln')
conv3 = layer_norm(conv2d(lrelu(conv2, 0.2), ngf*4, k_h=4, k_w=4, name='adv_g_enc3'), name='adv_g_enc3ln')
conv4 = layer_norm(conv2d(lrelu(conv3, 0.2), ngf*8, k_h=4, k_w=4, name='adv_g_enc4'), name='adv_g_enc4ln')
deconv1, _, _ = deconv2d(tf.nn.relu(conv4), [self.batch_size, s8, s8, ngf*4], k_h=4, k_w=4, name='adv_g_dec1', with_w=True)
deconv1 = layer_norm(deconv1, name="adv_g_dec1ln")
input = tf.concat([deconv1, conv3], axis=3)
deconv2, _, _ = deconv2d(tf.nn.relu(input), [self.batch_size, s4, s4, ngf*2], k_h=4, k_w=4, name='adv_g_dec2', with_w=True)
deconv2 = layer_norm(deconv2, name="adv_g_dec2ln")
input = tf.concat([deconv2, conv2], axis=3)
deconv3, _, _ = deconv2d(tf.nn.relu(input), [self.batch_size, s2, s2, ngf], k_h=4, k_w=4, name='adv_g_dec3', with_w=True)
deconv3 = layer_norm(deconv3, name="adv_g_dec3ln")
input = tf.concat([deconv3, conv1], axis=3)
deconv4, _, _ = deconv2d(tf.nn.relu(input), [self.batch_size, output_size, output_size, 3], k_h=4, k_w=4, name='adv_g_dec4', with_w=True)
return tf.tanh(deconv4)
def discriminator(self, x, reuse=None):
with tf.variable_scope('discriminator', reuse=reuse):
h0 = lrelu(conv2d(x, self.df_dim, name='d_h0_conv'))
h1 = lrelu(self.d_bn1(conv2d(h0, self.df_dim * 2, name='d_h1_conv')))
h2 = lrelu(self.d_bn2(conv2d(h1, self.df_dim * 4, name='d_h2_conv')))
h3 = lrelu(self.d_bn3(conv2d(h2, self.df_dim * 8, name='d_h3_conv')))
h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4)
def add_atari_layers(self, dims, var_dict):
x = tf.placeholder(tf.float32, shape=[None, dims[0], dims[1]*FRAME_STACK, 1])
conv1 = conv2d(x, 8, 4, 32, CONV1, var_dict=var_dict)
conv2 = conv2d(conv1, 4, 2, 64, CONV2, var_dict=var_dict)
conv3 = conv2d(conv2, 3, 1, 64, CONV3, var_dict=var_dict)
conv_shape = conv3.get_shape().as_list()
flatten = [-1, conv_shape[1]*conv_shape[2]*conv_shape[3]]
return x, tf.reshape(conv3, flatten)
def discriminator(image, reuse=False):
d_bn1 = ops.batch_norm(FLAGS.batch_size, name='d_bn1')
d_bn2 = ops.batch_norm(FLAGS.batch_size, name='d_bn2')
d_bn3 = ops.batch_norm(FLAGS.batch_size, name='d_bn3')
image = tf.reshape(image, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 1])
if reuse: tf.get_variable_scope().reuse_variables()
h0 = ops.lrelu(ops.conv2d(image, FLAGS.df_dim, name='d_h0_conv'))
h1 = ops.lrelu(d_bn1(ops.conv2d(h0, FLAGS.df_dim * 2, name='d_h1_conv')))
h2 = ops.lrelu(d_bn2(ops.conv2d(h1, FLAGS.df_dim * 4, name='d_h2_conv')))
h3 = ops.lrelu(d_bn3(ops.conv2d(h2, FLAGS.df_dim * 8, name='d_h3_conv')))
h4 = ops.linear(tf.reshape(h3, [FLAGS.batch_size, -1]), 1, 'd_h3_lin')
return tf.nn.sigmoid(h4)
def generator(self, images):
with tf.variable_scope("generator"):
g_h0 = tf.nn.relu(conv2d(images, 16, name='g_encode_0'))
g_h1 = tf.nn.relu(conv2d(g_h0, 32, name='g_encode_1'))
g_h2 = tf.nn.relu(conv2d(g_h1, 64, name='g_encode_2'))
g_flat = tf.reshape(g_h2, [self.batch_size, -1])
g_encode = linear(g_flat, 128, 'g_encode')
g_decode = linear(g_encode, 512 * 4 * 4, 'g_h0')
g_h3 = tf.nn.relu(tf.reshape(g_decode, [self.batch_size, 4, 4, 512]))
g_h4 = tf.nn.relu(conv2d_transpose(g_h3, [self.batch_size, 8, 8, 256], name='g_h1'))
g_h5 = tf.nn.relu(conv2d_transpose(g_h4, [self.batch_size, 16, 16, 128], name='g_h2'))
g_h6 = tf.nn.relu(conv2d_transpose(g_h5, [self.batch_size, 32, 32, 64], name='g_h3'))
g_h7 = conv2d_transpose(g_h6, [self.batch_size, 64, 64, 3], name='g_h4')
return tf.nn.tanh(g_h7)
def g_k(inputs, cond, filters, hps, channels, reuse=None, name=None):
"""
Three convolution layers for getting s_k,mu_k conditioning with x_{k-1} and condition h (if specified)
Args:
filters: the output channels of the first two convolution layers
channels: the output channels of s_k, mu_k
Returns:
shift, logs: 4D tensor
"""
with tf.variable_scope(name, "g_1", reuse=reuse):
inputs = convnet(inputs, cond, filters, hps, channels=2 * channels)
if cond is not None:
rank = len(cond.shape)
if rank == 2:
inputs += tf.reshape(tf.layers.dense(cond, 2 * channels, use_bias=False),
shape=[-1, 1, 1, 2 * channels])
elif rank == 4:
inputs += conv2d(cond, width=2*channels, name="conv2d")
shift = inputs[:, :, :, 0::2]
logs = inputs[:, :, :, 1::2]
# logs = alpha*tanh(logs)+beta, helpful for training stability
rescale = tf.get_variable("rescale", [], initializer=tf.constant_initializer(1.))
scale_shift = tf.get_variable("scale_shift", [], initializer=tf.constant_initializer(-3.))
logs = tf.tanh(logs) * rescale + scale_shift
return shift, logs
def encode_decode_1(self, x, reuse=False):
with tf.variable_scope("encode_decode_1") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(instance_norm(conv2d(x, output_dim=64, k_w=5, k_h=5, d_w=1, d_h=1, name='e_c1'), scope='e_in1'))
conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='e_c2'), scope='e_in2'))
conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='e_c3'), scope='e_in3'))
# for x_{1}
de_conv1 = lrelu(instance_norm(de_conv(conv3, output_shape=[self.batch_size, 64, 64, 128]
, name='e_d1', k_h=3, k_w=3), scope='e_in4'))
de_conv2 = lrelu(instance_norm(de_conv(de_conv1, output_shape=[self.batch_size, 128, 128, 64]
, name='e_d2', k_w=3, k_h=3), scope='e_in5'))
x_tilde1 = conv2d(de_conv2, output_dim=3, d_h=1, d_w=1, name='e_c4')
return x_tilde1
def tower(bn, suffix):
assert not self.y_dim
print "\ttower "+suffix
h0 = lrelu(bn(conv2d(noisy_image, self.df_dim, name='d_h0_conv' + suffix, d_h=2, d_w=2,
k_w=3, k_h=3), "d_bn_0" + suffix))
print "\th0 ", h0.get_shape()
h1 = lrelu(bn(conv2d(h0, self.df_dim * 2, name='d_h1_conv' + suffix, d_h=2, d_w=2,
k_w=3, k_h=3), "d_bn_1" + suffix))
print "\th1 ", h1.get_shape()
h2 = lrelu(bn(conv2d(h1, self.df_dim * 4, name='d_h2_conv' + suffix, d_h=2, d_w=2,
k_w=3, k_h=3), "d_bn_2" + suffix))
print "\th2 ", h2.get_shape()
h3 = lrelu(bn(conv2d(h2, self.df_dim*4, name='d_h3_conv' + suffix, d_h=1, d_w=1,
k_w=3, k_h=3), "d_bn_3" + suffix))
print "\th3 ", h3.get_shape()
h4 = lrelu(bn(conv2d(h3, self.df_dim*4, name='d_h4_conv' + suffix, d_h=1, d_w=1,
k_w=3, k_h=3), "d_bn_4" + suffix))
print "\th4 ", h4.get_shape()
h5 = lrelu(bn(conv2d(h4, self.df_dim*8, name='d_h5_conv' + suffix, d_h=2, d_w=2,
k_w=3, k_h=3), "d_bn_5" + suffix))
print "\th5 ", h5.get_shape()
h6 = lrelu(bn(conv2d(h5, self.df_dim*8, name='d_h6_conv' + suffix,
k_w=3, k_h=3), "d_bn_6" + suffix))
print "\th6 ", h6.get_shape()
# return tf.reduce_mean(h6, [1, 2])
h6_reshaped = tf.reshape(h6, [batch_size, -1])
print '\th6_reshaped: ', h6_reshaped.get_shape()
h7 = lrelu(bn(linear(h6_reshaped, self.df_dim * 40, scope="d_h7" + suffix), "d_bn_7" + suffix))
return h7
def naive_discriminator(self, image, y = None, reuse = False):
with tf.variable_scope("discriminator") as scope:
# image is 128 x 128 x (input_c_dim + output_c_dim)
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse == False
h0 = lrelu(conv2d(image, self.df_dim, name='adv_d_h0_conv'))
# h0 is (128 x 128 x self.df_dim)
h1 = lrelu(layer_norm((conv2d(h0, self.df_dim * 2, name='adv_d_h1_conv')), name="adv_d_ln1"))
# h1 is (64 x 64 x self.df_dim*2)
h2 = lrelu(layer_norm(conv2d(h1, self.df_dim * 4, name='adv_d_h2_conv'), name="adv_d_ln2"))
# h2 is (32x 32 x self.df_dim*4)
h3 = lrelu(layer_norm(conv2d(h2, self.df_dim * 8, d_h=1, d_w=1, name='adv_d_h3_conv'), name="adv_d_ln3"))
# h3 is (16 x 16 x self.df_dim*8)
h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'adv_d_h3_lin')
return tf.nn.sigmoid(h4), h4
def discriminator(self, images, image_size, reuse=False):
image_size /= 64
with tf.variable_scope('discriminator', reuse=reuse):
gd_h0 = lrelu(conv2d(images, 64, name="d_gd_h0_conv"))
gd_h1 = lrelu(conv2d(gd_h0, 128, name='d_gd_h1_conv'))
gd_h2 = lrelu(conv2d(gd_h1, 256, name='d_gd_h2_conv'))
gd_h3 = lrelu(conv2d(gd_h2, 512, name='d_gd_h3_conv'))
gd_h4 = lrelu(conv2d(gd_h3, 512, name='d_gd_h4_conv'))
gd_h5 = lrelu(conv2d(gd_h4, 512, name='d_gd_h5_conv'))
gd_h = linear(tf.reshape(
gd_h5, [self.batch_size, int(512 * image_size * image_size)]), 64 * image_size * image_size, 'd_gd_linear')
return linear(gd_h, 1, 'd_linear')
def g_0(cond, shape, name=None):
with tf.variable_scope(name, "g"):
channels = shape[-1]
inputs = tf.get_variable("h", [1, 1, 1, 2 * channels])
inputs = inputs + tf.zeros(shape[:-1] + [2 * channels]) # broadcasting
if cond is not None:
rank = len(cond.shape)
if rank == 2:
inputs += tf.reshape(tf.layers.dense(cond, 2 * channels, use_bias=False),
shape=[-1, 1, 1, 2 * channels])
elif rank == 4:
inputs += conv2d(cond, width=2*channels, name="conv2d")
shift = inputs[:, :, :, 0::2]
logs = inputs[:, :, :, 1::2]
# logs = alpha*tanh(logs)+beta, helpful for training stability
rescale = tf.get_variable("rescale", [], initializer=tf.constant_initializer(1.))
scale_shift = tf.get_variable("scale_shift", [], initializer=tf.constant_initializer(-3.))
logs = tf.tanh(logs) * rescale + scale_shift
return shift, logs
def discriminate(self, x_var, reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = lrelu(conv2d(x_var, output_dim=64, name='dis_conv1'))
conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='dis_conv2'), scope='dis_bn1'))
conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='dis_conv3'), scope='dis_bn2'))
conv4 = conv2d(conv3, output_dim=512, name='dis_conv4')
middle_conv = conv4
conv4 = lrelu(instance_norm(conv4, scope='dis_bn3'))
conv5 = lrelu(instance_norm(conv2d(conv4, output_dim=1024, name='dis_conv5'), scope='dis_bn4'))
conv6 = conv2d(conv5, output_dim=2, k_w=4, k_h=4, d_h=1, d_w=1, padding='VALID', name='dis_conv6')
return conv6, middle_conv
def discriminator(self, images, image_size, reuse=False):
image_size /= 64
with tf.variable_scope('discriminator', reuse=reuse):
gd_h0 = lrelu(conv2d(images, 64, name="d_gd_h0_conv"))
gd_h1 = lrelu(self.d_bns[0](conv2d(gd_h0, 128, name='d_gd_h1_conv')))
gd_h2 = lrelu(self.d_bns[1](conv2d(gd_h1, 256, name='d_gd_h2_conv')))
gd_h3 = lrelu(self.d_bns[2](conv2d(gd_h2, 512, name='d_gd_h3_conv')))
gd_h4 = lrelu(self.d_bns[3](conv2d(gd_h3, 512, name='d_gd_h4_conv')))
gd_h5 = lrelu(self.d_bns[4](conv2d(gd_h4, 512, name='d_gd_h5_conv')))
gd_h = linear(tf.reshape(
gd_h5, [self.batch_size, int(512 * image_size * image_size)]), 64 * image_size * image_size, 'd_gd_linear')
#ld_h0 = lrelu(conv2d(masked_images, 64, name="d_ld_h0_conv"))
#ld_h1 = lrelu(self.local_d_bns[0](conv2d(ld_h0, 128, name='d_ld_h1_conv')))
#ld_h2 = lrelu(self.local_d_bns[1](conv2d(ld_h1, 256, name='d_ld_h2_conv')))
#ld_h3 = lrelu(self.local_d_bns[2](conv2d(ld_h2, 512, name='d_ld_h3_conv')))
#ld_h4 = lrelu(self.local_d_bns[3](conv2d(ld_h3, 512, name='d_ld_h4_conv')))
#ld_h = linear(tf.reshape(
# ld_h4, [self.batch_size, int(512 * image_size * image_size)]), 64 * image_size * image_size, 'd_ld_linear')
#h = linear(tf.concat([gd_h, ld_h], 1), 1, 'd_linear')
h = linear(gd_h, 1, 'd_linear')
return tf.nn.sigmoid(h), h
def generator(images, options, reuse=False, name='gen'):
# reuse or not
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
def residule_block(x, dim, ks=3, s=1, name='res'):
p = int((ks - 1) / 2)
y = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT") #CONSTANT
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c1'),
name + '_in1')
y = tf.pad(tf.nn.relu(y), [[0, 0], [p, p], [p, p], [0, 0]],
"CONSTANT")
y = instance_norm(
conv2d(y, dim, ks, s, padding='VALID', name=name + '_c2'),
name + '_in2')
return y + x
# down sampling
c0 = tf.pad(images, [[0, 0], [3, 3], [3, 3], [0, 0]], "CONSTANT")
c1 = relu(
instance_norm(
conv2d(c0,
options.nf,
ks=7,
s=1,
padding='VALID',
name='gen_ds_conv1'), 'in1_1'))
c2 = relu(
instance_norm(
conv2d(c1, 2 * options.nf, ks=4, s=2, name='gen_ds_conv2'),
'in1_2'))
c3 = relu(
instance_norm(
conv2d(c2, 4 * options.nf, ks=4, s=2, name='gen_ds_conv3'),
'in1_3'))
# bottleneck
r1 = residule_block(c3, options.nf * 4, name='g_r1')
r2 = residule_block(r1, options.nf * 4, name='g_r2')
r3 = residule_block(r2, options.nf * 4, name='g_r3')
r4 = residule_block(r3, options.nf * 4, name='g_r4')
r5 = residule_block(r4, options.nf * 4, name='g_r5')
r6 = residule_block(r5, options.nf * 4, name='g_r6')
# up sampling
d1 = relu(
instance_norm(
deconv2d(r6, options.nf * 2, 4, 2, name='g_us_dconv1'),
'g_d1_in'))
d2 = relu(
instance_norm(deconv2d(d1, options.nf, 4, 2, name='g_us_dconv2'),
'g_d2_in'))
d2 = tf.pad(
d2, [[0, 0], [3, 3], [3, 3], [0, 0]],
"CONSTANT") #REFLECT instead of Padding with 0, [batch,h,w,c]
pred = tf.nn.tanh(conv2d(d2, 3, 7, 1, padding='VALID',
name='g_pred_c'))
return pred
def discriminate(self, conv, reuse=False, pg=1, t=False, alpha_trans=0.01):
#dis_as_v = []
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
if t:
# average pooling方法
conv_iden = downscale2d(conv)
#from RGB -->将RGB信息转换为图像特征信息，使用1*1卷积
conv_iden = lrelu(
conv2d(conv_iden,
output_dim=self.get_nf(pg - 2),
k_w=1,
k_h=1,
d_h=1,
d_w=1,
use_wscale=self.use_wscale,
name='dis_y_rgb_conv_{}'.format(
conv_iden.shape[1])))
# fromRGB
conv = lrelu(
conv2d(conv,
output_dim=self.get_nf(pg - 1),
k_w=1,
k_h=1,
d_w=1,
d_h=1,
use_wscale=self.use_wscale,
name='dis_y_rgb_conv_{}'.format(conv.shape[1])))
for i in range(pg - 1):
conv = lrelu(
conv2d(conv,
output_dim=self.get_nf(pg - 1 - i),
d_h=1,
d_w=1,
use_wscale=self.use_wscale,
name='dis_n_conv_1_{}'.format(conv.shape[1])))
conv = lrelu(
conv2d(conv,
output_dim=self.get_nf(pg - 2 - i),
d_h=1,
d_w=1,
use_wscale=self.use_wscale,
name='dis_n_conv_2_{}'.format(conv.shape[1])))
conv = downscale2d(conv)
if i == 0 and t:
conv = alpha_trans * conv + (1 - alpha_trans) * conv_iden
# MSD
conv = MinibatchstateConcat(conv)
conv = lrelu(
conv2d(conv,
output_dim=self.get_nf(1),
k_w=3,
k_h=3,
d_h=1,
d_w=1,
use_wscale=self.use_wscale,
name='dis_n_conv_1_{}'.format(conv.shape[1])))
conv = lrelu(
conv2d(conv,
output_dim=self.get_nf(1),
k_w=4,
k_h=4,
d_h=1,
d_w=1,
use_wscale=self.use_wscale,
padding='VALID',
name='dis_n_conv_2_{}'.format(conv.shape[1])))
conv = tf.reshape(conv, [self.batch_size, -1])
#for D
output = fully_connect(conv,
output_size=1,
use_wscale=self.use_wscale,
gain=1,
name='dis_n_fully')
return tf.nn.sigmoid(output), output
def generator_multiunet(image,
gf_dim,
reuse=False,
name="generator",
output_c_dim=-1,
istraining=True):
if istraining:
dropout_rate = 0.5
else:
dropout_rate = 1.0
with tf.variable_scope(name):
# image is 256 x 256 x input_c_dim
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
# image is (256 x 256 x input_c_dim)
e1 = instance_norm(conv2d(image, gf_dim, name='g_e1_conv'), 'g_bn_e1')
# e1 is (128 x 128 x self.gf_dim)
e2 = instance_norm(conv2d(lrelu(e1), gf_dim * 2, name='g_e2_conv'),
'g_bn_e2')
# e2 is (64 x 64 x self.gf_dim*2)
e3 = instance_norm(conv2d(lrelu(e2), gf_dim * 4, name='g_e3_conv'),
'g_bn_e3')
# e3 is (32 x 32 x self.gf_dim*4)
e4 = instance_norm(conv2d(lrelu(e3), gf_dim * 8, name='g_e4_conv'),
'g_bn_e4')
# e4 is (16 x 16 x self.gf_dim*8)
e5 = instance_norm(conv2d(lrelu(e4), gf_dim * 8, name='g_e5_conv'),
'g_bn_e5')
# e5 is (8 x 8 x self.gf_dim*8)
e6 = instance_norm(conv2d(lrelu(e5), gf_dim * 8, name='g_e6_conv'),
'g_bn_e6')
# e6 is (4 x 4 x self.gf_dim*8)
e7 = instance_norm(
conv2d(lrelu(e6),
gf_dim * 8,
ks=3,
s=1,
padding='VALID',
name='g_e7_conv'), 'g_bn_e7')
# e7 is (2 x 2 x self.gf_dim*8)
e8 = instance_norm(
conv2d(lrelu(e7),
gf_dim * 16,
ks=2,
s=1,
padding='VALID',
name='g_e8_conv'), 'g_bn_e8')
# e8 is (1 x 1 x self.gf_dim*8)
d1 = deconv2d(tf.nn.relu(e8),
gf_dim * 8,
ks=2,
s=1,
padding='VALID',
name='g_d1')
d1 = tf.nn.dropout(d1, dropout_rate)
d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3)
# d1 is (2 x 2 x self.gf_dim*8*2)
d2 = deconv2d(tf.nn.relu(d1),
gf_dim * 8,
ks=3,
s=1,
padding='VALID',
name='g_d2')
d2 = tf.nn.dropout(d2, dropout_rate)
d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3)
# d2 is (4 x 4 x self.gf_dim*8*2)
d3 = deconv2d(tf.nn.relu(d2), gf_dim * 8, name='g_d3')
d3 = tf.nn.dropout(d3, dropout_rate)
d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3)
# d3 is (8 x 8 x self.gf_dim*8*2)
d4 = deconv2d(tf.nn.relu(d3), gf_dim * 8, name='g_d4')
d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3)
# d4 is (16 x 16 x self.gf_dim*8*2)
d5 = deconv2d(tf.nn.relu(d4), gf_dim * 4, name='g_d5')
d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3)
# d5 is (32 x 32 x self.gf_dim*4*2)
d6 = deconv2d(tf.nn.relu(d5), gf_dim * 2, name='g_d6')
d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3)
# d6 is (64 x 64 x self.gf_dim*2*2)
d7 = deconv2d(tf.nn.relu(d6), gf_dim, name='g_d7')
d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3)
# d7 is (128 x 128 x self.gf_dim*1*2)
d6_pre = deconv2d(tf.nn.relu(d5), output_c_dim, name='g_d6_pre')
# d6_pre is (64 x 64 x output_c_dim)
d7_pre = deconv2d(tf.nn.relu(d6), output_c_dim, name='g_d7_pre')
# d7_pre is (128 x 128 x output_c_dim)
d8_pre = deconv2d(tf.nn.relu(d7), output_c_dim, name='g_d8_pre')
# d8_pre is (256 x 256 x output_c_dim)
return tf.nn.tanh(d8_pre), tf.nn.tanh(d7_pre), tf.nn.tanh(d6_pre)
def model(x):
# Current data input shape: (batch_size, n_steps, n_input)
x = tf.transpose(x, [1, 0, 2])
# (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, EMBEDDING_SIZE])
# get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, NUM_STEPS, x)
# B-directional LSTM
with tf.variable_scope("fw_cell"):
fw_cell = tf.nn.rnn_cell.LSTMCell(num_hidden,
forget_bias=1.0,
state_is_tuple=True)
# fw_cell = tf.nn.rnn_cell.DropoutWrapper(fw_cell, output_keep_prob=dropout_keep_prob)
if rnn_layer > 1:
fw_cell = tf.nn.rnn_cell.MultiRNNCell([fw_cell] * rnn_layer)
with tf.variable_scope("bw_cell"):
bw_cell = tf.nn.rnn_cell.LSTMCell(num_hidden,
forget_bias=1.0,
state_is_tuple=True)
# bw_cell = tf.nn.rnn_cell.DropoutWrapper(bw_cell, output_keep_prob=dropout_keep_prob)
if rnn_layer > 1:
bw_cell = tf.nn.rnn_cell.MultiRNNCell([bw_cell] * rnn_layer)
# output = [batch_size,num_hidden*2]
# outputs of Bi-directional LSTM to highway
with tf.variable_scope("rnn_def"):
outputs, fw_final_state, bw_final_state = tf.nn.bidirectional_rnn(
fw_cell, bw_cell, x, dtype=tf.float32)
# Highway
# convert to [batch_size,num_steps,num_hidden*2]
hw_input = tf.transpose(tf.pack(outputs, axis=0), [1, 0, 2])
# convert to [batch_size x num_steps,num_hidden*2]
hw_input = tf.reshape(hw_input, [-1, num_hidden * 2])
size = hw_input.get_shape()[1]
# size = num_hidden*2
# tf.tanh
# hw_output=[batch_size x num_steps,num_hidden*2]
hw_output = highways(hw_input, size)
# convert to [batch_size,num_steps,num_hidden*2]
hw_output = tf.reshape(hw_output, [-1, NUM_STEPS, num_hidden * 2])
# expand dim , cnn_input=[batch_size,num_steps,num_hidden*2,1]
cnn_input = tf.expand_dims(hw_output, -1)
# CNN
pooled_outputs = []
for idx, filter_size in enumerate(filter_sizes):
conv = conv2d(cnn_input,
filter_numbers[idx],
filter_size,
num_hidden * 2,
name="kernel%d" % idx)
# conv = batch_norm_conv2d(cnn_input,filter_numbers[idx], filter_size,idx,num_hidden*2,is_training,stddev=0.1, name="kernel%d" % idx)
# 1-max pooling,leave a tensor of shape[batch_size,1,1,num_filters]
pool = tf.nn.max_pool(
conv,
ksize=[1, max_document_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID')
pooled_outputs.append(tf.squeeze(pool))
if len(filter_sizes) > 1:
cnn_output = tf.concat(1, pooled_outputs)
else:
cnn_output = pooled_outputs[0]
# add dropout
cnn_output = tf.nn.dropout(cnn_output, dropout_keep_prob)
# fc1 layer
hidden = tf.matmul(cnn_output, fc1_weights)
# add batch normalization
# hidden = official_batch_norm_layer(tf.nn.bias_add(hidden,fc1_biases),100,is_training,False,scope="fc1_batch_norm")
fc1_output = tf.sigmoid(tf.nn.bias_add(hidden, fc1_biases))
# softmax linear layer , don't apply activation function
hidden = tf.matmul(fc1_output, fc2_weights)
fc2_output = tf.nn.bias_add(hidden, fc2_biases)
return fc2_output
def discriminator(self,
incom_x,
local_x_left,
local_x_right,
guided_fp_left,
guided_fp_right,
reuse=False):
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
# Global Discriminator Dg
x = incom_x
for i in range(6):
output_dim = np.minimum(16 * np.power(2, i + 1), 256)
x = lrelu(
conv2d(x,
output_dim=output_dim,
use_sp=self.use_sp,
name='dis_conv_global_{}'.format(i)))
x = tf.reshape(x, shape=[self.batch_size, -1])
ful_global = fully_connect(x,
output_size=output_dim,
use_sp=self.use_sp,
scope='dis_FC1')
if self.is_supervised:
with tf.variable_scope("local_d"):
# Local Discriminator Dl
x = local_x_left
for i in range(5):
output_dim = np.minimum(16 * np.power(2, i + 1), 256)
x = lrelu(
conv2d(x,
output_dim=output_dim,
use_sp=self.use_sp,
name='dis_conv_local_{}'.format(i)))
x = tf.reshape(x, shape=[self.batch_size, -1])
logits0 = fully_connect(x,
output_size=2,
use_sp=self.use_sp,
scope='dis_class')
ful_local_left = fully_connect(x,
output_size=output_dim,
use_sp=self.use_sp,
scope='dis_FC2')
with tf.variable_scope("local_d") as scope:
scope.reuse_variables()
x = local_x_right
for i in range(5):
output_dim = np.minimum(16 * np.power(2, i + 1), 256)
x = lrelu(
conv2d(x,
output_dim=output_dim,
use_sp=self.use_sp,
name='dis_conv_local_{}'.format(i)))
x = tf.reshape(x, shape=[self.batch_size, -1])
logits1 = fully_connect(x,
output_size=2,
use_sp=self.use_sp,
scope='dis_class')
ful_local_right = fully_connect(x,
output_size=output_dim,
use_sp=self.use_sp,
scope='dis_FC2')
ful_local = tf.concat([ful_local_left, ful_local_right],
axis=1)
else:
x = tf.concat([local_x_left, local_x_right], axis=3)
for i in range(5):
output_dim = np.minimum(16 * np.power(2, i + 1), 256)
x = lrelu(
conv2d(x,
output_dim=output_dim,
use_sp=self.use_sp,
name='dis_conv_local_{}'.format(i)))
x = tf.reshape(x, shape=[self.batch_size, -1])
ful_local = fully_connect(x,
output_size=output_dim * 2,
use_sp=self.use_sp,
scope='dis_FC2')
# Concatenation
ful = tf.concat(
[ful_global, ful_local, guided_fp_left, guided_fp_right],
axis=1)
ful = tf.nn.relu(
fully_connect(ful,
output_size=512,
use_sp=self.use_sp,
scope='dis_FC3'))
gan_logits = fully_connect(ful,
output_size=1,
use_sp=self.use_sp,
scope='dis_FC4')
if self.is_supervised:
return gan_logits, logits0, logits1
else:
return gan_logits
def inference1(img_l_batch,
img_l_gra_batch,
theme_ab_batch,
theme_mask_batch,
local_ab_batch,
local_mask_batch,
is_training=True,
scope_name='UserGuide'):
"""
:param img_l_batch: l channel of input image
:param img_l_gra_batch: sobel edge map of l channel of input image
:param theme_ab_batch: ab channel of input color theme
:param theme_mask_batch: theme mask
:param local_ab_batch: ab channel of input local points
:param local_mask_batch: local points mask
:param is_training: bool, indicate usage of model (training or testing)
:param scope_name: model name
:return: ab channel of output image
"""
assert img_l_batch.get_shape()[-1] == 1
assert img_l_gra_batch.get_shape(
)[-1] == 1 # horizontal and vertical direction
assert theme_ab_batch.get_shape()[-1] == 2
assert theme_mask_batch.get_shape()[-1] == 1
assert local_ab_batch.get_shape()[-1] == 2
assert local_mask_batch.get_shape()[-1] == 1
ngf = 64
theme_batch = tf.concat([theme_ab_batch, theme_mask_batch], axis=3)
local_batch = tf.concat([local_ab_batch, local_mask_batch], axis=3)
print('Image Inputs:', img_l_batch)
print('Theme Inputs:', theme_batch)
print('Points Inputs:', local_batch)
print()
with tf.variable_scope(scope_name, reuse=tf.AUTO_REUSE):
theme_batch = tf.reshape(theme_batch,
[img_l_batch.get_shape()[0], 1, 1, -1])
glob_conv1 = ops.conv2d(theme_batch,
ngf * 8,
1,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='glob_conv1')
glob_conv2 = ops.conv2d(glob_conv1,
ngf * 8,
1,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='glob_conv2')
glob_conv3 = ops.conv2d(glob_conv2,
ngf * 8,
1,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='glob_conv3')
glob_conv4 = ops.conv2d(glob_conv3,
ngf * 8,
1,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='glob_conv4')
print('ThemeBlock', glob_conv4)
ab_conv1_1 = ops.conv2d(local_batch,
ngf,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='ab_conv1_1')
bw_conv1_1 = ops.conv2d(img_l_batch,
ngf,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='bw_conv1_1')
gra_conv1_1 = ops.conv2d(img_l_gra_batch,
ngf,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='gra_conv1_1')
print('LocalBlock', gra_conv1_1)
conv1_1 = ab_conv1_1 + bw_conv1_1 + gra_conv1_1 # TODO: Merge Local Points and Gradient Maps
conv1_1 = ops.conv2d(conv1_1,
ngf,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv1_1')
conv1_2 = ops.conv2d(conv1_1,
ngf,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv1_2')
conv1_2_ss = ops.depth_wise_conv2d(conv1_2,
1,
1,
2,
activation_fn=None,
scope_name='conv1_2_ss')
print('ConvBlock 1', conv1_2_ss)
conv2_1 = ops.conv2d(conv1_2_ss,
ngf * 2,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv2_1')
conv2_2 = ops.conv2d(conv2_1,
ngf * 2,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv2_2')
conv2_2_ss = ops.depth_wise_conv2d(conv2_2,
1,
1,
2,
activation_fn=None,
scope_name='conv2_2_ss')
print('ConvBlock 2', conv2_2_ss)
conv3_1 = ops.conv2d(conv2_2_ss,
ngf * 4,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv3_1')
conv3_2 = ops.conv2d(conv3_1,
ngf * 4,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv3_2')
conv3_3 = ops.conv2d(conv3_2,
ngf * 4,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv3_3')
conv3_3_ss = ops.depth_wise_conv2d(conv3_3,
1,
1,
2,
activation_fn=None,
scope_name='conv3_3_ss')
print('ConvBlock 3', conv3_3_ss)
conv4_1 = ops.conv2d(conv3_3_ss,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv4_1')
conv4_2 = ops.conv2d(conv4_1,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv4_2')
conv4_3 = ops.conv2d(conv4_2,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv4_3')
print('ConvBlock 4', conv4_3)
conv4_3 = conv4_3 + glob_conv4 # TODO: Merge Color Theme
conv5_1 = ops.conv2d(conv4_3,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv5_1')
conv5_2 = ops.conv2d(conv5_1,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv5_2')
conv5_3 = ops.conv2d(conv5_2,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv5_3')
print('ConvBlock 5', conv5_3)
conv6_1 = ops.conv2d(conv5_3,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv6_1')
conv6_2 = ops.conv2d(conv6_1,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv6_2')
conv6_3 = ops.conv2d(conv6_2,
ngf * 8,
3,
1,
dilation=2,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv6_3')
print('ConvBlock 6', conv6_3)
conv7_1 = ops.conv2d(conv6_3,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv7_1')
conv7_2 = ops.conv2d(conv7_1,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv7_2')
conv7_3 = ops.conv2d(conv7_2,
ngf * 8,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv7_3')
print('ConvBlock 7', conv7_3)
conv3_3_short = ops.conv2d(conv3_3,
ngf * 4,
3,
1,
activation_fn=None,
is_training=is_training,
scope_name='conv3_3_short')
conv8_1 = ops.conv2d_transpose(conv7_3,
ngf * 4,
4,
2,
activation_fn=None,
is_training=is_training,
scope_name='conv8_1')
conv8_1_comb = tf.nn.relu(conv3_3_short + conv8_1)
conv8_2 = ops.conv2d(conv8_1_comb,
ngf * 4,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=None,
is_training=is_training,
scope_name='conv8_2')
conv8_3 = ops.conv2d(conv8_2,
ngf * 4,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv8_3')
print('ConvBlock 8', conv8_3)
conv2_2_short = ops.conv2d(conv2_2,
ngf * 2,
3,
1,
activation_fn=None,
is_training=is_training,
scope_name='conv2_2_short')
conv9_1 = ops.conv2d_transpose(conv8_3,
ngf * 2,
4,
2,
activation_fn=None,
is_training=is_training,
scope_name='conv9_1')
conv9_1_comb = tf.nn.relu(conv2_2_short + conv9_1)
conv9_2 = ops.conv2d(conv9_1_comb,
ngf * 2,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv9_2')
print('ConvBlock 9', conv9_2)
conv1_2_short = ops.conv2d(conv1_2,
ngf * 2,
3,
1,
activation_fn=None,
is_training=is_training,
scope_name='conv1_2_short')
conv10_1 = ops.conv2d_transpose(conv9_2,
ngf * 2,
4,
2,
activation_fn=None,
is_training=is_training,
scope_name='conv10_1')
conv10_1_comb = tf.nn.relu(conv1_2_short + conv10_1)
conv10_2 = ops.conv2d(conv10_1_comb,
ngf * 2,
3,
1,
activation_fn=tf.nn.relu,
norm_fn=tf.layers.batch_normalization,
is_training=is_training,
scope_name='conv10_2')
print('ConvBlock 10', conv10_2)
conv10_ab = ops.conv2d(conv10_2,
2,
1,
1,
activation_fn=tf.nn.tanh,
norm_fn=None,
is_training=is_training,
scope_name='conv10_ab')
print('OutputBlock', conv10_ab, end='\n\n')
return conv10_ab
def generate(self, z_var, pg=1, t=False, alpha_trans=0.0):
with tf.variable_scope('generator') as scope:
de = tf.reshape(
z_var,
[self.batch_size, 1, 1,
tf.cast(self.get_nf(1), tf.int32)])
de = conv2d(de,
output_dim=self.get_nf(1),
k_h=4,
k_w=4,
d_w=1,
d_h=1,
padding='Other',
name='gen_n_1_conv')
de = Pixl_Norm(lrelu(de))
de = tf.reshape(
de, [self.batch_size, 4, 4,
tf.cast(self.get_nf(1), tf.int32)])
de = conv2d(de,
output_dim=self.get_nf(1),
d_w=1,
d_h=1,
name='gen_n_2_conv')
de = Pixl_Norm(lrelu(de))
for i in range(pg - 1):
if i == pg - 2 and t:
#To RGB
de_iden = conv2d(de,
output_dim=3,
k_w=1,
k_h=1,
d_w=1,
d_h=1,
name='gen_y_rgb_conv_{}'.format(
de.shape[1]))
de_iden = upscale(de_iden, 2)
de = upscale(de, 2)
de = Pixl_Norm(
lrelu(
conv2d(de,
output_dim=self.get_nf(i + 1),
d_w=1,
d_h=1,
name='gen_n_conv_1_{}'.format(de.shape[1]))))
de = Pixl_Norm(
lrelu(
conv2d(de,
output_dim=self.get_nf(i + 1),
d_w=1,
d_h=1,
name='gen_n_conv_2_{}'.format(de.shape[1]))))
#To RGB
de = conv2d(de,
output_dim=3,
k_w=1,
k_h=1,
d_w=1,
d_h=1,
name='gen_y_rgb_conv_{}'.format(de.shape[1]))
if pg == 1:
return de
if t:
de = (1 - alpha_trans) * de_iden + alpha_trans * de
else:
de = de
return de
def convnet(inputs, cond, filters, hps, channels):
inputs = tf.nn.relu(conv2d(inputs, width=filters, name="conv2d_1"))
inputs = tf.nn.relu(conv2d(inputs, filters, filter_size=[1,1], name="conv2d_2"))
inputs = conv2d_zeros(inputs, channels, name="conv2d_3")
return inputs
def quat_inception(net, vp_mask):
net.quat_net = {}
with tf.variable_scope('Viewpoint_Net', reuse=tf.AUTO_REUSE):
vp_mask = tf.expand_dims(vp_mask, -1)
# Output (bs, 64, 64, ch)
conv1 = conv2d('conv1',
vp_mask,
3,
256,
stride=1,
norm=net.norm,
mode=net.mode,
act=None)
net.quat_net['conv1'] = conv1
conv2 = conv2d('conv2',
conv1,
1,
128,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv2'] = conv2
conv3 = conv2d('conv3',
conv2,
1,
128,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv3'] = conv3
# Output (bs, 32, 32, ch)
pool1 = tf.layers.max_pooling2d(conv3,
2,
2,
padding='same',
name='pool1')
net.quat_net['pool1'] = pool1
conv4 = conv2d('conv4',
pool1,
3,
512,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv4'] = conv4
conv5 = conv2d('conv5',
conv4,
1,
256,
stride=1,
norm=net.norm,
mode=net.mode)
conv5 = dropout(conv5, net.keep_prob)
net.quat_net['conv5'] = conv5
# Output (bs, 16, 16, ch)
pool2 = tf.layers.max_pooling2d(conv5,
2,
2,
padding='same',
name='pool2')
pool2 = dropout(pool2, net.keep_prob)
net.quat_net['pool2'] = pool2
fc1 = fully_connected('fc1', pool2, 1024)
net.quat_net['fc1'] = fc1
fc2 = fully_connected('fc2', fc1, 4 * net.num_classes)
# fc2 = tf.tanh(fc2)
net.quat_net['fc2'] = fc2
out = fc2
net.quat_net['out'] = out
return out
def im_vgg16(net, ims):
net.im_net = {}
bs, h, w, ch = tf_static_shape(ims)
with tf.variable_scope('ImNet_UNet', reuse=tf.AUTO_REUSE):
#VGG16 layers
conv1_1 = conv2d('conv1_1', ims, 3, 64, mode=net.mode, act=None)
net.im_net['conv1_1'] = conv1_1
conv1_2 = conv2d('conv1_2', conv1_1, 3, 64, mode=net.mode)
net.im_net['conv1_2'] = conv1_2
pool1 = tf.layers.max_pooling2d(conv1_2,
2,
2,
padding='same',
name='pool1')
conv2_1 = conv2d('conv2_1', pool1, 3, 128, mode=net.mode)
net.im_net['conv2_1'] = conv2_1
conv2_2 = conv2d('conv2_2', conv2_1, 3, 128, mode=net.mode)
net.im_net['conv2_2'] = conv2_2
pool2 = tf.layers.max_pooling2d(conv2_2,
2,
2,
padding='same',
name='pool2')
net.im_net['pool2'] = pool2
conv3_1 = conv2d('conv3_1', pool2, 3, 256, mode=net.mode)
net.im_net['conv3_1'] = conv3_1
conv3_2 = conv2d('conv3_2', conv3_1, 3, 256, mode=net.mode)
net.im_net['conv3_2'] = conv3_2
conv3_3 = conv2d('conv3_3', conv3_2, 3, 256, mode=net.mode)
net.im_net['conv3_3'] = conv3_3
pool3 = tf.layers.max_pooling2d(conv3_3,
2,
2,
padding='same',
name='pool3')
net.im_net['pool3'] = pool3
conv4_1 = conv2d('conv4_1', pool3, 3, 512, mode=net.mode)
net.im_net['conv4_1'] = conv4_1
conv4_2 = conv2d('conv4_2', conv4_1, 3, 512, mode=net.mode)
net.im_net['conv4_2'] = conv4_2
conv4_3 = conv2d('conv4_3', conv4_2, 3, 512, mode=net.mode)
net.im_net['conv4_3'] = conv4_3
pool4 = tf.layers.max_pooling2d(conv4_3,
2,
2,
padding='same',
name='pool4')
net.im_net['pool4'] = pool4
conv5_1 = conv2d('conv5_1', pool4, 3, 512, mode=net.mode)
net.im_net['conv5_1'] = conv5_1
conv5_2 = conv2d('conv5_2', conv5_1, 3, 512, mode=net.mode)
net.im_net['conv5_2'] = conv5_2
conv5_3 = conv2d('conv5_3', conv5_2, 3, 512, mode=net.mode)
net.im_net['conv5_3'] = conv5_3
#Deconv layers
feat_conv5 = conv2d('feat_conv5',
conv5_3,
1,
64,
norm=net.norm,
mode=net.mode)
net.im_net['feat_conv5'] = feat_conv5
upfeat_conv5 = deconv_pcnn(feat_conv5,
4,
4,
64,
2,
2,
name='upfeat_conv5',
trainable=False)
# upfeat_conv5 = deconv2d('upfeat_conv5', conv5_3, 4, 64, stride=2, padding="SAME", norm=net.norm, mode=net.mode)
net.im_net['upfeat_conv5'] = upfeat_conv5
feat_conv4 = conv2d('feat_conv4',
conv4_3,
1,
64,
norm=net.norm,
mode=net.mode)
net.im_net['feat_conv4'] = feat_conv4
add_feat = tf.add_n([upfeat_conv5, feat_conv4], name='add_feat')
add_feat = dropout(add_feat, net.keep_prob)
net.im_net['add_feat'] = add_feat
upfeat = deconv_pcnn(add_feat,
16,
16,
64,
8,
8,
name='upfeat',
trainable=False)
# upfeat = deconv2d('upfeat', add_feat, 16, 64, stride=8, padding="SAME", norm=net.norm, mode=net.mode)
net.im_net['upfeat'] = upfeat
return upfeat
def quat_res(net, vp_mask):
net.quat_net = {}
with tf.variable_scope('Quat_Net', reuse=tf.AUTO_REUSE):
vp_mask = tf.expand_dims(vp_mask, -1)
# Output (bs, 32, 32, 64)
conv1 = conv2d('conv1',
vp_mask,
7,
64,
stride=2,
norm=net.norm,
mode=net.mode,
act=None)
net.quat_net['conv1'] = conv1
# Output (bs, 16, 16, 64)
pool1 = tf.layers.max_pooling2d(conv1,
3,
2,
padding='same',
name='pool1')
net.quat_net['pool1'] = pool1
# Output (bs, 16, 16, 64)
conv2_1a = conv2d('conv2_1a',
pool1,
3,
64,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv2_1a'] = conv2_1a
conv2_2a = conv2d('conv2_2a',
conv2_1a,
3,
64,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv2_2a'] = conv2_2a
res_2a = tf.add_n([conv2_2a, pool1], name='res_2a')
conv2_1b = conv2d('conv2_1b',
res_2a,
3,
64,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv2_1b'] = conv2_1b
conv2_2b = conv2d('conv2_2b',
conv2_1b,
3,
64,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv2_2b'] = conv2_2b
res_2b = tf.add_n([conv2_2b, res_2a], name='res_2b')
# Output (bs, 8, 8, 128)
conv3_1a = conv2d('conv3_1a',
res_2b,
3,
128,
stride=2,
norm=net.norm,
mode=net.mode)
net.quat_net['conv3_1a'] = conv3_1a
conv3_2a = conv2d('conv3_2a',
conv3_1a,
3,
128,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv3_2a'] = conv3_2a
res_2b_skip = conv2d('res_2b_skip',
res_2b,
1,
128,
stride=2,
norm=net.norm,
mode=net.mode)
res_3a = tf.add_n([conv3_2a, res_2b_skip], name='res_3a')
conv3_1b = conv2d('conv3_1b',
res_3a,
3,
128,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv3_1b'] = conv3_1b
conv3_2b = conv2d('conv3_2b',
conv3_1b,
3,
128,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv3_2b'] = conv3_2b
res_3b = tf.add_n([conv3_2b, res_3a], name='res_3b')
# Output (bs, 4, 4, 256)
conv4_1a = conv2d('conv4_1a',
res_3b,
3,
256,
stride=2,
norm=net.norm,
mode=net.mode)
net.quat_net['conv4_1a'] = conv4_1a
conv4_2a = conv2d('conv4_2a',
conv4_1a,
3,
256,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv4_2a'] = conv4_2a
res_3b_skip = conv2d('res_3b_skip',
res_3b,
1,
256,
stride=2,
norm=net.norm,
mode=net.mode)
res_4a = tf.add_n([conv4_2a, res_3b_skip], name='res_4a')
conv4_1b = conv2d('conv4_1b',
res_4a,
3,
256,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv4_1b'] = conv4_1b
conv4_2b = conv2d('conv4_2b',
conv4_1b,
3,
256,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv4_2b'] = conv4_2b
res_4b = tf.add_n([conv4_2b, res_4a], name='res_4b')
# Output (bs, 2, 2, 512)
conv5_1a = conv2d('con5_1a',
res_4b,
3,
512,
stride=2,
norm=net.norm,
mode=net.mode)
net.quat_net['con5_1a'] = conv5_1a
conv5_2a = conv2d('con5_2a',
conv5_1a,
3,
512,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['con5_2a'] = conv5_2a
res_4b_skip = conv2d('res_4b_skip',
res_4b,
1,
512,
stride=2,
norm=net.norm,
mode=net.mode)
res_5a = tf.add_n([conv5_2a, res_4b_skip], name='res_5a')
conv5_1b = conv2d('conv5_1b',
res_5a,
3,
512,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv5_1b'] = conv5_1b
conv5_2b = conv2d('conv5_2b',
conv5_1b,
3,
512,
stride=1,
norm=net.norm,
mode=net.mode)
net.quat_net['conv5_2b'] = conv5_2b
res_5b = tf.add_n([conv5_2b, res_5a], name='res_5b')
res_5b = dropout(res_5b, net.keep_prob)
# Output (bs, 4*num_classes)
fc1 = fully_connected('fc1', res_5b, 512)
net.quat_net['fc1'] = fc1
fc2 = fully_connected('fc2', fc1, 4 * net.num_classes)
net.quat_net['fc2'] = fc2
# out = tf.tanh(fc2)
out = fc2
net.quat_net['out'] = out
return out
def network(depth,
rgb,
reuse=False,
dim1=56,
dim2=12,
sr_times=16,
name='network',
do_norm=False):
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
height, width = depth.get_shape()[1], depth.get_shape()[2]
depth = tf.image.resize_bicubic(
depth, size=[height * sr_times, width * sr_times])
depth_conv1 = conv2d(depth,
dim1,
3,
1,
0.02,
fn='prelu',
do_norm=do_norm,
name='depth_conv1',
name_bn='bn1',
name_prelu='alpha1')
depth_feature_resnet1 = build_feature_resnet_block(
depth_conv1, dim1, 'depth_feature_resnet1')
depth_pool1 = pool(depth_feature_resnet1, 'pool1')
depth_mapping_resnet1 = build_mapping_resnet_block(
depth_feature_resnet1, dim1, dim2, 'depth_mapping_resnet1')
depth_feature_resnet2 = build_feature_resnet_block(
depth_pool1, dim1, 'depth_feature_resnet2')
depth_pool2 = pool(depth_feature_resnet2, 'pool2')
depth_mapping_resnet2 = build_mapping_resnet_block(
depth_feature_resnet2, dim1, dim2, 'depth_mapping_resnet2')
depth_feature_resnet3 = build_feature_resnet_block(
depth_pool2, dim1, 'depth_feature_resnet3')
depth_pool3 = pool(depth_feature_resnet3, 'pool3')
depth_mapping_resnet3 = build_mapping_resnet_block(
depth_feature_resnet3, dim1, dim2, 'depth_mapping_resnet3')
depth_feature_resnet4 = build_feature_resnet_block(
depth_pool3, dim1, 'depth_feature_resnet4')
depth_mapping_resnet4 = build_mapping_resnet_block(
depth_feature_resnet4, dim1, dim2, 'depth_mapping_resnet4')
depth_deconv1 = deconv2d(depth_mapping_resnet4,
dim1,
do_norm=do_norm,
name='depth_deconv1',
name_bn='bn1',
name_prelu='alpha1')
depth_deconv1 = element_wise_linear_add(depth_deconv1,
depth_mapping_resnet3,
'alpha1')
depth_deconv2 = deconv2d(depth_deconv1,
dim1,
do_norm=do_norm,
name='depth_deconv2',
name_bn='bn1',
name_prelu='alpha1')
depth_deconv2 = element_wise_linear_add(depth_deconv2,
depth_mapping_resnet2,
'alpha2')
depth_deconv3 = deconv2d(depth_deconv2,
dim1,
do_norm=do_norm,
name='depth_deconv3',
name_bn='bn1',
name_prelu='alpha1')
depth_deconv3 = element_wise_linear_add(depth_deconv3,
depth_mapping_resnet1,
'alpha3')
rgb_conv1 = conv2d(rgb,
dim1,
3,
1,
0.02,
fn='prelu',
do_norm=do_norm,
name='rgb_conv1',
name_bn='bn1',
name_prelu='alpha1')
rgb_feature_resnet1 = build_feature_resnet_block(
rgb_conv1, dim1, 'rgb_feature_resnet1')
rgb_mapping_resnet1 = build_mapping_resnet_block(
rgb_feature_resnet1, dim1, dim2, 'rgb_mapping_resnet1')
rgb_pool1 = pool(rgb_feature_resnet1, 'pool1')
rgb_feature_resnet2 = build_feature_resnet_block(
rgb_pool1, dim1, 'rgb_feature_resnet2')
rgb_mapping_resnet2 = build_mapping_resnet_block(
rgb_feature_resnet2, dim1, dim2, 'rgb_mapping_resnet2')
rgb_pool2 = pool(rgb_feature_resnet2, 'pool2')
rgb_feature_resnet3 = build_feature_resnet_block(
rgb_pool2, dim1, 'rgb_feature_resnet3')
rgb_mapping_resnet3 = build_mapping_resnet_block(
rgb_feature_resnet3, dim1, dim2, 'rgb_mapping_resnet3')
rgb_pool3 = pool(rgb_feature_resnet3, 'pool3')
rgb_feature_resnet4 = build_feature_resnet_block(
rgb_pool3, dim1, 'rgb_feature_resnet4')
rgb_mapping_resnet4 = build_mapping_resnet_block(
rgb_feature_resnet4, dim1, dim2, 'rgb_mapping_resnet4')
rgb_deconv1 = deconv2d(rgb_mapping_resnet4,
dim1,
do_norm=do_norm,
name='rgb_deconv1',
name_bn='bn1',
name_prelu='alpha1')
rgb_deconv1 = element_wise_linear_add(rgb_deconv1, rgb_mapping_resnet3,
'alpha4')
rgb_deconv2 = deconv2d(rgb_deconv1,
dim1,
do_norm=do_norm,
name='rgb_deconv2',
name_bn='bn1',
name_prelu='alpha1')
rgb_deconv2 = element_wise_linear_add(rgb_deconv2, rgb_mapping_resnet2,
'alpha5')
rgb_deconv3 = deconv2d(rgb_deconv2,
dim1,
do_norm=do_norm,
name='rgb_deconv3',
name_bn='bn1',
name_prelu='alpha1')
rgb_deconv3 = element_wise_linear_add(rgb_deconv3, rgb_mapping_resnet1,
'alpha6')
out = tf.concat([depth_deconv3, rgb_deconv3], 3)
out = conv2d(out,
dim1,
3,
1,
0.02,
fn='prelu',
do_norm=do_norm,
name='out1',
name_bn='bn1',
name_prelu='alpha1')
out = conv2d(out,
1,
3,
1,
0.02,
do_norm=do_norm,
name='out2',
name_bn='bn1',
name_prelu='alpha1')
out = tf.concat([depth, out], 3)
out = PReLU(out, 'alpha7')
out = conv2d(out, 1, 3, 1, 0.02, do_norm=False, name='out')
print(out)
return out
def model(self, train_phase):
"""
Define the model - The network architecture
:param train_phase: tf.bool with True for train and False for test
"""
# Reshape the input for batchSize, dims_in[0] X dims_in[1] image, dims_in[2] channels
x_image = tf.reshape(self.input, [-1, self.dims_in[0], self.dims_in[1], self.dims_in[2]],
name='x_input_reshaped')
# Dump input image
tf.summary.image(self.get_name('x_input'), x_image)
# tf.image_summary(self.get_name('x_input'), x_image)
print('model - x_image:', x_image)
print('model - self.input:', self.input)
print('model - self.dims_in:', self.dims_in)
# Model convolutions
kh = 1 #averagepool
kw = 1
dh = 1
dw = 1
wind = [1,4,1,1]
stride = [1,2,2,1]
with tf.variable_scope('conv_1'):
conv_1, reg1 , weights1, biases1 = ops.conv2d(x_image, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_1")
# conv_1, reg1 , weights1, biases1 = ops.conv2d(x_image, output_dim=64, k_h=1, k_w=12, d_h=dh, d_w=dw, name="conv_1")
relu1 = tf.nn.relu(conv_1)
max_pool1 = tf.nn.max_pool(relu1, wind, stride, 'SAME')
print('if train_phase == True: ', train_phase)
if train_phase == True:
tf.get_variable_scope().reuse_variables() # tf.get_variable_scope().reuse_variables
tf.summary.histogram("weights1", weights1) # tf.histogram_summary("weights1", weights1)
tf.summary.histogram("biases1", biases1) # tf.histogram_summary("biases1", biases1)
tf.summary.merge_all() # tf.merge_all_summaries()
print('if train_phase == True: weights1.get_shape()',weights1.get_shape())
# def histogram(conv):
# return tf.summary.histogram("hist_conv_bn", conv) #tf.histogram_summary("hist_conv_bn", conv)
#
# train_phase1 = tf.constant(train_phase, dtype=tf.bool)
# conv_1 = batch_norm.batch_normalization(conv_1, n_out=128, train_phas=train_phase1, scope='bn1')
# histogram(conv_1)
# print('yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyees')
with tf.variable_scope('conv_2'):
# conv_2, reg2, weights2, biases2 = ops.conv2d(relu1, output_dim=32, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_2")
# conv_2, reg2, weights2, biases2 = ops.conv2d(relu1, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_2")
conv_2, reg2, weights2, biases2 = ops.conv2d(max_pool1, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_2")
# if train_phase == True:
# train_phase1 = tf.constant(train_phase, dtype=tf.bool)
# conv_2 = batch_norm.batch_normalization(conv_2, n_out=96, train_phas=train_phase1, scope='bn2')
relu2 = tf.nn.relu(conv_2)
max_pool2 = tf.nn.max_pool(relu2, wind, stride, 'SAME')
with tf.variable_scope('conv_3'):
# conv_3, reg3, weights3, biases3 = ops.conv2d(relu2, output_dim=16, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_3")
# conv_3, reg3, weights3, biases3 = ops.conv2d(relu2, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_3")
conv_3, reg3, weights3, biases3 = ops.conv2d(max_pool2, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_3")
# if train_phase == True:
# train_phase1 = tf.constant(train_phase, dtype=tf.bool)
# conv_3 = batch_norm.batch_normalization(conv_3, n_out=64, train_phas=train_phase1, scope='bn3')
relu3 = tf.nn.relu(conv_3)#+x_image)
with tf.variable_scope('conv_4'):
conv_4, reg4, weights4, biases4 = ops.conv2d(relu3, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_4")
# conv_4, reg4, weights4, biases4 = ops.conv2d(relu3, output_dim=32, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_4")
# if train_phase == True:
# train_phase1 = tf.constant(train_phase, dtype=tf.bool)
# conv_4 = batch_norm.batch_normalization(conv_4, n_out=32, train_phas=train_phase1, scope='bn4')
relu4 = tf.nn.relu(conv_4)
# repool = tf.image.resize_bilinear(relu4, [5000, 12], align_corners=None, name=None)
'''
print('relu_shape')
print(relu_shape)
print('relu_3')
print(relu_3)
reshape = tf.reshape(
relu_3,
[relu_shape[0], relu_shape[1] * relu_shape[2] * relu_shape[3]])
print('reshape')
print(reshape)
fc1_weights = tf.Variable( # fully connected, depth 512.
tf.truncated_normal([IMAGE_SIZE * IMAGE_SIZE * 64, NUM_CLASSES],
stddev=0.1,
seed=None,
dtype= tf.float32))
print('!!11111!!')
print(fc1_weights)
fc1_biases = tf.Variable(tf.constant(0.1, shape=[NUM_CLASSES], dtype = tf.float32))
predict = tf.matmul(reshape,fc1_weights)+fc1_biases
'''
with tf.variable_scope('conv_5'):
conv_5, reg5, weights5, biases5 = ops.conv2d(relu4, output_dim=64, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_5")
# conv_5, reg5, weights5, biases5 = ops.conv2d(repool, output_dim=1, k_h=kh, k_w=kw, d_h=dh, d_w=dw, name="conv_5")
print ('!!conv_5', conv_5.get_shape().as_list())
relu5 = tf.nn.relu(conv_5)
# ###########################################
# Reshape the feature map cuboid into a 2D matrix to feed it to the
# fully connected layers.
relu5_shape = relu5.get_shape().as_list()
print ('!!relu5_shape', relu5_shape)
reshape = tf.reshape(
relu5,
[relu5_shape[0], relu5_shape[1] * relu5_shape[2] * relu5_shape[3]])
print('!!!!!!!!!reshape',reshape)
print('!!!!!!!!!reshape.get_shape().as_list()',reshape.get_shape().as_list())
# predict = tf.nn.avg_pool(relu5, ksize=[1,2,2,1], strides=[1, 1, 1, 1], padding='SAME')
# # h = tf.nn.avg_pool(h, ksize=[1, 4, 4, 256], strides=[1, 1, 1, 1], padding='VALID')
# predict = tf.reduce_mean(predict, axis=[1, 2])
##########################################################################
# good
reshape_shape = reshape.get_shape().as_list()
with tf.variable_scope('fully_connected_1'):
fc1_weights = tf.Variable( # fully connected, depth 512.IMAGE_SIZE*2
tf.truncated_normal([ reshape_shape[1], NUM_OUT],
stddev=0.1,
seed=SEED,
dtype= tf.float32))
fc1_biases = tf.Variable(tf.constant(0.1, shape=[NUM_OUT], dtype = tf.float32))
predict = tf.nn.relu(tf.matmul(reshape,fc1_weights)+fc1_biases)
# predict = tf.nn.softmax(predict, name="softmax_tensor")
# predict_shape = predict.get_shape().as_list()
print('!!11111!!fc1_weights', fc1_weights)
print('!!!!predict_shape after fc1 predict.get_shape().as_list()', predict.get_shape().as_list())
# # Add a 50% dropout during training only. Dropout also scales
# # activations such that no rescaling is needed at evaluation time.
# if train_phase == True:
# predict = tf.nn.dropout(predict, 0.4, seed=SEED)
##########################################################################
# with tf.variable_scope('fully_connected_2'):
# fc2_weights = tf.Variable(tf.truncated_normal([predict_shape[1], NUM_OUT ],
# stddev=0.1,
# seed=SEED,
# dtype=tf.float32))
# fc2_biases = tf.Variable(tf.constant(
# 0.1, shape=[NUM_OUT], dtype=tf.float32))
# predict = tf.nn.relu(tf.matmul(predict, fc2_weights) + fc2_biases)
#
# print('!!11111!!predict after fc2', predict)
##########################################################################
# # predict = tf.nn.relu(tf.matmul(predict,fc1_weights)+fc1_biases)
# # predict = tf.nn.softmax(predict, name="softmax_tensor") #NUM_OUT
##########################################################################
# Fully connected layer. Note that the '+' operation automatically
# broadcasts the biases.
# fc1_weights = tf.Variable( # fully connected, depth 512.IMAGE_SIZE*2
# tf.truncated_normal([ reshape_shape[1], NUM_OUT],
# stddev=0.1,
# seed=SEED,
# dtype= tf.float32))
#
# print('!!11111!!fc1_weights', fc1_weights)
# fc1_biases = tf.Variable(tf.constant(0.1, shape=[NUM_OUT], dtype = tf.float32))
# predict = tf.nn.relu(tf.matmul(reshape,fc1_weights)+fc1_biases)
# # predict = tf.nn.softmax(tf.matmul(reshape,fc1_weights)+fc1_biases, name="softmax_tensor")
# predict_shape = predict.get_shape().as_list()
#
# fc2_weights = tf.Variable(tf.truncated_normal([2048, 1],
# stddev=0.1,
# seed=SEED,
# dtype=tf.float32))
# fc2_biases = tf.Variable(tf.constant(
# 0.1, shape=[2048], dtype=tf.float32))
# predict = tf.matmul(predict, fc2_weights) + fc2_biases
# Add a 50% dropout during training only. Dropout also scales
# activations such that no rescaling is needed at evaluation time.
# if train:
# if train_phase == True:
# reshape = tf.nn.dropout(reshape, 0.5, seed=SEED)
# predict = tf.matmul(reshape,fc1_weights)+fc1_biases
# predict = tf.nn.relu(tf.matmul(reshape,fc1_weights)+fc1_biases)
# predict = tf.nn.softmax(tf.matmul(reshape,fc1_weights)+fc1_biases, name="softmax_tensor")
# hidden = tf.nn.relu(tf.matmul(reshape,fc1_weights)+fc1_biases)
# fc2_weights = tf.Variable(tf.truncated_normal([2048, NUM_OUT],
# stddev=0.1,
# seed=SEED,
# dtype=tf.float32))
# fc2_biases = tf.Variable(tf.constant(
# 0.1, shape=[2048], dtype=tf.float32))
# Add a 50% dropout during training only. Dropout also scales
# activations such that no rescaling is needed at evaluation time.
# if train:
# hidden = tf.nn.dropout(hidden, 0.5, seed=SEED)
# predict = tf.matmul(hidden, fc2_weights) + fc2_biases
##########################################################################
# predict = x_image #tf.squeeze(relu5, 1) #relu5
predict = tf.squeeze(predict, 1) #relu5
reg = reg1+reg2+reg3+reg4+reg5
print('model - predict', predict)
print('model - conv_1', conv_1)
print('model - conv_2', conv_2)
print('model - conv_3', conv_3)
print('model - conv_4', conv_4)
print('model - conv_5', conv_5)
print('model - relu1', relu1)
print('model - relu2', relu2)
print('model - relu3', relu3)
print('model - relu4', relu4)
print('model - relu5', relu5)
print('model - reg', reg)
#################################################
# Reshape the output layer to a 1-dim Tensor to return predictions
# predict = tf.squeeze(conv_5, 1)
# reg = tf.squeeze(reg1+reg2+reg3+reg4+reg5, 1)
# predictions = tf.squeeze(output_layer, 1)
################################################
return predict, reg, conv_1
def generator(self, images):
with tf.variable_scope("generator"):
g_h0 = tf.nn.relu(conv2d(images, 64, 5, 5, 1, 1, name='g_h0'))
g_h1 = tf.nn.relu(self.g_bns[0](conv2d(g_h0, 128, 3, 3, 2, 2, name='g_h1')))
g_h2 = tf.nn.relu(self.g_bns[1](conv2d(g_h1, 128, 3, 3, 1, 1, name='g_h2')))
g_h3 = tf.nn.relu(self.g_bns[2](conv2d(g_h2, 256, 3, 3, 2, 2, name='g_h3')))
g_h4 = tf.nn.relu(self.g_bns[3](conv2d(g_h3, 256, 3, 3, 1, 1, name='g_h4')))
g_h5 = tf.nn.relu(self.g_bns[4](conv2d(g_h4, 256, 3, 3, 1, 1, name='g_h5')))
g_h6 = tf.nn.relu(self.g_bns[5](dilated_conv2d(g_h5, 256, 3, 3, 2, name='g_h6')))
g_h7 = tf.nn.relu(self.g_bns[6](dilated_conv2d(g_h6, 256, 3, 3, 4, name='g_h7')))
g_h8 = tf.nn.relu(self.g_bns[7](dilated_conv2d(g_h7, 256, 3, 3, 8, name='g_h8')))
g_h9 = tf.nn.relu(self.g_bns[8](dilated_conv2d(g_h8, 256, 3, 3, 16, name='g_h9')))
g_h10 = tf.nn.relu(self.g_bns[9](conv2d(g_h9, 256, 3, 3, 1, 1, name='g_h10')))
g_h11 = tf.nn.relu(self.g_bns[10](conv2d(g_h10, 256, 3, 3, 1, 1, name='g_h11')))
g_h12 = tf.nn.relu(self.g_bns[11](conv2d_transpose(
g_h11, [self.batch_size, int(self.image_size/2), int(self.image_size/2), 128], 4, 4, 2, 2, name='g_h12')))
g_h13 = tf.nn.relu(self.g_bns[12](conv2d(g_h12, 128, 3, 3, 1, 1, name='g_h13')))
g_h14 = tf.nn.relu(self.g_bns[13](conv2d_transpose(
g_h13, [self.batch_size, self.image_size, self.image_size, 64], 4, 4, 2, 2, name='g_h14')))
g_h15 = tf.nn.relu(self.g_bns[14](conv2d(g_h14, 32, 3, 3, 1, 1, name='g_h15')))
g_h16 = tf.nn.sigmoid(conv2d(g_h15, 3, 3, 3, 1, 1, name='g_h16'))
return g_h16
def build(self, input_, reuse=False, name="generator"):
with tf.variable_scope(name):
if reuse:
tf.get_variable_scope().reuse_variables()
else:
assert tf.get_variable_scope().reuse is False
e1 = instance_norm(
conv2d(input_, self.args.gf_dim, name='g_e1_conv'))
e2 = instance_norm(
conv2d(lrelu(e1), self.args.gf_dim * 2, name='g_e2_conv'),
'g_bn_e2')
e3 = instance_norm(
conv2d(lrelu(e2), self.args.gf_dim * 4, name='g_e3_conv'),
'g_bn_e3')
e4 = instance_norm(
conv2d(lrelu(e3), self.args.gf_dim * 8, name='g_e4_conv'),
'g_bn_e4')
e5 = instance_norm(
conv2d(lrelu(e4), self.args.gf_dim * 8, name='g_e5_conv'),
'g_bn_e5')
e6 = instance_norm(
conv2d(lrelu(e5), self.args.gf_dim * 8, name='g_e6_conv'),
'g_bn_e6')
e7 = instance_norm(
conv2d(lrelu(e6), self.args.gf_dim * 8, name='g_e7_conv'),
'g_bn_e7')
z_mean = instance_norm(
conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e8_conv'),
'mean')
z_logvar = instance_norm(
conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e9_conv'),
'logvar')
eps = tf.random_normal(shape=tf.shape(z_mean))
decoder_input = tf.add(z_mean,
tf.exp(z_logvar / 2) * eps,
name='decoder_input')
d1 = deconv2d(tf.nn.relu(decoder_input),
self.args.gf_dim * 8,
name='g_d1')
d1 = tf.nn.dropout(d1, self.dropout_rate)
d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3)
d2 = deconv2d(tf.nn.relu(d1), self.args.gf_dim * 8, name='g_d2')
d2 = tf.nn.dropout(d2, self.dropout_rate)
d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3)
d3 = deconv2d(tf.nn.relu(d2), self.args.gf_dim * 8, name='g_d3')
d3 = tf.nn.dropout(d3, self.dropout_rate)
d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3)
d4 = deconv2d(tf.nn.relu(d3), self.args.gf_dim * 8, name='g_d4')
d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3)
d5 = deconv2d(tf.nn.relu(d4), self.args.gf_dim * 4, name='g_d5')
d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3)
d6 = deconv2d(tf.nn.relu(d5), self.args.gf_dim * 2, name='g_d6')
d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3)
d7 = deconv2d(tf.nn.relu(d6), self.args.gf_dim, name='g_d7')
d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3)
d8 = deconv2d(tf.nn.relu(d7), self.args.out_dim, name='g_d8')
return z_mean, z_logvar, tf.nn.tanh(d8)
def model(x):
# Current data input shape: (batch_size, n_steps, n_input)
x = tf.transpose(x, [1, 0, 2])
# (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, EMBEDDING_SIZE])
# get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, NUM_STEPS, x)
# B-directional LSTM
fw_cell = tf.nn.rnn_cell.LSTMCell(num_hidden,
forget_bias=1.0,
state_is_tuple=True)
bw_cell = tf.nn.rnn_cell.LSTMCell(num_hidden,
forget_bias=1.0,
state_is_tuple=True)
if rnn_layer > 1:
fw_cell = tf.nn.rnn_cell.MultiRNNCell([fw_cell] * rnn_layer)
bw_cell = tf.nn.rnn_cell.MultiRNNCell([bw_cell] * rnn_layer)
# output = [batch_size,num_hidden*2]
# outputs of Bi-directional LSTM to highway
outputs, fw_final_state, bw_final_state = tf.nn.bidirectional_rnn(
fw_cell, bw_cell, x, dtype=tf.float32)
# Highway
# convert to [batch_size,num_steps,num_hidden*2]
hw_input = tf.transpose(tf.pack(outputs, axis=0), [1, 0, 2])
# add dropout for Bi-LSTM output
hw_input = tf.nn.dropout(hw_input, dropout_keep_prob)
# convert to [batch_size x num_steps,num_hidden*2]
hw_input = tf.reshape(hw_input, [-1, num_hidden * 2])
size = hw_input.get_shape()[1]
# size = num_hidden*2
# tf.tanh
# hw_output=[batch_size x num_steps,num_hidden*2]
hw_output = highways(hw_input, size)
# convert to [batch_size,num_steps,num_hidden*2]
hw_output = tf.reshape(hw_output, [-1, NUM_STEPS, num_hidden * 2])
# expand dim , cnn_input=[batch_size,num_steps,num_hidden*2,1]
cnn_input = tf.expand_dims(hw_output, -1)
# CNN
pooled_outputs = []
for idx, filter_size in enumerate(filter_sizes):
conv = conv2d(cnn_input,
filter_numbers[idx],
filter_size,
num_hidden * 2,
name="kernel%d" % idx)
# 1-max pooling,leave a tensor of shape[batch_size,1,1,num_filters]
pool = tf.nn.max_pool(
conv,
ksize=[1, max_document_length - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1],
padding='VALID')
pooled_outputs.append(tf.squeeze(pool))
if len(filter_sizes) > 1:
cnn_output = tf.concat(1, pooled_outputs)
else:
cnn_output = pooled_outputs[0]
# add dropout
cnn_output = tf.nn.dropout(cnn_output, dropout_keep_prob)
# fc1 layer
hidden = tf.matmul(cnn_output, fc1_weights) + fc1_biases
fc1_output = tf.sigmoid(hidden)
# fc2 layer
fc_output = tf.matmul(fc1_output, fc2_weights) + fc2_biases
return fc_output
def add_generator(self, name_scope='SoundNet'):
with tf.variable_scope(name_scope) as scope:
self.layers = {}
# Stream one: conv1 ~ conv7
self.layers[1] = conv2d(self.sound_input_placeholder,
1,
16,
k_h=64,
d_h=2,
p_h=32,
name_scope='conv1')
self.layers[2] = batch_norm(self.layers[1],
16,
self.config['eps'],
name_scope='conv1')
self.layers[3] = relu(self.layers[2], name_scope='conv1')
self.layers[4] = maxpool(self.layers[3],
k_h=8,
d_h=8,
name_scope='conv1')
self.layers[5] = conv2d(self.layers[4],
16,
32,
k_h=32,
d_h=2,
p_h=16,
name_scope='conv2')
self.layers[6] = batch_norm(self.layers[5],
32,
self.config['eps'],
name_scope='conv2')
self.layers[7] = relu(self.layers[6], name_scope='conv2')
self.layers[8] = maxpool(self.layers[7],
k_h=8,
d_h=8,
name_scope='conv2')
self.layers[9] = conv2d(self.layers[8],
32,
64,
k_h=16,
d_h=2,
p_h=8,
name_scope='conv3')
self.layers[10] = batch_norm(self.layers[9],
64,
self.config['eps'],
name_scope='conv3')
self.layers[11] = relu(self.layers[10], name_scope='conv3')
self.layers[12] = conv2d(self.layers[11],
64,
128,
k_h=8,
d_h=2,
p_h=4,
name_scope='conv4')
self.layers[13] = batch_norm(self.layers[12],
128,
self.config['eps'],
name_scope='conv4')
self.layers[14] = relu(self.layers[13], name_scope='conv4')
self.layers[15] = conv2d(self.layers[14],
128,
256,
k_h=4,
d_h=2,
p_h=2,
name_scope='conv5')
self.layers[16] = batch_norm(self.layers[15],
256,
self.config['eps'],
name_scope='conv5')
self.layers[17] = relu(self.layers[16], name_scope='conv5')
self.layers[18] = maxpool(self.layers[17],
k_h=4,
d_h=4,
name_scope='conv5')
self.layers[19] = conv2d(self.layers[18],
256,
512,
k_h=4,
d_h=2,
p_h=2,
name_scope='conv6')
self.layers[20] = batch_norm(self.layers[19],
512,
self.config['eps'],
name_scope='conv6')
self.layers[21] = relu(self.layers[20], name_scope='conv6')
self.layers[22] = conv2d(self.layers[21],
512,
1024,
k_h=4,
d_h=2,
p_h=2,
name_scope='conv7')
self.layers[23] = batch_norm(self.layers[22],
1024,
self.config['eps'],
name_scope='conv7')
self.layers[24] = relu(self.layers[23], name_scope='conv7')
# Split one: conv8, conv8_2
self.layers[25] = conv2d(self.layers[24],
1024,
1000,
k_h=8,
d_h=2,
name_scope='conv8')
self.layers[26] = conv2d(self.layers[24],
1024,
401,
k_h=8,
d_h=2,
name_scope='conv8_2')
def main(args=None):
print(args)
tf.reset_default_graph()
"""
Read dataset parser
"""
flags.network_name = args[0].split('/')[-1].split('.')[0].split('main_')[-1]
flags.logs_dir = './logs_' + flags.network_name
dataset_parser = GANParser(flags=flags)
"""
Transform data to TFRecord format (Only do once.)
"""
if False:
dataset_parser.load_paths(is_jpg=True, load_val=True)
dataset_parser.data2record(name='{}_train.tfrecords'.format(dataset_parser.dataset_name),
set_type='train', test_num=None)
dataset_parser.data2record(name='{}_val.tfrecords'.format(dataset_parser.dataset_name),
set_type='val', test_num=None)
# coco_parser.data2record_test(name='coco_stuff2017_test-dev_all_label.tfrecords', is_dev=True, test_num=None)
# coco_parser.data2record_test(name='coco_stuff2017_test_all_label.tfrecords', is_dev=False, test_num=None)
return
"""
Build Graph
"""
with tf.Graph().as_default():
"""
Input (TFRecord)
"""
with tf.name_scope('TFRecord'):
# DatasetA
training_a_dataset = dataset_parser.tfrecord_get_dataset(
name='{}_trainA.tfrecords'.format(dataset_parser.dataset_name), batch_size=flags.batch_size,
shuffle_size=None)
val_a_dataset = dataset_parser.tfrecord_get_dataset(
name='{}_valA.tfrecords'.format(dataset_parser.dataset_name), batch_size=flags.batch_size,
need_flip=(flags.mode == 'train'))
# DatasetB
training_b_dataset = dataset_parser.tfrecord_get_dataset(
name='{}_trainB.tfrecords'.format(dataset_parser.dataset_name), batch_size=flags.batch_size,
shuffle_size=None)
val_b_dataset = dataset_parser.tfrecord_get_dataset(
name='{}_valB.tfrecords'.format(dataset_parser.dataset_name), batch_size=flags.batch_size,
is_label=True, need_flip=(flags.mode == 'train'))
# A feed-able iterator
with tf.name_scope('RealA'):
handle_a = tf.placeholder(tf.string, shape=[])
iterator_a = tf.contrib.data.Iterator.from_string_handle(
handle_a, training_a_dataset.output_types, training_a_dataset.output_shapes)
real_a, real_a_name, real_a_shape = iterator_a.get_next()
with tf.name_scope('RealB'):
handle_b = tf.placeholder(tf.string, shape=[])
iterator_b = tf.contrib.data.Iterator.from_string_handle(
handle_b, training_b_dataset.output_types, training_b_dataset.output_shapes)
real_b, real_b_name, real_b_shape = iterator_b.get_next()
with tf.name_scope('InitialA_op'):
training_a_iterator = training_a_dataset.make_initializable_iterator()
validation_a_iterator = val_a_dataset.make_initializable_iterator()
with tf.name_scope('InitialB_op'):
training_b_iterator = training_b_dataset.make_initializable_iterator()
validation_b_iterator = val_b_dataset.make_initializable_iterator()
"""
Network (Computes predictions from the inference model)
"""
with tf.name_scope('Network'):
# Input
global_step = tf.Variable(0, trainable=False, name='global_step', dtype=tf.int32)
global_step_update_op = tf.assign_add(global_step, 1, name='global_step_update_op')
mean_rgb = tf.constant((123.68, 116.78, 103.94), dtype=tf.float32)
fake_b_pool = tf.placeholder(tf.float32,
shape=[None, flags.image_height, flags.image_width, flags.c_in_dim],
name='fake_B_pool')
image_linear_shape = tf.constant(flags.image_height * flags.image_width * flags.c_in_dim,
dtype=tf.int32, name='image_linear_shape')
# A -> B
'''
with tf.name_scope('Generator'):
with slim.arg_scope(vgg.vgg_arg_scope()):
net, end_points = vgg.vgg_16(real_a - mean_rgb, num_classes=1, is_training=True,
spatial_squeeze=False)
print(net)
return
with tf.variable_scope('Generator_A2B'):
pred = tf.layers.conv2d(tf.nn.relu(net), 1, 1, 1)
pred_upscale = tf.image.resize_bilinear(pred, (flags.image_height, flags.image_width),
name='up_scale')
segment_a = tf.nn.sigmoid(pred_upscale, name='segment_a')
# sigmoid cross entropy Loss
with tf.name_scope('loss_gen_a2b'):
loss_gen_a2b = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
logits=pred_upscale, labels=real_b/255.0, name='sigmoid'), name='mean')
'''
# A -> B
with tf.name_scope('Generator'):
with slim.arg_scope(resnet_v1.resnet_arg_scope()):
net, end_points = resnet_v1.resnet_v1_50(real_a - mean_rgb, num_classes=None, is_training=True,
global_pool=False, output_stride=8)
with tf.variable_scope('Generator_A2B'):
d1 = deconv2d(net, 256, 3, 2, name='g_d1_dc')
d1 = tf.nn.relu(instance_normalization(d1, 'g_d1_bn'))
d2 = deconv2d(d1, 128, 3, 2, name='g_d2_dc')
d2 = tf.nn.relu(instance_normalization(d2, 'g_d2_bn'))
d3 = deconv2d(d2, 64, 3, 2, name='g_d3_dc')
d3 = tf.nn.relu(instance_normalization(d3, 'g_d3_bn'))
d3 = tf.pad(d3, [[0, 0], [3, 3], [3, 3], [0, 0]], "REFLECT")
logits_a = conv2d(d3, 1, 7, 1, padding='VALID', name='g_pred_c')
# A -> B
# adjusted_a = tf.zeros_like(real_a, tf.float32, name='mask', optimize=True)
# logits_a = generator_resnet(real_a, flags, False, name="Generator_A2B")
adjusted_a = tf.layers.average_pooling2d(real_a, 11, strides=1, padding='same', name='adjusted_a')
segment_a = tf.nn.tanh(logits_a, name='segment_a')
logits_a_ori = tf.image.resize_bilinear(
logits_a, (real_a_shape[0][0], real_b_shape[0][1]), name='logits_a_ori')
segment_a_ori = tf.nn.tanh(logits_a_ori, name='segment_a_ori')
with tf.variable_scope('Fake_B'):
foreground = tf.multiply(real_a, segment_a, name='foreground')
background = tf.multiply(adjusted_a, (1 - segment_a), name='background')
fake_b_logits = tf.add(foreground, background, name='fake_b_logits')
fake_b = tf.clip_by_value(fake_b_logits, 0, 255, name='fake_b')
#
fake_b_f = tf.reshape(fake_b, [-1, image_linear_shape], name='fake_b_f')
fake_b_pool_f = tf.reshape(fake_b_pool, [-1, image_linear_shape], name='fake_b_pool_f')
real_b_f = tf.reshape(real_b, [-1, image_linear_shape], name='real_b_f')
dis_fake_b = discriminator_se_wgangp(fake_b_f, flags, reuse=False, name="Discriminator_B")
dis_fake_b_pool = discriminator_se_wgangp(fake_b_pool_f, flags, reuse=True, name="Discriminator_B")
dis_real_b = discriminator_se_wgangp(real_b_f, flags, reuse=True, name="Discriminator_B")
# WGAN Loss
with tf.name_scope('loss_gen_a2b'):
loss_gen_a2b = -tf.reduce_mean(dis_fake_b)
with tf.name_scope('loss_dis_b'):
loss_dis_b_adv_real = -tf.reduce_mean(dis_real_b)
loss_dis_b_adv_fake = tf.reduce_mean(dis_fake_b_pool)
loss_dis_b = tf.reduce_mean(dis_fake_b_pool) - tf.reduce_mean(dis_real_b)
with tf.name_scope('wgan-gp'):
alpha = tf.random_uniform(shape=[flags.batch_size, 1], minval=0., maxval=1.)
differences = fake_b_pool_f - real_b_f
interpolates = real_b_f + (alpha * differences)
gradients = tf.gradients(
discriminator_se_wgangp(interpolates, flags, reuse=True, name="Discriminator_B"),
[interpolates])[0]
slopes = tf.sqrt(tf.reduce_sum(tf.square(gradients), reduction_indices=[1]))
gradient_penalty = tf.reduce_mean((slopes - 1.) ** 2)
loss_dis_b += flags.lambda_gp * gradient_penalty
# Optimizer
trainable_var_resnet = tf.get_collection(
key=tf.GraphKeys.TRAINABLE_VARIABLES, scope='resnet_v1_50')
trainable_var_gen_a2b = tf.get_collection(
key=tf.GraphKeys.TRAINABLE_VARIABLES, scope='Generator_A2B') + trainable_var_resnet
# slim.model_analyzer.analyze_vars(trainable_var_gen_a2b, print_info=True)
# trainable_var_gen_a2b = tf.get_collection(
# key=tf.GraphKeys.TRAINABLE_VARIABLES, scope='Generator_A2B')
trainable_var_dis_b = tf.get_collection(
key=tf.GraphKeys.TRAINABLE_VARIABLES, scope='Discriminator_B')
with tf.name_scope('learning_rate_decay'):
decay = tf.maximum(0., 1. - (tf.cast(global_step, tf.float32) / flags.training_iter), name='decay')
learning_rate = tf.multiply(flags.learning_rate, decay, name='learning_rate')
train_op_gen_a2b = train_op(loss_gen_a2b, learning_rate, flags, trainable_var_gen_a2b, name='gen_a2b')
train_op_dis_b = train_op(loss_dis_b, learning_rate, flags, trainable_var_dis_b, name='dis_b')
saver = tf.train.Saver(max_to_keep=2)
# Graph Logs
with tf.name_scope('GEN_a2b'):
tf.summary.scalar("loss/gen_a2b/all", loss_gen_a2b)
with tf.name_scope('DIS_b'):
tf.summary.scalar("loss/dis_b/all", loss_dis_b)
tf.summary.scalar("loss/dis_b/adv_real", loss_dis_b_adv_real)
tf.summary.scalar("loss/dis_b/adv_fake", loss_dis_b_adv_fake)
summary_op = tf.summary.merge_all()
"""
Session
"""
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True
with tf.Session(config=tfconfig) as sess:
with tf.name_scope('Initial'):
ckpt = tf.train.get_checkpoint_state(dataset_parser.checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Model restored: {}".format(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("No Model found.")
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)
init_fn = slim.assign_from_checkpoint_fn('./pretrained/resnet_v1_50.ckpt',
slim.get_model_variables('resnet_v1_50'))
init_fn(sess)
summary_writer = tf.summary.FileWriter(dataset_parser.logs_dir, sess.graph)
"""
Training Mode
"""
if flags.mode == 'train':
print('Training mode! Batch size:{:d}'.format(flags.batch_size))
with tf.variable_scope('Input_port'):
training_a_handle = sess.run(training_a_iterator.string_handle())
training_b_handle = sess.run(training_b_iterator.string_handle())
# val_a_handle = sess.run(validation_a_iterator.string_handle())
# val_b_handle = sess.run(validation_b_iterator.string_handle())
image_pool_a, image_pool_b = ImagePool(flags.pool_size), ImagePool(flags.pool_size)
print('Start Training!')
start_time = time.time()
sess.run([training_a_iterator.initializer, training_b_iterator.initializer])
feed_dict_train = {handle_a: training_a_handle, handle_b: training_b_handle}
# feed_dict_valid = {is_training: False}
global_step_sess = sess.run(global_step)
while global_step_sess < flags.training_iter:
try:
# Update gen_A2B, gen_B2A
_, fake_b_sess = sess.run([train_op_gen_a2b, fake_b], feed_dict=feed_dict_train)
# _, loss_gen_a2b_sess = sess.run([train_op_gen_a2b, loss_gen_a2b], feed_dict=feed_dict_train)
# Update dis_B, dis_A
fake_b_pool_query = image_pool_b.query(fake_b_sess)
_ = sess.run(train_op_dis_b, feed_dict={
fake_b_pool: fake_b_pool_query, handle_b: training_b_handle})
sess.run(global_step_update_op)
global_step_sess, learning_rate_sess = sess.run([global_step, learning_rate])
print('global step:[{:d}/{:d}], learning rate:{:f}, time:{:4.4f}'.format(
global_step_sess, flags.training_iter, learning_rate_sess, time.time() - start_time))
# Logging the events
if global_step_sess % flags.log_freq == 1:
print('Logging the events')
summary_op_sess = sess.run(summary_op, feed_dict={
handle_a: training_a_handle, handle_b: training_b_handle,
fake_b_pool: fake_b_pool_query})
summary_writer.add_summary(summary_op_sess, global_step_sess)
# summary_writer.flush()
# Observe training situation (For debugging.)
if flags.debug and global_step_sess % flags.observe_freq == 1:
real_a_sess, real_b_sess, adjusted_a_sess, segment_a_sess, fake_b_sess, \
real_a_name_sess, real_b_name_sess = \
sess.run([real_a, real_b, adjusted_a, segment_a, fake_b,
real_a_name, real_b_name],
feed_dict={handle_a: training_a_handle, handle_b: training_b_handle})
print('Logging training images.')
dataset_parser.visualize_data(
real_a=real_a_sess, real_b=real_b_sess, adjusted_a=adjusted_a_sess,
segment_a=segment_a_sess, fake_b=fake_b_sess, shape=(1, 1),
global_step=global_step_sess, logs_dir=dataset_parser.logs_image_train_dir,
real_a_name=real_a_name_sess[0].decode(), real_b_name=real_b_name_sess[0].decode())
"""
Saving the checkpoint
"""
if global_step_sess % flags.save_freq == 0:
print('Saving model...')
saver.save(sess, dataset_parser.checkpoint_dir + '/model.ckpt',
global_step=global_step_sess)
except tf.errors.OutOfRangeError:
print('----------------One epochs finished!----------------')
sess.run([training_a_iterator.initializer, training_b_iterator.initializer])
elif flags.mode == 'test':
from PIL import Image
import scipy.ndimage.filters
import scipy.io as sio
import numpy as np
print('Start Testing!')
'''
with tf.variable_scope('Input_port'):
val_a_handle = sess.run(validation_a_iterator.string_handle())
val_b_handle = sess.run(validation_b_iterator.string_handle())
sess.run([validation_a_iterator.initializer, validation_b_iterator.initializer])
'''
with tf.variable_scope('Input_port'):
val_a_handle = sess.run(validation_a_iterator.string_handle())
val_b_handle = sess.run(validation_b_iterator.string_handle())
sess.run([validation_a_iterator.initializer, validation_b_iterator.initializer])
feed_dict_test = {handle_a: val_a_handle, handle_b: val_b_handle}
image_idx = 0
while True:
try:
segment_a_ori_sess, real_a_name_sess, real_b_sess, real_a_sess, fake_b_sess = \
sess.run([segment_a_ori, real_a_name, real_b, real_a, fake_b], feed_dict=feed_dict_test)
segment_a_np = (np.squeeze(segment_a_ori_sess) + 1.0) * 127.5
binary_a = np.zeros_like(segment_a_np, dtype=np.uint8)
# binary_a[segment_a_np > 127.5] = 255
binary_mean = np.mean(segment_a_np)
binary_a_high = np.mean(segment_a_np[segment_a_np > binary_mean])
binary_a_low = np.mean(segment_a_np[segment_a_np < binary_mean])
binary_a_ave = (binary_a_high + binary_a_low) / 2.0
segment_a_np_blur = scipy.ndimage.filters.gaussian_filter(segment_a_np, sigma=11)
binary_a[segment_a_np_blur > binary_a_ave] = 255
sio.savemat('{}/{}.mat'.format(
dataset_parser.logs_mat_output_dir, real_a_name_sess[0].decode()),
{'pred': segment_a_np, 'binary': binary_a})
# -----------------------------------------------------------------------------
if image_idx % 100 == 0:
real_a_sess = np.squeeze(real_a_sess)
x_png = Image.fromarray(real_a_sess.astype(np.uint8))
x_png.save('{}/{}_0_img.png'.format(dataset_parser.logs_image_val_dir,
real_a_name_sess[0].decode()), format='PNG')
x_png = Image.fromarray(segment_a_np.astype(np.uint8))
x_png.save('{}/{}_1_pred.png'.format(dataset_parser.logs_image_val_dir,
real_a_name_sess[0].decode()), format='PNG')
x_png = Image.fromarray(binary_a.astype(np.uint8))
x_png.save('{}/{}_2_binary.png'.format(dataset_parser.logs_image_val_dir,
real_a_name_sess[0].decode()), format='PNG')
fake_b_sess = np.squeeze(fake_b_sess)
x_png = Image.fromarray(fake_b_sess.astype(np.uint8))
x_png.save('{}/{}_3_fake.png'.format(dataset_parser.logs_image_val_dir,
real_a_name_sess[0].decode()), format='PNG')
real_b_sess = np.squeeze(real_b_sess)
x_png = Image.fromarray(real_b_sess.astype(np.uint8))
x_png.save('{}/{}_4_gt.png'.format(dataset_parser.logs_image_val_dir,
real_a_name_sess[0].decode()), format='PNG')
print(image_idx)
image_idx += 1
except tf.errors.OutOfRangeError:
print('----------------One epochs finished!----------------')
break
def disc_block(img, is_t, batch_size, sn, uc, senet):
with tf.variable_scope('dis'):
#block0 output_size=128
im = conv2d(img,
output_dim=32,
stride=2,
spectral_normed=sn,
update_collection=uc,
name='conv_1_32')
im = lrelu(im)
im = conv2d(im,
output_dim=32,
k_h=1,
k_w=1,
spectral_normed=sn,
update_collection=uc,
name='conv_2_32')
if senet == True:
im = se_layer(im, 32, 8, name='conv_se_32')
#block1 output_size=64
im = conv2d(im,
output_dim=64,
stride=2,
spectral_normed=sn,
update_collection=uc,
name='conv_1_64')
im = lrelu(im)
im = conv2d(im,
output_dim=64,
k_h=1,
k_w=1,
spectral_normed=sn,
update_collection=uc,
name='conv_2_64')
if senet == True:
im = se_layer(im, 64, 8, name='conv_se_64')
#block2 output_size=32
im = conv2d(im,
output_dim=128,
stride=2,
spectral_normed=sn,
update_collection=uc,
name='conv_1_128')
im = lrelu(im)
im = conv2d(im,
output_dim=128,
k_h=1,
k_w=1,
spectral_normed=sn,
update_collection=uc,
name='conv_2_128')
if senet == True:
im = se_layer(im, 128, 8, name='conv_se_128')
#block3 output_size=16
im = conv2d(im,
output_dim=256,
stride=2,
spectral_normed=sn,
update_collection=uc,
name='conv_1_256')
im = lrelu(im)
im = conv2d(im,
output_dim=256,
k_h=1,
k_w=1,
spectral_normed=sn,
update_collection=uc,
name='conv_2_256')
if senet == True:
im = se_layer(im, 256, 8, name='conv_se_256')
#block4 output_size=8
im = conv2d(im,
output_dim=512,
stride=2,
spectral_normed=sn,
update_collection=uc,
name='conv_1_512')
im = lrelu(im)
im = conv2d(im,
output_dim=512,
k_h=1,
k_w=1,
spectral_normed=sn,
update_collection=uc,
name='conv_2_512')
if senet == True:
im = se_layer(im, 512, 8, name='conv_se_512')
return im
def generator(self,
input_x,
img_mask,
guided_fp_left,
guided_fp_right,
use_sp=False,
reuse=False):
with tf.variable_scope("generator") as scope:
if reuse == True:
scope.reuse_variables()
x = tf.concat([input_x, img_mask], axis=3)
u_fp_list = []
for i in range(6):
c_dim = np.minimum(16 * np.power(2, i), 256)
if i == 0:
x = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=c_dim,
k_w=7,
k_h=7,
d_w=1,
d_h=1,
use_sp=use_sp,
name='conv_{}'.format(i)),
scope='conv_IN_{}'.format(i)))
else:
x = tf.nn.relu(
instance_norm(conv2d(x,
output_dim=c_dim,
k_w=4,
k_h=4,
d_w=2,
d_h=2,
use_sp=use_sp,
name='conv_{}'.format(i)),
scope='conv_IN_{}'.format(i)))
if i < 5:
u_fp_list.append(x)
bottleneck = tf.reshape(x, shape=[self.batch_size, -1])
bottleneck = fully_connect(bottleneck,
output_size=256,
use_sp=use_sp,
scope='FC1')
bottleneck = tf.concat(
[bottleneck, guided_fp_left, guided_fp_right], axis=1)
de_x = tf.nn.relu(
fully_connect(bottleneck,
output_size=256 * 8 * 8,
use_sp=use_sp,
scope='FC2'))
de_x = tf.reshape(de_x, shape=[self.batch_size, 8, 8, 256])
for i in range(5):
c_dim = np.maximum(256 / np.power(2, i), 16)
output_dim = 16 * np.power(2, i)
de_x = tf.nn.relu(
instance_norm(de_conv(de_x,
output_shape=[
self.batch_size, output_dim,
output_dim, c_dim
],
use_sp=use_sp,
name='deconv_{}'.format(i)),
scope='deconv_IN_{}'.format(i)))
if i < 4:
de_x = tf.concat(
[de_x, u_fp_list[len(u_fp_list) - (i + 1)]], axis=3)
recon_img1 = conv2d(de_x,
output_dim=3,
k_w=7,
k_h=7,
d_h=1,
d_w=1,
use_sp=use_sp,
name='output_conv')
return tf.nn.tanh(recon_img1)
def d_feature2(ipt,
use_depth_to_space=True,
name='d_feature2',
reuse=False,
is_training=True):
with tf.variable_scope(name):
if use_depth_to_space is True:
c3s1k256 = ops.conv2d(ipt,
512,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_1',
use_bias=True,
kernel_initializer=None)
c3s1k256 = ops.conv2d(c3s1k256,
512,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_2',
use_bias=True,
kernel_initializer=None)
return ops.conv2d(c3s1k256,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128',
use_bias=True,
kernel_initializer=None)
else:
c3s1k128 = ops.conv2d(ipt,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128_1',
use_bias=True,
kernel_initializer=None)
c3s1k128 = ops.conv2d(c3s1k128,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128_2',
use_bias=True,
kernel_initializer=None)
return c3s1k128
def __call__(self, ipt):
c2, c3, c4, c5 = ipt
with tf.variable_scope(self.name):
p5 = ops.conv2d(c5,
self.top_down_pyramid_size,
1,
0,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_c5p5')
p5_up = tf.tile(p5, multiples=[1, 2, 2, 1], name='fpn_p5upsampled')
p4 = tf.add(p5_up,
ops.conv2d(c4,
self.top_down_pyramid_size,
1,
0,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_c4p4'),
name='fpn_p4add')
p4_up = tf.tile(p4, multiples=[1, 2, 2, 1], name='fpn_p4upsampled')
p3 = tf.add(p4_up,
ops.conv2d(c3,
self.top_down_pyramid_size,
1,
0,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_c3p3'),
name='fpn_p3add')
p3_up = tf.tile(p3, multiples=[1, 2, 2, 1], name='fpn_p3upsampled')
p2 = tf.add(p3_up,
ops.conv2d(c2,
self.top_down_pyramid_size,
1,
0,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_c2p2'),
name='fpn_p2add')
p2 = ops.conv2d(p2,
self.top_down_pyramid_size,
3,
1,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_p2')
p3 = ops.conv2d(p3,
self.top_down_pyramid_size,
3,
1,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_p3')
p4 = ops.conv2d(p4,
self.top_down_pyramid_size,
3,
1,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_p4')
p5 = ops.conv2d(p5,
self.top_down_pyramid_size,
3,
1,
1,
norm=None,
activation=None,
reuse=self.reuse,
kernel_initializer='glorot_uniform_tanh',
use_bias=self.use_bias,
name='fpn_p5')
p6 = tf.nn.max_pool(p5, [1, 2, 2, 1], [1, 2, 2, 1],
padding='SAME',
name='fpn_p6')
self.var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.name)
self.reuse = True
return [p2, p3, p4, p5, p6]
def retina_bboxreg_subnet(ipt,
num_anchors,
name='retina_regbbox_subnet',
reuse=False,
is_training=True):
with tf.variable_scope(name):
c3s1k256 = ops.conv2d(ipt,
256,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_1',
use_bias=True,
kernel_initializer=None,
weights_std=0.01)
c3s1k256 = ops.conv2d(c3s1k256,
256,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_2',
use_bias=True,
kernel_initializer=None,
weights_std=0.01)
c3s1k256 = ops.conv2d(c3s1k256,
256,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_3',
use_bias=True,
kernel_initializer=None,
weights_std=0.01)
c3s1k256 = ops.conv2d(c3s1k256,
256,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_4',
use_bias=True,
kernel_initializer=None,
weights_std=0.01)
retina_bboxreg_subnet_output = ops.conv2d(c3s1k256,
num_anchors,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='output',
use_bias=True,
kernel_initializer=None,
weights_std=0.01,
bias_init=-np.log(99))
return retina_bboxreg_subnet_output
def tower(bn, suffix):
assert not self.y_dim
print "\ttower " + suffix
h0 = lrelu(
bn(
conv2d(noisy_image,
self.df_dim,
name='d_h0_conv' + suffix,
d_h=2,
d_w=2,
k_w=3,
k_h=3), "d_bn_0" + suffix))
print "\th0 ", h0.get_shape()
h1 = lrelu(
bn(
conv2d(h0,
self.df_dim * 2,
name='d_h1_conv' + suffix,
d_h=2,
d_w=2,
k_w=3,
k_h=3), "d_bn_1" + suffix))
print "\th1 ", h1.get_shape()
h2 = lrelu(
bn(
conv2d(h1,
self.df_dim * 4,
name='d_h2_conv' + suffix,
d_h=2,
d_w=2,
k_w=3,
k_h=3), "d_bn_2" + suffix))
print "\th2 ", h2.get_shape()
h3 = lrelu(
bn(
conv2d(h2,
self.df_dim * 4,
name='d_h3_conv' + suffix,
d_h=1,
d_w=1,
k_w=3,
k_h=3), "d_bn_3" + suffix))
print "\th3 ", h3.get_shape()
h4 = lrelu(
bn(
conv2d(h3,
self.df_dim * 4,
name='d_h4_conv' + suffix,
d_h=1,
d_w=1,
k_w=3,
k_h=3), "d_bn_4" + suffix))
print "\th4 ", h4.get_shape()
h5 = lrelu(
bn(
conv2d(h4,
self.df_dim * 8,
name='d_h5_conv' + suffix,
d_h=2,
d_w=2,
k_w=3,
k_h=3), "d_bn_5" + suffix))
print "\th5 ", h5.get_shape()
h6 = lrelu(
bn(
conv2d(h5,
self.df_dim * 8,
name='d_h6_conv' + suffix,
k_w=3,
k_h=3), "d_bn_6" + suffix))
print "\th6 ", h6.get_shape()
# return tf.reduce_mean(h6, [1, 2])
h6_reshaped = tf.reshape(h6, [batch_size, -1])
print '\th6_reshaped: ', h6_reshaped.get_shape()
h7 = lrelu(
bn(linear(h6_reshaped, self.df_dim * 40, scope="d_h7" + suffix),
"d_bn_7" + suffix))
return h7
def build(self,
inputs=None,
dropout_keep_prob=0.8,
num_classes=1000,
trainable=True,
restore_logits=True,
bs=128,
scope='',
devices=None,
device_placement=None):
"""Latest Inception from http://arxiv.org/abs/1512.00567.
"Rethinking the Inception Architecture for Computer Vision"
Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens,
Zbigniew Wojna
Args:
inputs: a tensor of size [batch_size, height, width, channels].
dropout_keep_prob: dropout keep_prob.
num_classes: number of predicted classes.
is_training: whether is training or not.
restore_logits: whether or not the logits layers should be restored.
Useful for fine-tuning a model with different num_classes.
scope: Optional scope for name_scope.
Returns:
a list containing 'logits', 'aux_logits' Tensors.
"""
# end_points will collect relevant activations for external use, for example
# summaries or losses.
def get_dev_id(i):
N = len(devices)
if device_placement == 'alternate':
dev_id = i % N
elif device_placement == 'random':
dev_id = random.randint(0, N - 1)
elif device_placement == 'expert':
dev_id = 0
else:
raise Exception(
'Dont use other than alternate with learned inception')
return dev_id
if inputs is None:
inputs = tf.ones((bs, 299, 299, 3))
end_points = {}
with tf.name_scope(scope, 'inception_v3', [inputs]):
with scopes.arg_scope(
[ops.conv2d, ops.fc, ops.batch_norm, ops.dropout],
is_training=trainable):
with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
stride=1,
padding='VALID'):
# with tf.device('/cpu:0') if devices else ExitStack() as gs:
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
# 299 x 299 x 3
end_points['conv0'] = ops.conv2d(inputs,
32, [3, 3],
stride=2,
scope='conv0')
# 149 x 149 x 32
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
end_points['conv1'] = ops.conv2d(end_points['conv0'],
32, [3, 3],
scope='conv1')
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
# 147 x 147 x 32
end_points['conv2'] = ops.conv2d(end_points['conv1'],
64, [3, 3],
padding='SAME',
scope='conv2')
with tf.device(devices[get_dev_id(
1)]) if devices else ExitStack() as gs:
# 147 x 147 x 64
end_points['pool1'] = ops.max_pool(end_points['conv2'],
[3, 3],
stride=2,
scope='pool1')
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
# 73 x 73 x 64
end_points['conv3'] = ops.conv2d(end_points['pool1'],
80, [1, 1],
scope='conv3')
# 73 x 73 x 80.
end_points['conv4'] = ops.conv2d(end_points['conv3'],
192, [3, 3],
scope='conv4')
# 71 x 71 x 192.
end_points['pool2'] = ops.max_pool(end_points['conv4'],
[3, 3],
stride=2,
scope='pool2')
# 35 x 35 x 192.
net = end_points['pool2']
# Inception blocks
with scopes.arg_scope([ops.conv2d, ops.max_pool, ops.avg_pool],
stride=1,
padding='SAME'):
# mixed: 35 x 35 x 256.
with tf.variable_scope('mixed_35x35x256a'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(net, 48, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 32, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch5x5,
branch3x3dbl, branch_pool
])
end_points['mixed_35x35x256a'] = net
# mixed_1: 35 x 35 x 288.
with tf.variable_scope('mixed_35x35x288a'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(net, 48, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 64, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch5x5,
branch3x3dbl, branch_pool
])
end_points['mixed_35x35x288a'] = net
# mixed_2: 35 x 35 x 288.
with tf.variable_scope('mixed_35x35x288b'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 64, [1, 1])
with tf.variable_scope('branch5x5'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(net, 48, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch5x5 = ops.conv2d(branch5x5, 64, [5, 5])
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 64, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch5x5,
branch3x3dbl, branch_pool
])
end_points['mixed_35x35x288b'] = net
# mixed_3: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768a'):
with tf.variable_scope('branch3x3'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3 = ops.conv2d(net,
384, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 64, [1, 1])
branch3x3dbl = ops.conv2d(
branch3x3dbl, 96, [3, 3])
branch3x3dbl = ops.conv2d(branch3x3dbl,
96, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.max_pool(net, [3, 3],
stride=2,
padding='VALID')
with tf.device(devices[get_dev_id(
1)]) if devices else ExitStack() as gs:
net = tf.concat(
axis=3,
values=[branch3x3, branch3x3dbl, branch_pool])
end_points['mixed_17x17x768a'] = net
# mixed4: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768b'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(net, 128, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(branch7x7, 128, [1, 7])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(net, 128, [1, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 128, [7, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 128, [1, 7])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 128, [7, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch7x7,
branch7x7dbl, branch_pool
])
end_points['mixed_17x17x768b'] = net
# mixed_5: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768c'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
1)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(net, 160, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(net, 160, [1, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [1, 7])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch7x7,
branch7x7dbl, branch_pool
])
end_points['mixed_17x17x768c'] = net
# mixed_6: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768d'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.variable_scope('branch7x7'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(net, 160, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch7x7 = ops.conv2d(branch7x7, 160, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(net, 160, [1, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [1, 7])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 160, [7, 1])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch7x7,
branch7x7dbl, branch_pool
])
end_points['mixed_17x17x768d'] = net
# mixed_7: 17 x 17 x 768.
with tf.variable_scope('mixed_17x17x768e'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 192, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
with tf.variable_scope('branch7x7'):
branch7x7 = ops.conv2d(net, 192, [1, 1])
branch7x7 = ops.conv2d(branch7x7, 192, [1, 7])
branch7x7 = ops.conv2d(branch7x7, 192, [7, 1])
with tf.variable_scope('branch7x7dbl'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(net, 192, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [7, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [1, 7])
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [7, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7dbl = ops.conv2d(
branch7x7dbl, 192, [1, 7])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch7x7,
branch7x7dbl, branch_pool
])
end_points['mixed_17x17x768e'] = net
# Auxiliary Head logits
aux_logits = tf.identity(end_points['mixed_17x17x768e'])
with tf.variable_scope('aux_logits'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
aux_logits = ops.avg_pool(aux_logits, [5, 5],
stride=3,
padding='VALID')
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
aux_logits = ops.conv2d(aux_logits,
128, [1, 1],
scope='proj')
# Shape of feature map before the final layer.
shape = aux_logits.get_shape()
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
aux_logits = ops.conv2d(aux_logits,
768,
shape[1:3],
stddev=0.01,
padding='VALID')
aux_logits = ops.flatten(aux_logits)
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
aux_logits = ops.fc(aux_logits,
num_classes,
activation=None,
stddev=0.001,
restore=restore_logits)
end_points['aux_logits'] = aux_logits
# mixed_8: 8 x 8 x 1280.
# Note that the scope below is not changed to not void previous
# checkpoints.
# (TODO) Fix the scope when appropriate.
with tf.variable_scope('mixed_17x17x1280a'):
with tf.variable_scope('branch3x3'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3 = ops.conv2d(net, 192, [1, 1])
branch3x3 = ops.conv2d(branch3x3,
320, [3, 3],
stride=2,
padding='VALID')
with tf.variable_scope('branch7x7x3'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch7x7x3 = ops.conv2d(net, 192, [1, 1])
branch7x7x3 = ops.conv2d(
branch7x7x3, 192, [1, 7])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch7x7x3 = ops.conv2d(
branch7x7x3, 192, [7, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch7x7x3 = ops.conv2d(branch7x7x3,
192, [3, 3],
stride=2,
padding='VALID')
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
with tf.variable_scope('branch_pool'):
branch_pool = ops.max_pool(net, [3, 3],
stride=2,
padding='VALID')
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
net = tf.concat(
axis=3,
values=[branch3x3, branch7x7x3, branch_pool])
end_points['mixed_17x17x1280a'] = net
# mixed_9: 8 x 8 x 2048.
with tf.variable_scope('mixed_8x8x2048a'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 320, [1, 1])
with tf.variable_scope('branch3x3'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3 = ops.conv2d(net, 384, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3 = tf.concat(
axis=3,
values=[
ops.conv2d(branch3x3, 384, [1, 3]),
ops.conv2d(branch3x3, 384, [3, 1])
])
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 448, [1, 1])
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 384, [3, 3])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3dbl = tf.concat(
axis=3,
values=[
ops.conv2d(branch3x3dbl, 384, [1, 3]),
ops.conv2d(branch3x3dbl, 384, [3, 1])
])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
with tf.variable_scope('branch_pool'):
branch_pool = ops.avg_pool(net, [3, 3])
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch3x3,
branch3x3dbl, branch_pool
])
end_points['mixed_8x8x2048a'] = net
# mixed_10: 8 x 8 x 2048.
with tf.variable_scope('mixed_8x8x2048b'):
with tf.variable_scope('branch1x1'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch1x1 = ops.conv2d(net, 320, [1, 1])
with tf.variable_scope('branch3x3'):
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3 = ops.conv2d(net, 384, [1, 1])
branch3x3 = tf.concat(
axis=3,
values=[
ops.conv2d(branch3x3, 384, [1, 3]),
ops.conv2d(branch3x3, 384, [3, 1])
])
with tf.variable_scope('branch3x3dbl'):
with tf.device(devices[get_dev_id(
2)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(net, 448, [1, 1])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch3x3dbl = ops.conv2d(
branch3x3dbl, 384, [3, 3])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch3x3dbl = tf.concat(
axis=3,
values=[
ops.conv2d(branch3x3dbl, 384, [1, 3]),
ops.conv2d(branch3x3dbl, 384, [3, 1])
])
with tf.variable_scope('branch_pool'):
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
branch_pool = ops.avg_pool(net, [3, 3])
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
branch_pool = ops.conv2d(
branch_pool, 192, [1, 1])
with tf.device(devices[get_dev_id(
3)]) if devices else ExitStack() as gs:
net = tf.concat(axis=3,
values=[
branch1x1, branch3x3,
branch3x3dbl, branch_pool
])
end_points['mixed_8x8x2048b'] = net
# Final pooling and prediction
with tf.device(devices[get_dev_id(
0)]) if devices else ExitStack() as gs:
with tf.variable_scope('logits'):
shape = net.get_shape()
net = ops.avg_pool(net,
shape[1:3],
padding='VALID',
scope='pool')
# 1 x 1 x 2048
net = ops.dropout(net,
dropout_keep_prob,
scope='dropout')
net = ops.flatten(net, scope='flatten')
# 2048
logits = ops.fc(net,
num_classes,
activation=None,
scope='logits',
restore=restore_logits)
# 1000
end_points['logits'] = logits
end_points['predictions'] = tf.nn.softmax(
logits, name='predictions')
if trainable:
predictions = NNModel.softmax_add_training_nodes(
bs, end_points['predictions'])
else:
predictions = end_points['predictions']
return tf.get_default_graph(), predictions
def d_block(self, inputs, filters):
h = lrelu(conv2d(inputs, filters, 3, 1))
h = conv2d(h, filters, 3, 1)
h = lrelu(tf.add(h, inputs))
return h
def discriminate(self, x_var, reuse=False, use_sp=False):
with tf.variable_scope("discriminator") as scope:
if reuse == True:
scope.reuse_variables()
conv1= lrelu(conv2d(x_var, spectural_normed=use_sp, output_dim=64, name='dis_conv1'))
conv2= lrelu(conv2d(conv1, spectural_normed=use_sp, output_dim=128, name='dis_conv2'))
conv3= lrelu(conv2d(conv2, spectural_normed=use_sp, output_dim=256, name='dis_conv3'))
conv4 = lrelu(conv2d(conv3, spectural_normed=use_sp, output_dim=512, name='dis_conv4'))
conv5 = lrelu(conv2d(conv4, spectural_normed=use_sp, output_dim=512, name='dis_conv5'))
conv6 = lrelu(conv2d(conv5, spectural_normed=use_sp, output_dim=1024, name='dis_conv6'))
#for gender
class_logits_1 = conv2d(conv6, spectural_normed=use_sp, output_dim=self.range_attri, k_h=2,
k_w=2, d_w=1, d_h=1, padding='VALID', name='dis_conv7')
#for smile
class_logits_2 = conv2d(conv6, spectural_normed=use_sp, output_dim=self.range_attri, k_h=2,
k_w=2, d_w=1, d_h=1, padding='VALID', name='dis_conv8')
#for hair color
class_logits_3 = conv2d(conv6, spectural_normed=use_sp, output_dim=self.range_attri, k_h=2,
k_w=2, d_w=1, d_h=1, padding='VALID', name='dis_conv9')
# for lipsticks
class_logits_4 = conv2d(conv6, spectural_normed=use_sp, output_dim=self.range_attri, k_h=2,
k_w=2, d_w=1, d_h=1, padding='VALID', name='dis_conv10')
#PatchGAN
gan_logits = conv2d(conv6, spectural_normed=use_sp, output_dim=1, k_h=1, k_w=1, d_w=1, d_h=1,
padding='VALID', name='dis_conv11')
return tf.squeeze(class_logits_1), tf.squeeze(class_logits_2), tf.squeeze(class_logits_3), tf.squeeze(class_logits_4), \
tf.squeeze(gan_logits)
def build_dqn(self):
self.w = {}
self.t_w = {}
# initializer = tf.contrib.layers.xavier_initializer()
initializer = tf.truncated_normal_initializer(0, 0.02)
activation_fn = tf.nn.relu
# training network
with tf.variable_scope('prediction'):
if self.cnn_format == 'NHWC':
self.s_t = tf.placeholder('float32', [
None, self.screen_height, self.screen_width,
self.history_length
],
name='s_t')
else:
self.s_t = tf.placeholder('float32', [
None, self.history_length, self.screen_height,
self.screen_width
],
name='s_t')
self.l1, self.w['l1_w'], self.w['l1_b'] = conv2d(self.s_t,
32, [8, 8],
[4, 4],
initializer,
activation_fn,
self.cnn_format,
name='l1')
self.l2, self.w['l2_w'], self.w['l2_b'] = conv2d(self.l1,
64, [4, 4],
[2, 2],
initializer,
activation_fn,
self.cnn_format,
name='l2')
self.l3, self.w['l3_w'], self.w['l3_b'] = conv2d(self.l2,
64, [3, 3],
[1, 1],
initializer,
activation_fn,
self.cnn_format,
name='l3')
shape = self.l3.get_shape().as_list()
self.l3_flat = tf.reshape(
self.l3, [-1, reduce(lambda x, y: x * y, shape[1:])])
if self.dueling:
self.value_hid, self.w['l4_val_w'], self.w['l4_val_b'] = \
linear(self.l3_flat, 512,
activation_fn=activation_fn, name='value_hid')
self.adv_hid, self.w['l4_adv_w'], self.w['l4_adv_b'] = \
linear(self.l3_flat, 512,
activation_fn=activation_fn, name='adv_hid')
self.value, self.w['val_w_out'], self.w['val_w_b'] = \
linear(self.value_hid, 1, name='value_out')
self.advantage, self.w['adv_w_out'], self.w['adv_w_b'] = \
linear(self.adv_hid, self.env.action_size, name='adv_out')
# Average Dueling
self.q = self.value + (self.advantage - tf.reduce_mean(
self.advantage, reduction_indices=1, keep_dims=True))
else:
self.l4, self.w['l4_w'], self.w['l4_b'] = linear(
self.l3_flat, 512, activation_fn=activation_fn, name='l4')
self.q, self.w['q_w'], self.w['q_b'] = linear(
self.l4, self.env.action_size, name='q')
self.q_action = tf.argmax(self.q, dimension=1)
q_summary = []
avg_q = tf.reduce_mean(self.q, 0)
for idx in range(self.env.action_size):
q_summary.append(tf.histogram_summary('q/%s' % idx,
avg_q[idx]))
self.q_summary = tf.merge_summary(q_summary, 'q_summary')
# target network
with tf.variable_scope('target'):
if self.cnn_format == 'NHWC':
self.target_s_t = tf.placeholder('float32', [
None, self.screen_height, self.screen_width,
self.history_length
],
name='target_s_t')
else:
self.target_s_t = tf.placeholder('float32', [
None, self.history_length, self.screen_height,
self.screen_width
],
name='target_s_t')
self.target_l1, self.t_w['l1_w'], self.t_w['l1_b'] = conv2d(
self.target_s_t,
32, [8, 8], [4, 4],
initializer,
activation_fn,
self.cnn_format,
name='target_l1')
self.target_l2, self.t_w['l2_w'], self.t_w['l2_b'] = conv2d(
self.target_l1,
64, [4, 4], [2, 2],
initializer,
activation_fn,
self.cnn_format,
name='target_l2')
self.target_l3, self.t_w['l3_w'], self.t_w['l3_b'] = conv2d(
self.target_l2,
64, [3, 3], [1, 1],
initializer,
activation_fn,
self.cnn_format,
name='target_l3')
shape = self.target_l3.get_shape().as_list()
self.target_l3_flat = tf.reshape(
self.target_l3, [-1, reduce(lambda x, y: x * y, shape[1:])])
if self.dueling:
self.t_value_hid, self.t_w['l4_val_w'], self.t_w['l4_val_b'] = \
linear(self.target_l3_flat, 512,
activation_fn=activation_fn, name='target_value_hid')
self.t_adv_hid, self.t_w['l4_adv_w'], self.t_w['l4_adv_b'] = \
linear(self.target_l3_flat, 512,
activation_fn=activation_fn, name='target_adv_hid')
self.t_value, self.t_w['val_w_out'], self.t_w['val_w_b'] = \
linear(self.t_value_hid, 1, name='target_value_out')
self.t_advantage, self.t_w['adv_w_out'], self.t_w['adv_w_b'] = \
linear(self.t_adv_hid, self.env.action_size,
name='target_adv_out')
# Average Dueling
self.target_q = self.t_value + (
self.t_advantage - tf.reduce_mean(
self.t_advantage, reduction_indices=1, keep_dims=True))
else:
self.target_l4, self.t_w['l4_w'], self.t_w['l4_b'] = \
linear(self.target_l3_flat, 512,
activation_fn=activation_fn, name='target_l4')
self.target_q, self.t_w['q_w'], self.t_w['q_b'] = \
linear(self.target_l4, self.env.action_size, name='target_q')
self.target_q_idx = tf.placeholder('int32', [None, None],
'outputs_idx')
self.target_q_with_idx = tf.gather_nd(self.target_q,
self.target_q_idx)
with tf.variable_scope('pred_to_target'):
self.t_w_input = {}
self.t_w_assign_op = {}
for name in self.w.keys():
self.t_w_input[name] = tf.placeholder(
'float32', self.t_w[name].get_shape().as_list(), name=name)
self.t_w_assign_op[name] = self.t_w[name].assign(
self.t_w_input[name])
# optimizer
with tf.variable_scope('optimizer'):
self.target_q_t = tf.placeholder('float32', [None],
name='target_q_t')
self.action = tf.placeholder('int64', [None], name='action')
action_one_hot = tf.one_hot(self.action,
self.env.action_size,
1.0,
0.0,
name='action_one_hot')
q_acted = tf.reduce_sum(self.q * action_one_hot,
reduction_indices=1,
name='q_acted')
self.delta = self.target_q_t - q_acted
self.clipped_delta = tf.clip_by_value(self.delta,
self.min_delta,
self.max_delta,
name='clipped_delta')
self.global_step = tf.Variable(0, trainable=False)
self.loss = tf.reduce_mean(tf.square(self.clipped_delta),
name='loss')
self.learning_rate_step = tf.placeholder('int64',
None,
name='learning_rate_step')
self.learning_rate_op = tf.maximum(
self.learning_rate_minimum,
tf.train.exponential_decay(self.learning_rate,
self.learning_rate_step,
self.learning_rate_decay_step,
self.learning_rate_decay,
staircase=True))
self.optim = tf.train.RMSPropOptimizer(self.learning_rate_op,
momentum=0.95,
epsilon=0.01).minimize(
self.loss)
with tf.variable_scope('summary'):
scalar_summary_tags = [
'average.reward', 'average.loss', 'average.q',
'episode.max reward', 'episode.min reward',
'episode.avg reward', 'episode.num of game',
'training.learning_rate'
]
self.summary_placeholders = {}
self.summary_ops = {}
for tag in scalar_summary_tags:
self.summary_placeholders[tag] = tf.placeholder(
'float32', None, name=tag.replace(' ', '_'))
self.summary_ops[tag] = tf.scalar_summary(
"%s/%s" % (self.env_name, tag),
self.summary_placeholders[tag])
histogram_summary_tags = ['episode.rewards', 'episode.actions']
for tag in histogram_summary_tags:
self.summary_placeholders[tag] = tf.placeholder(
'float32', None, name=tag.replace(' ', '_'))
self.summary_ops[tag] = tf.histogram_summary(
tag, self.summary_placeholders[tag])
self.writer = tf.train.SummaryWriter('./logs/%s' % self.model_dir,
self.sess.graph)
tf.initialize_all_variables().run()
self._saver = tf.train.Saver(list(self.w.values()) + [self.step_op],
max_to_keep=30)
self.load_model()
self.update_target_q_network()
def encode_decode(self, x, reuse=False):
with tf.variable_scope("encode_decode") as scope:
if reuse == True:
scope.reuse_variables()
conv1 = tf.nn.relu(
instance_norm(conv2d(x, output_dim=64, k_w=7, k_h=7, d_w=1, d_h=1, name='e_c1'), scope='e_in1'))
conv2 = tf.nn.relu(
instance_norm(conv2d(conv1, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c2'), scope='e_in2'))
conv3 = tf.nn.relu(
instance_norm(conv2d(conv2, output_dim=256, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c3'), scope='e_in3'))
r1 = Residual(conv3, residual_name='re_1')
r2 = Residual(r1, residual_name='re_2')
r3 = Residual(r2, residual_name='re_3')
r4 = Residual(r3, residual_name='re_4')
r5 = Residual(r4, residual_name='re_5')
r6 = Residual(r5, residual_name='re_6')
g_deconv1 = tf.nn.relu(instance_norm(de_conv(r6, output_shape=[self.batch_size,
self.output_size/2, self.output_size/2, 128], name='gen_deconv1'), scope="gen_in"))
# for 1
g_deconv_1_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1,
output_shape=[self.batch_size, self.output_size, self.output_size, 64], name='g_deconv_1_1'), scope='gen_in_1_1'))
#Refined Residual Image learning
g_deconv_1_1_x = tf.concat([g_deconv_1_1, x], axis=3)
x_tilde1 = conv2d(g_deconv_1_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_1_2')
# for 2
g_deconv_2_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1,
output_shape=[self.batch_size, self.output_size, self.output_size, 64]
, name='g_deconv_2_1'), scope='gen_in_2_1'))
g_deconv_2_1_x = tf.concat([g_deconv_2_1, x], axis=3)
x_tilde2 = conv2d(g_deconv_2_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_2_2')
# for 3
g_deconv_3_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1, output_shape=[self.batch_size,
self.output_size, self.output_size, 64], name='gen_deconv3_1'), scope='gen_in_3_1'))
g_deconv_3_1_x = tf.concat([g_deconv_3_1, x], axis=3)
g_deconv_3_2 = conv2d(g_deconv_3_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1,
name='gen_conv_3_2')
x_tilde3 = conv2d(g_deconv_3_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_3_3')
# for 4
g_deconv_4_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1, output_shape=[self.batch_size,
self.output_size, self.output_size, 64], name='gen_deconv4_1'), scope='gen_in_4_1'))
g_deconv_4_1_x = tf.concat([g_deconv_4_1, x], axis=3)
g_deconv_4_2 = conv2d(g_deconv_4_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1,
name='gen_conv_4_2')
x_tilde4 = conv2d(g_deconv_4_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_4_3')
# for 5
g_deconv_5_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1, output_shape=[self.batch_size,
self.output_size, self.output_size, 64], name='gen_deconv5_1'), scope='gen_in_5_1'))
g_deconv_5_1_x = tf.concat([g_deconv_5_1, x], axis=3)
g_deconv_5_2 = conv2d(g_deconv_5_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1,
name='gen_conv_5_2')
x_tilde5 = conv2d(g_deconv_5_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_5_3')
# for 6
g_deconv_6_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1, output_shape=[self.batch_size,
self.output_size, self.output_size, 64], name='gen_deconv6_1'), scope='gen_in_6_1'))
g_deconv_6_1_x = tf.concat([g_deconv_6_1, x], axis=3)
g_deconv_6_2 = conv2d(g_deconv_6_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1,
name='gen_conv_6_2')
x_tilde6 = conv2d(g_deconv_6_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_6_3')
# for 7
g_deconv_7_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1,
output_shape=[self.batch_size, self.output_size, self.output_size, 64], name='g_deconv_7_1'), scope='gen_in_7_1'))
g_deconv_7_1_x = tf.concat([g_deconv_7_1, x], axis=3)
x_tilde7 = conv2d(g_deconv_7_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_7_2')
# for 8
g_deconv_8_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1,
output_shape=[self.batch_size, self.output_size, self.output_size, 64]
, name='g_deconv_8_1'), scope='gen_in_8_1'))
g_deconv_8_1_x = tf.concat([g_deconv_8_1, x], axis=3)
x_tilde8 = conv2d(g_deconv_8_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_8_2')
return tf.nn.tanh(x_tilde1), tf.nn.tanh(x_tilde2), tf.nn.tanh(x_tilde3), \
tf.nn.tanh(x_tilde4), tf.nn.tanh(x_tilde5), tf.nn.tanh(x_tilde6), tf.nn.tanh(x_tilde7), tf.nn.tanh(x_tilde8)
def _vgg_conv_relu(self, x, n_in, n_out, scope):
with tf.variable_scope(scope):
conv = ops.conv2d(x, n_in, n_out, 3, 1, p='SAME')
relu = tf.nn.relu(conv)
return relu
def get_model(inputs, output_dim, wd=None):
""" inputs: (batch_size, num_point, 3)
returns: (batch_size, num_point, 3, output_dim)
"""
batch_size = inputs.get_shape()[0].value
num_point = inputs.get_shape()[1].value
inputs = tf.expand_dims(inputs, -1) # (batch_size, num_point, 3, 1)
#####################################
net = ops.conv2d("conv1", inputs, 64,
kernel_shape=[1, 3], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net = ops.conv2d("conv2", net, 64,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net = ops.conv2d("conv3", net, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_context = contextAwarenessOp("context3", net)
net = tf.concat([net, net_context], axis=-1) # (batch, num_pt, 1, ftr_dim3)
embed_ftr_dim = 128
net = ops.conv2d("conv4", net, embed_ftr_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net = ops.conv2d("conv5", net, embed_ftr_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
######### Self-Attention Context Awareness operation #########
# Using 1x1 conv to adjust feature channels.
fg_dim = int(embed_ftr_dim/3)
net = ops.conv2d("conv6", net, fg_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
# context awareness op
net_conf = ops.conv2d("confidence_input0", net, fg_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_conf = contextAwarenessOp("context_saca", net_conf)
net_conf = ops.conv2d("confidence_input1", net_conf, fg_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
# context
initial_context = tf.zeros([batch_size, 1, 1, fg_dim])
net_context = [initial_context]
for i in range(num_point-1):
conf = tf.concat([net[:, :i+1, :, :],
tf.tile(net_conf[:, i:i+1, :, :], [1, i+1, 1, 1])], axis=-1)
reuse = False
if i > 0: reuse = True
conf = ops.conv2d("conf1", conf, fg_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", reuse=reuse)
conf = ops.conv2d("conf2", conf, fg_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.tanh,
strides=[1, 1], padding="VALID", reuse=reuse)
curr_contxt = tf.reduce_sum(tf.multiply(net[:, :i+1, :, :], conf), axis=1, keepdims=True)
net_context.append(curr_contxt)
net_context = tf.concat(net_context, axis=1) # [batch_size, num_point, 1, fg_dim]
# Adjust feature channel using 1x1 conv.
net_context = ops.conv2d("context_output", net_context, embed_ftr_dim,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
###############################################################
# For first dimension (z)
net_pc_1 = ops.conv2d("pc_dim_1_0", net_context, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_1 = ops.conv2d("pc_dim_1_1", net_pc_1, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_1 = ops.conv2d("pc_dim_1_2", net_pc_1, output_dim,
kernel_shape=[1, 1], strides=[1, 1], padding="VALID")
# For second dimension (y)
net_pc_2 = ops.conv2d("pc_coord_2_0", inputs, 32,
kernel_shape=[1, 3], mask_type='B', activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_2 = ops.conv2d("pc_coord_2_1", net_pc_2, 32,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_2 = tf.concat([net_context, net_pc_2], axis=-1)
net_pc_2 = ops.conv2d("pc_dim_2_0", net_pc_2, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_2 = ops.conv2d("pc_dim_2_1", net_pc_2, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_2 = ops.conv2d("pc_dim_2_2", net_pc_2, output_dim,
kernel_shape=[1, 1], strides=[1, 1], padding="VALID")
# For third dimension (x)
net_pc_3 = ops.conv2d("pc_coord_3_0", inputs, 32,
kernel_shape=[1, 3], mask_type='C', activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_3 = ops.conv2d("pc_coord_3_1", net_pc_3, 32,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_3 = tf.concat([net_context, net_pc_3], axis=-1)
net_pc_3 = ops.conv2d("pc_dim_3_0", net_pc_3, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_3 = ops.conv2d("pc_dim_3_1", net_pc_3, 128,
kernel_shape=[1, 1], activation_fn=tf.nn.elu,
strides=[1, 1], padding="VALID", weights_regularizer=wd)
net_pc_3 = ops.conv2d("pc_dim_3_2", net_pc_3, output_dim,
kernel_shape=[1, 1], strides=[1, 1], padding="VALID")
return tf.concat([net_pc_1, net_pc_2, net_pc_3], axis=2)
def d_feature3(ipt,
use_depth_to_space=True,
name='d_feature3',
reuse=False,
is_training=True,
resize_method=tf.image.ResizeMethod.NEAREST_NEIGHBOR):
with tf.variable_scope(name):
if use_depth_to_space is True:
d_to_s_1 = tf.depth_to_space(ipt, 2)
c3s1k256 = ops.conv2d(d_to_s_1,
512,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_1',
use_bias=True,
kernel_initializer=None)
c3s1k256 = ops.conv2d(c3s1k256,
512,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k256_2',
use_bias=True,
kernel_initializer=None)
return ops.conv2d(c3s1k256,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128',
use_bias=True,
kernel_initializer=None)
else:
shape = ipt.get_shape().as_list()
c3s1k128 = ops.conv2d(ipt,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128_1',
use_bias=True,
kernel_initializer=None)
c3s1k128 = ops.conv2d(c3s1k128,
128,
3,
1,
1,
norm=None,
activation=tf.nn.relu,
reuse=reuse,
is_training=is_training,
name='c3s1k128_2',
use_bias=True,
kernel_initializer=None)
return tf.image.resize_images(c3s1k128,
[shape[1] * 2, shape[2] * 2],
resize_method)
def encoder_block(img, is_t, senet):
with tf.variable_scope('gen_downsample'):
with slim.arg_scope([slim.separable_conv2d], depth_multiplier=1):
skip = []
im = conv2d(img, output_dim=16, k_w=1, k_h=1, name='conv0_16')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv0_bn_16'))
skip.append(im)
# block1 output_size=128
im = conv2d(im, output_dim=32, stride=2, name='conv1_32')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv1_bn_32'))
im = conv2d(im, output_dim=64, name='conv1_64')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv1_bn_64'))
im_res = conv2d(im,
output_dim=128,
k_h=1,
k_w=1,
stride=2,
name='conv1_res_128')
im_res = bn(im_res, is_t=is_t, name='conv1_res_bn_128')
skip.append(im)
# block2 output_size=64
im = slim.separable_conv2d(im, 128, [3, 3], scope='conv2_128')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv2_bn_128'))
im = slim.separable_conv2d(im, 128, [3, 3], scope='conv2_128_2')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv2_bn_128_2'))
im = slim.separable_conv2d(im,
128, [3, 3],
stride=2,
scope='conv2_128_3')
im = bn(im, is_t=is_t, name='conv2_bn_128_3')
if senet == True:
im = se_layer(im, 128, 8, name='conv_se_128')
im = tf.add(im, im_res)
im_res = conv2d(im,
output_dim=256,
k_h=1,
k_w=1,
stride=2,
name='conv2_res_256')
im_res = bn(im_res, is_t=is_t, name='conv2_res_bn_256')
skip.append(im)
#block3 output_size=32
im = tf.nn.relu(im)
im = slim.separable_conv2d(im, 256, [3, 3], scope='conv3_256')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv3_bn_256'))
im = slim.separable_conv2d(im, 256, [3, 3], scope='conv3_256_2')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv3_bn_256_2'))
im = slim.separable_conv2d(im,
256, [3, 3],
stride=2,
scope='conv3_256_3')
im = bn(im, is_t=is_t, name='conv3_bn_256_3')
if senet == True:
im = se_layer(im, 256, 8, name='conv_se_256')
im = tf.add(im, im_res)
im_res = conv2d(im,
output_dim=728,
k_h=1,
k_w=1,
stride=2,
name='conv3_res_728')
im_res = bn(im_res, is_t=is_t, name='conv3_res_bn_728')
skip.append(im)
#block4 output_size=16
im = tf.nn.relu(im)
im = slim.separable_conv2d(im, 728, [3, 3], scope='conv4_728')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv4_bn_728'))
im = slim.separable_conv2d(im, 728, [3, 3], scope='conv4_728_2')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv4_bn_728_2'))
im = slim.separable_conv2d(im,
728, [3, 3],
stride=2,
scope='conv4_728_3')
im = bn(im, is_t=is_t, name='conv4_bn_728_3')
if senet == True:
im = se_layer(im, 728, 8, name='conv_se_728')
im = tf.add(im, im_res)
skip.append(im)
#Middle flow
for i in range(8):
im_res = im
im = tf.nn.relu(im)
im = slim.separable_conv2d(im,
728, [3, 3],
scope='conv_mf_sp1_{}'.format(i))
im = tf.nn.relu(
bn(im, is_t=is_t, name='conv_mf_bn1_{}'.format(i)))
im = slim.separable_conv2d(im,
728, [3, 3],
scope='conv_mf_sp2_{}'.format(i))
im = tf.nn.relu(
bn(im, is_t=is_t, name='conv_mf_bn2_{}'.format(i)))
im = slim.separable_conv2d(im,
728, [3, 3],
scope='conv_mf_sp3_{}'.format(i))
im = bn(im, is_t=is_t, name='conv_mf_bn3_{}'.format(i))
if senet == True:
im = se_layer(im, 728, 8, name='conv_se_728_{}'.format(i))
im = tf.add(im, im_res, name='con_mf_add_{}'.format(i))
#Exit flow
im_res = conv2d(im,
output_dim=1024,
k_w=1,
k_h=1,
stride=2,
name='conv_res_ex_1024')
im_res = bn(im_res, is_t=is_t, name='conv_res_ex_bn_1024')
im = tf.nn.relu(im, name='conv_exit_relu')
im = slim.separable_conv2d(im, 728, [3, 3], scope='conv_ex_728')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv_ex_bn_728'))
im = slim.separable_conv2d(im, 1024, [3, 3], scope='conv_ex1_1024')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv_ex1_bn_1024'))
im = slim.separable_conv2d(im,
1024, [3, 3],
stride=2,
scope='conv_ex2_1024')
im = bn(im, is_t=is_t, name='conv_ex2_bn_1024')
if senet == True:
im = se_layer(im, 1024, 8, name='conv_se_1024')
im = tf.add(im, im_res, name='conv5_add')
#Output
im = tf.nn.relu(im)
im = slim.separable_conv2d(im, 1536, [3, 3], scope='conv_out_1536')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv_out_bn_1536'))
im = slim.separable_conv2d(im,
1536, [3, 3],
scope='conv_out2_1536')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv_out2_bn_1536'))
im = slim.separable_conv2d(im, 2048, [3, 3], scope='conv_out_2048')
im = tf.nn.relu(bn(im, is_t=is_t, name='conv_out_bn_2048'))
if senet == True:
im = se_layer(im, 2048, 8, name='conv_se_2048')
return im, skip
