def encoder(self, x, is_training=True, reuse=False):
with tf.variable_scope("encoder", reuse=reuse):
if self.verbose: print(x.shape)
# Layer 1
net = layers.conv2d(x, self.conv_dim >> 1, name='en_conv1')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 2
net = layers.conv2d(net, self.conv_dim, name='en_conv2')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 3
net = layers.flatten(net)
if self.verbose: print(net.shape)
# Layer 4
net = layers.linear(net, self.linear_dim, scope='en_fc1')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 5
net = layers.linear(net, self.latent_dim, scope='en_fc2')
net = tf.identity(net)
if self.verbose: print(net.shape)
return net
def encoder_v2(self, x, is_training=True, reuse=False):
with tf.variable_scope("encoder_v2", reuse=reuse):
if self.verbose: print(x.shape)
# Layer 1
net = layers.conv2d(x, self.conv_dim >> 1, name='en_conv1')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 2
net = layers.conv2d(net, self.conv_dim, name='en_conv2')
net = layers.batch_norm(net,
is_training=is_training,
scope='en_bn2')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 3
net = layers.flatten(net)
if self.verbose: print(net.shape)
# Layer 4
net = layers.linear(net, self.linear_dim, scope='en_fc3')
net = layers.batch_norm(net,
is_training=is_training,
scope='en_bn3')
net = tf.nn.leaky_relu(net)
if self.verbose: print(net.shape)
# Layer 5
out_logit = layers.linear(net, self.latent_dim, scope='en_fc4')
out = tf.nn.sigmoid(out_logit, name="main_out")
if self.verbose: print(out.shape)
return out
def generator(self, batch):
grey_batch = rgb_to_grey(batch)
with tf.variable_scope('g_') as vs:
""" -----------------------------------------------------------------------------------
ENCODER
----------------------------------------------------------------------------------- """
self.en_h0 = conv2d(grey_batch, self.frame_size, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1")
self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0))
add_activation_summary(self.en_h0)
print(self.en_h0.get_shape().as_list())
self.en_h1 = conv2d(self.en_h0, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2")
self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2")
self.en_h1 = tf.nn.relu(self.en_h1)
add_activation_summary(self.en_h1)
print(self.en_h1.get_shape().as_list())
self.en_h2 = conv2d(self.en_h1, 256, 512, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3")
self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3")
self.en_h2 = tf.nn.relu(self.en_h2)
add_activation_summary(self.en_h2)
print(self.en_h2.get_shape().as_list())
self.en_h3 = conv2d(self.en_h2, 512, 1024, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4")
self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4")
self.en_h3 = tf.nn.relu(self.en_h3)
add_activation_summary(self.en_h3)
print(self.en_h3.get_shape().as_list())
""" -----------------------------------------------------------------------------------
GENERATOR
----------------------------------------------------------------------------------- """
self.fg_h0 = tf.reshape(self.en_h3, [-1, 2, 4, 4, 512])
print(self.fg_h0.get_shape().as_list())
self.fg_h1 = conv3d_transpose(self.fg_h0, 512, [self.batch_size, 4, 8, 8, 256], name='g_f_h1')
self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h1, scope='g_f_bn1'), name='g_f_relu1')
add_activation_summary(self.fg_h1)
print(self.fg_h1.get_shape().as_list())
self.fg_h2 = conv3d_transpose(self.fg_h1, 256, [self.batch_size, 8, 16, 16, 128], name='g_f_h2')
self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h2, scope='g_f_bn2'), name='g_f_relu2')
add_activation_summary(self.fg_h2)
print(self.fg_h2.get_shape().as_list())
self.fg_h3 = conv3d_transpose(self.fg_h2, 128, [self.batch_size, 16, 32, 32, 64], name='g_f_h3')
self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h3, scope='g_f_bn3'), name='g_f_relu3')
add_activation_summary(self.fg_h3)
print(self.fg_h3.get_shape().as_list())
self.fg_h4 = conv3d_transpose(self.fg_h3, 64, [self.batch_size, 32, 64, 64, 3], name='g_f_h4')
self.fg_fg = tf.nn.tanh(self.fg_h4, name='g_f_actvcation')
print(self.fg_fg.get_shape().as_list())
gen_reg = tf.reduce_mean(tf.square(grey_batch - rgb_to_grey(self.fg_fg)))
variables = tf.contrib.framework.get_variables(vs)
return self.fg_fg, gen_reg, variables
def __call__(self, x, reuse=True):
with tf.variable_scope(self.name, reuse=reuse) as vs:
bs = tf.shape(x)[0]
x = tf.reshape(x, [bs, 28, 28, 1])
conv1 = conv2d(x, 64, [4, 4], [2, 2], activation_fn=leaky_relu)
conv2 = conv2d(conv1, 128, [4, 4], [2, 2], activation_fn=leaky_relu)
conv2 = tf.contrib.layers.flatten(conv2)
fc1 = dense(conv2, 1024, activation_fn=leaky_relu)
fc2 = dense(fc1, 1, activation_fn=tf.identity)
return fc2, tf.sigmoid(fc2)
def ResNet_1_3_1(last):
is_training = False
channels = FLAGS.channels
l = 0
last = tf.identity(last, name='input')
# residual blocks
rb = 0
skip2 = last
while rb < FLAGS.res_blocks:
rb += 1
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=1,
out_channels=channels,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=FLAGS.activation,
init_factor=FLAGS.init_activation)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=FLAGS.activation,
init_factor=FLAGS.init_activation)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=1,
out_channels=channels,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_factor)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
last = tf.add(last, skip2, 'elementwise_sum')
skip2 = last
last = layers.apply_activation(last,
activation=FLAGS.activation,
data_format=FLAGS.data_format)
# return
last = tf.identity(last, name='output')
return last
def ForkNet(inputs, padding = 'VALID', name='ForkNet'):
'''
Built the ForkNet model.
Args:
inputs: mosaic polarized images
padding: padding mode of convolution
Returns:
s0: reconstructed s0 images
dolp: reconstructed dolp images
aop: reconstructed aop images
'''
# keep_prob = tf.where(is_training, 0.2, 1.0)
with tf.variable_scope(name):
# conventional layers
x_1 = conv2d(inputs, [4, 4], 96, activation=tf.nn.relu, padding=padding, name='conv_1')
tf.add_to_collection('feature_maps', x_1)
x_2 = conv2d(x_1, [3,3], 48, activation=tf.nn.relu, padding=padding, name='conv_2')
tf.add_to_collection('feature_maps', x_2)
x_3_1 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_1')
tf.add_to_collection('feature_maps', x_3_1)
s0 = conv2d(x_3_1, [5, 5], 1, activation=None, padding=padding, name='conv_4_1')
x_3_2 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_2')
tf.add_to_collection('feature_maps', x_3_2)
dolp = conv2d(x_3_2, [5, 5], 1, activation=None, padding=padding, name='conv_4_2')
x_3_3 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_3')
tf.add_to_collection('feature_maps', x_3_3)
aop = conv2d(x_3_3, [4, 4], 1, activation=None, padding=padding, name='conv_4_3')
aop = tf.atan(aop) / 2. + math.pi / 4
return s0, dolp, aop
def _build_feature_extractor(self, x, reuse=False):
"""
Build a CNN for convolutional feature extraction
Args:
x: input tensor
reuse: boolean, if true reuse already allocated variables, otherway make new ones
Returns:
x: activation of last convolutional layer
"""
if self.padding:
x = self.pad(x)
num_convolution = (self.receptive_field_size -
1) // (self.kernel_size - 1)
in_channel = x.get_shape()[-1].value
#Feature Extraction
for i in range(num_convolution - 1):
if i == 0:
kernel = [self.kernel_size, self.kernel_size, in_channel, 32]
elif i < 2:
kernel = [self.kernel_size, self.kernel_size, 32, 32]
elif i == 2:
kernel = [self.kernel_size, self.kernel_size, 32, 64]
else:
kernel = [self.kernel_size, self.kernel_size, 64, 64]
x = conv2d(x,
kernel,
activation=lambda x: tf.nn.relu(x),
padding='VALID',
reuse=reuse,
batch_norm=True,
training=self.is_training,
name='conv{}'.format(i))
#Final Layer
x = conv2d(x, [self.kernel_size, self.kernel_size, 64, 64],
activation=lambda x: x,
padding='VALID',
reuse=reuse,
batch_norm=True,
training=self.is_training,
name='conv_final')
return x
def fpn(self, features, out_channel):
last_inner = conv2d(
features[-1], out_channel, 1
) #Set up the last (except the top layer) fpn output. In additional, the first element of the resnet outputs will not be used in fpn
results = []
results.append(conv2d(last_inner, out_channel, 3, 1, padding=1))
assert len(
features
) - 2 > 0 #Please make sure that at least 3 features are feed in the fpn
for i in range(len(features) - 2):
inner_top_down = upsample_layer(last_inner, 2)
inner_lateral = conv2d(features[-2 - i], out_channel, 1)
last_inner = inner_lateral + inner_top_down
results.insert(
0, conv2d(last_inner, out_channel, 3, strides=1, padding=1))
with tf.variable_scope('top_blocks'):
extra_feas = self.fpn_extra_tops(results[-1], out_channel)
results.extend(extra_feas)
return results
def block(name, input, filters, ksz, stride, padding):
"""
:param name: a string
:param input: a tensor
:param filters: an integer, # filters
:param ksz: an integer, the size of filters
:param stride: an integer
:param padding: a string, all string need to be uppercase
:return:
"""
with tf.variable_scope(name):
conv = conv2d('conv', input, filters, ksz, stride, padding)
bn = batch_norm('bn', conv)
return relu('relu', bn)
def call(self, inputs):
# Bottom-up layers
c1 = ul.conv2d(inputs, 64, 7, strides=2)
c1 = ul.sep_conv2d(c1, 64, 3, strides=1)
c2 = ul.sep_conv2d(c1, 128, 3, strides=2)
c2 = ul.sep_conv2d(c2, 128, 3, strides=1)
c3 = ul.sep_conv2d(c2, 256, 3, strides=2)
c3 = ul.sep_conv2d(c3, 256, 3, strides=1)
c4 = ul.sep_conv2d(c3, 512, 3, strides=2)
c4 = ul.sep_conv2d(c4, 512, 3, strides=1)
c4 = ul.sep_conv2d(c4, 512, 3, strides=1)
c4 = ul.sep_conv2d(c4, 512, 3, strides=1)
c4 = ul.sep_conv2d(c4, 512, 3, strides=1)
p5 = ul.conv2d(c4, 256, 3, strides=2)
p6 = ul.conv2d(p5, 256, 3, strides=2)
# Top-down
p4 = ul.conv2d(c4, 256, 1, activation=None)
u4 = tf.layers.conv2d_transpose(p4,
256, [4, 4],
strides=[2, 2],
padding='same',
data_format='channels_first')
p3 = ul.add_forced(u4, ul.conv2d(c3, 256, 1, activation=None))
p3 = ul.conv2d(p3, 256, 3, activation=None)
u3 = tf.layers.conv2d_transpose(p3,
128, [4, 4],
strides=[2, 2],
padding='same',
data_format='channels_first')
p2 = ul.add_forced(u3, ul.conv2d(c2, 128, 1, activation=None))
p2 = ul.conv2d(p2, 256, 3, activation=None)
return p2, p3, p4, p5, p6
def inference(lefts, disps):
bs = int(lefts.shape[0])
print("Batch size: ", bs)
#256*512
# ->conv1->
#128*256
# ->conv2->
#64*128
# ->conv3->
#32*64
# ->conv4->
#16*32
# ->layers.conv_transpose4->
#32*64
# ->layers.conv_transpose5->
#64*128
# ->layers.conv_transpose6->
#128*256
# ->layers.conv_transpose7->
#256*512
#256x512x3 -> 256x512x32
with tf.variable_scope('left') as scope:
left_feat = layers.conv2d_relu('conv1', lefts, [3,3,3,32], [1,1,1,1])
#256x512x1 -> 256x512x32
with tf.variable_scope('disp') as scope:
disp_feat = layers.conv2d_relu('conv1', disps, [3,3,1,32], [1,1,1,1])
#x32 + x32 -> 256x512x64
concat = tf.concat([left_feat, disp_feat], axis=3)
#256x512x64 -> 256x512x64
net = layers.conv2d_relu('conv1_concat', concat, [3,3,64,64], [1,1,1,1])
scale1 = net
net = tf.contrib.layers.batch_norm(net)
#256x512x64 -> 128x256x64
net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool1')
def encoder_unit(name, batch, kernel_shape, strides, padding='SAME',
kernel_initializer=tf.contrib.layers.xavier_initializer(),
bias_initializer=tf.constant_initializer(0.0)):
with tf.variable_scope('encoder' + number):
with tf.variable_scope('conv'):
kernel = _variable_on_cpu('weights',
shape=kernel_shape,
initializer=kernel_initializer)
conv = tf.nn.conv2d(batch, kernel, strides, padding=padding)
biases = _variable_on_cpu('biases',
[kernel_shape[3]], bias_initializer)
pre_activation = tf.nn.bias_add(conv, biases)
print(net.name, net.shape)
# 128x256x64 -> 128x256x128
net = layers.conv2d_relu('conv2', net, [3,3,64,128], [1,1,1,1])
scale2 = net
net = tf.contrib.layers.batch_norm(net)
print(net.name, net.shape)
#128x256x128 -> 64x128x128
net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool2')
print(net.name, net.shape)
#64x128x128 ->64x128x128
net = layers.conv2d_relu('conv3', net, [3,3,128,256], [1,1,1,1])
scale3 = net
net = tf.contrib.layers.batch_norm(net)
print(net.name, net.shape)
#64x128x128 -> 32x64x256
net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool3')
print("scale3-->", net.name, net.shape)
#32x64x256 -> 32x64x512
net = layers.conv2d_relu('conv4', net, [3,3,256,512], [1,1,1,1])
net = tf.contrib.layers.batch_norm(net)
print(net.name, net.shape)
#32x64x512 -> 16x32x512
net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1],
padding='SAME', name='pool4')
print(net.name, net.shape)
#16x32x512 -> 32x64x256
up = layers.conv_transpose_2x_relu('transp1', net, [3,3,256,512], [1,2,2,1])
tf.add(up, scale3)
print(up.name, up.shape)
#32x64x256 -> 64x128x128
up = layers.conv_transpose_2x_relu('transp2', up, [3,3,128,256], [1,2,2,1])
tf.add(up, scale2)
print(up.name, up.shape)
#64x128x128 -> 128x256x64
up = layers.conv_transpose_2x_relu('transp3', up, [3,3,64,128], [1,2,2,1])
tf.add(up, scale1)
print(up.name, up.shape)
#128x256x64 -> 256x512x32
up = layers.conv_transpose_2x_relu('transp4', up, [3,3,32,64], [1,2,2,1])
tf.add(up, left_feat)
tf.add(up, disp_feat)
print(up.name, up.shape)
up = layers.conv2d('conv5', up, [3,3,32,1], [1,1,1,1])
print(up.name, up.shape)
return up
def ShuffleNet(last, group_num=8):
is_training = False
channels = FLAGS.channels
channel_index = -3 if FLAGS.data_format == 'NCHW' else -1
l = 0
last = tf.identity(last, name='input')
# residual blocks
rb = 0
skip2 = last
while rb < FLAGS.res_blocks:
rb += 1
l += 1
with tf.variable_scope('pointwise_group_conv{}'.format(l)) as scope:
group = tf.split(last, group_num, axis=channel_index)
for _ in range(group_num):
with tf.variable_scope('group{}'.format(_ + 1)) as scope:
group[_] = layers.conv2d(group[_],
ksize=1,
out_channels=channels //
group_num,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_activation)
last = tf.concat(group, axis=channel_index)
last = layers.apply_activation(last,
activation=FLAGS.activation,
data_format=FLAGS.data_format)
with tf.variable_scope('channel_shuffle{}'.format(l)) as scope:
shape = tf.shape(last)
if FLAGS.data_format == 'NCHW':
shape_divide = [
shape[0], group_num, channels // group_num, shape[2],
shape[3]
]
else:
shape_divide = [
shape[0], shape[1], shape[2], group_num,
channels // group_num
]
last = tf.reshape(last, shape_divide)
trans = (0, 2, 1, 3,
4) if FLAGS.data_format == 'NCHW' else (0, 1, 2, 4, 3)
last = tf.transpose(last, trans)
last = tf.reshape(last, shape)
l += 1
with tf.variable_scope('depthwise_conv{}'.format(l)) as scope:
last = layers.depthwise_conv2d(last,
ksize=3,
channel_multiplier=1,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_activation)
l += 1
with tf.variable_scope('pointwise_group_conv{}'.format(l)) as scope:
group = tf.split(last, group_num, axis=channel_index)
for _ in range(group_num):
with tf.variable_scope('group{}'.format(_ + 1)) as scope:
group[_] = layers.conv2d(group[_],
ksize=1,
out_channels=channels //
group_num,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_activation)
last = tf.concat(group, axis=channel_index)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
last = tf.add(last, skip2, 'elementwise_sum')
skip2 = last
last = layers.apply_activation(last, activation=FLAGS.activation)
# return
last = tf.identity(last, name='output')
return last
def resnet50(input):
"""
:param input: a 4-D tensor, NHWC
:return:
"""
# conv1, 7x7, 64, stride 2
conv1 = block('conv1', input, 64, 7, 2, 'SAME')
# conv2_x, 3x3 max pool, stride 2
maxpool2 = maxpool2d('pool2', conv1, 3, 2, 'SAME')
# conv2_1
# conv2_11, 1x1, 64
conv2_11 = block('conv2_11', maxpool2, 64, 1, 1, 'VALID')
# conv2_12, 3x3, 64
conv2_12 = block('conv2_12', conv2_11, 64, 3, 1, 'SAME')
# conv2_13, 1x1, 256
conv2_13 = block('conv2_13', conv2_12, 256, 1, 1, 'VALID')
r2_1 = tf.add_n([conv2d('pj2', maxpool2, 256, 1, 1, 'VALID'), conv2_13])
# conv2_2
# conv2_21, 1x1, 64
conv2_21 = block('conv2_21', conv2_13, 64, 1, 1, 'VALID')
# conv2_22, 3x3, 64
conv2_22 = block('conv2_22', conv2_21, 64, 3, 1, 'SAME')
# conv2_23, 1x1, 256
conv2_23 = block('conv2_23', conv2_22, 256, 1, 1, 'VALID')
r2_2 = tf.add_n([r2_1, conv2_23])
# conv2_3
# conv2_31, 1x1, 64
conv2_31 = block('conv2_31', r2_2, 64, 1, 1, 'VALID')
# conv2_32, 3x3, 64
conv2_32 = block('conv2_32', conv2_31, 64, 3, 1, 'SAME')
# conv2_33, 1x1, 256
conv2_33 = block('conv2_33', conv2_32, 256, 1, 1, 'VALID')
r2_3 = tf.add_n([r2_2, conv2_33])
# conv3_x
# conv3_1
# conv3_11, 1x1, 128
conv3_11 = block('conv3_11', r2_3, 128, 1, 2, 'VALID')
# conv3_12, 3x3, 128
conv3_12 = block('conv3_12', conv3_11, 128, 3, 1, 'SAME')
# conv3_13, 1x1, 512
conv3_13 = block('conv3_13', conv3_12, 512, 1, 1, 'VALID')
r3_1 = tf.add_n([conv2d('pj3', r2_3, 512, 1, 2, 'VALID'), conv3_13])
# conv3_2
# conv3_21, 1x1, 128
conv3_21 = block('conv3_21', r3_1, 128, 1, 1, 'VALID')
# conv3_22, 3x3, 128
conv3_22 = block('conv3_22', conv3_21, 128, 3, 1, 'SAME')
# conv3_23, 1x1, 512
conv3_23 = block('conv3_23', conv3_22, 512, 1, 1, 'VALID')
r3_2 = tf.add_n([r3_1, conv3_23])
# conv3_3
# conv3_31, 1x1, 128
conv3_31 = block('conv3_31', r3_2, 128, 1, 1, 'VALID')
# conv3_32, 3x3, 128
conv3_32 = block('conv3_32', conv3_31, 128, 3, 1, 'SAME')
# conv3_33, 1x1, 512
conv3_33 = block('conv3_33', conv3_32, 512, 1, 1, 'VALID')
r3_3 = tf.add_n([r3_2, conv3_33])
# conv3_4
# conv3_41, 1x1, 128
conv3_41 = block('conv3_41', r3_3, 128, 1, 1, 'VALID')
# conv3_42, 3x3, 128
conv3_42 = block('conv3_42', conv3_41, 128, 3, 1, 'SAME')
# conv3_43, 1x1, 512
conv3_43 = block('conv3_43', conv3_42, 512, 1, 1, 'VALID')
r3_4 = tf.add_n([r3_3, conv3_43])
# conv4_x
# conv4_1
# conv4_11, 1x1, 256
conv4_11 = block('conv4_11', r3_4, 256, 1, 2, 'VALID')
# conv4_12, 3x3, 256
conv4_12 = block('conv4_12', conv4_11, 256, 3, 1, 'SAME')
# conv4_13, 1x1, 1024
conv4_13 = block('conv4_13', conv4_12, 1024, 1, 1, 'VALID')
r4_1 = tf.add_n([conv2d('pj4', r3_4, 1024, 1, 2, 'VALID'), conv4_13])
# conv4_2
# conv4_21, 1x1, 256
conv4_21 = block('conv4_21', r4_1, 256, 1, 1, 'VALID')
# conv4_22, 3x3, 256
conv4_22 = block('conv4_22', conv4_21, 256, 3, 1, 'SAME')
# conv4_23, 1x1, 1024
conv4_23 = block('conv4_23', conv4_22, 1024, 1, 1, 'VALID')
r4_2 = tf.add_n([r4_1, conv4_23])
# conv4_3
# conv4_31, 1x1, 256
conv4_31 = block('conv4_31', r4_2, 256, 1, 1, 'VALID')
# conv4_32, 3x3, 256
conv4_32 = block('conv4_32', conv4_31, 256, 3, 1, 'SAME')
# conv4_33, 1x1, 1024
conv4_33 = block('conv4_33', conv4_32, 1024, 1, 1, 'VALID')
r4_3 = tf.add_n([r4_2, conv4_33])
# conv4_4
# conv4_41, 1x1, 256
conv4_41 = block('conv4_41', r4_3, 256, 1, 1, 'VALID')
# conv4_42, 3x3, 256
conv4_42 = block('conv4_42', conv4_41, 256, 3, 1, 'SAME')
# conv4_43, 1x1, 1024
conv4_43 = block('conv4_43', conv4_42, 1024, 1, 1, 'VALID')
r4_4 = tf.add_n([r4_3, conv4_43])
# conv4_5
# conv4_51, 1x1, 256
conv4_51 = block('conv4_51', r4_4, 256, 1, 1, 'VALID')
# conv4_52, 3x3, 256
conv4_52 = block('conv4_52', conv4_51, 256, 3, 1, 'SAME')
# conv4_53, 1x1, 1024
conv4_53 = block('conv4_53', conv4_52, 1024, 1, 1, 'VALID')
r4_5 = tf.add_n([r4_4, conv4_53])
# conv4_6
# conv4_61, 1x1, 256
conv4_61 = block('conv4_61', r4_5, 256, 1, 1, 'VALID')
# conv4_62, 3x3, 256
conv4_62 = block('conv4_62', conv4_61, 256, 3, 1, 'SAME')
# conv4_63, 1x1, 1024
conv4_63 = block('conv4_63', conv4_62, 1024, 1, 1, 'SAME')
r4_6 = tf.add_n([r4_5, conv4_63])
# conv5_x
# conv5_1
# conv5_11, 1x1, 512
conv5_11 = block('conv5_11', r4_6, 512, 1, 2, 'VALID')
# conv5_12, 3x3, 512
conv5_12 = block('conv5_12', conv5_11, 512, 3, 1, 'SAME')
# conv5_13, 1x1, 2048
conv5_13 = block('conv5_13', conv5_12, 2048, 1, 1, 'VALID')
r5_1 = tf.add_n([conv2d('pj5', r4_6, 2048, 1, 2, 'VALID'), conv5_13])
# conv5_2
# conv5_21, 1x1, 512
conv5_21 = block('conv5_21', r5_1, 512, 1, 1, 'VALID')
# conv5_22, 3x3, 512
conv5_22 = block('conv5_22', conv5_21, 512, 3, 1, 'SAME')
# conv5_23, 1x1, 2048
conv5_23 = block('conv5_23', conv5_22, 2048, 1, 1, 'VALID')
r5_2 = tf.add_n([r5_1, conv5_23])
# conv5_3
# conv5_31, 1x1, 512
conv5_31 = block('conv5_31', r5_2, 512, 1, 1, 'VALID')
# conv5_32, 3x3, 512
conv5_32 = block('conv5_32', conv5_31, 512, 3, 1, 'SAME')
# conv5_33, 1x1, 2048
conv5_33 = block('conv5_33', conv5_32, 2048, 1, 1, 'VALID')
r5_3 = tf.add_n([r5_2, conv5_33])
avgpool6 = avgpool2d('avgpool6', r5_3, 7, 1, 'VALID')
dense6 = tf.layers.dense(tf.layers.flatten(avgpool6), 12)
return dense6
def fpn_extra_tops(self, inputs,
out_channels): #Add extra 2 feature maps of fpn
f1 = conv2d(inputs, out_channels, 3, strides=2, padding=1)
f2 = conv2d(tf.nn.relu(f1), out_channels, 3, strides=2, padding=1)
return [f1, f2]
def generatorVid(self, img_batch, reuse=False):
with tf.variable_scope('gen_v', reuse=reuse) as vs:
""" -----------------------------------------------------------------------------------
ENCODER
----------------------------------------------------------------------------------- """
self.en_h0 = conv2d(img_batch, 3, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1")
self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0))
add_activation_summary(self.en_h0)
print(self.en_h0.get_shape().as_list())
self.en_h1 = conv2d(self.en_h0, 64, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2")
self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2")
self.en_h1 = tf.nn.relu(self.en_h1)
add_activation_summary(self.en_h1)
print(self.en_h1.get_shape().as_list())
output = tf.transpose(self.en_h1, [0, 3, 1, 2])
for i in xrange(3):
output = self.ResidualBlock('res1.16x16_{}'.format(i), 128, 128, 3, output, resample=None)
self.en_h1 = tf.transpose(output, [0, 2, 3, 1])
self.en_h2 = conv2d(self.en_h1, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3")
self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3")
self.en_h2 = tf.nn.relu(self.en_h2)
add_activation_summary(self.en_h2)
print(self.en_h2.get_shape().as_list())
output = tf.transpose(self.en_h2, [0, 3, 1, 2])
for i in xrange(3):
output = self.ResidualBlock('res1.16x16_2_{}'.format(i), 256, 256, 3, output, resample=None)
self.en_h2 = tf.transpose(output, [0, 2, 3, 1])
self.en_h3 = conv2d(self.en_h2, 256, 512, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4")
self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4")
self.en_h3 = tf.nn.relu(self.en_h3)
add_activation_summary(self.en_h3)
print(self.en_h3.get_shape().as_list())
output = tf.transpose(self.en_h3, [0, 3, 1, 2])
for i in xrange(3):
output = self.ResidualBlock('res1.16x16_3_{}'.format(i), 512, 512, 3, output, resample=None)
self.en_h3 = tf.transpose(output, [0, 2, 3, 1])
self.en_h4 = conv2d(self.en_h3, 512, 1024, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv5")
self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4, scope="enc_bn5")
self.en_h4 = tf.nn.relu(self.en_h4)
add_activation_summary(self.en_h4)
print(self.en_h4.get_shape().as_list())
""" -----------------------------------------------------------------------------------
GENERATOR
----------------------------------------------------------------------------------- """
self.z_ = tf.reshape(self.en_h4, [self.batch_size, 2, 4, 4, 512])
print(self.z_.get_shape().as_list())
self.fg_h1 = conv3d_transpose(self.z_, 512, [self.batch_size, 4, 8, 8, 256], name='g_f_h1')
self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h1, scope='g_f_bn1'), name='g_f_relu1')
add_activation_summary(self.fg_h1)
print(self.fg_h1.get_shape().as_list())
self.fg_h2 = conv3d_transpose(self.fg_h1, 256, [self.batch_size, 8, 16, 16, 128], name='g_f_h2')
self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h2, scope='g_f_bn2'), name='g_f_relu2')
add_activation_summary(self.fg_h2)
print(self.fg_h2.get_shape().as_list())
self.fg_h3 = conv3d_transpose(self.fg_h2, 128, [self.batch_size, 16, 32, 32, 64], name='g_f_h3')
self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h3, scope='g_f_bn3'), name='g_f_relu3')
add_activation_summary(self.fg_h3)
print(self.fg_h3.get_shape().as_list())
self.fg_h4 = conv3d_transpose(self.fg_h3, 64, [self.batch_size, 32, 64, 64, 3], name='g_f_h4')
self.fg_vid = tf.nn.tanh(self.fg_h4, name='g_f_actvcation')
print(self.fg_vid.get_shape().as_list())
# gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg[:, 0, :, :, :]))
# variables = tf.contrib.framework.get_variables(vs)
# # return self.fg_fg, gen_reg, variables
# return self.fg_vid, variables
return self.fg_vid
def discriminator(self, images, is_training=False, reuse=None):
# parameters
data_format = self.data_format
channels = self.d_channels
batch_norm = self.d_batch_norm
activation = self.d_activation
initializer = self.initializer
init_factor = self.init_factor
init_activation = self.init_activation
weight_key = self.discriminator_weight_key
channel_index = -3 if data_format == 'NCHW' else -1
reduce_strides = [1, 1, 2, 2
] if data_format == 'NCHW' else [1, 2, 2, 1]
# initialization
last = images
l = 0
with tf.variable_scope('discriminator', reuse=reuse) as scope:
# first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# residual blocks
depth = 0
skip2 = last
while depth < self.d_depth:
depth += 1
reduce_size = True
strides = reduce_strides if reduce_size else [1, 1, 1, 1]
double_channel = depth > 1
if double_channel: channels *= 2
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
last = layers.conv2d(last,
ksize=2,
out_channels=channels,
stride=strides,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if reduce_size:
skip2 = tf.nn.avg_pool(skip2,
ksize=reduce_strides,
strides=reduce_strides,
padding='SAME',
data_format=data_format)
if double_channel:
padding = [[0, 0] for _ in range(4)]
padding[channel_index] = [0, channels // 2]
skip2 = tf.pad(skip2, padding, mode='CONSTANT')
last = tf.add(last, skip2)
skip2 = last
# final conv layer
channels *= 2
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=2,
out_channels=channels,
stride=2,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=activation,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# dense layer (1024)
l += 1
print(last.get_shape())
with tf.variable_scope('dense{}'.format(l)) as scope:
last = tf.contrib.layers.flatten(last)
last = tf.contrib.layers.fully_connected(last,
1024,
activation_fn=None)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
# dense layer (1)
l += 1
with tf.variable_scope('dense{}'.format(l)) as scope:
last = tf.contrib.layers.fully_connected(last,
1,
activation_fn=None)
# return discriminating logits
print('Critic: totally {} convolutional/dense layers.'.format(l))
return last
def generator(self, img_batch):
with tf.variable_scope('g_') as vs:
""" -----------------------------------------------------------------------------------
ENCODER
----------------------------------------------------------------------------------- """
print('ENCODER')
self.en_h0 = conv2d(img_batch,
self.channels,
64,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv1")
self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0))
add_activation_summary(self.en_h0)
print(self.en_h0.get_shape().as_list())
self.en_h1 = conv2d(self.en_h0,
64,
128,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv2")
self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1,
scope="enc_bn2")
self.en_h1 = tf.nn.relu(self.en_h1)
add_activation_summary(self.en_h1)
print(self.en_h1.get_shape().as_list())
self.en_h2 = conv2d(self.en_h1,
128,
256,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv3")
self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2,
scope="enc_bn3")
self.en_h2 = tf.nn.relu(self.en_h2)
add_activation_summary(self.en_h2)
print(self.en_h2.get_shape().as_list())
self.en_h3 = conv2d(self.en_h1,
256,
512,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv4")
self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3,
scope="enc_bn4")
self.en_h3 = tf.nn.relu(self.en_h3)
add_activation_summary(self.en_h3)
print(self.en_h2.get_shape().as_list())
self.en_h4 = conv2d(self.en_h2,
512,
1024,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv5")
self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4,
scope="enc_bn5")
self.en_h4 = tf.nn.relu(self.en_h4)
add_activation_summary(self.en_h4)
print(self.en_h3.get_shape().as_list())
""" -----------------------------------------------------------------------------------
GENERATOR
----------------------------------------------------------------------------------- """
print('GENERATOR')
self.z_ = tf.reshape(self.en_h3, [self.batch_size, 2, 2, 1024])
print(self.z_.get_shape().as_list())
self.fg_h1 = tf.image.resize_images(
self.z_, [4, 4], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
self.fg_h1 = conv2d(self.fg_h1,
1024,
512,
d_h=1,
d_w=1,
name="gen_conv1")
self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h1, scope='g_f_bn1'),
name='g_f_relu1')
add_activation_summary(self.fg_h1)
print(self.fg_h1.get_shape().as_list())
self.fg_h2 = tf.image.resize_images(
self.fg_h1, [8, 8],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
self.fg_h2 = conv2d(self.fg_h2,
512,
256,
d_h=1,
d_w=1,
name="gen_conv2")
self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h2, scope='g_f_bn2'),
name='g_f_relu2')
add_activation_summary(self.fg_h2)
print(self.fg_h2.get_shape().as_list())
self.fg_h3 = tf.image.resize_images(
self.fg_h2, [16, 16],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
self.fg_h3 = conv2d(self.fg_h3,
256,
128,
d_h=1,
d_w=1,
name="gen_conv3")
self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h3, scope='g_f_bn3'),
name='g_f_relu3')
add_activation_summary(self.fg_h3)
print(self.fg_h3.get_shape().as_list())
self.fg_h4 = tf.image.resize_images(
self.fg_h3, [self.crop_size, self.crop_size],
method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
self.fg_h4 = conv2d(self.fg_h4,
128,
self.channels,
d_h=1,
d_w=1,
name="gen_conv4")
self.fg_fg = tf.nn.tanh(self.fg_h4, name='g_f_actication')
print(self.fg_fg.get_shape().as_list())
gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg))
variables = tf.contrib.framework.get_variables(vs)
return self.fg_fg, gen_reg, variables
def inference(self, spectrum, is_training=False, reuse=None):
print('k_first={}, res_blocks={}, channels={}'.format(
self.k_first, self.res_blocks, self.channels))
# parameters
data_format = self.data_format
channels = self.channels
batch_norm = self.batch_norm
activation = self.activation
initializer = self.initializer
init_factor = self.init_factor
init_activation = self.init_activation
weight_key = self.inference_weight_key
channel_index = -3 if self.data_format == 'NCHW' else -1
pool_ksize = [1, 1, 1, 3
] if self.data_format == 'NCHW' else [1, 1, 3, 1]
reduce_strides = [1, 1, 1, 2
] if self.data_format == 'NCHW' else [1, 1, 2, 1]
# initialization
last = spectrum
l = 0
with tf.variable_scope('inference', reuse=reuse) as scope:
# first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=[1, self.k_first],
out_channels=channels,
stride=[1, 1, 1, 1],
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
# residual blocks
rb = 0
skip2 = last
while rb < self.res_blocks:
rb += 1
reduce_size = rb % 1 == 0
strides = reduce_strides if reduce_size else [1, 1, 1, 1]
double_channel = rb % 3 == 1
if double_channel: channels *= 2
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
last = layers.conv2d(last,
ksize=[1, 1],
out_channels=channels // 2,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
last = layers.conv2d(last,
ksize=[1, 3],
out_channels=channels // 2,
stride=strides,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
last = layers.conv2d(last,
ksize=[1, 1],
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if reduce_size:
skip2 = tf.nn.avg_pool(skip2,
ksize=reduce_strides,
strides=reduce_strides,
padding='SAME',
data_format=data_format)
if double_channel:
padding = [[0, 0] for _ in range(4)]
padding[channel_index] = [0, channels // 2]
skip2 = tf.pad(skip2, padding, mode='CONSTANT')
last = tf.add(last, skip2)
skip2 = last
last = layers.apply_activation(last,
activation=activation,
data_format=data_format)
# final dense layer (num_labels)
l += 1
print(last.get_shape())
with tf.variable_scope('dense{}'.format(l)) as scope:
last = tf.contrib.layers.flatten(last)
last = tf.contrib.layers.fully_connected(last,
self.num_labels,
activation_fn=None)
# return predicted labels
print('Totally {} convolutional/dense layers.'.format(l))
return last
def generator(self,
images_src,
enc_src=None,
is_training=False,
reuse=None):
print(
'generator: k_first={}, k_last={}, e_depth={}, d_depth={}, channels={}'
.format(self.k_first, self.k_last, self.e_depth, self.d_depth,
self.channels))
# parameters
data_format = self.data_format
channels = self.channels
batch_norm = self.batch_norm
activation = self.activation
initializer = self.initializer
init_factor = self.init_factor
init_activation = self.init_activation
weight_key = self.generator_weight_key
# initialization
last = images_src
l = 0
with tf.variable_scope('generator', reuse=reuse) as scope:
# encoder
with tf.variable_scope('encoder') as scope:
# encoder - first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=self.k_first,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# encoder - residual blocks
depth = 0
while depth < self.e_depth:
downscale = False
depth += 1
skip2 = last
if downscale:
channels *= 4
with tf.variable_scope('trans{}'.format(l)) as scope:
if data_format == 'NCHW':
last = utils.image.NCHW2NHWC(last)
last = tf.space_to_depth(last, 2)
if data_format == 'NCHW':
last = utils.image.NHWC2NCHW(last)
channels //= 2
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope(
'skip_connection{}'.format(l)) as scope:
if downscale:
pool_size = [
1, 1, 2, 2
] if data_format == 'NCHW' else [1, 2, 2, 1]
skip2 = tf.nn.avg_pool(skip2,
pool_size,
pool_size,
padding='SAME',
data_format=data_format)
padding = [[0, 0], [0, 0], [0, 0], [0, 0]]
padding[-3 if data_format == 'NCHW' else -1] = [
0, channels // 2
]
skip2 = tf.pad(skip2, padding)
last = layers.SqueezeExcitation(
last,
channel_r=1,
data_format=data_format,
collection=weight_key)
last = tf.add(last, skip2)
# encoder - final conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=self.k_last,
out_channels=self.image_channels * 8,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
# encoded images
if not is_training:
last = tf.round(last) # {0, 1}
images_enc = last
if enc_src is not None: last = enc_src
# decoder
with tf.variable_scope('decoder') as scope:
# decoder - first conv layer
channels //= 1
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=self.k_first,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# decoder - residual blocks
depth = 0
while depth < self.d_depth:
depth += 1
skip2 = last
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope(
'skip_connection{}'.format(l)) as scope:
last = layers.SqueezeExcitation(
last,
channel_r=1,
data_format=data_format,
collection=weight_key)
last = tf.add(last, skip2)
'''
# sub-pixel conv layer
channels //= 2
l += 1
with tf.variable_scope('subpixel_conv{}'.format(l)) as scope:
last = layers.subpixel_conv2d(last, ksize=3, out_channels=channels,
scaling=2, padding='SAME', data_format=data_format,
batch_norm=None, is_training=is_training, activation=activation,
initializer=initializer, init_factor=init_activation,
collection=weight_key)
'''
# decoder - final conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=self.k_last,
out_channels=self.image_channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
images_dec = last
# return SR image
print('Generator: totally {} convolutional layers.'.format(l))
return images_enc, images_dec
def discriminator(self, images_enc, is_training=False, reuse=None):
# parameters
data_format = self.data_format
d_depth = 8
channels = 64
batch_norm = 0.99
activation = 'swish'
initializer = 4
init_factor = 1.0
init_activation = 2.0
weight_key = self.discriminator_weight_key
# initialization
last = images_enc
l = 0
with tf.variable_scope('discriminator', reuse=reuse) as scope:
# first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# residual blocks
depth = 0
while depth < d_depth:
downscale = depth % 2 == 1
depth += 1
skip2 = last
if downscale:
channels *= 4
with tf.variable_scope('trans{}'.format(l)) as scope:
if data_format == 'NCHW':
last = utils.image.NCHW2NHWC(last)
last = tf.space_to_depth(last, 2)
if data_format == 'NCHW':
last = utils.image.NHWC2NCHW(last)
channels //= 2
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=1,
out_channels=channels // 4,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=3,
out_channels=channels // 4,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
is_training=is_training,
data_format=data_format)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
ksize=1,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if downscale:
pool_size = [
1, 1, 2, 2
] if data_format == 'NCHW' else [1, 2, 2, 1]
skip2 = tf.nn.avg_pool(skip2,
pool_size,
pool_size,
padding='SAME',
data_format=data_format)
padding = [[0, 0], [0, 0], [0, 0], [0, 0]]
padding[-3 if data_format == 'NCHW' else -1] = [
0, channels // 2
]
skip2 = tf.pad(skip2, padding)
last = layers.SqueezeExcitation(last,
channel_r=4,
data_format=data_format,
collection=weight_key)
last = tf.add(last, skip2)
# global average pooling (1024)
with tf.variable_scope('trans{}'.format(l)) as scope:
last = tf.reduce_mean(
last, [-2, -1] if data_format == 'NCHW' else [-3, -2])
# dense layers (1024)
l += 1
with tf.variable_scope('dense{}'.format(l)) as scope:
last = tf.contrib.layers.fully_connected(last,
channels,
activation_fn=None)
last = layers.apply_activation(last,
activation='swish',
data_format=data_format,
collection=weight_key)
# dense layers (1)
channels = 1
l += 1
with tf.variable_scope('dense{}'.format(l)) as scope:
last = tf.contrib.layers.fully_connected(last,
channels,
activation_fn=None)
last = layers.apply_activation(last,
activation='sigmoid',
data_format=data_format,
collection=weight_key)
# return estimate
print('Discriminator: totally {} convolutional layers.'.format(l))
return last
def generator(self, images_lr, is_training=False, reuse=None):
print('k_first={}, k_last={}, g_depth={}, channels={}, channels2={}'.
format(self.k_first, self.k_last, self.g_depth, self.channels,
self.channels2))
# parameters
data_format = self.data_format
channels = self.channels
channels2 = self.channels2
batch_norm = self.batch_norm
activation = self.activation
initializer = self.initializer
init_factor = self.init_factor
init_activation = self.init_activation
weight_key = self.generator_weight_key
# initialization
last = images_lr
l = 0
with tf.variable_scope('generator', reuse=reuse) as scope:
skip0 = last
# first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=self.k_first,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=activation,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# hidden layers
depth = 0
while depth < self.g_depth:
depth += 1
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=3,
out_channels=channels,
stride=1,
padding='SAME',
data_format=data_format,
batch_norm=batch_norm,
is_training=is_training,
activation=activation,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# resize conv layer
l += 1
with tf.variable_scope('resize_conv{}'.format(l)) as scope:
last = layers.resize_conv2d(last,
ksize=self.k_last,
out_channels=self.image_channels,
scaling=self.scaling,
data_format=data_format,
batch_norm=None,
is_training=is_training,
activation=None,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
'''
# sub-pixel conv layer
l += 1
with tf.variable_scope('subpixel_conv{}'.format(l)) as scope:
last = layers.subpixel_conv2d(last, ksize=self.k_last, out_channels=self.image_channels,
scaling=self.scaling, data_format=data_format,
batch_norm=None, is_training=is_training, activation=None,
initializer=initializer, init_factor=init_factor,
collection=weight_key)
'''
# skip connection
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if data_format == 'NCHW':
skip0 = tf.transpose(skip0, (0, 2, 3, 1))
up_size = tf.shape(skip0)[-3:-1] * self.scaling
skip0 = tf.image.resize_nearest_neighbor(skip0, up_size)
if data_format == 'NCHW':
skip0 = tf.transpose(skip0, (0, 3, 1, 2))
last = tf.add(last, skip0)
# return SR image
print('Generator: totally {} convolutional layers.'.format(l))
return last
def generator(self, images_lr, train=False, reuse=None):
print('k_first={}, k_last={}, g_depth={}, channels={}, channels2={}'.
format(self.k_first, self.k_last, self.g_depth, self.channels,
self.channels2))
# parameters
data_format = self.data_format
channels = self.channels
channels2 = self.channels2
batch_norm = self.batch_norm
renorm = self.batch_renorm
activation = self.activation
initializer = self.initializer
init_factor = self.init_factor
init_activation = self.init_activation
weight_key = self.generator_weight_key
# initialization
last = images_lr
l = 0
with tf.variable_scope('generator', reuse=reuse) as scope:
skip0 = last
# first conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
self.k_first,
channels,
use_bias=True,
stride=1,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
skip1 = last
# residual blocks
depth = 0
while depth < self.g_depth:
depth += 1
skip2 = last
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
train=train,
data_format=data_format,
renorm=renorm)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
3,
channels,
use_bias=False,
stride=1,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
train=train,
data_format=data_format,
renorm=renorm)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
3,
channels,
use_bias=True,
stride=1,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
# skip connection - level 2
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if self.use_se == 1:
last = layers.SqueezeExcitation(
last,
channel_r=1,
data_format=data_format,
collection=weight_key)
elif self.use_se == 2:
last = layers.SqueezeExcitationLocal(
last,
channel_r=1,
ksize=16,
data_format=data_format,
collection=weight_key)
last = tf.add(last, skip2)
# skip connection - level 1
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.apply_batch_norm(last,
decay=batch_norm,
train=train,
data_format=data_format,
renorm=renorm)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
last = layers.conv2d(last,
3,
channels,
use_bias=True,
stride=1,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
if self.use_se == 1:
last = layers.SqueezeExcitation(last,
channel_r=1,
data_format=data_format,
collection=weight_key)
elif self.use_se == 2:
last = layers.SqueezeExcitationLocal(
last,
channel_r=1,
ksize=16,
data_format=data_format,
collection=weight_key)
last = tf.add(last, skip1)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
# resize conv layer
scale_num = 1
while 1 << scale_num <= self.scaling:
with tf.variable_scope(
'resize_conv{}'.format(scale_num)) as scope:
last = layers.resize_conv2d(last,
3,
channels2,
use_bias=True,
scaling=self.scaling,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_activation,
collection=weight_key)
last = layers.apply_activation(last,
activation=activation,
data_format=data_format,
collection=weight_key)
scale_num += 1
# final conv layer
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
self.k_last,
self.image_channels,
use_bias=True,
stride=1,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
# skip connection - level 0
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
scale_num = 1
while 1 << scale_num <= self.scaling:
with tf.variable_scope(
'resize_conv{}'.format(scale_num)) as scope:
skip0 = layers.resize_conv2d(skip0,
self.k_resize,
self.image_channels,
use_bias=True,
scaling=self.scaling,
padding='SAME',
data_format=data_format,
initializer=initializer,
init_factor=init_factor,
collection=weight_key)
scale_num += 1
last = tf.add(last, skip0)
# return SR image
print('Generator: totally {} convolutional layers.'.format(l))
return last
def generatorVid(self, img_batch, reuse=False):
with tf.variable_scope('gen_v', reuse=reuse) as vs:
""" -----------------------------------------------------------------------------------
ENCODER
----------------------------------------------------------------------------------- """
# self.en_h0 = conv2d(img_batch, 3, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1")
# self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0))
# add_activation_summary(self.en_h0)
# print(self.en_h0.get_shape().as_list())
self.en_h0 = conv2d(img_batch,
3,
32,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv1")
self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0))
add_activation_summary(self.en_h0)
print(self.en_h0.get_shape().as_list())
self.en_h1 = conv2d(self.en_h0,
32,
64,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv2")
self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1,
scope="enc_bn2")
self.en_h1 = tf.nn.relu(self.en_h1)
add_activation_summary(self.en_h1)
print(self.en_h1.get_shape().as_list())
#output = tf.transpose(self.en_h1, [0, 3, 1, 2])
#for i in xrange(3):
# output = self.ResidualBlock('res1.16x16_{}'.format(i), 256, 256, 3, output, resample=None)
#self.en_h1 = tf.transpose(output, [0, 2, 3, 1])
self.en_h2 = conv2d(self.en_h1,
64,
128,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv3")
self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2,
scope="enc_bn3")
self.en_h2 = tf.nn.relu(self.en_h2)
add_activation_summary(self.en_h2)
print(self.en_h2.get_shape().as_list())
#output = tf.transpose(self.en_h2, [0, 3, 1, 2])
#for i in xrange(3):
# output = self.ResidualBlock('res1.16x16_2_{}'.format(i), 512, 512, 3, output, resample=None)
#self.en_h2 = tf.transpose(output, [0, 2, 3, 1])
self.en_h3 = conv2d(self.en_h2,
128,
256,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv4")
self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3,
scope="enc_bn4")
self.en_h3 = tf.nn.relu(self.en_h3)
add_activation_summary(self.en_h3)
print(self.en_h3.get_shape().as_list())
self.en_h4 = conv2d(self.en_h3,
256,
256,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv5")
self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4,
scope="enc_bn5")
self.en_h4 = tf.nn.relu(self.en_h4)
add_activation_summary(self.en_h4)
print(self.en_h4.get_shape().as_list())
#self.en_h4 is [32, 4, 4, 256]
self.en_h5 = conv2d(self.en_h4,
256,
256,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv6")
self.en_h5 = tf.contrib.layers.batch_norm(self.en_h5,
scope="enc_bn6")
self.en_h5 = tf.nn.relu(self.en_h5)
add_activation_summary(self.en_h5)
print(self.en_h5.get_shape().as_list())
#self.en_h5 is [32,2,2,256]
self.en_h6 = conv2d(self.en_h5,
256,
256,
k_h=4,
k_w=4,
d_w=2,
d_h=2,
name="enc_conv7")
self.en_h6 = tf.contrib.layers.batch_norm(self.en_h6,
scope="enc_bn7")
self.en_h6 = tf.nn.relu(self.en_h6)
add_activation_summary(self.en_h6)
print(self.en_h6.get_shape().as_list())
#self.en_h6 is [32,1,1,256]
""" -----------------------------------------------------------------------------------
GENERATOR
----------------------------------------------------------------------------------- """
self.z_ = tf.reshape(self.en_h6, [self.batch_size, 1, 1, 1, 256])
print(self.z_.get_shape().as_list())
self.fg_h1 = conv3d_transpose(self.z_,
256, [self.batch_size, 1, 2, 2, 256],
name='g_f_h1')
self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h1, scope='g_f_bn1'),
name='g_f_relu1')
add_activation_summary(self.fg_h1)
print(self.fg_h1.get_shape().as_list())
#self.fg_h1 is [32, 1, 2, 2, 256]
encv5 = tf.reshape(self.en_h5, [self.batch_size, 1, 2, 2, 256])
encov5 = tf.concat([self.fg_h1, encv5], axis=4)
self.fg_h2 = conv3d_transpose(encov5,
512, [self.batch_size, 2, 4, 4, 256],
name='g_f_h2')
self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h2, scope='g_f_bn2'),
name='g_f_relu2')
add_activation_summary(self.fg_h2)
print(self.fg_h2.get_shape().as_list())
encv4 = tf.tile(tf.expand_dims(self.en_h4, axis=1),
[1, 2, 1, 1, 1])
encvo4 = tf.reshape(encv4, [self.batch_size, 2, 4, 4, 256])
encodv4 = tf.concat([self.fg_h2, encvo4], axis=4)
self.fg_h3 = conv3d_transpose(encodv4,
512, [self.batch_size, 4, 8, 8, 256],
name='g_f_h3')
self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h3, scope='g_f_bn3'),
name='g_f_relu3')
add_activation_summary(self.fg_h3)
print(self.fg_h3.get_shape().as_list())
encv3 = tf.tile(tf.expand_dims(self.en_h3, axis=1),
[1, 4, 1, 1, 1])
encvo3 = tf.reshape(encv3, [self.batch_size, 4, 8, 8, 256])
encodv3 = tf.concat([self.fg_h3, encvo3], axis=4)
self.fg_h4 = conv3d_transpose(encodv3,
512,
[self.batch_size, 8, 16, 16, 128],
name='g_f_h4')
self.fg_h4 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h4, scope='g_f_bn4'),
name='g_f_relu4')
add_activation_summary(self.fg_h4)
print(self.fg_h4.get_shape().as_list())
encv2 = tf.tile(tf.expand_dims(self.en_h2, axis=1),
[1, 8, 1, 1, 1])
encvo2 = tf.reshape(encv2, [self.batch_size, 8, 16, 16, 128])
#encvo2 is [32,8,16,16,128] and self.fg_h4 is [32,8,16,16,128] ans concat is [32,8,16,16,256]
encodv2 = tf.concat([self.fg_h4, encvo2], axis=4)
self.fg_h5 = conv3d_transpose(encodv2,
256,
[self.batch_size, 16, 32, 32, 64],
name='g_f_h5')
self.fg_h5 = tf.nn.relu(tf.contrib.layers.batch_norm(
self.fg_h5, scope='g_f_bn5'),
name='g_f_relu5')
add_activation_summary(self.fg_h5)
print(self.fg_h5.get_shape().as_list())
#self.fg_h5 is [32, 16, 32, 32, 64]
encv1 = tf.tile(tf.expand_dims(self.en_h1, axis=1),
[1, 16, 1, 1, 1])
encvo1 = tf.reshape(encv1, [self.batch_size, 16, 32, 32, 64])
encodv1 = tf.concat([self.fg_h5, encvo1], axis=4)
self.fg_h6 = conv3d_transpose(encodv1,
128,
[self.batch_size, 32, 64, 64, 3],
name='g_f_h6')
self.fg_vid = tf.nn.tanh(self.fg_h6, name='g_f_actvcation')
print(self.fg_vid.get_shape().as_list())
# gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg[:, 0, :, :, :]))
# variables = tf.contrib.framework.get_variables(vs)
# # return self.fg_fg, gen_reg, variables
# return self.fg_vid, variables
return self.fg_vid
def ResNeXt(last, group_num=16):
is_training = False
channels = FLAGS.channels
channel_index = -3 if FLAGS.data_format == 'NCHW' else -1
l = 0
last = tf.identity(last, name='input')
# residual blocks
rb = 0
skip2 = last
while rb < FLAGS.res_blocks:
rb += 1
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=1,
out_channels=channels,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=FLAGS.activation,
init_factor=FLAGS.init_activation)
'''
# original version - individual weights for each group
l += 1
with tf.variable_scope('group_conv{}'.format(l)) as scope:
group = tf.split(last, group_num, axis=channel_index)
for _ in range(group_num):
with tf.variable_scope('group{}'.format(_ + 1)) as scope:
group[_] = layers.conv2d(group[_], ksize=3, out_channels=channels // group_num,
stride=1, padding='SAME', data_format=FLAGS.data_format,
bn=FLAGS.batch_norm, train=is_training, activation=None,
init_factor=FLAGS.init_activation)
last = tf.concat(group, axis=channel_index)
last = layers.apply_activation(last, activation=FLAGS.activation,
data_format=FLAGS.data_format)
'''
# modified version - shared weights between groups
l += 1
with tf.variable_scope('group_conv{}'.format(l)) as scope:
# channel en-batching
shape = tf.shape(last)
if FLAGS.data_format == 'NCHW':
shape_batch = [
shape[0] * group_num, channels // group_num, shape[2],
shape[3]
]
else:
shape_divide = [
shape[0], shape[1], shape[2], group_num,
channels // group_num
]
last = tf.reshape(last, shape_divide)
last = tf.transpose(last, (0, 3, 1, 2, 4))
shape_batch = [
shape[0] * group_num, shape[1], shape[2],
channels // group_num
]
last = tf.reshape(last, shape_batch)
# convolution
last = layers.conv2d(last,
ksize=3,
out_channels=channels // group_num,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_activation)
# channel de-batching
if FLAGS.data_format != 'NCHW':
shape_divide = [
shape[0], group_num, shape[1], shape[2],
channels // group_num
]
last = tf.reshape(last, shape_divide)
last = tf.transpose(last, (0, 2, 3, 1, 4))
last = tf.reshape(last, shape)
# activation
last = layers.apply_activation(last,
activation=FLAGS.activation,
data_format=FLAGS.data_format)
l += 1
with tf.variable_scope('conv{}'.format(l)) as scope:
last = layers.conv2d(last,
ksize=1,
out_channels=channels,
stride=1,
padding='SAME',
data_format=FLAGS.data_format,
bn=FLAGS.batch_norm,
train=is_training,
activation=None,
init_factor=FLAGS.init_factor)
with tf.variable_scope('skip_connection{}'.format(l)) as scope:
last = tf.add(last, skip2, 'elementwise_sum')
skip2 = last
last = layers.apply_activation(last,
activation=FLAGS.activation,
data_format=FLAGS.data_format)
# return
last = tf.identity(last, name='output')
return last
