def discriminator(self, mri, pet, reuse = False, disc_type=1):
with tf.variable_scope('discriminator') as scope:
if reuse:
scope.reuse_variables()
inputs = tf.concat(
[mri, pet], self.channel_axis, name='/concat')
# print('input',' ',inputs.shape)
conv1 = ops.conv(inputs, 16, 3 , '/conv1', self.conf.data_type)
conv1 = ops.batch_norm(conv1, '/batch1',ac_fn = ops.leaky_relu)
# print('conv1 ',conv1.shape)
conv2 = ops.conv(conv1, 32, 3, '/conv2', self.conf.data_type,2)
conv2 = ops.batch_norm(conv2, '/batch2',ac_fn = ops.leaky_relu)
# print('conv2 ',conv2.shape)
conv3 = ops.conv(conv2, 64, 3, '/conv3', self.conf.data_type)
conv3 = ops.batch_norm(conv3, '/batch3',ac_fn = ops.leaky_relu)
# print('conv3 ',conv3.shape)
conv4 = ops.conv(conv3, 128, 3, '/conv4', self.conf.data_type,2)
conv4 = ops.batch_norm(conv4, '/batch4',ac_fn = ops.leaky_relu)
# print('conv4 ',conv4.shape)
flatten = tf.contrib.layers.flatten(conv4)
# print('flatten ',flatten.shape)
logits = tf.contrib.layers.fully_connected(flatten, 1, activation_fn=None, scope = '/fully1')
# print('logits ',logits.shape)
features = tf.contrib.layers.fully_connected(flatten, 20, activation_fn=None, scope = '/fully2')
if self.conf.model_option.endswith('metric'):
features = ops.batch_norm(features, '/batch5', ac_fn=None)
return logits, features
def discriminator(self, mri, pet, reuse = False):
with tf.variable_scope('discriminator') as scope:
if reuse:
scope.reuse_variables()
inputs = tf.concat(
[mri, pet], self.channel_axis, name='/concat')
print('input',' ',inputs.shape)
conv1 = ops.conv(inputs, 16, (3, 3, 3), '/conv1', self.conf.data_type)
conv1 = ops.batch_norm(conv1, '/batch1',ac_fn = ops.leaky_relu)
print('conv1 ',conv1.shape)
conv1 = tf.nn.dropout(conv1, 0.5, name='drop1')
conv2 = ops.conv(conv1, 32, (3, 3, 3), '/conv2', self.conf.data_type,2)
conv2 = ops.batch_norm(conv2, '/batch2',ac_fn = ops.leaky_relu)
print('conv2 ',conv2.shape)
conv2 = tf.nn.dropout(conv2, 0.5, name = 'drop2')
conv3 = ops.conv(conv2, 64, (3, 3, 3), '/conv3', self.conf.data_type)
conv3 = ops.batch_norm(conv3, '/batch3',ac_fn = ops.leaky_relu)
print('conv3 ',conv3.shape)
conv3 = tf.nn.dropout(conv3, 0.5, name = 'drop3')
conv4 = ops.conv(conv3, 128, (3, 3, 3), '/conv4', self.conf.data_type,2)
conv4 = ops.batch_norm(conv4, '/batch4',ac_fn = ops.leaky_relu)
conv4 = tf.nn.dropout(conv4, 0.5, name = 'drop4')
print('conv4 ',conv4.shape)
flatten = tf.contrib.layers.flatten(conv4)
logits = tf.contrib.layers.fully_connected(flatten, 5, activation_fn=None, scope = '/fully')
print('flatten ',flatten.shape)
print('logits ',logits.shape)
d = tf.nn.sigmoid(logits)
return d[:,0], logits[:,0], d[:,1:5], logits[:,1:5]
def extract_features_resnet50(self,
im,
name,
is_training=True,
reuse=False):
use_global_pool = True
num_classes = 4096 if use_global_pool else 512
with tf.name_scope(name):
with slim.arg_scope(resnet_utils.resnet_arg_scope()):
out, _ = resnet_v2.resnet_v2_50(inputs=im,
num_classes=num_classes,
global_pool=use_global_pool,
is_training=self.is_training,
spatial_squeeze=True,
scope='resnet_v2_50',
reuse=reuse)
if not use_global_pool:
args = {
'reuse': reuse,
'norm': None,
'activation': tf.nn.relu,
'padding': 'SAME',
'is_training': is_training
}
out_args = copy.deepcopy(args)
out_args['activation'] = None
out = ops.conv(out, 1024, 3, 2, name='conv1', **args)
out = slim.batch_norm(out)
out = ops.conv(out, 2048, 3, 2, name='conv2', **args)
out = slim.batch_norm(out)
out = ops.conv(out, 4096, 3, 2, name='conv3', **out_args)
out = slim.batch_norm(out)
out = tf.squeeze(out, [1, 2], name='SpatialSqueeze')
return out
def build_bottom_block(self, inputs, name):
out_num = inputs.shape[self.channel_axis].value
conv1 = ops.conv(inputs, 2 * out_num, self.conv_size, name + '/conv1',
self.conf.data_type)
conv2 = ops.conv(conv1, out_num, self.conv_size, name + '/conv2',
self.conf.data_type)
return conv2
def pixel_discriminator(self,
input,
channels=3,
ndf=64,
norm_type='instance',
init_type='normal',
init_gain=1.0,
is_training=True,
sigmoid=None):
"""
1x1 PatchGAN Discriminator (pixelGAN)
"""
# 1x1 convolution with 64 filters and no normalization
layer = ops.conv(input,
in_channels=channels,
out_channels=ndf,
filter_size=1,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
use_bias=False,
norm_type=None,
activation_type='LeakyReLU',
is_training=is_training,
scope='layer0',
reuse=self.reuse)
# 1x1 convolution with 128 filters and instance normalization
layer = ops.conv(layer,
in_channels=ndf,
out_channels=2 * ndf,
filter_size=1,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='layer1',
reuse=self.reuse)
# produces a single channel prediction map
layer = ops.conv(layer,
in_channels=2 * ndf,
out_channels=1,
filter_size=1,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=sigmoid,
is_training=is_training,
scope='layer2',
reuse=self.reuse)
return layer
def build_down_block(self, inputs, name, down_outputs, first=False):
out_num = self.conf.start_channel_num if first else 2 * \
inputs.shape[self.channel_axis].value
conv1 = ops.conv(inputs, out_num, self.conv_size,
name+'/conv1', self.conf.data_type)
conv1 = ops.batch_norm(conv1, name+'/batch1')
conv2 = ops.conv(conv1, out_num, self.conv_size,
name+'/conv2', self.conf.data_type,2)
conv2 = ops.batch_norm(conv2, name+'/batch2')
down_outputs.append(conv1)
return conv2
def build_down_block(self, inputs, name, down_outputs, first=False):
out_num = self.conf.start_channel_num if first else 2 * \
inputs.shape[self.channel_axis].value
conv1 = ops.conv(inputs, out_num, self.conv_size, name + '/conv1',
self.conf.data_type)
conv2 = ops.conv(conv1, out_num, self.conv_size, name + '/conv2',
self.conf.data_type)
down_outputs.append(conv2)
pool = ops.pool(conv2, self.pool_size, name + '/pool',
self.conf.data_type)
return pool
def resnet_block(input,
channels,
filter_size=3,
stride=1,
norm_type='instance',
init_type='normal',
init_gain=1.0,
activation_type='ReLU',
is_training=True,
dropout=False,
scope=None,
reuse=False):
"""
Residual block that contains two 3x3 convolution layers with the same number
of filters on both layer.
"""
with tf.variable_scope(scope, reuse=reuse):
conv1 = ops.conv(input,
channels,
channels,
filter_size=filter_size,
stride=stride,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type=activation_type,
is_training=is_training,
scope='res_conv1',
reuse=reuse)
if dropout:
conv1 = tf.nn.dropout(conv1, keep_prob=0.5)
conv2 = ops.conv(conv1,
channels,
channels,
filter_size=filter_size,
stride=stride,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type=None,
is_training=is_training,
scope='res_conv2',
reuse=reuse)
layer = input + conv2
return layer
def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance',
is_training=True, scope=None, reuse=False):
"""
Residual block that contains two 3x3 convolution layers with the same number
of filters on both layer.
"""
with tf.variable_scope(scope, reuse=reuse):
conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride,
padding_type='REFLECT', norm_type=norm_type,
is_training=is_training, scope='res_conv1', reuse=reuse)
conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride,
padding_type='REFLECT', norm_type=norm_type, activation_type=None,
is_training=is_training, scope='res_conv2', reuse=reuse)
layer = input + conv2
return layer
def build_up_block(self, inputs, down_inputs, name, final=False):
out_num = inputs.shape[self.channel_axis].value
conv1 = self.deconv_func()(
inputs, out_num, self.conv_size, name+'/conv1',
self.conf.data_type, action=self.conf.action)
conv1 = ops.batch_norm(conv1, name+'/batch1')
conv1 = tf.concat(
[conv1, down_inputs], self.channel_axis, name=name+'/concat')
conv2 = self.conv_func()(
conv1, out_num, self.conv_size, name+'/conv2', self.conf.data_type)
conv2 = ops.batch_norm(conv2, name+'/batch2')
out_num = self.conf.class_num if final else out_num/2
if final:
conv3 = ops.conv(conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type)
else:
conv3 = ops.conv(conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type)
conv3 = ops.batch_norm(conv3, name+'/batch3')
return conv3
def n_layer_discriminator(self, input, channels=3, n_layers=3, ndf=64,
norm_type='instance', init_type='normal',
init_gain=1.0, is_training=True, sigmoid=None):
"""
N-layer PatchGAN Discriminator
"""
# first layer does not use instance normalization
layer = ops.conv(input, in_channels=channels, out_channels=ndf, filter_size=4,
stride=2, weight_init_type=init_type, weight_init_gain=init_gain,
use_bias=False, norm_type=None, activation_type='LeakyReLU',
is_training=is_training, scope='layer0', reuse=self.reuse)
nf_mult = 1
nf_mult_prev = 1
for idx in range(1, n_layers):
nf_mult_prev = nf_mult
nf_mult = min(2 ** idx, 8)
# perform 4x4 convolutions for n layers with a max of 512 filters
layer = ops.conv(layer, in_channels=ndf*nf_mult_prev, out_channels=ndf*nf_mult,
filter_size=4, stride=2, weight_init_type=init_type, weight_init_gain=init_gain,
norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training,
scope='layer'+str(idx), reuse=self.reuse)
# nth layer of 4x4 convolutions uses a stride of 1
nf_mult_prev = nf_mult
nf_mult = min(2 ** n_layers, 8)
layer = ops.conv(layer, in_channels=ndf*nf_mult_prev, out_channels=ndf*nf_mult,
filter_size=4, stride=1, weight_init_type=init_type, weight_init_gain=init_gain,
norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training,
scope='layer'+str(n_layers), reuse=self.reuse)
# produces a single channel prediction map
layer = ops.conv(layer, in_channels=ndf*nf_mult, out_channels=1, filter_size=4, stride=1,
weight_init_type=init_type, weight_init_gain=init_gain, use_bias=False, norm_type=None,
activation_type=sigmoid, is_training=is_training, scope='d_out', reuse=self.reuse)
return layer
def build_up_block(self, inputs, down_inputs, name, final=False,Decoder=False):
out_num = inputs.shape[self.channel_axis].value
conv1 = self.deconv_func()(
inputs, out_num, self.conv_size, name+'/conv1',
self.conf.data_type)
conv1 = tf.concat(
[conv1, down_inputs], self.channel_axis, name=name+'/concat')
conv2 = self.conv_func()(
conv1, out_num, self.conv_size, name+'/conv2', self.conf.data_type)
if Decoder == True:
conv2 = self.transform.Decoder(conv2,conv2)
out_num = self.conf.class_num if final else out_num/2
conv3 = ops.conv(
conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type)
return conv3
def resnet_generator(self,
input,
channels=3,
ngf=64,
norm_type='instance',
init_type='normal',
init_gain=1.0,
dropout=False,
is_training=True,
n_blocks=16):
"""
Resnet-based generator that contains Resnet blocks in between
some downsampling and upsampling layers.
"""
def resnet_block(input,
channels,
filter_size=3,
stride=1,
norm_type='instance',
init_type='normal',
init_gain=1.0,
activation_type='ReLU',
is_training=True,
dropout=False,
scope=None,
reuse=False):
"""
Residual block that contains two 3x3 convolution layers with the same number
of filters on both layer.
"""
with tf.variable_scope(scope, reuse=reuse):
conv1 = ops.conv(input,
channels,
channels,
filter_size=filter_size,
stride=stride,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type=activation_type,
is_training=is_training,
scope='res_conv1',
reuse=reuse)
if dropout:
conv1 = tf.nn.dropout(conv1, keep_prob=0.5)
conv2 = ops.conv(conv1,
channels,
channels,
filter_size=filter_size,
stride=stride,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type=None,
is_training=is_training,
scope='res_conv2',
reuse=reuse)
layer = input + conv2
return layer
# 9x9 convolution-prelu layer with 64 filters and stride 1
conv1 = ops.conv(input,
in_channels=channels,
out_channels=ngf,
filter_size=9,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type='ParametricReLU',
is_training=is_training,
scope='conv1',
reuse=self.reuse)
res_block = conv1
# n residual blocks
for idx in range(n_blocks):
# residual block that contains two 3x3 convolution layers with 64 filters and stride 1
res_block = resnet_block(res_block,
channels=ngf,
filter_size=3,
stride=1,
norm_type=norm_type,
init_type=init_type,
init_gain=init_gain,
activation_type='ParametricReLU',
is_training=is_training,
dropout=dropout,
scope='res_block' + str(idx),
reuse=self.reuse)
# 3x3 convolution-prelu layer with 64 filters and stride 1
conv2 = ops.conv(res_block,
in_channels=ngf,
out_channels=ngf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type=None,
is_training=is_training,
scope='conv2',
reuse=self.reuse)
res = conv1 + conv2
# 3x3 convolution layer with 256 filters and stride 1
conv3 = ops.conv(res,
in_channels=ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='conv3',
reuse=self.reuse)
# pixel shuffle upsample by factor of 2
upsample1 = ops.pixel_shuffle(conv3, block_size=2)
upsample1 = ops.__parametric_relu(upsample1, name='parametricrelu1')
# 3x3 convolution layer with 256 filters and stride 1
conv4 = ops.conv(upsample1,
in_channels=ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='conv4',
reuse=self.reuse)
# pixel shuffle upsample by factor of 2
upsample2 = ops.pixel_shuffle(conv4, block_size=2)
upsample2 = ops.__parametric_relu(upsample2, name='parametricrelu2')
# 9x9 convolution layer with 3 filters and stride 1
output = ops.conv(upsample2,
in_channels=4 * ngf,
out_channels=channels,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type='tanh',
is_training=is_training,
scope='output',
reuse=self.reuse)
return output
def __call__(self, input, mask=None):
"""
Resnet-based generator that contains Resnet blocks in between
some downsampling and upsampling layers.
"""
def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance',
is_training=True, scope=None, reuse=False):
"""
Residual block that contains two 3x3 convolution layers with the same number
of filters on both layer.
"""
with tf.variable_scope(scope, reuse=reuse):
conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride,
padding_type='REFLECT', norm_type=norm_type,
is_training=is_training, scope='res_conv1', reuse=reuse)
conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride,
padding_type='REFLECT', norm_type=norm_type, activation_type=None,
is_training=is_training, scope='res_conv2', reuse=reuse)
layer = input + conv2
return layer
with tf.variable_scope(self.name):
channels = self.channels+1 if mask is not None else self.channels
concat_input = tf.concat([input, mask], axis=-1) if mask is not None else input
# 7x7 convolution-instance norm-relu layer with 64 filters and stride 1
c7s1_64 = ops.conv(concat_input, in_channels=channels, out_channels=self.ngf, filter_size=7, stride=1,
padding_type='REFLECT', weight_init_type=self.init_type, weight_init_gain=self.init_gain,
norm_type=self.norm_type, is_training=self.is_training, scope='c7s1-64', reuse=self.reuse)
# 3x3 convolution-instance norm-relu layer with 128 filters and stride 2
d128 = ops.conv(c7s1_64, in_channels=self.ngf, out_channels=2*self.ngf, filter_size=3,
stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain,
norm_type=self.norm_type, is_training=self.is_training, scope='d128', reuse=self.reuse)
# 3x3 convolution-instance norm-relu layer with 256 filters and stride 2
d256 = ops.conv(d128, in_channels=2*self.ngf, out_channels=4*self.ngf, filter_size=3,
stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain,
norm_type=self.norm_type, is_training=self.is_training, scope='d256', reuse=self.reuse)
r256 = d256
# Resnet blocks with 256 filters
for idx in range(self.n_blocks):
# residual block that contains two 3x3 convolution layers with 256 filters and stride 1
r256 = resnet_block(r256, channels=4*self.ngf, filter_size=3, stride=1, norm_type=self.norm_type,
is_training=self.is_training, scope='r256-'+str(idx), reuse=self.reuse)
# 3x3 fractional strided convolution-instance norm-relu layer with 128 filters and stride 1/2
u128 = ops.transpose_conv(r256, in_channels=4*self.ngf, out_channels=2*self.ngf, filter_size=3,
stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain,
norm_type=self.norm_type, is_training=self.is_training, scope='u128', reuse=self.reuse)
# 3x3 fractional strided convolution-instance norm-relu layer with 64 filters and stride 1/2
u64 = ops.transpose_conv(u128, in_channels=2*self.ngf, out_channels=self.ngf, filter_size=3,
stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain,
norm_type=self.norm_type, is_training=self.is_training, scope='u64', reuse=self.reuse)
# 7x7 convolution-instance norm-relu layer with 3 filters and stride 1
c7s1_3 = ops.conv(u64, in_channels=self.ngf, out_channels=self.channels, filter_size=7, stride=1,
padding_type='REFLECT', weight_init_type=self.init_type, weight_init_gain=self.init_gain,
use_bias=False, norm_type=None, activation_type=None, is_training=self.is_training,
scope='c7s1-3', reuse=self.reuse)
output = tf.math.tanh(c7s1_3+input, name='gen_out')
# set reuse to True for next call
self.reuse = True
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name)
return output
def build_discriminator(self,
input,
channels=3,
ndf=64,
norm_type='batch',
init_type='normal',
init_gain=0.02,
is_training=True):
"""
SRGAN Discriminator
"""
conv_block1 = ops.conv(input,
in_channels=channels,
out_channels=ndf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv1',
reuse=self.reuse)
conv_block2 = ops.conv(conv_block1,
in_channels=ndf,
out_channels=ndf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv2',
reuse=self.reuse)
conv_block3 = ops.conv(conv_block2,
in_channels=ndf,
out_channels=2 * ndf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv3',
reuse=self.reuse)
conv_block4 = ops.conv(conv_block3,
in_channels=2 * ndf,
out_channels=2 * ndf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv4',
reuse=self.reuse)
conv_block5 = ops.conv(conv_block4,
in_channels=2 * ndf,
out_channels=4 * ndf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv5',
reuse=self.reuse)
conv_block6 = ops.conv(conv_block5,
in_channels=4 * ndf,
out_channels=4 * ndf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv6',
reuse=self.reuse)
conv_block7 = ops.conv(conv_block6,
in_channels=4 * ndf,
out_channels=8 * ndf,
filter_size=3,
stride=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv7',
reuse=self.reuse)
conv_block8 = ops.conv(conv_block7,
in_channels=8 * ndf,
out_channels=8 * ndf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv8',
reuse=self.reuse)
x = ops.flatten(conv_block8)
dense = ops.dense(x,
in_size=x.get_shape().as_list()[1],
out_size=1024,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type='LeakyReLU',
is_training=is_training,
scope='dense',
reuse=self.reuse)
output = ops.dense(dense,
in_size=1024,
out_size=1,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type='sigmoid',
is_training=is_training,
scope='output',
reuse=self.reuse)
return output
def resnet_generator(self,
input,
channels=3,
ngf=64,
norm_type='instance',
init_type='normal',
init_gain=1.0,
dropout=False,
is_training=True,
n_blocks=6):
"""
Resnet-based generator that contains Resnet blocks in between
some downsampling and upsampling layers.
"""
def resnet_block(input,
channels,
filter_size=3,
stride=1,
norm_type='instance',
activation_type='ReLU',
is_training=True,
dropout=False,
scope=None,
reuse=False):
"""
Residual block that contains two 3x3 convolution layers with the same number
of filters on both layer.
"""
with tf.variable_scope(scope, reuse=reuse):
conv1 = ops.conv(input,
channels,
channels,
filter_size=filter_size,
stride=stride,
padding_type='REFLECT',
norm_type=norm_type,
activation_type=activation_type,
is_training=is_training,
scope='res_conv1',
reuse=reuse)
if dropout:
conv1 = tf.nn.dropout(conv1, keep_prob=0.5)
conv2 = ops.conv(conv1,
channels,
channels,
filter_size=filter_size,
stride=stride,
padding_type='REFLECT',
norm_type=norm_type,
activation_type=None,
is_training=is_training,
scope='res_conv2',
reuse=reuse)
layer = input + conv2
return layer
# 7x7 convolution-instance norm-relu layer with 64 filters and stride 1
c7s1_64 = ops.conv(input,
in_channels=channels,
out_channels=ngf,
filter_size=7,
stride=1,
padding_type='REFLECT',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
is_training=is_training,
scope='c7s1-64',
reuse=self.reuse)
# 3x3 convolution-instance norm-relu layer with 128 filters and stride 2
d128 = ops.conv(c7s1_64,
in_channels=ngf,
out_channels=2 * ngf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
is_training=is_training,
scope='d128',
reuse=self.reuse)
# 3x3 convolution-instance norm-relu layer with 256 filters and stride 2
d256 = ops.conv(d128,
in_channels=2 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
is_training=is_training,
scope='d256',
reuse=self.reuse)
r256 = d256
# Resnet blocks with 256 filters
for idx in range(n_blocks):
# residual block that contains two 3x3 convolution layers with 256 filters and stride 1
r256 = resnet_block(r256,
channels=4 * ngf,
filter_size=3,
stride=1,
norm_type=norm_type,
is_training=is_training,
dropout=dropout,
scope='r256-' + str(idx),
reuse=self.reuse)
# 3x3 fractional strided convolution-instance norm-relu layer with 128 filters and stride 1/2
u128 = ops.transpose_conv(r256,
in_channels=4 * ngf,
out_channels=2 * ngf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
is_training=is_training,
scope='u128',
reuse=self.reuse)
# 3x3 fractional strided convolution-instance norm-relu layer with 64 filters and stride 1/2
u64 = ops.transpose_conv(u128,
in_channels=2 * ngf,
out_channels=ngf,
filter_size=3,
stride=2,
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
is_training=is_training,
scope='u64',
reuse=self.reuse)
# 7x7 convolution-instance norm-relu layer with 3 filters and stride 1
c7s1_3 = ops.conv(u64,
in_channels=ngf,
out_channels=channels,
filter_size=7,
stride=1,
padding_type='REFLECT',
weight_init_type=init_type,
weight_init_gain=init_gain,
use_bias=False,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='c7s1-3',
reuse=self.reuse)
return tf.math.tanh(c7s1_3 + input, name='gen_out')
def unet_resize_conv(self,
input,
gray,
channels=3,
ngf=64,
norm_type='instance',
init_type='normal',
init_gain=1.0,
dropout=False,
is_training=True,
self_attention=True,
times_residual=True,
skip=0.0):
"""
Unet-based generator that contains Resnet blocks in between
some downsampling and upsampling layers.
"""
input, pad_left, pad_right, pad_top, pad_bottom = ops.pad_tensor(input)
gray, pad_left, pad_right, pad_top, pad_bottom = ops.pad_tensor(gray)
if self_attention:
gray_2 = ops.max_pooling(gray)
gray_3 = ops.max_pooling(gray_2)
gray_4 = ops.max_pooling(gray_3)
gray_5 = ops.max_pooling(gray_4)
in_channels = channels + 1 if self_attention else channels
x = tf.concat([input, gray], -1) if self_attention else input
x = ops.conv(x,
in_channels=in_channels,
out_channels=ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv1',
reuse=self.reuse)
conv_block1 = ops.conv(x,
in_channels=ngf,
out_channels=ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv2',
reuse=self.reuse)
x = ops.max_pooling(conv_block1)
x = ops.conv(x,
in_channels=ngf,
out_channels=2 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv3',
reuse=self.reuse)
conv_block2 = ops.conv(x,
in_channels=2 * ngf,
out_channels=2 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv4',
reuse=self.reuse)
x = ops.max_pooling(conv_block2)
x = ops.conv(x,
in_channels=2 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv5',
reuse=self.reuse)
conv_block3 = ops.conv(x,
in_channels=4 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv6',
reuse=self.reuse)
x = ops.max_pooling(conv_block3)
x = ops.conv(x,
in_channels=4 * ngf,
out_channels=8 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv7',
reuse=self.reuse)
conv_block4 = ops.conv(x,
in_channels=8 * ngf,
out_channels=8 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv8',
reuse=self.reuse)
x = ops.max_pooling(conv_block4)
x = ops.conv(x,
in_channels=8 * ngf,
out_channels=16 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv9',
reuse=self.reuse)
x = x * gray_5 if self_attention else x
conv_block5 = ops.conv(x,
in_channels=16 * ngf,
out_channels=16 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv10',
reuse=self.reuse)
upsample_block1 = ops.upsample(conv_block5,
rescale_factor=2,
in_channels=16 * ngf,
out_channels=8 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='deconv1',
reuse=self.reuse)
x = conv_block4 * gray_4 if self_attention else conv_block4
x = tf.concat([upsample_block1, x], -1)
x = ops.conv(x,
in_channels=16 * ngf,
out_channels=8 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv11',
reuse=self.reuse)
conv_block6 = ops.conv(x,
in_channels=8 * ngf,
out_channels=8 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv12',
reuse=self.reuse)
upsample_block2 = ops.upsample(conv_block6,
rescale_factor=2,
in_channels=8 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='deconv2',
reuse=self.reuse)
x = conv_block3 * gray_3 if self_attention else conv_block3
x = tf.concat([upsample_block2, x], -1)
x = ops.conv(x,
in_channels=8 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv13',
reuse=self.reuse)
conv_block7 = ops.conv(x,
in_channels=4 * ngf,
out_channels=4 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv14',
reuse=self.reuse)
upsample_block3 = ops.upsample(conv_block7,
rescale_factor=2,
in_channels=4 * ngf,
out_channels=2 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='deconv3',
reuse=self.reuse)
x = conv_block2 * gray_2 if self_attention else conv_block2
x = tf.concat([upsample_block3, x], -1)
x = ops.conv(x,
in_channels=4 * ngf,
out_channels=2 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv15',
reuse=self.reuse)
conv_block8 = ops.conv(x,
in_channels=2 * ngf,
out_channels=2 * ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv16',
reuse=self.reuse)
upsample_block4 = ops.upsample(conv_block8,
rescale_factor=2,
in_channels=2 * ngf,
out_channels=ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='deconv4',
reuse=self.reuse)
x = conv_block1 * gray if self_attention else conv_block1
x = tf.concat([upsample_block4, x], -1)
x = ops.conv(x,
in_channels=2 * ngf,
out_channels=ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv17',
reuse=self.reuse)
conv_block9 = ops.conv(x,
in_channels=ngf,
out_channels=ngf,
filter_size=3,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=norm_type,
activation_type='LeakyReLU',
is_training=is_training,
scope='conv18',
reuse=self.reuse)
latent = ops.conv(conv_block9,
in_channels=ngf,
out_channels=3,
filter_size=1,
stride=1,
padding_type='SAME',
weight_init_type=init_type,
weight_init_gain=init_gain,
norm_type=None,
activation_type=None,
is_training=is_training,
scope='conv19',
reuse=self.reuse)
latent = latent * gray if times_residual else latent
output = latent + input * skip if skip > 0. else latent
output = ops.pad_tensor_back(output, pad_left, pad_right, pad_top,
pad_bottom)
latent = ops.pad_tensor_back(latent, pad_left, pad_right, pad_top,
pad_bottom)
gray = ops.pad_tensor_back(gray, pad_left, pad_right, pad_top,
pad_bottom)
if skip > 0.:
return output, latent
else:
return output
def __call__(self, input):
"""
70x70 PatchGAN discriminator w/ global average pooling at final layer
"""
with tf.variable_scope(self.name):
output = ops.conv(input,
in_channels=self.channels,
out_channels=self.ndf,
filter_size=4,
stride=2,
weight_init_type=self.init_type,
weight_init_gain=self.init_gain,
use_bias=False,
norm_type=None,
activation_type='LeakyReLU',
is_training=self.is_training,
scope='layer0',
reuse=self.reuse)
output = ops.conv(output,
in_channels=self.ndf,
out_channels=2 * self.ndf,
filter_size=4,
stride=2,
weight_init_type=self.init_type,
weight_init_gain=self.init_gain,
norm_type=self.norm_type,
activation_type='LeakyReLU',
is_training=self.is_training,
scope='layer1',
reuse=self.reuse)
output = ops.conv(output,
in_channels=2 * self.ndf,
out_channels=4 * self.ndf,
filter_size=4,
stride=2,
weight_init_type=self.init_type,
weight_init_gain=self.init_gain,
norm_type=self.norm_type,
activation_type='LeakyReLU',
is_training=self.is_training,
scope='layer2',
reuse=self.reuse)
output = ops.conv(output,
in_channels=4 * self.ndf,
out_channels=8 * self.ndf,
filter_size=4,
stride=1,
weight_init_type=self.init_type,
weight_init_gain=self.init_gain,
norm_type=self.norm_type,
activation_type='LeakyReLU',
is_training=self.is_training,
scope='layer3',
reuse=self.reuse)
output = ops.conv(output,
in_channels=8 * self.ndf,
out_channels=1,
filter_size=4,
stride=1,
weight_init_type=self.init_type,
weight_init_gain=self.init_gain,
use_bias=False,
norm_type=None,
activation_type=None,
is_training=self.is_training,
scope='out',
reuse=self.reuse)
output = ops.global_average_pooling(output)
# set reuse to True for next call
self.reuse = True
self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.name)
return output
