def discriminator(self, mri, pet, reuse = False, disc_type=1): with tf.variable_scope('discriminator') as scope: if reuse: scope.reuse_variables() inputs = tf.concat( [mri, pet], self.channel_axis, name='/concat') # print('input',' ',inputs.shape) conv1 = ops.conv(inputs, 16, 3 , '/conv1', self.conf.data_type) conv1 = ops.batch_norm(conv1, '/batch1',ac_fn = ops.leaky_relu) # print('conv1 ',conv1.shape) conv2 = ops.conv(conv1, 32, 3, '/conv2', self.conf.data_type,2) conv2 = ops.batch_norm(conv2, '/batch2',ac_fn = ops.leaky_relu) # print('conv2 ',conv2.shape) conv3 = ops.conv(conv2, 64, 3, '/conv3', self.conf.data_type) conv3 = ops.batch_norm(conv3, '/batch3',ac_fn = ops.leaky_relu) # print('conv3 ',conv3.shape) conv4 = ops.conv(conv3, 128, 3, '/conv4', self.conf.data_type,2) conv4 = ops.batch_norm(conv4, '/batch4',ac_fn = ops.leaky_relu) # print('conv4 ',conv4.shape) flatten = tf.contrib.layers.flatten(conv4) # print('flatten ',flatten.shape) logits = tf.contrib.layers.fully_connected(flatten, 1, activation_fn=None, scope = '/fully1') # print('logits ',logits.shape) features = tf.contrib.layers.fully_connected(flatten, 20, activation_fn=None, scope = '/fully2') if self.conf.model_option.endswith('metric'): features = ops.batch_norm(features, '/batch5', ac_fn=None) return logits, features
def discriminator(self, mri, pet, reuse = False): with tf.variable_scope('discriminator') as scope: if reuse: scope.reuse_variables() inputs = tf.concat( [mri, pet], self.channel_axis, name='/concat') print('input',' ',inputs.shape) conv1 = ops.conv(inputs, 16, (3, 3, 3), '/conv1', self.conf.data_type) conv1 = ops.batch_norm(conv1, '/batch1',ac_fn = ops.leaky_relu) print('conv1 ',conv1.shape) conv1 = tf.nn.dropout(conv1, 0.5, name='drop1') conv2 = ops.conv(conv1, 32, (3, 3, 3), '/conv2', self.conf.data_type,2) conv2 = ops.batch_norm(conv2, '/batch2',ac_fn = ops.leaky_relu) print('conv2 ',conv2.shape) conv2 = tf.nn.dropout(conv2, 0.5, name = 'drop2') conv3 = ops.conv(conv2, 64, (3, 3, 3), '/conv3', self.conf.data_type) conv3 = ops.batch_norm(conv3, '/batch3',ac_fn = ops.leaky_relu) print('conv3 ',conv3.shape) conv3 = tf.nn.dropout(conv3, 0.5, name = 'drop3') conv4 = ops.conv(conv3, 128, (3, 3, 3), '/conv4', self.conf.data_type,2) conv4 = ops.batch_norm(conv4, '/batch4',ac_fn = ops.leaky_relu) conv4 = tf.nn.dropout(conv4, 0.5, name = 'drop4') print('conv4 ',conv4.shape) flatten = tf.contrib.layers.flatten(conv4) logits = tf.contrib.layers.fully_connected(flatten, 5, activation_fn=None, scope = '/fully') print('flatten ',flatten.shape) print('logits ',logits.shape) d = tf.nn.sigmoid(logits) return d[:,0], logits[:,0], d[:,1:5], logits[:,1:5]
def extract_features_resnet50(self, im, name, is_training=True, reuse=False): use_global_pool = True num_classes = 4096 if use_global_pool else 512 with tf.name_scope(name): with slim.arg_scope(resnet_utils.resnet_arg_scope()): out, _ = resnet_v2.resnet_v2_50(inputs=im, num_classes=num_classes, global_pool=use_global_pool, is_training=self.is_training, spatial_squeeze=True, scope='resnet_v2_50', reuse=reuse) if not use_global_pool: args = { 'reuse': reuse, 'norm': None, 'activation': tf.nn.relu, 'padding': 'SAME', 'is_training': is_training } out_args = copy.deepcopy(args) out_args['activation'] = None out = ops.conv(out, 1024, 3, 2, name='conv1', **args) out = slim.batch_norm(out) out = ops.conv(out, 2048, 3, 2, name='conv2', **args) out = slim.batch_norm(out) out = ops.conv(out, 4096, 3, 2, name='conv3', **out_args) out = slim.batch_norm(out) out = tf.squeeze(out, [1, 2], name='SpatialSqueeze') return out
def build_bottom_block(self, inputs, name): out_num = inputs.shape[self.channel_axis].value conv1 = ops.conv(inputs, 2 * out_num, self.conv_size, name + '/conv1', self.conf.data_type) conv2 = ops.conv(conv1, out_num, self.conv_size, name + '/conv2', self.conf.data_type) return conv2
def pixel_discriminator(self, input, channels=3, ndf=64, norm_type='instance', init_type='normal', init_gain=1.0, is_training=True, sigmoid=None): """ 1x1 PatchGAN Discriminator (pixelGAN) """ # 1x1 convolution with 64 filters and no normalization layer = ops.conv(input, in_channels=channels, out_channels=ndf, filter_size=1, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, use_bias=False, norm_type=None, activation_type='LeakyReLU', is_training=is_training, scope='layer0', reuse=self.reuse) # 1x1 convolution with 128 filters and instance normalization layer = ops.conv(layer, in_channels=ndf, out_channels=2 * ndf, filter_size=1, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='layer1', reuse=self.reuse) # produces a single channel prediction map layer = ops.conv(layer, in_channels=2 * ndf, out_channels=1, filter_size=1, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=sigmoid, is_training=is_training, scope='layer2', reuse=self.reuse) return layer
def build_down_block(self, inputs, name, down_outputs, first=False): out_num = self.conf.start_channel_num if first else 2 * \ inputs.shape[self.channel_axis].value conv1 = ops.conv(inputs, out_num, self.conv_size, name+'/conv1', self.conf.data_type) conv1 = ops.batch_norm(conv1, name+'/batch1') conv2 = ops.conv(conv1, out_num, self.conv_size, name+'/conv2', self.conf.data_type,2) conv2 = ops.batch_norm(conv2, name+'/batch2') down_outputs.append(conv1) return conv2
def build_down_block(self, inputs, name, down_outputs, first=False): out_num = self.conf.start_channel_num if first else 2 * \ inputs.shape[self.channel_axis].value conv1 = ops.conv(inputs, out_num, self.conv_size, name + '/conv1', self.conf.data_type) conv2 = ops.conv(conv1, out_num, self.conv_size, name + '/conv2', self.conf.data_type) down_outputs.append(conv2) pool = ops.pool(conv2, self.pool_size, name + '/pool', self.conf.data_type) return pool
def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance', init_type='normal', init_gain=1.0, activation_type='ReLU', is_training=True, dropout=False, scope=None, reuse=False): """ Residual block that contains two 3x3 convolution layers with the same number of filters on both layer. """ with tf.variable_scope(scope, reuse=reuse): conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type=activation_type, is_training=is_training, scope='res_conv1', reuse=reuse) if dropout: conv1 = tf.nn.dropout(conv1, keep_prob=0.5) conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type=None, is_training=is_training, scope='res_conv2', reuse=reuse) layer = input + conv2 return layer
def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance', is_training=True, scope=None, reuse=False): """ Residual block that contains two 3x3 convolution layers with the same number of filters on both layer. """ with tf.variable_scope(scope, reuse=reuse): conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, is_training=is_training, scope='res_conv1', reuse=reuse) conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, activation_type=None, is_training=is_training, scope='res_conv2', reuse=reuse) layer = input + conv2 return layer
def build_up_block(self, inputs, down_inputs, name, final=False): out_num = inputs.shape[self.channel_axis].value conv1 = self.deconv_func()( inputs, out_num, self.conv_size, name+'/conv1', self.conf.data_type, action=self.conf.action) conv1 = ops.batch_norm(conv1, name+'/batch1') conv1 = tf.concat( [conv1, down_inputs], self.channel_axis, name=name+'/concat') conv2 = self.conv_func()( conv1, out_num, self.conv_size, name+'/conv2', self.conf.data_type) conv2 = ops.batch_norm(conv2, name+'/batch2') out_num = self.conf.class_num if final else out_num/2 if final: conv3 = ops.conv(conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type) else: conv3 = ops.conv(conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type) conv3 = ops.batch_norm(conv3, name+'/batch3') return conv3
def n_layer_discriminator(self, input, channels=3, n_layers=3, ndf=64, norm_type='instance', init_type='normal', init_gain=1.0, is_training=True, sigmoid=None): """ N-layer PatchGAN Discriminator """ # first layer does not use instance normalization layer = ops.conv(input, in_channels=channels, out_channels=ndf, filter_size=4, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, use_bias=False, norm_type=None, activation_type='LeakyReLU', is_training=is_training, scope='layer0', reuse=self.reuse) nf_mult = 1 nf_mult_prev = 1 for idx in range(1, n_layers): nf_mult_prev = nf_mult nf_mult = min(2 ** idx, 8) # perform 4x4 convolutions for n layers with a max of 512 filters layer = ops.conv(layer, in_channels=ndf*nf_mult_prev, out_channels=ndf*nf_mult, filter_size=4, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='layer'+str(idx), reuse=self.reuse) # nth layer of 4x4 convolutions uses a stride of 1 nf_mult_prev = nf_mult nf_mult = min(2 ** n_layers, 8) layer = ops.conv(layer, in_channels=ndf*nf_mult_prev, out_channels=ndf*nf_mult, filter_size=4, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='layer'+str(n_layers), reuse=self.reuse) # produces a single channel prediction map layer = ops.conv(layer, in_channels=ndf*nf_mult, out_channels=1, filter_size=4, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, use_bias=False, norm_type=None, activation_type=sigmoid, is_training=is_training, scope='d_out', reuse=self.reuse) return layer
def build_up_block(self, inputs, down_inputs, name, final=False,Decoder=False): out_num = inputs.shape[self.channel_axis].value conv1 = self.deconv_func()( inputs, out_num, self.conv_size, name+'/conv1', self.conf.data_type) conv1 = tf.concat( [conv1, down_inputs], self.channel_axis, name=name+'/concat') conv2 = self.conv_func()( conv1, out_num, self.conv_size, name+'/conv2', self.conf.data_type) if Decoder == True: conv2 = self.transform.Decoder(conv2,conv2) out_num = self.conf.class_num if final else out_num/2 conv3 = ops.conv( conv2, out_num, self.conv_size, name+'/conv3', self.conf.data_type) return conv3
def resnet_generator(self, input, channels=3, ngf=64, norm_type='instance', init_type='normal', init_gain=1.0, dropout=False, is_training=True, n_blocks=16): """ Resnet-based generator that contains Resnet blocks in between some downsampling and upsampling layers. """ def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance', init_type='normal', init_gain=1.0, activation_type='ReLU', is_training=True, dropout=False, scope=None, reuse=False): """ Residual block that contains two 3x3 convolution layers with the same number of filters on both layer. """ with tf.variable_scope(scope, reuse=reuse): conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type=activation_type, is_training=is_training, scope='res_conv1', reuse=reuse) if dropout: conv1 = tf.nn.dropout(conv1, keep_prob=0.5) conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type=None, is_training=is_training, scope='res_conv2', reuse=reuse) layer = input + conv2 return layer # 9x9 convolution-prelu layer with 64 filters and stride 1 conv1 = ops.conv(input, in_channels=channels, out_channels=ngf, filter_size=9, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type='ParametricReLU', is_training=is_training, scope='conv1', reuse=self.reuse) res_block = conv1 # n residual blocks for idx in range(n_blocks): # residual block that contains two 3x3 convolution layers with 64 filters and stride 1 res_block = resnet_block(res_block, channels=ngf, filter_size=3, stride=1, norm_type=norm_type, init_type=init_type, init_gain=init_gain, activation_type='ParametricReLU', is_training=is_training, dropout=dropout, scope='res_block' + str(idx), reuse=self.reuse) # 3x3 convolution-prelu layer with 64 filters and stride 1 conv2 = ops.conv(res_block, in_channels=ngf, out_channels=ngf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type=None, is_training=is_training, scope='conv2', reuse=self.reuse) res = conv1 + conv2 # 3x3 convolution layer with 256 filters and stride 1 conv3 = ops.conv(res, in_channels=ngf, out_channels=4 * ngf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='conv3', reuse=self.reuse) # pixel shuffle upsample by factor of 2 upsample1 = ops.pixel_shuffle(conv3, block_size=2) upsample1 = ops.__parametric_relu(upsample1, name='parametricrelu1') # 3x3 convolution layer with 256 filters and stride 1 conv4 = ops.conv(upsample1, in_channels=ngf, out_channels=4 * ngf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='conv4', reuse=self.reuse) # pixel shuffle upsample by factor of 2 upsample2 = ops.pixel_shuffle(conv4, block_size=2) upsample2 = ops.__parametric_relu(upsample2, name='parametricrelu2') # 9x9 convolution layer with 3 filters and stride 1 output = ops.conv(upsample2, in_channels=4 * ngf, out_channels=channels, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type='tanh', is_training=is_training, scope='output', reuse=self.reuse) return output
def __call__(self, input, mask=None): """ Resnet-based generator that contains Resnet blocks in between some downsampling and upsampling layers. """ def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance', is_training=True, scope=None, reuse=False): """ Residual block that contains two 3x3 convolution layers with the same number of filters on both layer. """ with tf.variable_scope(scope, reuse=reuse): conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, is_training=is_training, scope='res_conv1', reuse=reuse) conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, activation_type=None, is_training=is_training, scope='res_conv2', reuse=reuse) layer = input + conv2 return layer with tf.variable_scope(self.name): channels = self.channels+1 if mask is not None else self.channels concat_input = tf.concat([input, mask], axis=-1) if mask is not None else input # 7x7 convolution-instance norm-relu layer with 64 filters and stride 1 c7s1_64 = ops.conv(concat_input, in_channels=channels, out_channels=self.ngf, filter_size=7, stride=1, padding_type='REFLECT', weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, is_training=self.is_training, scope='c7s1-64', reuse=self.reuse) # 3x3 convolution-instance norm-relu layer with 128 filters and stride 2 d128 = ops.conv(c7s1_64, in_channels=self.ngf, out_channels=2*self.ngf, filter_size=3, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, is_training=self.is_training, scope='d128', reuse=self.reuse) # 3x3 convolution-instance norm-relu layer with 256 filters and stride 2 d256 = ops.conv(d128, in_channels=2*self.ngf, out_channels=4*self.ngf, filter_size=3, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, is_training=self.is_training, scope='d256', reuse=self.reuse) r256 = d256 # Resnet blocks with 256 filters for idx in range(self.n_blocks): # residual block that contains two 3x3 convolution layers with 256 filters and stride 1 r256 = resnet_block(r256, channels=4*self.ngf, filter_size=3, stride=1, norm_type=self.norm_type, is_training=self.is_training, scope='r256-'+str(idx), reuse=self.reuse) # 3x3 fractional strided convolution-instance norm-relu layer with 128 filters and stride 1/2 u128 = ops.transpose_conv(r256, in_channels=4*self.ngf, out_channels=2*self.ngf, filter_size=3, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, is_training=self.is_training, scope='u128', reuse=self.reuse) # 3x3 fractional strided convolution-instance norm-relu layer with 64 filters and stride 1/2 u64 = ops.transpose_conv(u128, in_channels=2*self.ngf, out_channels=self.ngf, filter_size=3, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, is_training=self.is_training, scope='u64', reuse=self.reuse) # 7x7 convolution-instance norm-relu layer with 3 filters and stride 1 c7s1_3 = ops.conv(u64, in_channels=self.ngf, out_channels=self.channels, filter_size=7, stride=1, padding_type='REFLECT', weight_init_type=self.init_type, weight_init_gain=self.init_gain, use_bias=False, norm_type=None, activation_type=None, is_training=self.is_training, scope='c7s1-3', reuse=self.reuse) output = tf.math.tanh(c7s1_3+input, name='gen_out') # set reuse to True for next call self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return output
def build_discriminator(self, input, channels=3, ndf=64, norm_type='batch', init_type='normal', init_gain=0.02, is_training=True): """ SRGAN Discriminator """ conv_block1 = ops.conv(input, in_channels=channels, out_channels=ndf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type='LeakyReLU', is_training=is_training, scope='conv1', reuse=self.reuse) conv_block2 = ops.conv(conv_block1, in_channels=ndf, out_channels=ndf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv2', reuse=self.reuse) conv_block3 = ops.conv(conv_block2, in_channels=ndf, out_channels=2 * ndf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv3', reuse=self.reuse) conv_block4 = ops.conv(conv_block3, in_channels=2 * ndf, out_channels=2 * ndf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv4', reuse=self.reuse) conv_block5 = ops.conv(conv_block4, in_channels=2 * ndf, out_channels=4 * ndf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv5', reuse=self.reuse) conv_block6 = ops.conv(conv_block5, in_channels=4 * ndf, out_channels=4 * ndf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv6', reuse=self.reuse) conv_block7 = ops.conv(conv_block6, in_channels=4 * ndf, out_channels=8 * ndf, filter_size=3, stride=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv7', reuse=self.reuse) conv_block8 = ops.conv(conv_block7, in_channels=8 * ndf, out_channels=8 * ndf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv8', reuse=self.reuse) x = ops.flatten(conv_block8) dense = ops.dense(x, in_size=x.get_shape().as_list()[1], out_size=1024, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type='LeakyReLU', is_training=is_training, scope='dense', reuse=self.reuse) output = ops.dense(dense, in_size=1024, out_size=1, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type='sigmoid', is_training=is_training, scope='output', reuse=self.reuse) return output
def resnet_generator(self, input, channels=3, ngf=64, norm_type='instance', init_type='normal', init_gain=1.0, dropout=False, is_training=True, n_blocks=6): """ Resnet-based generator that contains Resnet blocks in between some downsampling and upsampling layers. """ def resnet_block(input, channels, filter_size=3, stride=1, norm_type='instance', activation_type='ReLU', is_training=True, dropout=False, scope=None, reuse=False): """ Residual block that contains two 3x3 convolution layers with the same number of filters on both layer. """ with tf.variable_scope(scope, reuse=reuse): conv1 = ops.conv(input, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, activation_type=activation_type, is_training=is_training, scope='res_conv1', reuse=reuse) if dropout: conv1 = tf.nn.dropout(conv1, keep_prob=0.5) conv2 = ops.conv(conv1, channels, channels, filter_size=filter_size, stride=stride, padding_type='REFLECT', norm_type=norm_type, activation_type=None, is_training=is_training, scope='res_conv2', reuse=reuse) layer = input + conv2 return layer # 7x7 convolution-instance norm-relu layer with 64 filters and stride 1 c7s1_64 = ops.conv(input, in_channels=channels, out_channels=ngf, filter_size=7, stride=1, padding_type='REFLECT', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, is_training=is_training, scope='c7s1-64', reuse=self.reuse) # 3x3 convolution-instance norm-relu layer with 128 filters and stride 2 d128 = ops.conv(c7s1_64, in_channels=ngf, out_channels=2 * ngf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, is_training=is_training, scope='d128', reuse=self.reuse) # 3x3 convolution-instance norm-relu layer with 256 filters and stride 2 d256 = ops.conv(d128, in_channels=2 * ngf, out_channels=4 * ngf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, is_training=is_training, scope='d256', reuse=self.reuse) r256 = d256 # Resnet blocks with 256 filters for idx in range(n_blocks): # residual block that contains two 3x3 convolution layers with 256 filters and stride 1 r256 = resnet_block(r256, channels=4 * ngf, filter_size=3, stride=1, norm_type=norm_type, is_training=is_training, dropout=dropout, scope='r256-' + str(idx), reuse=self.reuse) # 3x3 fractional strided convolution-instance norm-relu layer with 128 filters and stride 1/2 u128 = ops.transpose_conv(r256, in_channels=4 * ngf, out_channels=2 * ngf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, is_training=is_training, scope='u128', reuse=self.reuse) # 3x3 fractional strided convolution-instance norm-relu layer with 64 filters and stride 1/2 u64 = ops.transpose_conv(u128, in_channels=2 * ngf, out_channels=ngf, filter_size=3, stride=2, weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, is_training=is_training, scope='u64', reuse=self.reuse) # 7x7 convolution-instance norm-relu layer with 3 filters and stride 1 c7s1_3 = ops.conv(u64, in_channels=ngf, out_channels=channels, filter_size=7, stride=1, padding_type='REFLECT', weight_init_type=init_type, weight_init_gain=init_gain, use_bias=False, norm_type=None, activation_type=None, is_training=is_training, scope='c7s1-3', reuse=self.reuse) return tf.math.tanh(c7s1_3 + input, name='gen_out')
def unet_resize_conv(self, input, gray, channels=3, ngf=64, norm_type='instance', init_type='normal', init_gain=1.0, dropout=False, is_training=True, self_attention=True, times_residual=True, skip=0.0): """ Unet-based generator that contains Resnet blocks in between some downsampling and upsampling layers. """ input, pad_left, pad_right, pad_top, pad_bottom = ops.pad_tensor(input) gray, pad_left, pad_right, pad_top, pad_bottom = ops.pad_tensor(gray) if self_attention: gray_2 = ops.max_pooling(gray) gray_3 = ops.max_pooling(gray_2) gray_4 = ops.max_pooling(gray_3) gray_5 = ops.max_pooling(gray_4) in_channels = channels + 1 if self_attention else channels x = tf.concat([input, gray], -1) if self_attention else input x = ops.conv(x, in_channels=in_channels, out_channels=ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv1', reuse=self.reuse) conv_block1 = ops.conv(x, in_channels=ngf, out_channels=ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv2', reuse=self.reuse) x = ops.max_pooling(conv_block1) x = ops.conv(x, in_channels=ngf, out_channels=2 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv3', reuse=self.reuse) conv_block2 = ops.conv(x, in_channels=2 * ngf, out_channels=2 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv4', reuse=self.reuse) x = ops.max_pooling(conv_block2) x = ops.conv(x, in_channels=2 * ngf, out_channels=4 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv5', reuse=self.reuse) conv_block3 = ops.conv(x, in_channels=4 * ngf, out_channels=4 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv6', reuse=self.reuse) x = ops.max_pooling(conv_block3) x = ops.conv(x, in_channels=4 * ngf, out_channels=8 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv7', reuse=self.reuse) conv_block4 = ops.conv(x, in_channels=8 * ngf, out_channels=8 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv8', reuse=self.reuse) x = ops.max_pooling(conv_block4) x = ops.conv(x, in_channels=8 * ngf, out_channels=16 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv9', reuse=self.reuse) x = x * gray_5 if self_attention else x conv_block5 = ops.conv(x, in_channels=16 * ngf, out_channels=16 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv10', reuse=self.reuse) upsample_block1 = ops.upsample(conv_block5, rescale_factor=2, in_channels=16 * ngf, out_channels=8 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='deconv1', reuse=self.reuse) x = conv_block4 * gray_4 if self_attention else conv_block4 x = tf.concat([upsample_block1, x], -1) x = ops.conv(x, in_channels=16 * ngf, out_channels=8 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv11', reuse=self.reuse) conv_block6 = ops.conv(x, in_channels=8 * ngf, out_channels=8 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv12', reuse=self.reuse) upsample_block2 = ops.upsample(conv_block6, rescale_factor=2, in_channels=8 * ngf, out_channels=4 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='deconv2', reuse=self.reuse) x = conv_block3 * gray_3 if self_attention else conv_block3 x = tf.concat([upsample_block2, x], -1) x = ops.conv(x, in_channels=8 * ngf, out_channels=4 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv13', reuse=self.reuse) conv_block7 = ops.conv(x, in_channels=4 * ngf, out_channels=4 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv14', reuse=self.reuse) upsample_block3 = ops.upsample(conv_block7, rescale_factor=2, in_channels=4 * ngf, out_channels=2 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='deconv3', reuse=self.reuse) x = conv_block2 * gray_2 if self_attention else conv_block2 x = tf.concat([upsample_block3, x], -1) x = ops.conv(x, in_channels=4 * ngf, out_channels=2 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv15', reuse=self.reuse) conv_block8 = ops.conv(x, in_channels=2 * ngf, out_channels=2 * ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv16', reuse=self.reuse) upsample_block4 = ops.upsample(conv_block8, rescale_factor=2, in_channels=2 * ngf, out_channels=ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='deconv4', reuse=self.reuse) x = conv_block1 * gray if self_attention else conv_block1 x = tf.concat([upsample_block4, x], -1) x = ops.conv(x, in_channels=2 * ngf, out_channels=ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv17', reuse=self.reuse) conv_block9 = ops.conv(x, in_channels=ngf, out_channels=ngf, filter_size=3, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=norm_type, activation_type='LeakyReLU', is_training=is_training, scope='conv18', reuse=self.reuse) latent = ops.conv(conv_block9, in_channels=ngf, out_channels=3, filter_size=1, stride=1, padding_type='SAME', weight_init_type=init_type, weight_init_gain=init_gain, norm_type=None, activation_type=None, is_training=is_training, scope='conv19', reuse=self.reuse) latent = latent * gray if times_residual else latent output = latent + input * skip if skip > 0. else latent output = ops.pad_tensor_back(output, pad_left, pad_right, pad_top, pad_bottom) latent = ops.pad_tensor_back(latent, pad_left, pad_right, pad_top, pad_bottom) gray = ops.pad_tensor_back(gray, pad_left, pad_right, pad_top, pad_bottom) if skip > 0.: return output, latent else: return output
def __call__(self, input): """ 70x70 PatchGAN discriminator w/ global average pooling at final layer """ with tf.variable_scope(self.name): output = ops.conv(input, in_channels=self.channels, out_channels=self.ndf, filter_size=4, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, use_bias=False, norm_type=None, activation_type='LeakyReLU', is_training=self.is_training, scope='layer0', reuse=self.reuse) output = ops.conv(output, in_channels=self.ndf, out_channels=2 * self.ndf, filter_size=4, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, activation_type='LeakyReLU', is_training=self.is_training, scope='layer1', reuse=self.reuse) output = ops.conv(output, in_channels=2 * self.ndf, out_channels=4 * self.ndf, filter_size=4, stride=2, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, activation_type='LeakyReLU', is_training=self.is_training, scope='layer2', reuse=self.reuse) output = ops.conv(output, in_channels=4 * self.ndf, out_channels=8 * self.ndf, filter_size=4, stride=1, weight_init_type=self.init_type, weight_init_gain=self.init_gain, norm_type=self.norm_type, activation_type='LeakyReLU', is_training=self.is_training, scope='layer3', reuse=self.reuse) output = ops.conv(output, in_channels=8 * self.ndf, out_channels=1, filter_size=4, stride=1, weight_init_type=self.init_type, weight_init_gain=self.init_gain, use_bias=False, norm_type=None, activation_type=None, is_training=self.is_training, scope='out', reuse=self.reuse) output = ops.global_average_pooling(output) # set reuse to True for next call self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return output