def __call__(self, feature, reuse=False): with tf.variable_scope(self.name): if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False h0 = lrelu(conv2d(feature, 64, name='d_h0_conv')) # h0 is (16 x 16 x 64) h1 = lrelu( instance_norm( conv2d(h0, self.hidden_dim * 2, name='d_h1_conv'), 'd_bn1')) # h1 is (8 x 8 x self.hidden_dim*2) h2 = lrelu( instance_norm( conv2d(h1, self.hidden_dim * 4, name='d_h2_conv'), 'd_bn2')) # h2 is (4 x 4 x self.hidden_dim*4) h3 = lrelu( instance_norm( conv2d(h2, self.hidden_dim * 8, s=1, name='d_h3_conv'), 'd_bn3')) # h3 is (4 x 4 x self.hidden_dim*8) h4 = tf.nn.sigmoid(conv2d(h3, 1, s=1, name='d_h3_pred')) # h4 is (4 x 4 x 1) return tf.identity(h4, name=self.name + '_output')
def discriminate(self, x_var, reuse=False): with tf.variable_scope("discriminator") as scope: if reuse == True: scope.reuse_variables() conv1 = lrelu(conv2d(x_var, output_dim=64, name='dis_conv1')) conv2 = lrelu( instance_norm(conv2d(conv1, output_dim=128, name='dis_conv2'), scope='dis_bn1')) conv3 = lrelu( instance_norm(conv2d(conv2, output_dim=256, name='dis_conv3'), scope='dis_bn2')) conv4 = conv2d(conv3, output_dim=512, name='dis_conv4') middle_conv = conv4 conv4 = lrelu(instance_norm(conv4, scope='dis_bn3')) conv5 = lrelu( instance_norm(conv2d(conv4, output_dim=1024, name='dis_conv5'), scope='dis_bn4')) conv6 = conv2d(conv5, output_dim=2, k_w=4, k_h=4, d_h=1, d_w=1, padding='VALID', name='dis_conv6') return conv6, middle_conv
def encode_decode(self, x_var, x_exemplar, reuse=False): with tf.variable_scope("encode_decode") as scope: if reuse == True: scope.reuse_variables() x_var = tf.concat([x_var, x_exemplar], axis=3) conv1 = tf.nn.relu( instance_norm(conv2d(x_var, output_dim=64, k_w=7, k_h=7, d_w=1, d_h=1, name='e_c1'), scope='e_in1')) conv2 = tf.nn.relu( instance_norm(conv2d(conv1, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c2'), scope='e_in2')) conv3 = tf.nn.relu( instance_norm(conv2d(conv2, output_dim=256, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c3'), scope='e_in3')) r1 = Residual(conv3, residual_name='re_1') r2 = Residual(r1, residual_name='re_2') r3 = Residual(r2, residual_name='re_3') r4 = Residual(r3, residual_name='re_4') r5 = Residual(r4, residual_name='re_5') r6 = Residual(r5, residual_name='re_6') g_deconv1 = tf.nn.relu(instance_norm(de_conv(r6, output_shape=[self.batch_size, self.output_size/2, self.output_size/2, 128], name='gen_deconv1'), scope="gen_in")) # for 1 g_deconv_1_1 = tf.nn.relu(instance_norm(de_conv(g_deconv1, output_shape=[self.batch_size, self.output_size, self.output_size, 32], name='g_deconv_1_1'), scope='gen_in_1_1')) g_deconv_1_1_x = tf.concat([g_deconv_1_1, x_var], axis=3) x_tilde1 = conv2d(g_deconv_1_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_1_2') return tf.nn.tanh(x_tilde1)
def Kdiscriminator(self, x, reuse=False, name='Kd_'): with tf.variable_scope(name_or_scope=name, reuse=reuse): x = Conv(name='conv1', x=x, filter_size=4, in_filters=self.channel, out_filters=self.n_feats, strides=2, padding="SAME") x = instance_norm(name='inst_norm1', x=x, dim=self.n_feats) x = tf.nn.leaky_relu(x) prev = 1 n = 1 for i in range(self.Kdiscrim_blocks): prev = n n = min(2**(i + 1), 8) x = Conv(name='conv%02d' % i, x=x, filter_size=4, in_filters=self.n_feats * prev, out_filters=self.n_feats * n, strides=2, padding="SAME") x = instance_norm(name='instance_norm%02d' % i, x=x, dim=self.n_feats * n) x = tf.nn.leaky_relu(x) prev = n n = min(2**self.Kdiscrim_blocks, 8) x = Conv(name='conv_d1', x=x, filter_size=4, in_filters=self.n_feats * prev, out_filters=self.n_feats * n, strides=1, padding="SAME") x = instance_norm(name='instance_norm_d1', x=x, dim=self.n_feats * n) x = tf.nn.leaky_relu(x) x = Conv(name='conv_d2', x=x, filter_size=4, in_filters=self.n_feats * n, out_filters=1, strides=1, padding="SAME") x = tf.nn.sigmoid(x) return x
def residule_block(x, dim, ks=3, s=1, name='res'): p = int((ks - 1) / 2) y = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]], "CONSTANT") #CONSTANT y = instance_norm( conv2d(y, dim, ks, s, padding='VALID', name=name + '_c1'), name + '_in1') y = tf.pad(tf.nn.relu(y), [[0, 0], [p, p], [p, p], [0, 0]], "CONSTANT") y = instance_norm( conv2d(y, dim, ks, s, padding='VALID', name=name + '_c2'), name + '_in2') return y + x
def encode(self, x, reuse=False): with tf.variable_scope("encode") as scope: if reuse == True: scope.reuse_variables() conv1 = tf.nn.relu( instance_norm(conv2d(x, output_dim=32, k_w=7, k_h=7, d_w=1, d_h=1, name='e_c1'), scope='e_in1')) conv2 = tf.nn.relu( instance_norm(conv2d(conv1, output_dim=64, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c2'), scope='e_in2')) conv3 = tf.nn.relu( instance_norm(conv2d(conv2, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c3'), scope='e_in3')) conv4 = tf.nn.relu( instance_norm(conv2d(conv3, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c4'), scope='e_in4')) bottleneck = tf.reshape(conv4, [self.batch_size, -1]) content = fully_connect(bottleneck, output_size=128, scope='e_ful1') return content
def encode_decode_2(self, x, reuse=False): with tf.variable_scope("encode_decode_2") as scope: if reuse == True: scope.reuse_variables() conv1 = lrelu( instance_norm( conv2d(x, output_dim=64, k_w=5, k_h=5, d_w=1, d_h=1, name='e_c1'), scope='e_in1', )) conv2 = lrelu( instance_norm(conv2d(conv1, output_dim=128, name='e_c2'), scope='e_in2')) conv3 = lrelu( instance_norm(conv2d(conv2, output_dim=256, name='e_c3'), scope='e_in3')) # for x_{1} de_conv1 = lrelu( instance_norm( de_conv(conv3, output_shape=[self.batch_size, 64, 64, 128], name='e_d1', k_h=3, k_w=3), scope='e_in4', )) de_conv2 = lrelu( instance_norm( de_conv(de_conv1, output_shape=[self.batch_size, 128, 128, 64], name='e_d2', k_w=3, k_h=3), scope='e_in5', )) x_tilde1 = conv2d(de_conv2, output_dim=3, d_h=1, d_w=1, name='e_c4') return x_tilde1
def __call__(self, input, reuse=False): with tf.variable_scope(self.name): if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False act = tf.nn.relu _, h, w, c = input.shape.as_list() # Justin Johnson's model from https://github.com/jcjohnson/fast-neural-style/ # The network with 9 blocks consists of: c7s1-32, d64, d128, R128, R128, R128, # R128, R128, R128, R128, R128, R128, u64, u32, c7s1-3 # input shape == (32 x 32 x 128) c1 = act( instance_norm(conv2d(input, c, 7, 1, name='g_e1_c'), 'g_e1_bn')) # c1 shape == (32 x 32 x 128) c2 = act( instance_norm(conv2d(c1, c * 2, 3, 2, name='g_e2_c'), 'g_e2_bn')) # c2 shape == (16 x 16 x 256) c3 = act( instance_norm(conv2d(c2, c * 4, 3, 2, name='g_e3_c'), 'g_e3_bn')) # c3 shape == (8 x 8 x 512) # define G network with 9 resnet blocks r1 = residule_block(c3, c * 4, name='g_r1') r2 = residule_block(r1, c * 4, name='g_r2') r3 = residule_block(r2, c * 4, name='g_r3') r4 = residule_block(r3, c * 4, name='g_r4') r5 = residule_block(r4, c * 4, name='g_r5') r6 = residule_block(r5, c * 4, name='g_r6') r7 = residule_block(r6, c * 4, name='g_r7') r8 = residule_block(r7, c * 4, name='g_r8') r9 = residule_block(r8, c * 4, name='g_r9') d1 = act( instance_norm(deconv2d(r9, c * 2, 3, 2, name='g_d1_dc'), 'g_d1_bn')) # d1 shape == (16 x 16 x 256) d2 = act( instance_norm(deconv2d(d1, c, 3, 2, name='g_d2_dc'), 'g_d2_bn')) # d1 shape == (32 x 32 x 128) output = tf.nn.tanh(deconv2d(d2, c, 7, 1, name='g_output_dc')) # output shape == (32, 32, 128) return tf.identity(output, name=self.name + '_output')
def encode_decode_1(self, x, reuse=False): with tf.variable_scope("encode_decode_1") as scope: if reuse == True: scope.reuse_variables() conv1 = lrelu(instance_norm(conv2d(x, output_dim=64, k_w=5, k_h=5, d_w=1, d_h=1, name='e_c1'), scope='e_in1')) conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='e_c2'), scope='e_in2')) conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='e_c3'), scope='e_in3')) # for x_{1} de_conv1 = lrelu(instance_norm(de_conv(conv3, output_shape=[self.batch_size, 64, 64, 128] , name='e_d1', k_h=3, k_w=3), scope='e_in4')) de_conv2 = lrelu(instance_norm(de_conv(de_conv1, output_shape=[self.batch_size, 128, 128, 64] , name='e_d2', k_w=3, k_h=3), scope='e_in5')) x_tilde1 = conv2d(de_conv2, output_dim=3, d_h=1, d_w=1, name='e_c4') return x_tilde1
def discriminate(self, x_var, reuse=False): with tf.variable_scope("discriminator") as scope: if reuse == True: scope.reuse_variables() conv1 = lrelu(conv2d(x_var, output_dim=64, name='dis_conv1')) conv2 = lrelu(instance_norm(conv2d(conv1, output_dim=128, name='dis_conv2'), scope='dis_bn1')) conv3 = lrelu(instance_norm(conv2d(conv2, output_dim=256, name='dis_conv3'), scope='dis_bn2')) conv4 = conv2d(conv3, output_dim=512, name='dis_conv4') middle_conv = conv4 conv4 = lrelu(instance_norm(conv4, scope='dis_bn3')) conv5 = lrelu(instance_norm(conv2d(conv4, output_dim=1024, name='dis_conv5'), scope='dis_bn4')) conv6 = conv2d(conv5, output_dim=2, k_w=4, k_h=4, d_h=1, d_w=1, padding='VALID', name='dis_conv6') return conv6, middle_conv
def generator(self, x, c, reuse=False): print("Generator ...........") with tf.variable_scope('generator') as scope: if (reuse): scope.reuse_variables() c = tf.reshape(c, [-1, 1, 1, self.feature_length]) c = tf.tile(c, [1, 64, 64, 1]) inputs = tf.concat([x, c], axis=3) # print(inputs) # Iinitial concat conv layer # pad 3 => https://arxiv.org/pdf/1711.09020.pdf outputs = tf.pad(inputs, [[0, 0], [3, 3], [3, 3], [0, 0]]) outputs = ops.conv2d(outputs, 64, 7, 1, 'VALID', scope="g_init") # Downsize twice outputs = ops.conv2d(outputs, 128, 4, 2, scope="g_down_sample_1") outputs = ops.conv2d(outputs, 256, 4, 2, scope="g_down_sample_2") # Resblock CONV-(N256, K3x3, S1, P1), IN, ReLU outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_1") outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_2") outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_3") outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_4") outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_5") outputs = residule_block(outputs, 256, 3, 1, scope="g_resblock_6") # Upsampling twice outputs = ops.instance_norm( ops.relu( ops.deconv2d(outputs, 128, 4, 2, scope='g_upsampling_1')), 'g_in_1') outputs = ops.instance_norm( ops.relu( ops.deconv2d(outputs, 64, 4, 2, scope='g_upsampling_2')), 'g_in_2') # pad 3 => https://arxiv.org/pdf/1711.09020.pdf outputs = tf.pad(outputs, [[0, 0], [3, 3], [3, 3], [0, 0]]) outputs = ops.tanh( ops.conv2d(outputs, 3, 7, 1, 'VALID', scope='g_out')) return outputs
def residule_block(inputs, filters, kernel_size, stride, scope): with tf.variable_scope(scope): outputs = ops.conv2d(inputs, filters, kernel_size, stride, scope='conv_1') outputs = ops.instance_norm(outputs, 'in_1') outputs = ops.relu(outputs) outputs = ops.conv2d(outputs, filters, kernel_size, stride, scope='conv_2') outputs = ops.instance_norm(outputs, 'in_2') return outputs + inputs
def spade(x, condition, num_hidden=128, use_spectral_norm=False, scope="spade"): """Spatially Adaptive Instance Norm implementation. Given x, applies a normalization that is conditioned on condition. Args: x: [B, H, W, C] A tensor to apply normalization condition: [B, H', W', C'] A tensor to condition the normalization parameters num_hidden: (int) The number of intermediate channels to create the SPADE layer with use_spectral_norm: (bool) If true, creates convolutions with spectral normalization applied to its weights scope: (str) The variable scope Returns: A tensor that has been normalized by parameters estimated by cond. """ channel = x.shape[-1] with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE): x_normed = ops.instance_norm(x) # Produce affine parameters from conditioning image. # First resize. height, width = x.get_shape().as_list()[1:3] condition = diff_resize_area(condition, [height, width]) condition = ops.sn_conv(condition, num_hidden, kernel_size=3, use_spectral_norm=use_spectral_norm, scope="conv_cond") condition = tf.nn.relu(condition) gamma = ops.sn_conv(condition, channel, kernel_size=3, use_spectral_norm=use_spectral_norm, scope="gamma", pad_type="CONSTANT") beta = ops.sn_conv(condition, channel, kernel_size=3, use_spectral_norm=use_spectral_norm, scope="beta", pad_type="CONSTANT") out = x_normed * (1 + gamma) + beta return out
def encode_decode(self, input_x, img_mask, guided_fp_left, guided_fp_right, use_sp=False, reuse=False): with tf.variable_scope("ed") as scope: if reuse == True: scope.reuse_variables() #encode x = tf.concat([input_x, img_mask], axis=3) for i in range(6): c_dim = np.minimum(16 * np.power(2, i), 256) if i == 0: x = tf.nn.relu( instance_norm(conv2d(x, output_dim=c_dim, k_w=7, k_h=7, d_w=1, d_h=1, use_sp=use_sp, name='e_c{}'.format(i)) , scope='e_in_{}'.format(i))) else: x = tf.nn.relu( instance_norm(conv2d(x, output_dim=c_dim, k_w=4, k_h=4, d_w=2, d_h=2, use_sp=use_sp, name='e_c{}'.format(i)) , scope='e_in_{}'.format(i))) bottleneck = tf.reshape(x, shape=[self.batch_size, -1]) bottleneck = fully_connect(bottleneck, output_size=256, use_sp=use_sp, scope='e_ful1') bottleneck = tf.concat([bottleneck, guided_fp_left, guided_fp_right], axis=1) de_x = tf.nn.relu(fully_connect(bottleneck, output_size=256*8*8, use_sp=use_sp, scope='d_ful1')) de_x = tf.reshape(de_x, shape=[self.batch_size, 8, 8, 256]) #de_x = tf.tile(de_x, (1, 8, 8, 1), name='tile') #decode for i in range(5): c_dim = np.maximum(256 / np.power(2, i), 16) output_dim = 16 * np.power(2, i) print de_x de_x = tf.nn.relu(instance_norm(de_conv(de_x, output_shape=[self.batch_size, output_dim, output_dim, c_dim], use_sp=use_sp, name='g_deconv_{}'.format(i)), scope='g_in_{}'.format(i))) #de_x = tf.concat([de_x, input_x], axis=3) x_tilde1 = conv2d(de_x, output_dim=3, k_w=7, k_h=7, d_h=1, d_w=1, use_sp=use_sp, name='g_conv1') return tf.nn.tanh(x_tilde1)
def encoder(x, scope="spade_encoder"): """Encoder that outputs global N(mu, sig) parameters. Args: x: [B, H, W, 4] an RGBD image (usually the initial image) which is used to sample noise from a distirbution to feed into the refinement network. Range [0, 1]. scope: (str) variable scope Returns: (mu, logvar) are [B, 256] tensors of parameters defining a normal distribution to sample from. """ x = 2 * x - 1 num_channel = 16 with tf.compat.v1.variable_scope(scope, reuse=tf.compat.v1.AUTO_REUSE): x = ops.sn_conv(x, num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_0") x = ops.instance_norm(x, scope="inst_norm_0") x = ops.leaky_relu(x, 0.2) x = ops.sn_conv(x, 2 * num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_1") x = ops.instance_norm(x, scope="inst_norm_1") x = ops.leaky_relu(x, 0.2) x = ops.sn_conv(x, 4 * num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_2") x = ops.instance_norm(x, scope="inst_norm_2") x = ops.leaky_relu(x, 0.2) x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_3") x = ops.instance_norm(x, scope="inst_norm_3") x = ops.leaky_relu(x, 0.2) x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_4") x = ops.instance_norm(x, scope="inst_norm_4") x = ops.leaky_relu(x, 0.2) x = ops.sn_conv(x, 8 * num_channel, kernel_size=3, stride=2, use_bias=True, use_spectral_norm=True, scope="conv_5") x = ops.instance_norm(x, scope="inst_norm_5") x = ops.leaky_relu(x, 0.2) mu = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING, scope="linear_mu") logvar = ops.fully_connected(x, config.DIM_OF_STYLE_EMBEDDING, scope="linear_logvar") return mu, logvar
def encode_decode2(self, x, img_mask, pg=1, is_trans=False, alpha_trans=0.01, reuse=False): with tf.variable_scope("ed") as scope: if reuse == True: scope.reuse_variables() x = tf.concat([x, img_mask], axis=3) if is_trans: x_trans = downscale2d(x) #fromrgb x_trans = tf.nn.relu(instance_norm(conv2d(x_trans, output_dim=self.get_nf(pg - 2), k_w=1, k_h=1, d_h=1, d_w=1, name='gen_rgb_e_{}'.format(x_trans.shape[1])), scope='gen_rgb_e_in_{}'.format(x_trans.shape[1]))) #fromrgb x = tf.nn.relu(instance_norm(conv2d(x, output_dim=self.get_nf(pg - 1), k_w=1, k_h=1, d_h=1, d_w=1, name='gen_rgb_e_{}'.format(x.shape[1])), scope='gen_rgb_e_in_{}'.format(x.shape[1]))) for i in range(pg - 1): print "encode", x.shape x = tf.nn.relu(instance_norm(conv2d(x, output_dim=self.get_nf(pg - 2 - i), d_h=1, d_w=1, name='gen_conv_e_{}'.format(x.shape[1])), scope='gen_conv_e_in_{}'.format(x.shape[1]))) x = downscale2d(x) if i == 0 and is_trans: x = alpha_trans * x + (1 - alpha_trans) * x_trans up_x = tf.nn.relu( instance_norm(dilated_conv2d(x, output_dim=512, k_w=3, k_h=3, rate=4, name='gen_conv_dilated'), scope='gen_conv_in')) up_x = tf.nn.relu(instance_norm(conv2d(up_x, output_dim=self.get_nf(1), d_w=1, d_h=1, name='gen_conv_d'), scope='gen_conv_d_in_{}'.format(x.shape[1]))) for i in range(pg - 1): print "decode", up_x.shape if i == pg - 2 and is_trans: #torgb up_x_trans = conv2d(up_x, output_dim=self.channel, k_w=1, k_h=1, d_w=1, d_h=1, name='gen_rgb_d_{}'.format(up_x.shape[1])) up_x_trans = upscale(up_x_trans, 2) up_x = upscale(up_x, 2) up_x = tf.nn.relu(instance_norm(conv2d(up_x, output_dim=self.get_nf(i + 1), d_w=1, d_h=1, name='gen_conv_d_{}'.format(up_x.shape[1])), scope='gen_conv_d_in_{}'.format(up_x.shape[1]))) #torgb up_x = conv2d(up_x, output_dim=self.channel, k_w=1, k_h=1, d_w=1, d_h=1, name='gen_rgb_d_{}'.format(up_x.shape[1])) if pg == 1: up_x = up_x else: if is_trans: up_x = (1 - alpha_trans) * up_x_trans + alpha_trans * up_x else: up_x = up_x return up_x
def patch_discriminator(rgbd_sequence, scope="spade_discriminator"): """Creates a patch discriminator to process RGBD values. Args: rgbd_sequence: [B, H, W, 4] A batch of RGBD images. scope: (str) variable scope Returns: (list of features, logits) """ num_channel = 64 num_layers = 4 features = [] with tf.variable_scope(scope, reuse=tf.AUTO_REUSE): x = ops.sn_conv(rgbd_sequence, num_channel, kernel_size=4, stride=2, sn=False) channel = num_channel for i in range(1, num_layers): stride = 1 if i == num_layers - 1 else 2 channel = min(channel * 2, 512) x = ops.sn_conv(x, channel, kernel_size=4, stride=stride, sn=True, scope="conv_{}".format(i)) x = ops.instance_norm(x, scope="inst_norm_{}".format(i)) x = tf.nn.lrelu(x, 0.2) features.append(x) logit = ops.sn_conv(x, 1, kernel_size=4, stride=1, sn=False, scope="D_logit") return features, logit
def generator(self, input_x, img_mask, guided_fp_left, guided_fp_right, use_sp=False, reuse=False): with tf.variable_scope("generator") as scope: if reuse == True: scope.reuse_variables() x = tf.concat([input_x, img_mask], axis=3) u_fp_list = [] for i in range(6): c_dim = np.minimum(16 * np.power(2, i), 256) if i == 0: x = tf.nn.relu( instance_norm(conv2d(x, output_dim=c_dim, k_w=7, k_h=7, d_w=1, d_h=1, use_sp=use_sp, name='conv_{}'.format(i)), scope='conv_IN_{}'.format(i))) else: x = tf.nn.relu( instance_norm(conv2d(x, output_dim=c_dim, k_w=4, k_h=4, d_w=2, d_h=2, use_sp=use_sp, name='conv_{}'.format(i)), scope='conv_IN_{}'.format(i))) if i < 5: u_fp_list.append(x) bottleneck = tf.reshape(x, shape=[self.batch_size, -1]) bottleneck = fully_connect(bottleneck, output_size=256, use_sp=use_sp, scope='FC1') bottleneck = tf.concat( [bottleneck, guided_fp_left, guided_fp_right], axis=1) de_x = tf.nn.relu( fully_connect(bottleneck, output_size=256 * 8 * 8, use_sp=use_sp, scope='FC2')) de_x = tf.reshape(de_x, shape=[self.batch_size, 8, 8, 256]) for i in range(5): c_dim = np.maximum(256 / np.power(2, i), 16) output_dim = 16 * np.power(2, i) de_x = tf.nn.relu( instance_norm(de_conv(de_x, output_shape=[ self.batch_size, output_dim, output_dim, c_dim ], use_sp=use_sp, name='deconv_{}'.format(i)), scope='deconv_IN_{}'.format(i))) if i < 4: de_x = tf.concat( [de_x, u_fp_list[len(u_fp_list) - (i + 1)]], axis=3) recon_img1 = conv2d(de_x, output_dim=3, k_w=7, k_h=7, d_h=1, d_w=1, use_sp=use_sp, name='output_conv') return tf.nn.tanh(recon_img1)
def generator(images, options, reuse=False, name='gen'): # reuse or not with tf.variable_scope(name): if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False def residule_block(x, dim, ks=3, s=1, name='res'): p = int((ks - 1) / 2) y = tf.pad(x, [[0, 0], [p, p], [p, p], [0, 0]], "CONSTANT") #CONSTANT y = instance_norm( conv2d(y, dim, ks, s, padding='VALID', name=name + '_c1'), name + '_in1') y = tf.pad(tf.nn.relu(y), [[0, 0], [p, p], [p, p], [0, 0]], "CONSTANT") y = instance_norm( conv2d(y, dim, ks, s, padding='VALID', name=name + '_c2'), name + '_in2') return y + x # down sampling c0 = tf.pad(images, [[0, 0], [3, 3], [3, 3], [0, 0]], "CONSTANT") c1 = relu( instance_norm( conv2d(c0, options.nf, ks=7, s=1, padding='VALID', name='gen_ds_conv1'), 'in1_1')) c2 = relu( instance_norm( conv2d(c1, 2 * options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2')) c3 = relu( instance_norm( conv2d(c2, 4 * options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3')) # bottleneck r1 = residule_block(c3, options.nf * 4, name='g_r1') r2 = residule_block(r1, options.nf * 4, name='g_r2') r3 = residule_block(r2, options.nf * 4, name='g_r3') r4 = residule_block(r3, options.nf * 4, name='g_r4') r5 = residule_block(r4, options.nf * 4, name='g_r5') r6 = residule_block(r5, options.nf * 4, name='g_r6') # up sampling d1 = relu( instance_norm( deconv2d(r6, options.nf * 2, 4, 2, name='g_us_dconv1'), 'g_d1_in')) d2 = relu( instance_norm(deconv2d(d1, options.nf, 4, 2, name='g_us_dconv2'), 'g_d2_in')) d2 = tf.pad(d2, [[0, 0], [3, 3], [3, 3], [0, 0]], "CONSTANT") pred = tf.nn.tanh(conv2d(d2, 3, 7, 1, padding='VALID', name='g_pred_c')) return pred
def generator(images, options, reuse=False, name='gen'): # reuse or not with tf.variable_scope(name): if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False # down sampling x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1')) x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3')) # bottleneck x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6')) # up sampling x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1')) x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2')) x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3')) return x
def encode_decode(self, x, reuse=False): with tf.variable_scope("encode_decode") as scope: if reuse == True: scope.reuse_variables() conv1 = tf.nn.relu( instance_norm(conv2d(x, output_dim=64, k_w=7, k_h=7, d_w=1, d_h=1, name='e_c1'), scope='e_in1')) conv2 = tf.nn.relu( instance_norm(conv2d(conv1, output_dim=128, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c2'), scope='e_in2')) conv3 = tf.nn.relu( instance_norm(conv2d(conv2, output_dim=256, k_w=4, k_h=4, d_w=2, d_h=2, name='e_c3'), scope='e_in3')) r1 = Residual(conv3, residual_name='re_1') r2 = Residual(r1, residual_name='re_2') r3 = Residual(r2, residual_name='re_3') r4 = Residual(r3, residual_name='re_4') r5 = Residual(r4, residual_name='re_5') r6 = Residual(r5, residual_name='re_6') g_deconv1 = tf.nn.relu( instance_norm(de_conv(r6, output_shape=[ self.batch_size, self.output_size / 2, self.output_size / 2, 128 ], name='gen_deconv1'), scope="gen_in")) # for 1 g_deconv_1_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='g_deconv_1_1'), scope='gen_in_1_1')) g_deconv_1_1_x = tf.concat([g_deconv_1_1, x], axis=3) x_tilde1 = conv2d(g_deconv_1_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_1_2') # for 2 g_deconv_2_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='g_deconv_2_1'), scope='gen_in_2_1')) g_deconv_2_1_x = tf.concat([g_deconv_2_1, x], axis=3) x_tilde2 = conv2d(g_deconv_2_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_2_2') # for 3 g_deconv_3_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='gen_deconv3_1'), scope='gen_in_3_1')) g_deconv_3_1_x = tf.concat([g_deconv_3_1, x], axis=3) g_deconv_3_2 = conv2d(g_deconv_3_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1, name='gen_conv_3_2') x_tilde3 = conv2d(g_deconv_3_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_3_3') # for 4 g_deconv_4_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='gen_deconv4_1'), scope='gen_in_4_1')) g_deconv_4_1_x = tf.concat([g_deconv_4_1, x], axis=3) g_deconv_4_2 = conv2d(g_deconv_4_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1, name='gen_conv_4_2') x_tilde4 = conv2d(g_deconv_4_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_4_3') # for 5 g_deconv_5_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='gen_deconv5_1'), scope='gen_in_5_1')) g_deconv_5_1_x = tf.concat([g_deconv_5_1, x], axis=3) g_deconv_5_2 = conv2d(g_deconv_5_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1, name='gen_conv_5_2') x_tilde5 = conv2d(g_deconv_5_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_5_3') # for 6 g_deconv_6_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='gen_deconv6_1'), scope='gen_in_6_1')) g_deconv_6_1_x = tf.concat([g_deconv_6_1, x], axis=3) g_deconv_6_2 = conv2d(g_deconv_6_1_x, output_dim=32, k_w=3, k_h=3, d_h=1, d_w=1, name='gen_conv_6_2') x_tilde6 = conv2d(g_deconv_6_2, output_dim=3, k_h=3, k_w=3, d_h=1, d_w=1, name='gen_conv_6_3') # for 7 g_deconv_7_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='g_deconv_7_1'), scope='gen_in_7_1')) g_deconv_7_1_x = tf.concat([g_deconv_7_1, x], axis=3) x_tilde7 = conv2d(g_deconv_7_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_7_2') # for 8 g_deconv_8_1 = tf.nn.relu( instance_norm(de_conv(g_deconv1, output_shape=[ self.batch_size, self.output_size, self.output_size, 64 ], name='g_deconv_8_1'), scope='gen_in_8_1')) g_deconv_8_1_x = tf.concat([g_deconv_8_1, x], axis=3) x_tilde8 = conv2d(g_deconv_8_1_x, output_dim=self.channel, k_w=7, k_h=7, d_h=1, d_w=1, name='gen_conv_8_2') return tf.nn.tanh(x_tilde1), tf.nn.tanh(x_tilde2), tf.nn.tanh(x_tilde3), \ tf.nn.tanh(x_tilde4), tf.nn.tanh(x_tilde5), tf.nn.tanh(x_tilde6), tf.nn.tanh(x_tilde7), tf.nn.tanh(x_tilde8)
def generator(images, options, reuse=False, name='gen'): # down sampling x = relu(instance_norm(conv2d(images, options.nf, ks=7, s=1, name='gen_ds_conv1'), 'in1_1')) x = relu(instance_norm(conv2d(x, 2*options.nf, ks=4, s=2, name='gen_ds_conv2'), 'in1_2')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=4, s=2, name='gen_ds_conv3'), 'in1_3')) # bottleneck x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv1'), 'in2_1')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv2'), 'in2_2')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv3'), 'in2_3')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv4'), 'in2_4')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv5'), 'in2_5')) x = relu(instance_norm(conv2d(x, 4*options.nf, ks=3, s=1, name='gen_bn_conv6'), 'in2_6')) # up sampling x = relu(instance_norm(deconv2d(x, 2*options.nf, ks=4, s=2, name='gen_us_deconv1'), 'in3_1')) x = relu(instance_norm(deconv2d(x, options.nf, ks=4, s=2, name='gen_us_deconv2'), 'in3_2')) x = tanh(deconv2d(x, 3, ks=7, s=1, name='gen_us_dwconv3')) return x
def generator_multiunet(image, gf_dim, reuse=False, name="generator", output_c_dim=-1, istraining=True): if istraining: dropout_rate = 0.5 else: dropout_rate = 1.0 with tf.variable_scope(name): # image is 256 x 256 x input_c_dim if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False # image is (256 x 256 x input_c_dim) e1 = instance_norm(conv2d(image, gf_dim, name='g_e1_conv'), 'g_bn_e1') # e1 is (128 x 128 x self.gf_dim) e2 = instance_norm(conv2d(lrelu(e1), gf_dim * 2, name='g_e2_conv'), 'g_bn_e2') # e2 is (64 x 64 x self.gf_dim*2) e3 = instance_norm(conv2d(lrelu(e2), gf_dim * 4, name='g_e3_conv'), 'g_bn_e3') # e3 is (32 x 32 x self.gf_dim*4) e4 = instance_norm(conv2d(lrelu(e3), gf_dim * 8, name='g_e4_conv'), 'g_bn_e4') # e4 is (16 x 16 x self.gf_dim*8) e5 = instance_norm(conv2d(lrelu(e4), gf_dim * 8, name='g_e5_conv'), 'g_bn_e5') # e5 is (8 x 8 x self.gf_dim*8) e6 = instance_norm(conv2d(lrelu(e5), gf_dim * 8, name='g_e6_conv'), 'g_bn_e6') # e6 is (4 x 4 x self.gf_dim*8) e7 = instance_norm( conv2d(lrelu(e6), gf_dim * 8, ks=3, s=1, padding='VALID', name='g_e7_conv'), 'g_bn_e7') # e7 is (2 x 2 x self.gf_dim*8) e8 = instance_norm( conv2d(lrelu(e7), gf_dim * 16, ks=2, s=1, padding='VALID', name='g_e8_conv'), 'g_bn_e8') # e8 is (1 x 1 x self.gf_dim*8) d1 = deconv2d(tf.nn.relu(e8), gf_dim * 8, ks=2, s=1, padding='VALID', name='g_d1') d1 = tf.nn.dropout(d1, dropout_rate) d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3) # d1 is (2 x 2 x self.gf_dim*8*2) d2 = deconv2d(tf.nn.relu(d1), gf_dim * 8, ks=3, s=1, padding='VALID', name='g_d2') d2 = tf.nn.dropout(d2, dropout_rate) d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3) # d2 is (4 x 4 x self.gf_dim*8*2) d3 = deconv2d(tf.nn.relu(d2), gf_dim * 8, name='g_d3') d3 = tf.nn.dropout(d3, dropout_rate) d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3) # d3 is (8 x 8 x self.gf_dim*8*2) d4 = deconv2d(tf.nn.relu(d3), gf_dim * 8, name='g_d4') d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3) # d4 is (16 x 16 x self.gf_dim*8*2) d5 = deconv2d(tf.nn.relu(d4), gf_dim * 4, name='g_d5') d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3) # d5 is (32 x 32 x self.gf_dim*4*2) d6 = deconv2d(tf.nn.relu(d5), gf_dim * 2, name='g_d6') d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3) # d6 is (64 x 64 x self.gf_dim*2*2) d7 = deconv2d(tf.nn.relu(d6), gf_dim, name='g_d7') d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3) # d7 is (128 x 128 x self.gf_dim*1*2) d6_pre = deconv2d(tf.nn.relu(d5), output_c_dim, name='g_d6_pre') # d6_pre is (64 x 64 x output_c_dim) d7_pre = deconv2d(tf.nn.relu(d6), output_c_dim, name='g_d7_pre') # d7_pre is (128 x 128 x output_c_dim) d8_pre = deconv2d(tf.nn.relu(d7), output_c_dim, name='g_d8_pre') # d8_pre is (256 x 256 x output_c_dim) return tf.nn.tanh(d8_pre), tf.nn.tanh(d7_pre), tf.nn.tanh(d6_pre)
def build(self, input_, reuse=False, name="generator"): with tf.variable_scope(name): if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse is False e1 = instance_norm( conv2d(input_, self.args.gf_dim, name='g_e1_conv')) e2 = instance_norm( conv2d(lrelu(e1), self.args.gf_dim * 2, name='g_e2_conv'), 'g_bn_e2') e3 = instance_norm( conv2d(lrelu(e2), self.args.gf_dim * 4, name='g_e3_conv'), 'g_bn_e3') e4 = instance_norm( conv2d(lrelu(e3), self.args.gf_dim * 8, name='g_e4_conv'), 'g_bn_e4') e5 = instance_norm( conv2d(lrelu(e4), self.args.gf_dim * 8, name='g_e5_conv'), 'g_bn_e5') e6 = instance_norm( conv2d(lrelu(e5), self.args.gf_dim * 8, name='g_e6_conv'), 'g_bn_e6') e7 = instance_norm( conv2d(lrelu(e6), self.args.gf_dim * 8, name='g_e7_conv'), 'g_bn_e7') z_mean = instance_norm( conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e8_conv'), 'mean') z_logvar = instance_norm( conv2d(lrelu(e7), self.args.gf_dim * 8, name='g_e9_conv'), 'logvar') eps = tf.random_normal(shape=tf.shape(z_mean)) decoder_input = tf.add(z_mean, tf.exp(z_logvar / 2) * eps, name='decoder_input') d1 = deconv2d(tf.nn.relu(decoder_input), self.args.gf_dim * 8, name='g_d1') d1 = tf.nn.dropout(d1, self.dropout_rate) d1 = tf.concat([instance_norm(d1, 'g_bn_d1'), e7], 3) d2 = deconv2d(tf.nn.relu(d1), self.args.gf_dim * 8, name='g_d2') d2 = tf.nn.dropout(d2, self.dropout_rate) d2 = tf.concat([instance_norm(d2, 'g_bn_d2'), e6], 3) d3 = deconv2d(tf.nn.relu(d2), self.args.gf_dim * 8, name='g_d3') d3 = tf.nn.dropout(d3, self.dropout_rate) d3 = tf.concat([instance_norm(d3, 'g_bn_d3'), e5], 3) d4 = deconv2d(tf.nn.relu(d3), self.args.gf_dim * 8, name='g_d4') d4 = tf.concat([instance_norm(d4, 'g_bn_d4'), e4], 3) d5 = deconv2d(tf.nn.relu(d4), self.args.gf_dim * 4, name='g_d5') d5 = tf.concat([instance_norm(d5, 'g_bn_d5'), e3], 3) d6 = deconv2d(tf.nn.relu(d5), self.args.gf_dim * 2, name='g_d6') d6 = tf.concat([instance_norm(d6, 'g_bn_d6'), e2], 3) d7 = deconv2d(tf.nn.relu(d6), self.args.gf_dim, name='g_d7') d7 = tf.concat([instance_norm(d7, 'g_bn_d7'), e1], 3) d8 = deconv2d(tf.nn.relu(d7), self.args.out_dim, name='g_d8') return z_mean, z_logvar, tf.nn.tanh(d8)