def encoder(self, x, is_training=True, reuse=False): with tf.variable_scope("encoder", reuse=reuse): if self.verbose: print(x.shape) # Layer 1 net = layers.conv2d(x, self.conv_dim >> 1, name='en_conv1') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 2 net = layers.conv2d(net, self.conv_dim, name='en_conv2') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 3 net = layers.flatten(net) if self.verbose: print(net.shape) # Layer 4 net = layers.linear(net, self.linear_dim, scope='en_fc1') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 5 net = layers.linear(net, self.latent_dim, scope='en_fc2') net = tf.identity(net) if self.verbose: print(net.shape) return net
def encoder_v2(self, x, is_training=True, reuse=False): with tf.variable_scope("encoder_v2", reuse=reuse): if self.verbose: print(x.shape) # Layer 1 net = layers.conv2d(x, self.conv_dim >> 1, name='en_conv1') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 2 net = layers.conv2d(net, self.conv_dim, name='en_conv2') net = layers.batch_norm(net, is_training=is_training, scope='en_bn2') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 3 net = layers.flatten(net) if self.verbose: print(net.shape) # Layer 4 net = layers.linear(net, self.linear_dim, scope='en_fc3') net = layers.batch_norm(net, is_training=is_training, scope='en_bn3') net = tf.nn.leaky_relu(net) if self.verbose: print(net.shape) # Layer 5 out_logit = layers.linear(net, self.latent_dim, scope='en_fc4') out = tf.nn.sigmoid(out_logit, name="main_out") if self.verbose: print(out.shape) return out
def generator(self, batch): grey_batch = rgb_to_grey(batch) with tf.variable_scope('g_') as vs: """ ----------------------------------------------------------------------------------- ENCODER ----------------------------------------------------------------------------------- """ self.en_h0 = conv2d(grey_batch, self.frame_size, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1") self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0)) add_activation_summary(self.en_h0) print(self.en_h0.get_shape().as_list()) self.en_h1 = conv2d(self.en_h0, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2") self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2") self.en_h1 = tf.nn.relu(self.en_h1) add_activation_summary(self.en_h1) print(self.en_h1.get_shape().as_list()) self.en_h2 = conv2d(self.en_h1, 256, 512, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3") self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3") self.en_h2 = tf.nn.relu(self.en_h2) add_activation_summary(self.en_h2) print(self.en_h2.get_shape().as_list()) self.en_h3 = conv2d(self.en_h2, 512, 1024, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4") self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4") self.en_h3 = tf.nn.relu(self.en_h3) add_activation_summary(self.en_h3) print(self.en_h3.get_shape().as_list()) """ ----------------------------------------------------------------------------------- GENERATOR ----------------------------------------------------------------------------------- """ self.fg_h0 = tf.reshape(self.en_h3, [-1, 2, 4, 4, 512]) print(self.fg_h0.get_shape().as_list()) self.fg_h1 = conv3d_transpose(self.fg_h0, 512, [self.batch_size, 4, 8, 8, 256], name='g_f_h1') self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h1, scope='g_f_bn1'), name='g_f_relu1') add_activation_summary(self.fg_h1) print(self.fg_h1.get_shape().as_list()) self.fg_h2 = conv3d_transpose(self.fg_h1, 256, [self.batch_size, 8, 16, 16, 128], name='g_f_h2') self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h2, scope='g_f_bn2'), name='g_f_relu2') add_activation_summary(self.fg_h2) print(self.fg_h2.get_shape().as_list()) self.fg_h3 = conv3d_transpose(self.fg_h2, 128, [self.batch_size, 16, 32, 32, 64], name='g_f_h3') self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h3, scope='g_f_bn3'), name='g_f_relu3') add_activation_summary(self.fg_h3) print(self.fg_h3.get_shape().as_list()) self.fg_h4 = conv3d_transpose(self.fg_h3, 64, [self.batch_size, 32, 64, 64, 3], name='g_f_h4') self.fg_fg = tf.nn.tanh(self.fg_h4, name='g_f_actvcation') print(self.fg_fg.get_shape().as_list()) gen_reg = tf.reduce_mean(tf.square(grey_batch - rgb_to_grey(self.fg_fg))) variables = tf.contrib.framework.get_variables(vs) return self.fg_fg, gen_reg, variables
def __call__(self, x, reuse=True): with tf.variable_scope(self.name, reuse=reuse) as vs: bs = tf.shape(x)[0] x = tf.reshape(x, [bs, 28, 28, 1]) conv1 = conv2d(x, 64, [4, 4], [2, 2], activation_fn=leaky_relu) conv2 = conv2d(conv1, 128, [4, 4], [2, 2], activation_fn=leaky_relu) conv2 = tf.contrib.layers.flatten(conv2) fc1 = dense(conv2, 1024, activation_fn=leaky_relu) fc2 = dense(fc1, 1, activation_fn=tf.identity) return fc2, tf.sigmoid(fc2)
def ResNet_1_3_1(last): is_training = False channels = FLAGS.channels l = 0 last = tf.identity(last, name='input') # residual blocks rb = 0 skip2 = last while rb < FLAGS.res_blocks: rb += 1 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=1, out_channels=channels, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=FLAGS.activation, init_factor=FLAGS.init_activation) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=FLAGS.activation, init_factor=FLAGS.init_activation) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=1, out_channels=channels, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_factor) with tf.variable_scope('skip_connection{}'.format(l)) as scope: last = tf.add(last, skip2, 'elementwise_sum') skip2 = last last = layers.apply_activation(last, activation=FLAGS.activation, data_format=FLAGS.data_format) # return last = tf.identity(last, name='output') return last
def ForkNet(inputs, padding = 'VALID', name='ForkNet'): ''' Built the ForkNet model. Args: inputs: mosaic polarized images padding: padding mode of convolution Returns: s0: reconstructed s0 images dolp: reconstructed dolp images aop: reconstructed aop images ''' # keep_prob = tf.where(is_training, 0.2, 1.0) with tf.variable_scope(name): # conventional layers x_1 = conv2d(inputs, [4, 4], 96, activation=tf.nn.relu, padding=padding, name='conv_1') tf.add_to_collection('feature_maps', x_1) x_2 = conv2d(x_1, [3,3], 48, activation=tf.nn.relu, padding=padding, name='conv_2') tf.add_to_collection('feature_maps', x_2) x_3_1 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_1') tf.add_to_collection('feature_maps', x_3_1) s0 = conv2d(x_3_1, [5, 5], 1, activation=None, padding=padding, name='conv_4_1') x_3_2 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_2') tf.add_to_collection('feature_maps', x_3_2) dolp = conv2d(x_3_2, [5, 5], 1, activation=None, padding=padding, name='conv_4_2') x_3_3 = conv2d(x_2, [3, 3], 32, activation=tf.nn.relu, padding=padding, name='conv_3_3') tf.add_to_collection('feature_maps', x_3_3) aop = conv2d(x_3_3, [4, 4], 1, activation=None, padding=padding, name='conv_4_3') aop = tf.atan(aop) / 2. + math.pi / 4 return s0, dolp, aop
def _build_feature_extractor(self, x, reuse=False): """ Build a CNN for convolutional feature extraction Args: x: input tensor reuse: boolean, if true reuse already allocated variables, otherway make new ones Returns: x: activation of last convolutional layer """ if self.padding: x = self.pad(x) num_convolution = (self.receptive_field_size - 1) // (self.kernel_size - 1) in_channel = x.get_shape()[-1].value #Feature Extraction for i in range(num_convolution - 1): if i == 0: kernel = [self.kernel_size, self.kernel_size, in_channel, 32] elif i < 2: kernel = [self.kernel_size, self.kernel_size, 32, 32] elif i == 2: kernel = [self.kernel_size, self.kernel_size, 32, 64] else: kernel = [self.kernel_size, self.kernel_size, 64, 64] x = conv2d(x, kernel, activation=lambda x: tf.nn.relu(x), padding='VALID', reuse=reuse, batch_norm=True, training=self.is_training, name='conv{}'.format(i)) #Final Layer x = conv2d(x, [self.kernel_size, self.kernel_size, 64, 64], activation=lambda x: x, padding='VALID', reuse=reuse, batch_norm=True, training=self.is_training, name='conv_final') return x
def fpn(self, features, out_channel): last_inner = conv2d( features[-1], out_channel, 1 ) #Set up the last (except the top layer) fpn output. In additional, the first element of the resnet outputs will not be used in fpn results = [] results.append(conv2d(last_inner, out_channel, 3, 1, padding=1)) assert len( features ) - 2 > 0 #Please make sure that at least 3 features are feed in the fpn for i in range(len(features) - 2): inner_top_down = upsample_layer(last_inner, 2) inner_lateral = conv2d(features[-2 - i], out_channel, 1) last_inner = inner_lateral + inner_top_down results.insert( 0, conv2d(last_inner, out_channel, 3, strides=1, padding=1)) with tf.variable_scope('top_blocks'): extra_feas = self.fpn_extra_tops(results[-1], out_channel) results.extend(extra_feas) return results
def block(name, input, filters, ksz, stride, padding): """ :param name: a string :param input: a tensor :param filters: an integer, # filters :param ksz: an integer, the size of filters :param stride: an integer :param padding: a string, all string need to be uppercase :return: """ with tf.variable_scope(name): conv = conv2d('conv', input, filters, ksz, stride, padding) bn = batch_norm('bn', conv) return relu('relu', bn)
def call(self, inputs): # Bottom-up layers c1 = ul.conv2d(inputs, 64, 7, strides=2) c1 = ul.sep_conv2d(c1, 64, 3, strides=1) c2 = ul.sep_conv2d(c1, 128, 3, strides=2) c2 = ul.sep_conv2d(c2, 128, 3, strides=1) c3 = ul.sep_conv2d(c2, 256, 3, strides=2) c3 = ul.sep_conv2d(c3, 256, 3, strides=1) c4 = ul.sep_conv2d(c3, 512, 3, strides=2) c4 = ul.sep_conv2d(c4, 512, 3, strides=1) c4 = ul.sep_conv2d(c4, 512, 3, strides=1) c4 = ul.sep_conv2d(c4, 512, 3, strides=1) c4 = ul.sep_conv2d(c4, 512, 3, strides=1) p5 = ul.conv2d(c4, 256, 3, strides=2) p6 = ul.conv2d(p5, 256, 3, strides=2) # Top-down p4 = ul.conv2d(c4, 256, 1, activation=None) u4 = tf.layers.conv2d_transpose(p4, 256, [4, 4], strides=[2, 2], padding='same', data_format='channels_first') p3 = ul.add_forced(u4, ul.conv2d(c3, 256, 1, activation=None)) p3 = ul.conv2d(p3, 256, 3, activation=None) u3 = tf.layers.conv2d_transpose(p3, 128, [4, 4], strides=[2, 2], padding='same', data_format='channels_first') p2 = ul.add_forced(u3, ul.conv2d(c2, 128, 1, activation=None)) p2 = ul.conv2d(p2, 256, 3, activation=None) return p2, p3, p4, p5, p6
def inference(lefts, disps): bs = int(lefts.shape[0]) print("Batch size: ", bs) #256*512 # ->conv1-> #128*256 # ->conv2-> #64*128 # ->conv3-> #32*64 # ->conv4-> #16*32 # ->layers.conv_transpose4-> #32*64 # ->layers.conv_transpose5-> #64*128 # ->layers.conv_transpose6-> #128*256 # ->layers.conv_transpose7-> #256*512 #256x512x3 -> 256x512x32 with tf.variable_scope('left') as scope: left_feat = layers.conv2d_relu('conv1', lefts, [3,3,3,32], [1,1,1,1]) #256x512x1 -> 256x512x32 with tf.variable_scope('disp') as scope: disp_feat = layers.conv2d_relu('conv1', disps, [3,3,1,32], [1,1,1,1]) #x32 + x32 -> 256x512x64 concat = tf.concat([left_feat, disp_feat], axis=3) #256x512x64 -> 256x512x64 net = layers.conv2d_relu('conv1_concat', concat, [3,3,64,64], [1,1,1,1]) scale1 = net net = tf.contrib.layers.batch_norm(net) #256x512x64 -> 128x256x64 net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool1') def encoder_unit(name, batch, kernel_shape, strides, padding='SAME', kernel_initializer=tf.contrib.layers.xavier_initializer(), bias_initializer=tf.constant_initializer(0.0)): with tf.variable_scope('encoder' + number): with tf.variable_scope('conv'): kernel = _variable_on_cpu('weights', shape=kernel_shape, initializer=kernel_initializer) conv = tf.nn.conv2d(batch, kernel, strides, padding=padding) biases = _variable_on_cpu('biases', [kernel_shape[3]], bias_initializer) pre_activation = tf.nn.bias_add(conv, biases) print(net.name, net.shape) # 128x256x64 -> 128x256x128 net = layers.conv2d_relu('conv2', net, [3,3,64,128], [1,1,1,1]) scale2 = net net = tf.contrib.layers.batch_norm(net) print(net.name, net.shape) #128x256x128 -> 64x128x128 net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool2') print(net.name, net.shape) #64x128x128 ->64x128x128 net = layers.conv2d_relu('conv3', net, [3,3,128,256], [1,1,1,1]) scale3 = net net = tf.contrib.layers.batch_norm(net) print(net.name, net.shape) #64x128x128 -> 32x64x256 net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool3') print("scale3-->", net.name, net.shape) #32x64x256 -> 32x64x512 net = layers.conv2d_relu('conv4', net, [3,3,256,512], [1,1,1,1]) net = tf.contrib.layers.batch_norm(net) print(net.name, net.shape) #32x64x512 -> 16x32x512 net = tf.nn.max_pool(net, ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], padding='SAME', name='pool4') print(net.name, net.shape) #16x32x512 -> 32x64x256 up = layers.conv_transpose_2x_relu('transp1', net, [3,3,256,512], [1,2,2,1]) tf.add(up, scale3) print(up.name, up.shape) #32x64x256 -> 64x128x128 up = layers.conv_transpose_2x_relu('transp2', up, [3,3,128,256], [1,2,2,1]) tf.add(up, scale2) print(up.name, up.shape) #64x128x128 -> 128x256x64 up = layers.conv_transpose_2x_relu('transp3', up, [3,3,64,128], [1,2,2,1]) tf.add(up, scale1) print(up.name, up.shape) #128x256x64 -> 256x512x32 up = layers.conv_transpose_2x_relu('transp4', up, [3,3,32,64], [1,2,2,1]) tf.add(up, left_feat) tf.add(up, disp_feat) print(up.name, up.shape) up = layers.conv2d('conv5', up, [3,3,32,1], [1,1,1,1]) print(up.name, up.shape) return up
def ShuffleNet(last, group_num=8): is_training = False channels = FLAGS.channels channel_index = -3 if FLAGS.data_format == 'NCHW' else -1 l = 0 last = tf.identity(last, name='input') # residual blocks rb = 0 skip2 = last while rb < FLAGS.res_blocks: rb += 1 l += 1 with tf.variable_scope('pointwise_group_conv{}'.format(l)) as scope: group = tf.split(last, group_num, axis=channel_index) for _ in range(group_num): with tf.variable_scope('group{}'.format(_ + 1)) as scope: group[_] = layers.conv2d(group[_], ksize=1, out_channels=channels // group_num, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_activation) last = tf.concat(group, axis=channel_index) last = layers.apply_activation(last, activation=FLAGS.activation, data_format=FLAGS.data_format) with tf.variable_scope('channel_shuffle{}'.format(l)) as scope: shape = tf.shape(last) if FLAGS.data_format == 'NCHW': shape_divide = [ shape[0], group_num, channels // group_num, shape[2], shape[3] ] else: shape_divide = [ shape[0], shape[1], shape[2], group_num, channels // group_num ] last = tf.reshape(last, shape_divide) trans = (0, 2, 1, 3, 4) if FLAGS.data_format == 'NCHW' else (0, 1, 2, 4, 3) last = tf.transpose(last, trans) last = tf.reshape(last, shape) l += 1 with tf.variable_scope('depthwise_conv{}'.format(l)) as scope: last = layers.depthwise_conv2d(last, ksize=3, channel_multiplier=1, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_activation) l += 1 with tf.variable_scope('pointwise_group_conv{}'.format(l)) as scope: group = tf.split(last, group_num, axis=channel_index) for _ in range(group_num): with tf.variable_scope('group{}'.format(_ + 1)) as scope: group[_] = layers.conv2d(group[_], ksize=1, out_channels=channels // group_num, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_activation) last = tf.concat(group, axis=channel_index) with tf.variable_scope('skip_connection{}'.format(l)) as scope: last = tf.add(last, skip2, 'elementwise_sum') skip2 = last last = layers.apply_activation(last, activation=FLAGS.activation) # return last = tf.identity(last, name='output') return last
def resnet50(input): """ :param input: a 4-D tensor, NHWC :return: """ # conv1, 7x7, 64, stride 2 conv1 = block('conv1', input, 64, 7, 2, 'SAME') # conv2_x, 3x3 max pool, stride 2 maxpool2 = maxpool2d('pool2', conv1, 3, 2, 'SAME') # conv2_1 # conv2_11, 1x1, 64 conv2_11 = block('conv2_11', maxpool2, 64, 1, 1, 'VALID') # conv2_12, 3x3, 64 conv2_12 = block('conv2_12', conv2_11, 64, 3, 1, 'SAME') # conv2_13, 1x1, 256 conv2_13 = block('conv2_13', conv2_12, 256, 1, 1, 'VALID') r2_1 = tf.add_n([conv2d('pj2', maxpool2, 256, 1, 1, 'VALID'), conv2_13]) # conv2_2 # conv2_21, 1x1, 64 conv2_21 = block('conv2_21', conv2_13, 64, 1, 1, 'VALID') # conv2_22, 3x3, 64 conv2_22 = block('conv2_22', conv2_21, 64, 3, 1, 'SAME') # conv2_23, 1x1, 256 conv2_23 = block('conv2_23', conv2_22, 256, 1, 1, 'VALID') r2_2 = tf.add_n([r2_1, conv2_23]) # conv2_3 # conv2_31, 1x1, 64 conv2_31 = block('conv2_31', r2_2, 64, 1, 1, 'VALID') # conv2_32, 3x3, 64 conv2_32 = block('conv2_32', conv2_31, 64, 3, 1, 'SAME') # conv2_33, 1x1, 256 conv2_33 = block('conv2_33', conv2_32, 256, 1, 1, 'VALID') r2_3 = tf.add_n([r2_2, conv2_33]) # conv3_x # conv3_1 # conv3_11, 1x1, 128 conv3_11 = block('conv3_11', r2_3, 128, 1, 2, 'VALID') # conv3_12, 3x3, 128 conv3_12 = block('conv3_12', conv3_11, 128, 3, 1, 'SAME') # conv3_13, 1x1, 512 conv3_13 = block('conv3_13', conv3_12, 512, 1, 1, 'VALID') r3_1 = tf.add_n([conv2d('pj3', r2_3, 512, 1, 2, 'VALID'), conv3_13]) # conv3_2 # conv3_21, 1x1, 128 conv3_21 = block('conv3_21', r3_1, 128, 1, 1, 'VALID') # conv3_22, 3x3, 128 conv3_22 = block('conv3_22', conv3_21, 128, 3, 1, 'SAME') # conv3_23, 1x1, 512 conv3_23 = block('conv3_23', conv3_22, 512, 1, 1, 'VALID') r3_2 = tf.add_n([r3_1, conv3_23]) # conv3_3 # conv3_31, 1x1, 128 conv3_31 = block('conv3_31', r3_2, 128, 1, 1, 'VALID') # conv3_32, 3x3, 128 conv3_32 = block('conv3_32', conv3_31, 128, 3, 1, 'SAME') # conv3_33, 1x1, 512 conv3_33 = block('conv3_33', conv3_32, 512, 1, 1, 'VALID') r3_3 = tf.add_n([r3_2, conv3_33]) # conv3_4 # conv3_41, 1x1, 128 conv3_41 = block('conv3_41', r3_3, 128, 1, 1, 'VALID') # conv3_42, 3x3, 128 conv3_42 = block('conv3_42', conv3_41, 128, 3, 1, 'SAME') # conv3_43, 1x1, 512 conv3_43 = block('conv3_43', conv3_42, 512, 1, 1, 'VALID') r3_4 = tf.add_n([r3_3, conv3_43]) # conv4_x # conv4_1 # conv4_11, 1x1, 256 conv4_11 = block('conv4_11', r3_4, 256, 1, 2, 'VALID') # conv4_12, 3x3, 256 conv4_12 = block('conv4_12', conv4_11, 256, 3, 1, 'SAME') # conv4_13, 1x1, 1024 conv4_13 = block('conv4_13', conv4_12, 1024, 1, 1, 'VALID') r4_1 = tf.add_n([conv2d('pj4', r3_4, 1024, 1, 2, 'VALID'), conv4_13]) # conv4_2 # conv4_21, 1x1, 256 conv4_21 = block('conv4_21', r4_1, 256, 1, 1, 'VALID') # conv4_22, 3x3, 256 conv4_22 = block('conv4_22', conv4_21, 256, 3, 1, 'SAME') # conv4_23, 1x1, 1024 conv4_23 = block('conv4_23', conv4_22, 1024, 1, 1, 'VALID') r4_2 = tf.add_n([r4_1, conv4_23]) # conv4_3 # conv4_31, 1x1, 256 conv4_31 = block('conv4_31', r4_2, 256, 1, 1, 'VALID') # conv4_32, 3x3, 256 conv4_32 = block('conv4_32', conv4_31, 256, 3, 1, 'SAME') # conv4_33, 1x1, 1024 conv4_33 = block('conv4_33', conv4_32, 1024, 1, 1, 'VALID') r4_3 = tf.add_n([r4_2, conv4_33]) # conv4_4 # conv4_41, 1x1, 256 conv4_41 = block('conv4_41', r4_3, 256, 1, 1, 'VALID') # conv4_42, 3x3, 256 conv4_42 = block('conv4_42', conv4_41, 256, 3, 1, 'SAME') # conv4_43, 1x1, 1024 conv4_43 = block('conv4_43', conv4_42, 1024, 1, 1, 'VALID') r4_4 = tf.add_n([r4_3, conv4_43]) # conv4_5 # conv4_51, 1x1, 256 conv4_51 = block('conv4_51', r4_4, 256, 1, 1, 'VALID') # conv4_52, 3x3, 256 conv4_52 = block('conv4_52', conv4_51, 256, 3, 1, 'SAME') # conv4_53, 1x1, 1024 conv4_53 = block('conv4_53', conv4_52, 1024, 1, 1, 'VALID') r4_5 = tf.add_n([r4_4, conv4_53]) # conv4_6 # conv4_61, 1x1, 256 conv4_61 = block('conv4_61', r4_5, 256, 1, 1, 'VALID') # conv4_62, 3x3, 256 conv4_62 = block('conv4_62', conv4_61, 256, 3, 1, 'SAME') # conv4_63, 1x1, 1024 conv4_63 = block('conv4_63', conv4_62, 1024, 1, 1, 'SAME') r4_6 = tf.add_n([r4_5, conv4_63]) # conv5_x # conv5_1 # conv5_11, 1x1, 512 conv5_11 = block('conv5_11', r4_6, 512, 1, 2, 'VALID') # conv5_12, 3x3, 512 conv5_12 = block('conv5_12', conv5_11, 512, 3, 1, 'SAME') # conv5_13, 1x1, 2048 conv5_13 = block('conv5_13', conv5_12, 2048, 1, 1, 'VALID') r5_1 = tf.add_n([conv2d('pj5', r4_6, 2048, 1, 2, 'VALID'), conv5_13]) # conv5_2 # conv5_21, 1x1, 512 conv5_21 = block('conv5_21', r5_1, 512, 1, 1, 'VALID') # conv5_22, 3x3, 512 conv5_22 = block('conv5_22', conv5_21, 512, 3, 1, 'SAME') # conv5_23, 1x1, 2048 conv5_23 = block('conv5_23', conv5_22, 2048, 1, 1, 'VALID') r5_2 = tf.add_n([r5_1, conv5_23]) # conv5_3 # conv5_31, 1x1, 512 conv5_31 = block('conv5_31', r5_2, 512, 1, 1, 'VALID') # conv5_32, 3x3, 512 conv5_32 = block('conv5_32', conv5_31, 512, 3, 1, 'SAME') # conv5_33, 1x1, 2048 conv5_33 = block('conv5_33', conv5_32, 2048, 1, 1, 'VALID') r5_3 = tf.add_n([r5_2, conv5_33]) avgpool6 = avgpool2d('avgpool6', r5_3, 7, 1, 'VALID') dense6 = tf.layers.dense(tf.layers.flatten(avgpool6), 12) return dense6
def fpn_extra_tops(self, inputs, out_channels): #Add extra 2 feature maps of fpn f1 = conv2d(inputs, out_channels, 3, strides=2, padding=1) f2 = conv2d(tf.nn.relu(f1), out_channels, 3, strides=2, padding=1) return [f1, f2]
def generatorVid(self, img_batch, reuse=False): with tf.variable_scope('gen_v', reuse=reuse) as vs: """ ----------------------------------------------------------------------------------- ENCODER ----------------------------------------------------------------------------------- """ self.en_h0 = conv2d(img_batch, 3, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1") self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0)) add_activation_summary(self.en_h0) print(self.en_h0.get_shape().as_list()) self.en_h1 = conv2d(self.en_h0, 64, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2") self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2") self.en_h1 = tf.nn.relu(self.en_h1) add_activation_summary(self.en_h1) print(self.en_h1.get_shape().as_list()) output = tf.transpose(self.en_h1, [0, 3, 1, 2]) for i in xrange(3): output = self.ResidualBlock('res1.16x16_{}'.format(i), 128, 128, 3, output, resample=None) self.en_h1 = tf.transpose(output, [0, 2, 3, 1]) self.en_h2 = conv2d(self.en_h1, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3") self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3") self.en_h2 = tf.nn.relu(self.en_h2) add_activation_summary(self.en_h2) print(self.en_h2.get_shape().as_list()) output = tf.transpose(self.en_h2, [0, 3, 1, 2]) for i in xrange(3): output = self.ResidualBlock('res1.16x16_2_{}'.format(i), 256, 256, 3, output, resample=None) self.en_h2 = tf.transpose(output, [0, 2, 3, 1]) self.en_h3 = conv2d(self.en_h2, 256, 512, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4") self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4") self.en_h3 = tf.nn.relu(self.en_h3) add_activation_summary(self.en_h3) print(self.en_h3.get_shape().as_list()) output = tf.transpose(self.en_h3, [0, 3, 1, 2]) for i in xrange(3): output = self.ResidualBlock('res1.16x16_3_{}'.format(i), 512, 512, 3, output, resample=None) self.en_h3 = tf.transpose(output, [0, 2, 3, 1]) self.en_h4 = conv2d(self.en_h3, 512, 1024, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv5") self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4, scope="enc_bn5") self.en_h4 = tf.nn.relu(self.en_h4) add_activation_summary(self.en_h4) print(self.en_h4.get_shape().as_list()) """ ----------------------------------------------------------------------------------- GENERATOR ----------------------------------------------------------------------------------- """ self.z_ = tf.reshape(self.en_h4, [self.batch_size, 2, 4, 4, 512]) print(self.z_.get_shape().as_list()) self.fg_h1 = conv3d_transpose(self.z_, 512, [self.batch_size, 4, 8, 8, 256], name='g_f_h1') self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h1, scope='g_f_bn1'), name='g_f_relu1') add_activation_summary(self.fg_h1) print(self.fg_h1.get_shape().as_list()) self.fg_h2 = conv3d_transpose(self.fg_h1, 256, [self.batch_size, 8, 16, 16, 128], name='g_f_h2') self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h2, scope='g_f_bn2'), name='g_f_relu2') add_activation_summary(self.fg_h2) print(self.fg_h2.get_shape().as_list()) self.fg_h3 = conv3d_transpose(self.fg_h2, 128, [self.batch_size, 16, 32, 32, 64], name='g_f_h3') self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm(self.fg_h3, scope='g_f_bn3'), name='g_f_relu3') add_activation_summary(self.fg_h3) print(self.fg_h3.get_shape().as_list()) self.fg_h4 = conv3d_transpose(self.fg_h3, 64, [self.batch_size, 32, 64, 64, 3], name='g_f_h4') self.fg_vid = tf.nn.tanh(self.fg_h4, name='g_f_actvcation') print(self.fg_vid.get_shape().as_list()) # gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg[:, 0, :, :, :])) # variables = tf.contrib.framework.get_variables(vs) # # return self.fg_fg, gen_reg, variables # return self.fg_vid, variables return self.fg_vid
def discriminator(self, images, is_training=False, reuse=None): # parameters data_format = self.data_format channels = self.d_channels batch_norm = self.d_batch_norm activation = self.d_activation initializer = self.initializer init_factor = self.init_factor init_activation = self.init_activation weight_key = self.discriminator_weight_key channel_index = -3 if data_format == 'NCHW' else -1 reduce_strides = [1, 1, 2, 2 ] if data_format == 'NCHW' else [1, 2, 2, 1] # initialization last = images l = 0 with tf.variable_scope('discriminator', reuse=reuse) as scope: # first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) # residual blocks depth = 0 skip2 = last while depth < self.d_depth: depth += 1 reduce_size = True strides = reduce_strides if reduce_size else [1, 1, 1, 1] double_channel = depth > 1 if double_channel: channels *= 2 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format) last = layers.conv2d(last, ksize=2, out_channels=channels, stride=strides, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format) last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope('skip_connection{}'.format(l)) as scope: if reduce_size: skip2 = tf.nn.avg_pool(skip2, ksize=reduce_strides, strides=reduce_strides, padding='SAME', data_format=data_format) if double_channel: padding = [[0, 0] for _ in range(4)] padding[channel_index] = [0, channels // 2] skip2 = tf.pad(skip2, padding, mode='CONSTANT') last = tf.add(last, skip2) skip2 = last # final conv layer channels *= 2 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=2, out_channels=channels, stride=2, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=activation, initializer=initializer, init_factor=init_activation, collection=weight_key) # dense layer (1024) l += 1 print(last.get_shape()) with tf.variable_scope('dense{}'.format(l)) as scope: last = tf.contrib.layers.flatten(last) last = tf.contrib.layers.fully_connected(last, 1024, activation_fn=None) last = layers.apply_activation(last, activation=activation, data_format=data_format) # dense layer (1) l += 1 with tf.variable_scope('dense{}'.format(l)) as scope: last = tf.contrib.layers.fully_connected(last, 1, activation_fn=None) # return discriminating logits print('Critic: totally {} convolutional/dense layers.'.format(l)) return last
def generator(self, img_batch): with tf.variable_scope('g_') as vs: """ ----------------------------------------------------------------------------------- ENCODER ----------------------------------------------------------------------------------- """ print('ENCODER') self.en_h0 = conv2d(img_batch, self.channels, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1") self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0)) add_activation_summary(self.en_h0) print(self.en_h0.get_shape().as_list()) self.en_h1 = conv2d(self.en_h0, 64, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2") self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2") self.en_h1 = tf.nn.relu(self.en_h1) add_activation_summary(self.en_h1) print(self.en_h1.get_shape().as_list()) self.en_h2 = conv2d(self.en_h1, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3") self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3") self.en_h2 = tf.nn.relu(self.en_h2) add_activation_summary(self.en_h2) print(self.en_h2.get_shape().as_list()) self.en_h3 = conv2d(self.en_h1, 256, 512, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4") self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4") self.en_h3 = tf.nn.relu(self.en_h3) add_activation_summary(self.en_h3) print(self.en_h2.get_shape().as_list()) self.en_h4 = conv2d(self.en_h2, 512, 1024, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv5") self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4, scope="enc_bn5") self.en_h4 = tf.nn.relu(self.en_h4) add_activation_summary(self.en_h4) print(self.en_h3.get_shape().as_list()) """ ----------------------------------------------------------------------------------- GENERATOR ----------------------------------------------------------------------------------- """ print('GENERATOR') self.z_ = tf.reshape(self.en_h3, [self.batch_size, 2, 2, 1024]) print(self.z_.get_shape().as_list()) self.fg_h1 = tf.image.resize_images( self.z_, [4, 4], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) self.fg_h1 = conv2d(self.fg_h1, 1024, 512, d_h=1, d_w=1, name="gen_conv1") self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h1, scope='g_f_bn1'), name='g_f_relu1') add_activation_summary(self.fg_h1) print(self.fg_h1.get_shape().as_list()) self.fg_h2 = tf.image.resize_images( self.fg_h1, [8, 8], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) self.fg_h2 = conv2d(self.fg_h2, 512, 256, d_h=1, d_w=1, name="gen_conv2") self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h2, scope='g_f_bn2'), name='g_f_relu2') add_activation_summary(self.fg_h2) print(self.fg_h2.get_shape().as_list()) self.fg_h3 = tf.image.resize_images( self.fg_h2, [16, 16], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) self.fg_h3 = conv2d(self.fg_h3, 256, 128, d_h=1, d_w=1, name="gen_conv3") self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h3, scope='g_f_bn3'), name='g_f_relu3') add_activation_summary(self.fg_h3) print(self.fg_h3.get_shape().as_list()) self.fg_h4 = tf.image.resize_images( self.fg_h3, [self.crop_size, self.crop_size], method=tf.image.ResizeMethod.NEAREST_NEIGHBOR) self.fg_h4 = conv2d(self.fg_h4, 128, self.channels, d_h=1, d_w=1, name="gen_conv4") self.fg_fg = tf.nn.tanh(self.fg_h4, name='g_f_actication') print(self.fg_fg.get_shape().as_list()) gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg)) variables = tf.contrib.framework.get_variables(vs) return self.fg_fg, gen_reg, variables
def inference(self, spectrum, is_training=False, reuse=None): print('k_first={}, res_blocks={}, channels={}'.format( self.k_first, self.res_blocks, self.channels)) # parameters data_format = self.data_format channels = self.channels batch_norm = self.batch_norm activation = self.activation initializer = self.initializer init_factor = self.init_factor init_activation = self.init_activation weight_key = self.inference_weight_key channel_index = -3 if self.data_format == 'NCHW' else -1 pool_ksize = [1, 1, 1, 3 ] if self.data_format == 'NCHW' else [1, 1, 3, 1] reduce_strides = [1, 1, 1, 2 ] if self.data_format == 'NCHW' else [1, 1, 2, 1] # initialization last = spectrum l = 0 with tf.variable_scope('inference', reuse=reuse) as scope: # first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=[1, self.k_first], out_channels=channels, stride=[1, 1, 1, 1], padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_factor, collection=weight_key) # residual blocks rb = 0 skip2 = last while rb < self.res_blocks: rb += 1 reduce_size = rb % 1 == 0 strides = reduce_strides if reduce_size else [1, 1, 1, 1] double_channel = rb % 3 == 1 if double_channel: channels *= 2 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format) last = layers.conv2d(last, ksize=[1, 1], out_channels=channels // 2, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format) last = layers.conv2d(last, ksize=[1, 3], out_channels=channels // 2, stride=strides, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format) last = layers.conv2d(last, ksize=[1, 1], out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope('skip_connection{}'.format(l)) as scope: if reduce_size: skip2 = tf.nn.avg_pool(skip2, ksize=reduce_strides, strides=reduce_strides, padding='SAME', data_format=data_format) if double_channel: padding = [[0, 0] for _ in range(4)] padding[channel_index] = [0, channels // 2] skip2 = tf.pad(skip2, padding, mode='CONSTANT') last = tf.add(last, skip2) skip2 = last last = layers.apply_activation(last, activation=activation, data_format=data_format) # final dense layer (num_labels) l += 1 print(last.get_shape()) with tf.variable_scope('dense{}'.format(l)) as scope: last = tf.contrib.layers.flatten(last) last = tf.contrib.layers.fully_connected(last, self.num_labels, activation_fn=None) # return predicted labels print('Totally {} convolutional/dense layers.'.format(l)) return last
def generator(self, images_src, enc_src=None, is_training=False, reuse=None): print( 'generator: k_first={}, k_last={}, e_depth={}, d_depth={}, channels={}' .format(self.k_first, self.k_last, self.e_depth, self.d_depth, self.channels)) # parameters data_format = self.data_format channels = self.channels batch_norm = self.batch_norm activation = self.activation initializer = self.initializer init_factor = self.init_factor init_activation = self.init_activation weight_key = self.generator_weight_key # initialization last = images_src l = 0 with tf.variable_scope('generator', reuse=reuse) as scope: # encoder with tf.variable_scope('encoder') as scope: # encoder - first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=self.k_first, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) # encoder - residual blocks depth = 0 while depth < self.e_depth: downscale = False depth += 1 skip2 = last if downscale: channels *= 4 with tf.variable_scope('trans{}'.format(l)) as scope: if data_format == 'NCHW': last = utils.image.NCHW2NHWC(last) last = tf.space_to_depth(last, 2) if data_format == 'NCHW': last = utils.image.NHWC2NCHW(last) channels //= 2 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope( 'skip_connection{}'.format(l)) as scope: if downscale: pool_size = [ 1, 1, 2, 2 ] if data_format == 'NCHW' else [1, 2, 2, 1] skip2 = tf.nn.avg_pool(skip2, pool_size, pool_size, padding='SAME', data_format=data_format) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] padding[-3 if data_format == 'NCHW' else -1] = [ 0, channels // 2 ] skip2 = tf.pad(skip2, padding) last = layers.SqueezeExcitation( last, channel_r=1, data_format=data_format, collection=weight_key) last = tf.add(last, skip2) # encoder - final conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=self.k_last, out_channels=self.image_channels * 8, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_factor, collection=weight_key) # encoded images if not is_training: last = tf.round(last) # {0, 1} images_enc = last if enc_src is not None: last = enc_src # decoder with tf.variable_scope('decoder') as scope: # decoder - first conv layer channels //= 1 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=self.k_first, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) # decoder - residual blocks depth = 0 while depth < self.d_depth: depth += 1 skip2 = last l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope( 'skip_connection{}'.format(l)) as scope: last = layers.SqueezeExcitation( last, channel_r=1, data_format=data_format, collection=weight_key) last = tf.add(last, skip2) ''' # sub-pixel conv layer channels //= 2 l += 1 with tf.variable_scope('subpixel_conv{}'.format(l)) as scope: last = layers.subpixel_conv2d(last, ksize=3, out_channels=channels, scaling=2, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=activation, initializer=initializer, init_factor=init_activation, collection=weight_key) ''' # decoder - final conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=self.k_last, out_channels=self.image_channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_factor, collection=weight_key) images_dec = last # return SR image print('Generator: totally {} convolutional layers.'.format(l)) return images_enc, images_dec
def discriminator(self, images_enc, is_training=False, reuse=None): # parameters data_format = self.data_format d_depth = 8 channels = 64 batch_norm = 0.99 activation = 'swish' initializer = 4 init_factor = 1.0 init_activation = 2.0 weight_key = self.discriminator_weight_key # initialization last = images_enc l = 0 with tf.variable_scope('discriminator', reuse=reuse) as scope: # first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) # residual blocks depth = 0 while depth < d_depth: downscale = depth % 2 == 1 depth += 1 skip2 = last if downscale: channels *= 4 with tf.variable_scope('trans{}'.format(l)) as scope: if data_format == 'NCHW': last = utils.image.NCHW2NHWC(last) last = tf.space_to_depth(last, 2) if data_format == 'NCHW': last = utils.image.NHWC2NCHW(last) channels //= 2 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=1, out_channels=channels // 4, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=3, out_channels=channels // 4, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, is_training=is_training, data_format=data_format) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, ksize=1, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope('skip_connection{}'.format(l)) as scope: if downscale: pool_size = [ 1, 1, 2, 2 ] if data_format == 'NCHW' else [1, 2, 2, 1] skip2 = tf.nn.avg_pool(skip2, pool_size, pool_size, padding='SAME', data_format=data_format) padding = [[0, 0], [0, 0], [0, 0], [0, 0]] padding[-3 if data_format == 'NCHW' else -1] = [ 0, channels // 2 ] skip2 = tf.pad(skip2, padding) last = layers.SqueezeExcitation(last, channel_r=4, data_format=data_format, collection=weight_key) last = tf.add(last, skip2) # global average pooling (1024) with tf.variable_scope('trans{}'.format(l)) as scope: last = tf.reduce_mean( last, [-2, -1] if data_format == 'NCHW' else [-3, -2]) # dense layers (1024) l += 1 with tf.variable_scope('dense{}'.format(l)) as scope: last = tf.contrib.layers.fully_connected(last, channels, activation_fn=None) last = layers.apply_activation(last, activation='swish', data_format=data_format, collection=weight_key) # dense layers (1) channels = 1 l += 1 with tf.variable_scope('dense{}'.format(l)) as scope: last = tf.contrib.layers.fully_connected(last, channels, activation_fn=None) last = layers.apply_activation(last, activation='sigmoid', data_format=data_format, collection=weight_key) # return estimate print('Discriminator: totally {} convolutional layers.'.format(l)) return last
def generator(self, images_lr, is_training=False, reuse=None): print('k_first={}, k_last={}, g_depth={}, channels={}, channels2={}'. format(self.k_first, self.k_last, self.g_depth, self.channels, self.channels2)) # parameters data_format = self.data_format channels = self.channels channels2 = self.channels2 batch_norm = self.batch_norm activation = self.activation initializer = self.initializer init_factor = self.init_factor init_activation = self.init_activation weight_key = self.generator_weight_key # initialization last = images_lr l = 0 with tf.variable_scope('generator', reuse=reuse) as scope: skip0 = last # first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=self.k_first, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=None, is_training=is_training, activation=activation, initializer=initializer, init_factor=init_activation, collection=weight_key) # hidden layers depth = 0 while depth < self.g_depth: depth += 1 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=3, out_channels=channels, stride=1, padding='SAME', data_format=data_format, batch_norm=batch_norm, is_training=is_training, activation=activation, initializer=initializer, init_factor=init_activation, collection=weight_key) # resize conv layer l += 1 with tf.variable_scope('resize_conv{}'.format(l)) as scope: last = layers.resize_conv2d(last, ksize=self.k_last, out_channels=self.image_channels, scaling=self.scaling, data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_factor, collection=weight_key) ''' # sub-pixel conv layer l += 1 with tf.variable_scope('subpixel_conv{}'.format(l)) as scope: last = layers.subpixel_conv2d(last, ksize=self.k_last, out_channels=self.image_channels, scaling=self.scaling, data_format=data_format, batch_norm=None, is_training=is_training, activation=None, initializer=initializer, init_factor=init_factor, collection=weight_key) ''' # skip connection with tf.variable_scope('skip_connection{}'.format(l)) as scope: if data_format == 'NCHW': skip0 = tf.transpose(skip0, (0, 2, 3, 1)) up_size = tf.shape(skip0)[-3:-1] * self.scaling skip0 = tf.image.resize_nearest_neighbor(skip0, up_size) if data_format == 'NCHW': skip0 = tf.transpose(skip0, (0, 3, 1, 2)) last = tf.add(last, skip0) # return SR image print('Generator: totally {} convolutional layers.'.format(l)) return last
def generator(self, images_lr, train=False, reuse=None): print('k_first={}, k_last={}, g_depth={}, channels={}, channels2={}'. format(self.k_first, self.k_last, self.g_depth, self.channels, self.channels2)) # parameters data_format = self.data_format channels = self.channels channels2 = self.channels2 batch_norm = self.batch_norm renorm = self.batch_renorm activation = self.activation initializer = self.initializer init_factor = self.init_factor init_activation = self.init_activation weight_key = self.generator_weight_key # initialization last = images_lr l = 0 with tf.variable_scope('generator', reuse=reuse) as scope: skip0 = last # first conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, self.k_first, channels, use_bias=True, stride=1, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_activation, collection=weight_key) skip1 = last # residual blocks depth = 0 while depth < self.g_depth: depth += 1 skip2 = last l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, train=train, data_format=data_format, renorm=renorm) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, 3, channels, use_bias=False, stride=1, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_activation, collection=weight_key) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, train=train, data_format=data_format, renorm=renorm) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, 3, channels, use_bias=True, stride=1, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_activation, collection=weight_key) # skip connection - level 2 with tf.variable_scope('skip_connection{}'.format(l)) as scope: if self.use_se == 1: last = layers.SqueezeExcitation( last, channel_r=1, data_format=data_format, collection=weight_key) elif self.use_se == 2: last = layers.SqueezeExcitationLocal( last, channel_r=1, ksize=16, data_format=data_format, collection=weight_key) last = tf.add(last, skip2) # skip connection - level 1 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.apply_batch_norm(last, decay=batch_norm, train=train, data_format=data_format, renorm=renorm) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) last = layers.conv2d(last, 3, channels, use_bias=True, stride=1, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_activation, collection=weight_key) with tf.variable_scope('skip_connection{}'.format(l)) as scope: if self.use_se == 1: last = layers.SqueezeExcitation(last, channel_r=1, data_format=data_format, collection=weight_key) elif self.use_se == 2: last = layers.SqueezeExcitationLocal( last, channel_r=1, ksize=16, data_format=data_format, collection=weight_key) last = tf.add(last, skip1) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) # resize conv layer scale_num = 1 while 1 << scale_num <= self.scaling: with tf.variable_scope( 'resize_conv{}'.format(scale_num)) as scope: last = layers.resize_conv2d(last, 3, channels2, use_bias=True, scaling=self.scaling, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_activation, collection=weight_key) last = layers.apply_activation(last, activation=activation, data_format=data_format, collection=weight_key) scale_num += 1 # final conv layer l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, self.k_last, self.image_channels, use_bias=True, stride=1, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_factor, collection=weight_key) # skip connection - level 0 with tf.variable_scope('skip_connection{}'.format(l)) as scope: scale_num = 1 while 1 << scale_num <= self.scaling: with tf.variable_scope( 'resize_conv{}'.format(scale_num)) as scope: skip0 = layers.resize_conv2d(skip0, self.k_resize, self.image_channels, use_bias=True, scaling=self.scaling, padding='SAME', data_format=data_format, initializer=initializer, init_factor=init_factor, collection=weight_key) scale_num += 1 last = tf.add(last, skip0) # return SR image print('Generator: totally {} convolutional layers.'.format(l)) return last
def generatorVid(self, img_batch, reuse=False): with tf.variable_scope('gen_v', reuse=reuse) as vs: """ ----------------------------------------------------------------------------------- ENCODER ----------------------------------------------------------------------------------- """ # self.en_h0 = conv2d(img_batch, 3, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1") # self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0)) # add_activation_summary(self.en_h0) # print(self.en_h0.get_shape().as_list()) self.en_h0 = conv2d(img_batch, 3, 32, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv1") self.en_h0 = tf.nn.relu(tf.contrib.layers.batch_norm(self.en_h0)) add_activation_summary(self.en_h0) print(self.en_h0.get_shape().as_list()) self.en_h1 = conv2d(self.en_h0, 32, 64, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv2") self.en_h1 = tf.contrib.layers.batch_norm(self.en_h1, scope="enc_bn2") self.en_h1 = tf.nn.relu(self.en_h1) add_activation_summary(self.en_h1) print(self.en_h1.get_shape().as_list()) #output = tf.transpose(self.en_h1, [0, 3, 1, 2]) #for i in xrange(3): # output = self.ResidualBlock('res1.16x16_{}'.format(i), 256, 256, 3, output, resample=None) #self.en_h1 = tf.transpose(output, [0, 2, 3, 1]) self.en_h2 = conv2d(self.en_h1, 64, 128, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv3") self.en_h2 = tf.contrib.layers.batch_norm(self.en_h2, scope="enc_bn3") self.en_h2 = tf.nn.relu(self.en_h2) add_activation_summary(self.en_h2) print(self.en_h2.get_shape().as_list()) #output = tf.transpose(self.en_h2, [0, 3, 1, 2]) #for i in xrange(3): # output = self.ResidualBlock('res1.16x16_2_{}'.format(i), 512, 512, 3, output, resample=None) #self.en_h2 = tf.transpose(output, [0, 2, 3, 1]) self.en_h3 = conv2d(self.en_h2, 128, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv4") self.en_h3 = tf.contrib.layers.batch_norm(self.en_h3, scope="enc_bn4") self.en_h3 = tf.nn.relu(self.en_h3) add_activation_summary(self.en_h3) print(self.en_h3.get_shape().as_list()) self.en_h4 = conv2d(self.en_h3, 256, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv5") self.en_h4 = tf.contrib.layers.batch_norm(self.en_h4, scope="enc_bn5") self.en_h4 = tf.nn.relu(self.en_h4) add_activation_summary(self.en_h4) print(self.en_h4.get_shape().as_list()) #self.en_h4 is [32, 4, 4, 256] self.en_h5 = conv2d(self.en_h4, 256, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv6") self.en_h5 = tf.contrib.layers.batch_norm(self.en_h5, scope="enc_bn6") self.en_h5 = tf.nn.relu(self.en_h5) add_activation_summary(self.en_h5) print(self.en_h5.get_shape().as_list()) #self.en_h5 is [32,2,2,256] self.en_h6 = conv2d(self.en_h5, 256, 256, k_h=4, k_w=4, d_w=2, d_h=2, name="enc_conv7") self.en_h6 = tf.contrib.layers.batch_norm(self.en_h6, scope="enc_bn7") self.en_h6 = tf.nn.relu(self.en_h6) add_activation_summary(self.en_h6) print(self.en_h6.get_shape().as_list()) #self.en_h6 is [32,1,1,256] """ ----------------------------------------------------------------------------------- GENERATOR ----------------------------------------------------------------------------------- """ self.z_ = tf.reshape(self.en_h6, [self.batch_size, 1, 1, 1, 256]) print(self.z_.get_shape().as_list()) self.fg_h1 = conv3d_transpose(self.z_, 256, [self.batch_size, 1, 2, 2, 256], name='g_f_h1') self.fg_h1 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h1, scope='g_f_bn1'), name='g_f_relu1') add_activation_summary(self.fg_h1) print(self.fg_h1.get_shape().as_list()) #self.fg_h1 is [32, 1, 2, 2, 256] encv5 = tf.reshape(self.en_h5, [self.batch_size, 1, 2, 2, 256]) encov5 = tf.concat([self.fg_h1, encv5], axis=4) self.fg_h2 = conv3d_transpose(encov5, 512, [self.batch_size, 2, 4, 4, 256], name='g_f_h2') self.fg_h2 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h2, scope='g_f_bn2'), name='g_f_relu2') add_activation_summary(self.fg_h2) print(self.fg_h2.get_shape().as_list()) encv4 = tf.tile(tf.expand_dims(self.en_h4, axis=1), [1, 2, 1, 1, 1]) encvo4 = tf.reshape(encv4, [self.batch_size, 2, 4, 4, 256]) encodv4 = tf.concat([self.fg_h2, encvo4], axis=4) self.fg_h3 = conv3d_transpose(encodv4, 512, [self.batch_size, 4, 8, 8, 256], name='g_f_h3') self.fg_h3 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h3, scope='g_f_bn3'), name='g_f_relu3') add_activation_summary(self.fg_h3) print(self.fg_h3.get_shape().as_list()) encv3 = tf.tile(tf.expand_dims(self.en_h3, axis=1), [1, 4, 1, 1, 1]) encvo3 = tf.reshape(encv3, [self.batch_size, 4, 8, 8, 256]) encodv3 = tf.concat([self.fg_h3, encvo3], axis=4) self.fg_h4 = conv3d_transpose(encodv3, 512, [self.batch_size, 8, 16, 16, 128], name='g_f_h4') self.fg_h4 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h4, scope='g_f_bn4'), name='g_f_relu4') add_activation_summary(self.fg_h4) print(self.fg_h4.get_shape().as_list()) encv2 = tf.tile(tf.expand_dims(self.en_h2, axis=1), [1, 8, 1, 1, 1]) encvo2 = tf.reshape(encv2, [self.batch_size, 8, 16, 16, 128]) #encvo2 is [32,8,16,16,128] and self.fg_h4 is [32,8,16,16,128] ans concat is [32,8,16,16,256] encodv2 = tf.concat([self.fg_h4, encvo2], axis=4) self.fg_h5 = conv3d_transpose(encodv2, 256, [self.batch_size, 16, 32, 32, 64], name='g_f_h5') self.fg_h5 = tf.nn.relu(tf.contrib.layers.batch_norm( self.fg_h5, scope='g_f_bn5'), name='g_f_relu5') add_activation_summary(self.fg_h5) print(self.fg_h5.get_shape().as_list()) #self.fg_h5 is [32, 16, 32, 32, 64] encv1 = tf.tile(tf.expand_dims(self.en_h1, axis=1), [1, 16, 1, 1, 1]) encvo1 = tf.reshape(encv1, [self.batch_size, 16, 32, 32, 64]) encodv1 = tf.concat([self.fg_h5, encvo1], axis=4) self.fg_h6 = conv3d_transpose(encodv1, 128, [self.batch_size, 32, 64, 64, 3], name='g_f_h6') self.fg_vid = tf.nn.tanh(self.fg_h6, name='g_f_actvcation') print(self.fg_vid.get_shape().as_list()) # gen_reg = tf.reduce_mean(tf.square(img_batch - self.fg_fg[:, 0, :, :, :])) # variables = tf.contrib.framework.get_variables(vs) # # return self.fg_fg, gen_reg, variables # return self.fg_vid, variables return self.fg_vid
def ResNeXt(last, group_num=16): is_training = False channels = FLAGS.channels channel_index = -3 if FLAGS.data_format == 'NCHW' else -1 l = 0 last = tf.identity(last, name='input') # residual blocks rb = 0 skip2 = last while rb < FLAGS.res_blocks: rb += 1 l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=1, out_channels=channels, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=FLAGS.activation, init_factor=FLAGS.init_activation) ''' # original version - individual weights for each group l += 1 with tf.variable_scope('group_conv{}'.format(l)) as scope: group = tf.split(last, group_num, axis=channel_index) for _ in range(group_num): with tf.variable_scope('group{}'.format(_ + 1)) as scope: group[_] = layers.conv2d(group[_], ksize=3, out_channels=channels // group_num, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_activation) last = tf.concat(group, axis=channel_index) last = layers.apply_activation(last, activation=FLAGS.activation, data_format=FLAGS.data_format) ''' # modified version - shared weights between groups l += 1 with tf.variable_scope('group_conv{}'.format(l)) as scope: # channel en-batching shape = tf.shape(last) if FLAGS.data_format == 'NCHW': shape_batch = [ shape[0] * group_num, channels // group_num, shape[2], shape[3] ] else: shape_divide = [ shape[0], shape[1], shape[2], group_num, channels // group_num ] last = tf.reshape(last, shape_divide) last = tf.transpose(last, (0, 3, 1, 2, 4)) shape_batch = [ shape[0] * group_num, shape[1], shape[2], channels // group_num ] last = tf.reshape(last, shape_batch) # convolution last = layers.conv2d(last, ksize=3, out_channels=channels // group_num, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_activation) # channel de-batching if FLAGS.data_format != 'NCHW': shape_divide = [ shape[0], group_num, shape[1], shape[2], channels // group_num ] last = tf.reshape(last, shape_divide) last = tf.transpose(last, (0, 2, 3, 1, 4)) last = tf.reshape(last, shape) # activation last = layers.apply_activation(last, activation=FLAGS.activation, data_format=FLAGS.data_format) l += 1 with tf.variable_scope('conv{}'.format(l)) as scope: last = layers.conv2d(last, ksize=1, out_channels=channels, stride=1, padding='SAME', data_format=FLAGS.data_format, bn=FLAGS.batch_norm, train=is_training, activation=None, init_factor=FLAGS.init_factor) with tf.variable_scope('skip_connection{}'.format(l)) as scope: last = tf.add(last, skip2, 'elementwise_sum') skip2 = last last = layers.apply_activation(last, activation=FLAGS.activation, data_format=FLAGS.data_format) # return last = tf.identity(last, name='output') return last