def generator(self, x_data, name='g_'): with tf.variable_scope(name): x_data, output = flatten(x_data), None if self.flags.dataset == 'mnist': g0 = tf.nn.relu(tf_utils.linear(x_data, self.num_hiddens[0], name='fc1'), name='relu1') output = tf_utils.linear( g0, self.image_size[0] * self.image_size[1] * self.image_size[2]) elif self.flags.dataset == 'cifar10': g0 = tf.nn.relu(tf_utils.linear(x_data, self.num_hiddens[0], name='fc1'), name='relu1') g1 = tf.nn.relu(tf_utils.linear(g0, self.num_hiddens[1], name='fc2'), name='relu2') output = tf_utils.linear( g1, self.image_size[0] * self.image_size[1] * self.image_size[2]) else: raise NotImplementedError return self.out_func(output)
def discriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() # 64 -> 32 or 32 -> 16 h0_conv = tf_utils.conv2d(data, self.dis_c[0], name='h0_conv2d') h0_lrelu = tf_utils.lrelu(h0_conv, name='h0_lrelu') # 32 -> 16 or 16 -> 8 h1_conv = tf_utils.conv2d(h0_lrelu, self.dis_c[1], name='h1_conv2d') h1_batchnorm = tf_utils.batch_norm(h1_conv, name='h1_batchnorm', _ops=self._dis_train_ops) h1_lrelu = tf_utils.lrelu(h1_batchnorm, name='h1_lrelu') # 16 -> 8 or 8 -> 4 h2_conv = tf_utils.conv2d(h1_lrelu, self.dis_c[2], name='h2_conv2d') h2_batchnorm = tf_utils.batch_norm(h2_conv, name='h2_batchnorm', _ops=self._dis_train_ops) h2_lrelu = tf_utils.lrelu(h2_batchnorm, name='h2_lrelu') if (self.flags.dataset == 'mnist') or (self.flags.dataset == 'cifar10'): h2_flatten = flatten(h2_lrelu) h3_linear = tf_utils.linear(h2_flatten, 1, name='h3_linear') return tf.nn.sigmoid(h3_linear), h3_linear else: # 8 -> 4 h3_conv = tf_utils.conv2d(h2_lrelu, self.dis_c[3], name='h3_conv2d') h3_batchnorm = tf_utils.batch_norm(h3_conv, name='h3_batchnorm', _ops=self._dis_train_ops) h3_lrelu = tf_utils.lrelu(h3_batchnorm, name='h3_lrelu') h3_flatten = flatten(h3_lrelu) h4_linear = tf_utils.linear(h3_flatten, 1, name='h4_linear') return tf.nn.sigmoid(h4_linear), h4_linear
def decoder(self, z, name='decoder', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() tf_utils.print_activations(z) # 1st hidden layer h0_linear = tf_utils.linear(z, self.n_hidden, name='h0_linear') h0_tanh = tf_utils.tanh(h0_linear, name='h0_tanh') h0_drop = tf.nn.dropout(h0_tanh, keep_prob=self.keep_prob_tfph, name='h0_drop') tf_utils.print_activations(h0_drop) # 2nd hidden layer h1_linear = tf_utils.linear(h0_drop, self.n_hidden, name='h1_linear') h1_elu = tf_utils.elu(h1_linear, name='h1_elu') h1_drop = tf.nn.dropout(h1_elu, keep_prob=self.keep_prob_tfph, name='h1_drop') tf_utils.print_activations(h1_drop) # 3rd hidden layer h2_linear = tf_utils.linear(h1_drop, self.output_dim, name='h2_linear') h2_sigmoid = tf_utils.sigmoid(h2_linear, name='h2_sigmoid') tf_utils.print_activations(h2_sigmoid) output = tf.reshape(h2_sigmoid, [-1, *self.image_size]) tf_utils.print_activations(output) return output
def forward_network(self, input_img, reuse=False): with tf.compat.v1.variable_scope(self.name, reuse=reuse): tf_utils.print_activations(input_img, logger=None) inputs = self.conv2d_fixed_padding(inputs=input_img, filters=64, kernel_size=7, strides=2, name='conv1') inputs = tf_utils.max_pool(inputs, name='3x3_maxpool', ksize=[1, 3, 3, 1], strides=[1, 2, 2, 1], logger=None) inputs = self.block_layer(inputs=inputs, filters=64, block_fn=self.bottleneck_block, blocks=self.layers[0], strides=1, train_mode=False, name='block_layer1') inputs = self.block_layer(inputs=inputs, filters=128, block_fn=self.bottleneck_block, blocks=self.layers[1], strides=2, train_mode=False, name='block_layer2') inputs = self.block_layer(inputs=inputs, filters=256, block_fn=self.bottleneck_block, blocks=self.layers[2], strides=2, train_mode=False, name='block_layer3') inputs = self.block_layer(inputs=inputs, filters=512, block_fn=self.bottleneck_block, blocks=self.layers[3], strides=2, train_mode=False, name='block_layer4') inputs = tf_utils.relu(inputs, name='before_flatten_relu', logger=None) # _, h, w, _ = inputs.get_shape().as_list() # inputs = tf_utils.avg_pool(inputs, name='gap', ksize=[1, h, w, 1], strides=[1, 1, 1, 1], logger=self.logger) # Flatten & FC1 inputs = tf_utils.flatten(inputs, name='flatten', logger=None) inputs = tf_utils.linear(inputs, 512, name='FC1') inputs = tf_utils.relu(inputs, name='FC1_relu', logger=None) inputs = tf_utils.linear(inputs, 256, name='FC2') inputs = tf_utils.relu(inputs, name='FC2_relu', logger=None) logits = tf_utils.linear(inputs, self.num_attribute, name='Out') return logits
def discriminator(self, y_data, name='d_', is_reuse=False): with tf.variable_scope(name, reuse=is_reuse): y_data = flatten(y_data) d0 = tf.nn.relu( tf_utils.linear(y_data, self.num_hidden, name='fc1')) d1 = tf_utils.linear(d0, 1, name='fc2') return tf.nn.sigmoid(d1), d1
def generator(self, x_data, name='g_'): with tf.variable_scope(name): x_data = flatten(x_data) g0 = tf.nn.relu(tf_utils.linear(x_data, self.num_hidden, name='fc1'), name='relu1') g1 = tf_utils.linear( g0, self.image_size[0] * self.image_size[1] * self.image_size[2]) return tf.nn.tanh(g1)
def Discriminator(inputs, is_reuse=True, name='disc'): with tf.variable_scope(name, reuse=is_reuse): print('is_reuse: {}'.format(is_reuse)) output01 = tf_utils.linear(inputs, DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, DIS_DIM, name='fc-4') return output04
def generator(self, data, name='g_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() data_flatten = flatten(data) # 4 x 4 h0_linear = tf_utils.linear(data_flatten, 4*4*self.gen_c[0], name='h0_linear') h0_reshape = tf.reshape(h0_linear, [tf.shape(h0_linear)[0], 4, 4, self.gen_c[0]]) h0_batchnorm = tf_utils.batch_norm(h0_reshape, name='h0_batchnorm', _ops=self._gen_train_ops) h0_relu = tf.nn.relu(h0_batchnorm, name='h0_relu') # 8 x 8 h1_deconv = tf_utils.deconv2d(h0_relu, self.gen_c[1], name='h1_deconv2d') h1_batchnorm = tf_utils.batch_norm(h1_deconv, name='h1_batchnorm', _ops=self._gen_train_ops) h1_relu = tf.nn.relu(h1_batchnorm, name='h1_relu') # 16 x 16 h2_deconv = tf_utils.deconv2d(h1_relu, self.gen_c[2], name='h2_deconv2d') h2_batchnorm = tf_utils.batch_norm(h2_deconv, name='h2_batchnorm', _ops=self._gen_train_ops) h2_relu = tf.nn.relu(h2_batchnorm, name='h2_relu') # 32 x 32 h3_deconv = tf_utils.deconv2d(h2_relu, self.gen_c[3], name='h3_deconv2d') h3_batchnorm = tf_utils.batch_norm(h3_deconv, name='h3_batchnorm', _ops=self._gen_train_ops) h3_relu = tf.nn.relu(h3_batchnorm, name='h3_relu') # 64 x 64 output = tf_utils.deconv2d(h3_relu, self.image_size[2], name='h4_deconv2d') return tf.nn.tanh(output)
def input_to_feature(self, z): with tf.variable_scope("g_", reuse=True): data_flatten = flatten(z) # 4 x 4 h0_linear = tf_utils.linear(data_flatten, 4 * 4 * self.gen_c[0], name='h0_linear') h0_reshape = tf.reshape( h0_linear, [tf.shape(h0_linear)[0], 4, 4, self.gen_c[0]]) h0_batchnorm = tf_utils.batch_norm(h0_reshape, name='h0_batchnorm', _ops=self._gen_train_ops) h0_relu = tf.nn.relu(h0_batchnorm, name='h0_relu') # 8 x 8 h1_deconv = tf_utils.deconv2d(h0_relu, self.gen_c[1], name='h1_deconv2d') h1_batchnorm = tf_utils.batch_norm(h1_deconv, name='h1_batchnorm', _ops=self._gen_train_ops) h1_relu = tf.nn.relu(h1_batchnorm, name='h1_relu') # 16 x 16 h2_deconv = tf_utils.deconv2d(h1_relu, self.gen_c[2], name='h2_deconv2d') h2_batchnorm = tf_utils.batch_norm(h2_deconv, name='h2_batchnorm', _ops=self._gen_train_ops) h2_relu = tf.nn.relu(h2_batchnorm, name='h2_relu') return h2_relu
def encoder(self,data,name='enc_'): with tf.variable_scope(name): # 64 -> 32 or 32 -> 16 h0_conv = tf_utils.conv3d(data, self.dis_c[0], name='h0_conv3d') h0_lrelu = tf_utils.lrelu(h0_conv, name='h0_lrelu') # 32 -> 16 or 16 -> 8 h1_conv = tf_utils.conv3d(h0_lrelu, self.dis_c[1], name='h1_conv3d') h1_batchnorm = tf_utils.batch_norm(h1_conv, name='h1_batchnorm', _ops=self._dis_train_ops) h1_lrelu = tf_utils.lrelu(h1_batchnorm, name='h1_lrelu') # 16 -> 8 or 8 -> 4 h2_conv = tf_utils.conv3d(h1_lrelu, self.dis_c[2], name='h2_conv3d') h2_batchnorm = tf_utils.batch_norm(h2_conv, name='h2_batchnorm', _ops=self._dis_train_ops) h2_lrelu = tf_utils.lrelu(h2_batchnorm, name='h2_lrelu') h3_conv = tf_utils.conv3d(h2_lrelu, self.dis_c[3], name='h3_conv3d') h3_batchnorm = tf_utils.batch_norm(h3_conv, name='h3_batchnorm', _ops=self._dis_train_ops) h3_lrelu = tf_utils.lrelu(h3_batchnorm, name='h3_lrelu') h4_conv = tf_utils.conv3d(h3_lrelu, self.dis_c[3],k_d=1,k_h=2,k_w=2, name='h4_conv3d') h4_batchnorm = tf_utils.batch_norm(h4_conv, name='h4_batchnorm', _ops=self._dis_train_ops) h4_lrelu = tf_utils.lrelu(h4_batchnorm, name='h4_lrelu') h4_flatten = flatten(h4_lrelu) h5_linear = tf_utils.linear(h4_flatten, 1024, name='h4_linear') return tf.nn.sigmoid(h5_linear), h5_linear
def decoder(self,data,name='dec_'): with tf.variable_scope(name): h0_linear = tf_utils.linear(data, 5*8*self.dis_c[3],input_size=1024, name='h0_linear') h0_reshape = tf.reshape(h0_linear, [tf.shape(h0_linear)[0],1, 5, 8, self.dis_c[3]]) h0_batchnorm = tf_utils.batch_norm(h0_reshape, name='h0_batchnorm', _ops=self._gen_train_ops) h0_relu = tf.nn.relu(h0_batchnorm, name='h0_relu') # 8 x 8 h1_deconv = tf_utils.deconv3d(h0_relu, self.dis_c[3],output_size=[10,15,256],k_t=4,d_t=5, d_h=2, d_w=2,padding_='SAME', name='h1_deconv3d') h1_batchnorm = tf_utils.batch_norm(h1_deconv, name='h1_batchnorm', _ops=self._gen_train_ops) h1_relu = tf.nn.relu(h1_batchnorm, name='h1_relu') # 16 x 16 h2_deconv = tf_utils.deconv3d(h1_relu, self.dis_c[2],output_size=[20,30,256],stepup_out=1, name='h2_deconv3d') h2_batchnorm = tf_utils.batch_norm(h2_deconv, name='h2_batchnorm', _ops=self._gen_train_ops) h2_relu = tf.nn.relu(h2_batchnorm, name='h2_relu') h3_deconv = tf_utils.deconv3d(h2_relu, self.dis_c[1],output_size=[40,60,128],stepup_out=2,name='h3_deconv3d') h3_batchnorm = tf_utils.batch_norm(h3_deconv, name='h3_batchnorm', _ops=self._gen_train_ops) h3_relu = tf.nn.relu(h3_batchnorm, name='h3_relu') h4_deconv = tf_utils.deconv3d(h3_relu, self.dis_c[0],output_size=[79,119,64],stepup_out=2,d_t=1, d_h=2, d_w=2,padding_='SAME' ,name='h4_deconv3d') h4_batchnorm = tf_utils.batch_norm(h4_deconv, name='h4_batchnorm', _ops=self._gen_train_ops) h4_relu = tf.nn.relu(h4_batchnorm, name='h4_relu') # 64 x 64 output = tf_utils.deconv3d(h4_relu, self.image_size[2],output_size=self.image_size, d_t=1, d_h=2, d_w=2,padding_='SAME',name='h5_deconv3d') return tf.nn.tanh(output)
def generator(self, data, name='g_'): with tf.variable_scope(name): data_flatten = flatten(data) # 4 x 4 h0_linear = tf_utils.linear(data_flatten, 4*4*self.gen_c[0], name='h0_linear') h0_reshape = tf.reshape(h0_linear, [tf.shape(h0_linear)[0], 4, 4, self.gen_c[0]]) h0_batchnorm = tf_utils.batch_norm(h0_reshape, name='h0_batchnorm', _ops=self._gen_train_ops) h0_relu = tf.nn.relu(h0_batchnorm, name='h0_relu') # 8 x 8 h1_deconv = tf_utils.deconv2d(h0_relu, self.gen_c[1], name='h1_deconv2d') h1_batchnorm = tf_utils.batch_norm(h1_deconv, name='h1_batchnorm', _ops=self._gen_train_ops) h1_relu = tf.nn.relu(h1_batchnorm, name='h1_relu') # 16 x 16 h2_deconv = tf_utils.deconv2d(h1_relu, self.gen_c[2], name='h2_deconv2d') h2_batchnorm = tf_utils.batch_norm(h2_deconv, name='h2_batchnorm', _ops=self._gen_train_ops) h2_relu = tf.nn.relu(h2_batchnorm, name='h2_relu') # 32 x 32 if (self.flags.dataset == 'mnist') or (self.flags.dataset == 'cifar10'): output = tf_utils.deconv2d(h2_relu, self.image_size[2], name='h3_deconv2d') return tf.nn.tanh(output) else: h3_deconv = tf_utils.deconv2d(h2_relu, self.gen_c[3], name='h3_deconv2d') h3_batchnorm = tf_utils.batch_norm(h3_deconv, name='h3_batchnorm', _ops=self._gen_train_ops) h3_relu = tf.nn.relu(h3_batchnorm, name='h3_relu') # 64 x 64 output = tf_utils.deconv2d(h3_relu, self.image_size[2], name='h4_deconv2d') return tf.nn.tanh(output)
def basicDiscriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() tf_utils.print_activations(data) # from (N, 32, 32, 1) to (N, 16, 16, 64) h0_conv = tf_utils.conv2d(data, self.dis_c[0], k_h=5, k_w=5, name='h0_conv2d') h0_lrelu = tf_utils.lrelu(h0_conv, name='h0_lrelu') # from (N, 16, 16, 64) to (N, 8, 8, 128) h1_conv = tf_utils.conv2d(h0_lrelu, self.dis_c[1], k_h=5, k_w=5, name='h1_conv2d') h1_lrelu = tf_utils.lrelu(h1_conv, name='h1_lrelu') # from (N, 8, 8, 128) to (N, 4, 4, 256) h2_conv = tf_utils.conv2d(h1_lrelu, self.dis_c[2], k_h=5, k_w=5, name='h2_conv2d') h2_lrelu = tf_utils.lrelu(h2_conv, name='h2_lrelu') # from (N, 4, 4, 256) to (N, 4096) and to (N, 1) h2_flatten = flatten(h2_lrelu) h3_linear = tf_utils.linear(h2_flatten, 1, name='h3_linear') return tf.nn.sigmoid(h3_linear), h3_linear
def discriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() # (128, 256) -> (64, 128) h0_conv = tf_utils.conv2d(data, self.dis_c[0], name='h0_conv2d') h0_lrelu = tf_utils.lrelu(h0_conv, name='h0_lrelu') # (64, 128) -> (32, 64) h1_conv = tf_utils.conv2d(h0_lrelu, self.dis_c[1], name='h1_conv2d') h1_batchnorm = tf_utils.batch_norm(h1_conv, name='h1_batchnorm', _ops=self._dis_train_ops) h1_lrelu = tf_utils.lrelu(h1_batchnorm, name='h1_lrelu') # (32, 64) -> (16, 32) h2_conv = tf_utils.conv2d(h1_lrelu, self.dis_c[2], name='h2_conv2d') h2_batchnorm = tf_utils.batch_norm(h2_conv, name='h2_batchnorm', _ops=self._dis_train_ops) h2_lrelu = tf_utils.lrelu(h2_batchnorm, name='h2_lrelu') # (16, 32) -> (8, 16) h3_conv = tf_utils.conv2d(h2_lrelu, self.dis_c[3], name='h3_conv2d') h3_batchnorm = tf_utils.batch_norm(h3_conv, name='h3_batchnorm', _ops=self._dis_train_ops) h3_lrelu = tf_utils.lrelu(h3_batchnorm, name='h3_lrelu') # (8, 16) -> (4, 8) h4_conv = tf_utils.conv2d(h3_lrelu, self.dis_c[4], name='h4_conv2d') h4_batchnorm = tf_utils.batch_norm(h4_conv, name='h4_batchnorm', _ops=self._dis_train_ops) h4_lrelu = tf_utils.lrelu(h4_batchnorm, name='h4_lrelu') # (4, 8) -> (2, 4) h5_conv = tf_utils.conv2d(h4_lrelu, self.dis_c[5], name='h5_conv2d') h5_batchnorm = tf_utils.batch_norm(h5_conv, name='h5_batchnorm', _ops=self._dis_train_ops) h5_lrelu = tf_utils.lrelu(h5_batchnorm, name='h5_lrelu') h5_flatten = flatten(h5_lrelu) h6_linear = tf_utils.linear(h5_flatten, 1, name='h6_linear') return tf.nn.sigmoid(h6_linear), h6_linear
def basicGenerator(self, data, name='g_'): with tf.variable_scope(name): data_flatten = flatten(data) tf_utils.print_activations(data_flatten) # from (N, 128) to (N, 4, 4, 256) h0_linear = tf_utils.linear(data_flatten, self.gen_c[0], name='h0_linear') if self.flags.dataset == 'cifar10': h0_linear = tf.reshape(h0_linear, [ tf.shape(h0_linear)[0], 4, 4, int(self.gen_c[0] / (4 * 4)) ]) h0_linear = tf_utils.norm(h0_linear, _type='batch', _ops=self.gen_train_ops, name='h0_norm') h0_relu = tf.nn.relu(h0_linear, name='h0_relu') h0_reshape = tf.reshape( h0_relu, [tf.shape(h0_relu)[0], 4, 4, int(self.gen_c[0] / (4 * 4))]) # from (N, 4, 4, 256) to (N, 8, 8, 128) h1_deconv = tf_utils.deconv2d(h0_reshape, self.gen_c[1], k_h=5, k_w=5, name='h1_deconv2d') if self.flags.dataset == 'cifar10': h1_deconv = tf_utils.norm(h1_deconv, _type='batch', _ops=self.gen_train_ops, name='h1_norm') h1_relu = tf.nn.relu(h1_deconv, name='h1_relu') # from (N, 8, 8, 128) to (N, 16, 16, 64) h2_deconv = tf_utils.deconv2d(h1_relu, self.gen_c[2], k_h=5, k_w=5, name='h2_deconv2d') if self.flags.dataset == 'cifar10': h2_deconv = tf_utils.norm(h2_deconv, _type='batch', _ops=self.gen_train_ops, name='h2_norm') h2_relu = tf.nn.relu(h2_deconv, name='h2_relu') # from (N, 16, 16, 64) to (N, 32, 32, 1) output = tf_utils.deconv2d(h2_relu, self.image_size[2], k_h=5, k_w=5, name='h3_deconv2d') return tf_utils.tanh(output)
def discriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() h2_lrelu, _= self.encoder(data) # 8 -> 4 h4_linear = tf_utils.linear(h2_lrelu, 1, name='h4_linear') return tf.nn.sigmoid(h4_linear), h4_linear
def Generator(n_samples, real_data_, name='gen'): if FIXED_GENERATOR: return real_data_ + (1. * tf.random_normal(tf.shape(real_data_))) else: with tf.variable_scope(name): noise = tf.random_normal([n_samples, 100]) output01 = tf_utils.linear(noise, DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, GEN_DIM, name='fc-4') return output04
def discriminator(self, y_data, name='d_', is_reuse=False): with tf.variable_scope(name, reuse=is_reuse): y_data, output = flatten(y_data), None if self.flags.dataset == 'mnist': d0 = tf.nn.relu( tf_utils.linear(y_data, self.num_hiddens[0], name='fc1')) output = tf_utils.linear(d0, 1, name='fc2') elif self.flags.dataset == 'cifar10': d0 = tf.nn.relu( tf_utils.linear(y_data, self.num_hiddens[0], name='fc1')) d1 = tf.nn.relu( tf_utils.linear(d0, self.num_hiddens[1], name='fc2')) output = tf_utils.linear(d1, 1, name='fc3') else: raise NotImplementedError return tf.nn.sigmoid(output), output
def Generator_Softmax(n_samples, name='gen'): with tf.variable_scope(name): noise = tf.random_normal([n_samples, GEN_DIM]) output01 = tf_utils.linear(noise, 2*DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, 2*DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, 2*DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, GEN_DIM, name='fc-4') # Reminder: a logit can be modeled as a linear function of the predictors output05 = tf.nn.softmax(output04, name = 'softmax-1') return output05
def encoder(self, data, name='encoder'): with tf.variable_scope(name): data_flatten = flatten(data) tf_utils.print_activations(data_flatten) # 1st hidden layer h0_linear = tf_utils.linear(data_flatten, self.n_hidden, name='h0_linear') h0_elu = tf_utils.elu(h0_linear, name='h0_elu') h0_drop = tf.nn.dropout(h0_elu, keep_prob=self.keep_prob_tfph, name='h0_drop') tf_utils.print_activations(h0_drop) # 2nd hidden layer h1_linear = tf_utils.linear(h0_drop, self.n_hidden, name='h1_linear') h1_tanh = tf_utils.tanh(h1_linear, name='h1_tanh') h1_drop = tf.nn.dropout(h1_tanh, keep_prob=self.keep_prob_tfph, name='h1_drop') tf_utils.print_activations(h1_drop) # 3rd hidden layer h2_linear = tf_utils.linear(h1_drop, 2 * self.flags.z_dim, name='h2_linear') tf_utils.print_activations(h2_linear) # The mean parameter is unconstrained mean = h2_linear[:, :self.flags.z_dim] # The standard deviation must be positive. # Parameterize with a softplus and add a small epsilon for numerical stability stddev = 1e-6 + tf.nn.softplus(h2_linear[:, self.flags.z_dim:]) tf_utils.print_activations(mean) tf_utils.print_activations(stddev) return mean, stddev
def Generator(n_samples, real_data_, name='gen'): if FIXED_GENERATOR: return real_data_ + (1. * tf.random_normal(tf.shape(real_data_))) else: with tf.variable_scope(name): noise = tf.random_normal([n_samples, GEN_DIM]) output01 = tf_utils.linear(noise, DIM, name='fc-1') output01 = tf_utils.relu(output01, name='relu-1') output02 = tf_utils.linear(output01, DIM, name='fc-2') output02 = tf_utils.relu(output02, name='relu-2') output03 = tf_utils.linear(output02, DIM, name='fc-3') output03 = tf_utils.relu(output03, name='relu-3') output04 = tf_utils.linear(output03, GEN_DIM, name='fc-4') # Reminder: a logit can be modeled as a linear function of the predictors output05 = tf.nn.softmax(output04, name='softmax-1') return output05
def resnetDiscriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name) as scope: if is_reuse is True: scope.reuse_variables() tf_utils.print_activations(data) # (N, 64, 64, 64) conv_0 = tf_utils.conv2d(data, output_dim=self.dis_c[0], k_h=3, k_w=3, d_h=1, d_w=1, name='conv_0') # (N, 32, 32, 128) resblock_1 = tf_utils.res_block_v2(conv_0, self.dis_c[1], filter_size=3, _ops=self.dis_train_ops, norm_='layer', resample='down', name='res_block_1') # (N, 16, 16, 256) resblock_2 = tf_utils.res_block_v2(resblock_1, self.dis_c[2], filter_size=3, _ops=self.dis_train_ops, norm_='layer', resample='down', name='res_block_2') # (N, 8, 8, 512) resblock_3 = tf_utils.res_block_v2(resblock_2, self.dis_c[3], filter_size=3, _ops=self.dis_train_ops, norm_='layer', resample='down', name='res_block_3') # (N, 4, 4, 512) resblock_4 = tf_utils.res_block_v2(resblock_3, self.dis_c[4], filter_size=3, _ops=self.dis_train_ops, norm_='layer', resample='down', name='res_block_4') # (N, 4*4*512) flatten_5 = flatten(resblock_4) output = tf_utils.linear(flatten_5, 1, name='output') return tf.nn.sigmoid(output), output
def __init__(self, input_dim, output_dim=1, optimizer=None, use_dropout=True, lr=0.001, random_seed=123, is_train=True, log_dir=None, name=None): self.name = name self.is_train = is_train self.log_dir = log_dir self.cur_lr = None self.logger, self.file_handler, self.stream_handler = utils.init_logger(log_dir=self.log_dir, name=self.name, is_train=self.is_train) with tf.variable_scope(self.name): # Placeholders for inputs self.X = tf.placeholder(dtype=tf.float32, shape=[None, input_dim], name='X') self.y = tf.placeholder(dtype=tf.float32, shape=[None, output_dim], name='y') self.keep_prob = tf.placeholder(tf.float32, name='keep_prob') tf_utils.print_activations(self.X, logger=self.logger if self.is_train else None) # Placeholders for TensorBoard self.train_acc = tf.placeholder(tf.float32, name='train_acc') self.val_acc = tf.placeholder(tf.float32, name='val_acc') net = self.X if use_dropout: net = tf_utils.dropout(x=net, keep_prob=self.keep_prob, seed=random_seed, name='dropout', logger=self.logger if self.is_train else None) # Network, loss, and optimizer self.y_pred = tf_utils.linear(net, output_size=output_dim) tf_utils.print_activations(self.y_pred, logger=self.logger if self.is_train else None) self.loss = tf.math.reduce_mean(tf.nn.l2_loss(self.y_pred - self.y)) self.train_op, self.cur_lr = optimizer_fn(optimizer, lr=lr, loss=self.loss, name=self.name) # Accuracy etc self.y_pred_round = tf.math.round(x=self.y_pred, name='rounded_pred') accuracy = tf.equal(tf.cast(x=self.y_pred_round, dtype=tf.int32), tf.cast(x=self.y, dtype=tf.int32)) self.accuracy = tf.reduce_mean(tf.cast(x=accuracy, dtype=tf.float32)) * 100. self._tensorboard() tf_utils.show_all_variables(logger=self.logger if self.is_train else None)
def discriminator(self, data, name='d_', is_reuse=False): with tf.variable_scope(name, reuse=is_reuse) as scope: # 64 -> 32 h0_conv = tf_utils.conv2d(data, self.dis_c[0], name='h0_conv2d') h0_lrelu = tf_utils.lrelu(h0_conv, name='h0_lrelu') # 32 -> 16 h1_conv = tf_utils.conv2d(h0_lrelu, self.dis_c[1], name='h1_conv2d') h1_batchnorm = tf_utils.batch_norm(h1_conv, name='h1_batchnorm', _ops=self._dis_train_ops) h1_lrelu = tf_utils.lrelu(h1_batchnorm, name='h1_lrelu') # 16 -> 8 h2_conv = tf_utils.conv2d(h1_lrelu, self.dis_c[2], name='h2_conv2d') h2_batchnorm = tf_utils.batch_norm(h2_conv, name='h2_batchnorm', _ops=self._dis_train_ops) h2_lrelu = tf_utils.lrelu(h2_batchnorm, name='h2_lrelu') # 8 -> 4 h3_conv = tf_utils.conv2d(h2_lrelu, self.dis_c[3], name='h3_conv2d') h3_batchnorm = tf_utils.batch_norm(h3_conv, name='h3_batchnorm', _ops=self._dis_train_ops) h3_lrelu = tf_utils.lrelu(h3_batchnorm, name='h3_lrelu') h3_flatten = flatten(h3_lrelu) h4_linear = tf_utils.linear(h3_flatten, 1, name='h4_linear') return tf.nn.sigmoid(h4_linear), h4_linear
def __call__(self, x, is_train=True): with tf.variable_scope(self.name, reuse=self.reuse): tf_utils.print_activations(x) # (N, 120, 160, 1) -> (N, 60, 80, 64) h0_conv = tf_utils.conv2d( x, output_dim=self.dims[0], initializer='he', name='h0_conv', logger=self.logger if is_train is True else None) h0_lrelu = tf_utils.lrelu( h0_conv, name='h0_lrelu', logger=self.logger if is_train is True else None) # (N, 60, 80, 64) -> (N, 30, 40, 128) h1_conv = tf_utils.conv2d( h0_lrelu, output_dim=self.dims[1], initializer='he', name='h1_conv', logger=self.logger if is_train is True else None) h1_norm = tf_utils.norm( h1_conv, name='h1_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h1_lrelu = tf_utils.lrelu( h1_norm, name='h1_lrelu', logger=self.logger if is_train is True else None) # (N, 30, 40, 128) -> (N, 15, 20, 256) h2_conv = tf_utils.conv2d( h1_lrelu, output_dim=self.dims[2], initializer='he', name='h2_conv', logger=self.logger if is_train is True else None) h2_norm = tf_utils.norm( h2_conv, name='h2_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h2_lrelu = tf_utils.lrelu( h2_norm, name='h2_lrelu', logger=self.logger if is_train is True else None) # (N, 15, 20, 256) -> (N, 8, 10, 512) h3_conv = tf_utils.conv2d( h2_lrelu, output_dim=self.dims[3], initializer='he', name='h3_conv', logger=self.logger if is_train is True else None) h3_norm = tf_utils.norm( h3_conv, name='h3_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h3_lrelu = tf_utils.lrelu( h3_norm, name='h3_lrelu', logger=self.logger if is_train is True else None) # (N, 8, 10, 512) -> (N, 4, 5, 1024) h4_conv = tf_utils.conv2d( h3_lrelu, output_dim=self.dims[4], initializer='he', name='h4_conv', logger=self.logger if is_train is True else None) h4_norm = tf_utils.norm( h4_conv, name='h4_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h4_lrelu = tf_utils.lrelu( h4_norm, name='h4_lrelu', logger=self.logger if is_train is True else None) # (N, 4, 5, 1024) -> (N, 4*5*1024) h4_flatten = tf_utils.flatten( h4_lrelu, name='h4_flatten', logger=self.logger if is_train is True else None) # (N, 4*5*1024) -> (N, 1) output = tf_utils.linear( h4_flatten, output_size=self.dims[5], initializer='he', name='output', logger=self.logger if is_train is True else None) # Set reuse=True for next call self.reuse = True self.variables = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return output
def __call__(self, x, is_train=True): with tf.variable_scope(self.name, reuse=self.reuse): tf_utils.print_activations(x) # (N, 100) -> (N, 4, 5, 512) h0_linear = tf_utils.linear( x, 4 * 5 * self.dims[0], name='h0_linear', initializer='He', logger=self.logger if is_train is True else None) h0_reshape = tf.reshape( h0_linear, [tf.shape(h0_linear)[0], 4, 5, self.dims[0]]) # (N, 4, 5, 512) -> (N, 8, 10, 512) resblock_1 = tf_utils.res_block_v2( x=h0_reshape, k=self.dims[1], filter_size=3, _ops=self._ops, norm_='batch', resample='up', name='res_block_1', logger=self.logger if is_train is True else None) # (N, 8, 10, 512) -> (N, 16, 20, 256) resblock_2 = tf_utils.res_block_v2( x=resblock_1, k=self.dims[2], filter_size=3, _ops=self._ops, norm_='batch', resample='up', name='res_block_2', logger=self.logger if is_train is True else None) # (N, 16, 20, 256) -> (N, 15, 20, 256) resblock_2_split, _ = tf.split(resblock_2, [15, 1], axis=1, name='resblock_2_split') tf_utils.print_activations( resblock_2_split, logger=self.logger if is_train is True else None) # (N, 15, 20, 256) -> (N, 30, 40, 128) resblock_3 = tf_utils.res_block_v2( x=resblock_2_split, k=self.dims[3], filter_size=3, _ops=self._ops, norm_='batch', resample='up', name='res_block_3', logger=self.logger if is_train is True else None) # (N, 30, 40, 128) -> (N, 60, 80, 64) resblock_4 = tf_utils.res_block_v2( x=resblock_3, k=self.dims[4], filter_size=3, _ops=self._ops, norm_='batch', resample='up', name='res_block_4', logger=self.logger if is_train is True else None) # (N, 60, 80, 64) -> (N, 120, 160, 64) resblock_5 = tf_utils.res_block_v2( x=resblock_4, k=self.dims[5], filter_size=3, _ops=self._ops, norm_='batch', resample='up', name='res_block_5', logger=self.logger if is_train is True else None) norm_5 = tf_utils.norm( resblock_5, name='norm_5', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) relu_5 = tf_utils.relu( norm_5, name='relu_5', logger=self.logger if is_train is True else None) # (N, 120, 160, 64) -> (N, 120, 160, 3) conv_6 = tf_utils.conv2d( relu_5, output_dim=self.dims[6], k_h=3, k_w=3, d_h=1, d_w=1, name='conv_6', logger=self.logger if is_train is True else None) output = tf_utils.tanh( conv_6, name='output', logger=self.logger if is_train is True else None) # Set reuse=True for next call self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return output
def generator(self, data, name='g_'): with tf.variable_scope(name): data_flatten = flatten(data) tf_utils.print_activations(data_flatten) # from (N, 128) to (N, 2, 4, 512) h0_linear = tf_utils.linear(data_flatten, self.gen_c[0], name='h0_linear') h0_reshape = tf.reshape( h0_linear, [tf.shape(h0_linear)[0], 2, 4, int(self.gen_c[0] / (2 * 4))]) # (N, 4, 8, 512) resblock_1 = tf_utils.res_block_v2(h0_reshape, self.gen_c[1], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_1') # (N, 8, 16, 256) resblock_2 = tf_utils.res_block_v2(resblock_1, self.gen_c[2], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_2') # (N, 16, 32, 128) resblock_3 = tf_utils.res_block_v2(resblock_2, self.gen_c[3], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_3') # (N, 32, 64, 64) resblock_4 = tf_utils.res_block_v2(resblock_3, self.gen_c[4], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_4') # (N, 64, 128, 32) resblock_5 = tf_utils.res_block_v2(resblock_4, self.gen_c[5], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_5') # (N, 128, 256, 32) resblock_6 = tf_utils.res_block_v2(resblock_5, self.gen_c[6], filter_size=3, _ops=self.gen_train_ops, norm_='batch', resample='up', name='res_block_6') norm_7 = tf_utils.norm(resblock_6, _type='batch', _ops=self.gen_train_ops, name='norm_7') relu_7 = tf_utils.relu(norm_7, name='relu_7') # (N, 128, 256, 3) output = tf_utils.conv2d(relu_7, output_dim=self.image_size[2], k_w=3, k_h=3, d_h=1, d_w=1, name='output') return tf_utils.tanh(output)
def __call__(self, x, is_train=True): with tf.variable_scope(self.name, reuse=self.reuse): tf_utils.print_activations(x) # (N, 100) -> (N, 4, 5, 1024) h0_linear = tf_utils.linear( x, 4 * 5 * self.dims[0], name='h0_linear', initializer='He', logger=self.logger if is_train is True else None) h0_reshape = tf.reshape( h0_linear, [tf.shape(h0_linear)[0], 4, 5, self.dims[0]]) h0_norm = tf_utils.norm( h0_reshape, name='h0_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h0_relu = tf_utils.relu( h0_norm, name='h0_relu', logger=self.logger if is_train is True else None) # (N, 4, 5, 1024) -> (N, 8, 10, 512) h1_deconv = tf_utils.deconv2d( h0_relu, output_dim=self.dims[1], name='h1_deconv2d', initializer='He', logger=self.logger if is_train is True else None) h1_norm = tf_utils.norm( h1_deconv, name='h1_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h1_relu = tf_utils.relu( h1_norm, name='h1_relu', logger=self.logger if is_train is True else None) # (N, 8, 10, 512) -> (N, 16, 20, 256) h2_deconv = tf_utils.deconv2d( h1_relu, output_dim=self.dims[2], name='h2_deconv2d', initializer='He', logger=self.logger if is_train is True else None) h2_norm = tf_utils.norm( h2_deconv, name='h2_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h2_relu = tf_utils.relu( h2_norm, name='h2_relu', logger=self.logger if is_train is True else None) # (N, 16, 20, 256) -> (N, 15, 20, 256) h2_split, _ = tf.split(h2_relu, [15, 1], axis=1, name='h2_split') tf_utils.print_activations( h2_split, logger=self.logger if is_train is True else None) # (N, 15, 20, 256) -> (N, 30, 40, 128) h3_deconv = tf_utils.deconv2d( h2_split, output_dim=self.dims[3], name='h3_deconv2d', initializer='He', logger=self.logger if is_train is True else None) h3_norm = tf_utils.norm( h3_deconv, name='h3_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h3_relu = tf_utils.relu( h3_norm, name='h3_relu', logger=self.logger if is_train is True else None) # (N, 30, 40, 128) -> (N, 60, 80, 64) h4_deconv = tf_utils.deconv2d( h3_relu, output_dim=self.dims[4], name='h4_deconv2d', initializer='He', logger=self.logger if is_train is True else None) h4_norm = tf_utils.norm( h4_deconv, name='h4_batch', _type='batch', _ops=self._ops, is_train=is_train, logger=self.logger if is_train is True else None) h4_relu = tf_utils.relu( h4_norm, name='h4_relu', logger=self.logger if is_train is True else None) # (N, 60, 80, 64) -> (N, 120, 160, 1) h5_deconv = tf_utils.deconv2d( h4_relu, output_dim=self.dims[5], name='h5_deconv', initializer='He', logger=self.logger if is_train is True else None) output = tf_utils.tanh( h5_deconv, name='output', logger=self.logger if is_train is True else None) # Set reuse=True for next call self.reuse = True self.variables = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return output
def generator(self, data, name='g_'): with tf.variable_scope(name): data_flatten = flatten(data) # 2 x 4 h0_linear = tf_utils.linear(data_flatten, 2 * 4 * self.gen_c[0], name='h0_linear') h0_reshape = tf.reshape( h0_linear, [tf.shape(h0_linear)[0], 2, 4, self.gen_c[0]]) h0_batchnorm = tf_utils.batch_norm(h0_reshape, name='h0_batchnorm', _ops=self._gen_train_ops) h0_relu = tf.nn.relu(h0_batchnorm, name='h0_relu') # 4 x 8 h1_deconv = tf_utils.deconv2d(h0_relu, self.gen_c[1], name='h1_deconv2d') h1_batchnorm = tf_utils.batch_norm(h1_deconv, name='h1_batchnorm', _ops=self._gen_train_ops) h1_relu = tf.nn.relu(h1_batchnorm, name='h1_relu') # 8 x 16 h2_deconv = tf_utils.deconv2d(h1_relu, self.gen_c[2], name='h2_deconv2d') h2_batchnorm = tf_utils.batch_norm(h2_deconv, name='h2_batchnorm', _ops=self._gen_train_ops) h2_relu = tf.nn.relu(h2_batchnorm, name='h2_relu') # 16 x 32 h3_deconv = tf_utils.deconv2d(h2_relu, self.gen_c[3], name='h3_deconv2d') h3_batchnorm = tf_utils.batch_norm(h3_deconv, name='h3_batchnorm', _ops=self._gen_train_ops) h3_relu = tf.nn.relu(h3_batchnorm, name='h3_relu') # 32 x 64 h4_deconv = tf_utils.deconv2d(h3_relu, self.gen_c[4], name='h4_deconv2d') h4_batchnorm = tf_utils.batch_norm(h4_deconv, name='h4_batchnorm', _ops=self._gen_train_ops) h4_relu = tf.nn.relu(h4_batchnorm, name='h4_relu') # 64 x 128 h5_deconv = tf_utils.deconv2d(h4_relu, self.gen_c[5], name='h5_deconv2d') h5_batchnorm = tf_utils.batch_norm(h5_deconv, name='h5_batchnorm', _ops=self._gen_train_ops) h5_relu = tf.nn.relu(h5_batchnorm, name='h5_relu') # 128 x 256 output = tf_utils.deconv2d(h5_relu, self.image_size[2], name='h6_deconv2d') return tf.nn.tanh(output)
def KGenerator(self, data, reuse=False, name='Kg_'): with tf.variable_scope(name, reuse=reuse): data_flatten = flatten(data) #tf_utils.print_activations(data_flatten) # from (N, 64) to (N, 4, 4, 128) h0_linear = tf_utils.linear(data_flatten, 4 * 4 * 128, name='h0_linear') h0_linear = tf.reshape(h0_linear, [tf.shape(h0_linear)[0], 4, 4, 128]) h0_linear = tf_utils.norm(h0_linear, _type='instance', name='h0_norm') h0_relu = tf.nn.relu(h0_linear, name='h0_relu') h0_reshape = tf.reshape(h0_relu, [tf.shape(h0_relu)[0], 4, 4, 128]) # from (N, 4, 4, 128) to (N, 8, 8, 128) h1_deconv = tf_utils.deconv2d(h0_reshape, 128, k_h=5, k_w=5, name='h1_deconv2d', is_print=False) h1_deconv = tf_utils.norm(h1_deconv, _type='instance', name='h1_norm') h1_relu = tf.nn.relu(h1_deconv, name='h1_relu') # from (N, 8, 8, 128) to (N, 16, 16, 64) h2_deconv = tf_utils.deconv2d(h1_relu, 64, k_h=5, k_w=5, name='h2_deconv2d', is_print=False) h2_deconv = tf_utils.norm(h2_deconv, _type='instance', name='h2_norm') h2_relu = tf.nn.relu(h2_deconv, name='h2_relu') # from (N, 16, 16, 64) to (N, 32, 32, 32) h3_deconv = tf_utils.deconv2d(h2_relu, 32, k_h=5, k_w=5, name='h3_deconv2d', is_print=False) h3_deconv = tf_utils.norm(h3_deconv, _type='instance', name='h3_norm') h3_relu = tf.nn.relu(h3_deconv, name='h3_relu') # from (N, 32, 32, 32) to (N, 15,15, 1) h4 = tf_utils.conv2d(h3_relu, k=1, k_h=4, k_w=4, d_h=2, d_w=2, stddev=0.02, padding='VALID', name='h4_conv2d', is_print=False) x = (h4 - tf.reduce_min(h4)) / (tf.reduce_max(h4) - tf.reduce_sum(h4)) x = x / tf.reduce_sum(x) return x