def generator(hparams, z, train, reuse): with tf.variable_scope("generator") as scope: if reuse: tf.get_variable_scope().reuse_variables() s_h, s_w = hparams.output_height, hparams.output_width s_h2, s_h4, s_h8, s_h16 = int(s_h/2), int(s_h/4), int(s_h/8), int(s_h/16) s_w2, s_w4, s_w8, s_w16 = int(s_w/2), int(s_w/4), int(s_w/8), int(s_w/16) g_bn0 = ops.batch_norm(name='g_bn0') g_bn1 = ops.batch_norm(name='g_bn1') g_bn2 = ops.batch_norm(name='g_bn2') g_bn3 = ops.batch_norm(name='g_bn3') # project `z` and reshape h0 = tf.reshape(ops.linear(z, hparams.gf_dim*8*s_h16*s_w16, 'g_h0_lin'), [-1, s_h16, s_w16, hparams.gf_dim * 8]) h0 = tf.nn.relu(g_bn0(h0, train=train)) h1 = ops.deconv2d(h0, [hparams.batch_size, s_h8, s_w8, hparams.gf_dim*4], name='g_h1') h1 = tf.nn.relu(g_bn1(h1, train=train)) h2 = ops.deconv2d(h1, [hparams.batch_size, s_h4, s_w4, hparams.gf_dim*2], name='g_h2') h2 = tf.nn.relu(g_bn2(h2, train=train)) h3 = ops.deconv2d(h2, [hparams.batch_size, s_h2, s_w2, hparams.gf_dim*1], name='g_h3') h3 = tf.nn.relu(g_bn3(h3, train=train)) h4 = ops.deconv2d(h3, [hparams.batch_size, s_h, s_w, hparams.c_dim], name='g_h4') x_gen = tf.nn.tanh(h4) return x_gen
def __call__(self, x, is_reuse=False, is_train=True): with tf.variable_scope('generator') as scope: if is_reuse: scope.reuse_variables() unit_size = self.img_size[0] // (2 ** self.layer_n) unit_n = self.smallest_hidden_unit_n * (2 ** (self.layer_n - 1)) batch_size = int(x.shape[0]) with tf.variable_scope('pre'): x = linear(x, unit_size * unit_size * unit_n) x = tf.reshape(x, (batch_size, unit_size, unit_size, unit_n)) if self.is_bn: x = batch_norm(x, is_train) x = tf.nn.relu(x) for i in range(self.layer_n): with tf.variable_scope('layer{}'.format(i)): if i == self.layer_n - 1: unit_n = self.img_dim else: unit_n = self.smallest_hidden_unit_n * (2 ** (self.layer_n - i - 2)) x_shape = x.get_shape().as_list() x = tf.image.resize_bilinear(x, (x_shape[1] * 2, x_shape[2] * 2)) x = conv2d(x, unit_n, self.k_size, 1, 'SAME') if i != self.layer_n - 1: if self.is_bn: x = batch_norm(x, is_train) x = tf.nn.relu(x) x = tf.nn.tanh(x) return x
def discriminator(hparams, x, train, reuse): if reuse: tf.get_variable_scope().reuse_variables() d_bn1 = ops.batch_norm(name='d_bn1') d_bn2 = ops.batch_norm(name='d_bn2') d_bn3 = ops.batch_norm(name='d_bn3') h0 = ops.lrelu(ops.conv2d(x, hparams.df_dim, name='d_h0_conv')) h1 = ops.conv2d(h0, hparams.df_dim * 2, name='d_h1_conv') h1 = ops.lrelu(d_bn1(h1, train=train)) h2 = ops.conv2d(h1, hparams.df_dim * 4, name='d_h2_conv') h2 = ops.lrelu(d_bn2(h2, train=train)) h3 = ops.conv2d(h2, hparams.df_dim * 8, name='d_h3_conv') h3 = ops.lrelu(d_bn3(h3, train=train)) h4 = ops.linear(tf.reshape(h3, [hparams.batch_size, -1]), 1, 'd_h3_lin') d_logit = h4 d = tf.nn.sigmoid(d_logit) return d, d_logit
def eve_model(self, data_input=None, reuse=False): ####### Eve's network ####### with tf.variable_scope("eve") as scope: if reuse: scope.reuse_variables() self.eve_input = data_input print(self.eve_input) self.eve0 = lrelu( conv2d(self.eve_input, self.output_size, name='eve_h0_conv')) self.eve_bn1 = batch_norm(name='eve_bn1') self.eve1 = lrelu( self.eve_bn1( conv2d(self.eve0, self.output_size * 2, name='eve_h1_conv'))) self.eve_bn2 = batch_norm(name='eve_bn2') self.eve2 = lrelu( self.eve_bn2( conv2d(self.eve1, self.output_size * 4, name='eve_h2_conv'))) self.eve_bn3 = batch_norm(name='eve_bn3') self.eve3 = lrelu( self.eve_bn3( conv2d(self.eve2, self.output_size * 8, name='eve_h3_conv'))) self.eve4 = linear(tf.reshape(self.eve0, [self.batch_size, -1]), 1, 'eve_h3_lin') return self.eve4, self.eve4
def bob_model(self, data_input_image=None): ####### Bob's network ####### # bob's input self.bob_input = data_input_image print(self.bob_input) self.bob0 = lrelu( conv2d(self.bob_input, self.output_size, name='bob_h0_conv')) self.bob_bn1 = batch_norm(name='bob_bn1') self.bob1 = lrelu( self.bob_bn1( conv2d(self.bob0, self.output_size * 2, name='bob_h1_conv'))) self.bob_bn2 = batch_norm(name='bob_bn2') self.bob2 = lrelu( self.bob_bn2( conv2d(self.bob1, self.output_size * 4, name='bob_h2_conv'))) self.bob_bn3 = batch_norm(name='bob_bn3') self.bob3 = lrelu( self.bob_bn3( conv2d(self.bob2, self.output_size * 8, name='bob_h3_conv'))) self.bob4 = linear(tf.reshape(self.bob3, [self.batch_size, -1]), self.msg_len, 'bob_h3_lin') return tf.nn.tanh(self.bob4)
def generator(input_z, t_txt=None, is_train=True, reuse=False, batch_size=batch_size): g_bn0 = ops.batch_norm(name='g_bn0') g_bn1 = ops.batch_norm(name='g_bn1') g_bn2 = ops.batch_norm(name='g_bn2') g_bn3 = ops.batch_norm(name='g_bn3') s = image_size # output image size [64] s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) gf_dim = 128 with tf.variable_scope("generator", reuse=reuse): tl.layers.set_name_reuse(reuse) z_concat = tf.concat([input_z, t_txt], 1) z_ = ops.linear(z_concat, gf_dim * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, gf_dim * 8]) h0 = tf.nn.relu(g_bn0(h0)) h1 = ops.deconv2d(h0, [batch_size, s8, s8, gf_dim * 4], name='g_h1') h1 = tf.nn.relu(g_bn1(h1)) h2 = ops.deconv2d(h1, [batch_size, s4, s4, gf_dim * 2], name='g_h2') h2 = tf.nn.relu(g_bn2(h2)) h3 = ops.deconv2d(h2, [batch_size, s2, s2, gf_dim * 1], name='g_h3') h3 = tf.nn.relu(g_bn3(h3)) h4 = ops.deconv2d(h3, [batch_size, s, s, 3], name='g_h4') return h4, tf.tanh(h4)
def out_block(input_anc, input_pos, channels, laxer_idx, stride_input=1, k_size=8, padding_type='VALID'): # Last conv layer, flatten the output weights = ops.weight([k_size, k_size, k_size, channels[0], channels[1]], layer_name='wcnn' + str(laxer_idx + 1)) bias = ops.bias([1, 1, 1, channels[1]], layer_name='bcnn' + str(laxer_idx + 1)) conv_output_anc = tf.add( ops.conv3d(input_anc, weights, stride=[stride_input, stride_input, stride_input], padding=padding_type), bias) conv_output_pos = tf.add( ops.conv3d(input_pos, weights, stride=[stride_input, stride_input, stride_input], padding=padding_type), bias) conv_output_anc = ops.batch_norm(conv_output_anc) conv_output_pos = ops.batch_norm(conv_output_pos) conv_output_anc = tf.contrib.layers.flatten(conv_output_anc) conv_output_pos = tf.contrib.layers.flatten(conv_output_pos) return conv_output_anc, conv_output_pos
def generator(self, z, y, is_training=True, reuse=False): with tf.variable_scope('generator', values=[z], reuse=reuse): z = tf.concat([z, y], axis=1) net = tf.layers.dense(z, units=self.s_size * self.s_size * self.g_depth) net = tf.reshape(net, [-1, self.s_size, self.s_size, self.g_depth]) net = tf.nn.relu(batch_norm(net, is_training)) shortcut = net #define 16 res block, do not change the feature map size and number for i in range(16): net = self.g_block(net, is_training) net = tf.nn.relu(batch_norm(net, is_training)) net = tf.add(net, shortcut) #sub-pixel, convolution, upsampling for i in range(self.num_layers): net = conv2d(net, self.g_depth * 4, 3, 1, use_bias=False) net = tf.depth_to_space( net, 2) #change from imgH * imgW * 256 to 2*imgH * 2*imgW * 64 net = tf.nn.relu(batch_norm(net, is_training)) net = tf.layers.conv2d(net, filters=self.c_dim, kernel_size=5, strides=1, padding='SAME', activation=tf.nn.tanh, use_bias=True) return net
def batch_norm_test(): with tf.device('/' + FLAGS.device + ":0"): input_tensor = tf.Variable(initial_value=tf.truncated_normal([4, 3, 3, 2], mean=0.1, dtype=dtype)) mean = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype)) variance = tf.Variable(initial_value=tf.square(tf.truncated_normal([2], mean=0.1, dtype=dtype))) offset = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype)) scale = tf.Variable(initial_value=tf.truncated_normal([2], mean=0.1, dtype=dtype)) param_1 = tf.constant(np.random.rand(4, 3, 3, 2), dtype=dtype) param_2 = tf.constant(np.random.rand(4 * 3 * 3, 2), dtype=dtype) output_tensor = ops.batch_norm(input_tensor, mean, variance, offset, scale, 1e-6) output_tensor = tf.reshape(output_tensor * param_1, [-1, 2]) output_tensor = ops.batch_norm(output_tensor, mean, variance, offset, scale, 1e-6) recv = tf.reduce_sum(output_tensor * param_2) grads = tf.gradients(recv, [input_tensor, mean, variance, offset, scale]) output_tensor_ = tf.nn.batch_normalization(input_tensor, mean, variance, offset, scale, 1e-6) output_tensor_ = tf.reshape(output_tensor_ * param_1, [-1, 2]) output_tensor_ = ops.batch_norm(output_tensor_, mean, variance, offset, scale, 1e-6) recv_ = tf.reduce_sum(output_tensor_ * param_2) grads_ = tf.gradients(recv_, [input_tensor, mean, variance, offset, scale]) with tf.Session() as sess: tf.global_variables_initializer().run() assert np.max(np.abs(sess.run(output_tensor) - sess.run(output_tensor_))) < 1e-5 for g, g_ in zip(grads, grads_): assert np.max(np.abs(sess.run(g) - sess.run(g_))) < 2e-4
def get_discriminator_net(self, name, images, reuse=False): batch_size = self.get_batch_size() with tf.variable_scope("discriminator", reuse=reuse) as scope: # concat yb = tf.reshape(self.labels, shape=[batch_size, 1, 1, -1]) net = ops.conv_cond_concat(images, yb) # TODO: why is output_dim is self.data.get_number_of_labels() here?? net, w1 = ops.conv2d( net, self.data.get_number_of_labels(), 4, 4, 2, 2, scope='dis_conv1', weights_initializer=self.get_weights_initializer()) tf.add_to_collection('weight_1', w1) net = ops.lrelu(net) net = ops.conv_cond_concat(net, yb) tf.add_to_collection('ac_1', net) net, w2 = ops.conv2d( net, batch_size, 4, 4, 2, 2, scope='dis_conv2', weights_initializer=self.get_weights_initializer()) tf.add_to_collection('weight_2', w2) net = ops.lrelu( ops.batch_norm(net, self.is_training, scope='dis_bn1')) tf.add_to_collection('ac_2', net) net = tf.reshape(net, [batch_size, -1]) net = tf.concat([net, self.labels], 1) # TODO: figure out what implications the change in "output_size" will have. net = ops.linear( net, output_size=1024, scope='dis_fully1', weights_initializer=self.get_weights_initializer()) net = ops.batch_norm(net, self.is_training, scope='dis_bn2', reuse=reuse) net = ops.lrelu(net) net = tf.concat([net, self.labels], 1) out = ops.linear(net, output_size=1, scope='dis_fully2', weights_initializer=xavier_initializer()) return tf.nn.sigmoid(out, name=name), out
def naive_discriminator(self, image, y=None, reuse=False): with tf.variable_scope("discriminator") as scope: # image is 128 x 128 x (input_c_dim + output_c_dim) if reuse: tf.get_variable_scope().reuse_variables() else: assert tf.get_variable_scope().reuse == False h0 = lrelu(conv2d(image, self.df_dim, name='adv_d_h0_conv')) # h0 is (128 x 128 x self.df_dim) h1 = lrelu( batch_norm((conv2d(h0, self.df_dim * 2, name='adv_d_h1_conv')), name="adv_d_bn1")) # h1 is (64 x 64 x self.df_dim*2) h2 = lrelu( batch_norm(conv2d(h1, self.df_dim * 4, name='adv_d_h2_conv'), name="adv_d_bn2")) # h2 is (32x 32 x self.df_dim*4) h3 = lrelu( batch_norm(conv2d(h2, self.df_dim * 8, d_h=1, d_w=1, name='adv_d_h3_conv'), name="adv_d_bn3")) # h3 is (16 x 16 x self.df_dim*8) h4 = linear(tf.reshape(h3, [self.batch_size, -1]), 1, 'adv_d_h3_lin') return tf.nn.sigmoid(h4), h4
def discriminator(self, opts, input_, is_training, prefix='DISCRIMINATOR', reuse=False): """Encoder function, suitable for simple toy experiments. """ num_filters = opts['d_num_filters'] with tf.variable_scope(prefix, reuse=reuse): h0 = ops.conv2d(opts, input_, num_filters / 8, scope='h0_conv') h0 = ops.batch_norm(opts, h0, is_training, reuse, scope='bn_layer1') h0 = tf.nn.relu(h0) h1 = ops.conv2d(opts, h0, num_filters / 4, scope='h1_conv') h1 = ops.batch_norm(opts, h1, is_training, reuse, scope='bn_layer2') h1 = tf.nn.relu(h1) h2 = ops.conv2d(opts, h1, num_filters / 2, scope='h2_conv') h2 = ops.batch_norm(opts, h2, is_training, reuse, scope='bn_layer3') h2 = tf.nn.relu(h2) h3 = ops.conv2d(opts, h2, num_filters, scope='h3_conv') h3 = ops.batch_norm(opts, h3, is_training, reuse, scope='bn_layer4') h3 = tf.nn.relu(h3) # Already has NaNs!! latent_mean = ops.linear(opts, h3, opts['latent_space_dim'], scope='h3_lin') log_latent_sigmas = ops.linear(opts, h3, opts['latent_space_dim'], scope='h3_lin_sigma') return latent_mean, log_latent_sigmas
def __init__(self, sess, batch_size=32, image_size=256, lam=0.8, checkpoint_dir=None): self.sess = sess self.batch_size = batch_size self.image_size = image_size self.image_shape = [image_size, image_size, 3] self.lam = lam self.checkpoint_dir = checkpoint_dir self.global_step = tf.Variable(0, trainable=False) self.images = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='images') self.images_summary = tf.summary.image("image", self.images) self.d_bns = [batch_norm(name='d_bn{}'.format(i)) for i in range(5)] self.local_d_bns = [batch_norm(name='d_local_bn{}'.format(i)) for i in range(4)] self.g_bns = [batch_norm(name='g_bn{}'.format(i, )) for i in range(15)] self.D, self.D_logits = self.discriminator(self.images, self.image_size) self.d_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=self.D_logits, labels=tf.ones_like(self.D))) self.D_summary = tf.summary.histogram("d", self.D) self.d_loss_real_summary = tf.summary.scalar("d_loss_real", self.d_loss_real) self.masks = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='masks') self.MG = tf.multiply(self.images, self.masks) self.G = self.generator(self.MG) self.MG_summary = tf.summary.image("mg", self.MG) self.G_summary = tf.summary.image("g", self.G) self.D_fake, self.D_fake_logits = self.discriminator(self.G, self.image_size, reuse=True) self.d_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=self.D_fake_logits, labels=tf.zeros_like(self.D_fake))) self.D_fake_summary = tf.summary.histogram("d_fake", self.D_fake) self.d_loss_fake_summary = tf.summary.scalar("d_loss_fake", self.d_loss_fake) self.d_loss = self.d_loss_real + self.d_loss_fake self.g_loss_d = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits( logits=self.D_fake_logits, labels=tf.ones_like(self.D_fake))) self.g_loss_l = tf.reduce_mean(tf.contrib.layers.flatten( tf.multiply(self.G - self.images, self.G - self.images))) self.g_loss = (1 - self.lam) * self.g_loss_d + self.lam * self.g_loss_l self.g_loss_d_summary = tf.summary.scalar("g_loss_d", self.g_loss_d) self.g_loss_l_summary = tf.summary.scalar("g_loss_l", self.g_loss_l) self.g_loss_summary = tf.summary.scalar("g_loss", self.g_loss) t_vars = tf.trainable_variables() self.d_vars = [var for var in t_vars if 'd_' in var.name] self.g_vars = [var for var in t_vars if 'g_' in var.name] self.saver = tf.train.Saver(max_to_keep=10) self.g_summary = tf.summary.merge([ self.G_summary, self.MG_summary, self.D_fake_summary, self.d_loss_fake_summary, self.g_loss_summary, self.g_loss_d_summary, self.g_loss_l_summary]) self.d_summary = tf.summary.merge([ self.images_summary, self.D_summary, self.d_loss_real_summary]) self.writer = tf.summary.FileWriter(os.path.join(self.checkpoint_dir, "logs"), self.sess.graph)
def _build_model(self): with tf.variable_scope('discriminator', reuse=self.reuse): df_dim = self.df_dim h0 = ops.lrelu(ops.conv2d( self.image, df_dim, name='d_h0_conv')) # 32 h1 = ops.lrelu(ops.batch_norm(ops.conv2d( h0, df_dim * 2, name='d_h1_conv'))) # 16 h2 = ops.lrelu(ops.batch_norm(ops.conv2d( h1, df_dim * 4, name='d_h2_conv'))) # 8 h3 = ops.lrelu(ops.batch_norm(ops.conv2d( h2, df_dim * 8, name='d_h3_conv'))) # 4 # ADD TEXT EMBEDDING TO THE NETWORK reduced_text_embeddings = ops.lrelu(ops.linear( self.text, self.hparas['TEXT_DIM'], 'd_embedding')) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 1) reduced_text_embeddings = tf.expand_dims(reduced_text_embeddings, 2) tiled_embeddings = tf.tile(reduced_text_embeddings, [ 1, 4, 4, 1], name='tiled_embeddings') h3_concat = tf.concat([h3, tiled_embeddings], 3, name='h3_concat') h3_new = ops.lrelu(ops.batch_norm(ops.conv2d( h3_concat, df_dim * 8, 1, 1, 1, 1, name='d_h3_conv_new'))) # 4 h4 = ops.linear(tf.reshape( h3_new, [self.hparas['BATCH_SIZE'], -1]), 1, 'd_h3_lin') self.logits = h4 self.discriminator_net = tf.nn.sigmoid(h4) self.outputs = tf.nn.sigmoid(h4)
def generator(hparams, z, scope_name, train, reuse): with tf.variable_scope(scope_name) as scope: if reuse: scope.reuse_variables() output_size = 64 s = output_size s2, s4, s8, s16 = int(s/2), int(s/4), int(s/8), int(s/16) g_bn0 = ops.batch_norm(name='g_bn0') g_bn1 = ops.batch_norm(name='g_bn1') g_bn2 = ops.batch_norm(name='g_bn2') g_bn3 = ops.batch_norm(name='g_bn3') # project `z` and reshape h0 = tf.reshape(ops.linear(z, hparams.gf_dim*8*s16*s16, 'g_h0_lin'), [-1, s16, s16, hparams.gf_dim * 8]) h0 = tf.nn.relu(g_bn0(h0, train=train)) h1 = ops.deconv2d(h0, [hparams.batch_size, s8, s8, hparams.gf_dim*4], name='g_h1') h1 = tf.nn.relu(g_bn1(h1, train=train)) h2 = ops.deconv2d(h1, [hparams.batch_size, s4, s4, hparams.gf_dim*2], name='g_h2') h2 = tf.nn.relu(g_bn2(h2, train=train)) h3 = ops.deconv2d(h2, [hparams.batch_size, s2, s2, hparams.gf_dim*1], name='g_h3') h3 = tf.nn.relu(g_bn3(h3, train=train)) h4 = ops.deconv2d(h3, [hparams.batch_size, s, s, hparams.c_dim], name='g_h4') x_gen = tf.nn.tanh(h4) return x_gen
def _build_model(self): with tf.variable_scope('generator', reuse=self.reuse): s = self.hparas['IMAGE_SIZE'][0] s2, s4, s8, s16 = int(s / 2), int(s / 4), int(s / 8), int(s / 16) gf_dim = self.gf_dim reduced_text_embedding = ops.lrelu(ops.linear( self.text, self.hparas['TEXT_DIM'], 'g_embedding')) z_concat = tf.concat([self.z, reduced_text_embedding], 1) z_ = ops.linear(z_concat, gf_dim * 8 * s16 * s16, 'g_h0_lin') h0 = tf.reshape(z_, [-1, s16, s16, gf_dim * 8]) h0 = tf.nn.relu(ops.batch_norm(h0)) h1 = ops.deconv2d(h0, [self.hparas['BATCH_SIZE'], s8, s8, gf_dim * 4], name='g_h1') h1 = tf.nn.relu(ops.batch_norm(h1)) h2 = ops.deconv2d(h1, [self.hparas['BATCH_SIZE'], s4, s4, gf_dim * 2], name='g_h2') h2 = tf.nn.relu(ops.batch_norm(h2)) h3 = ops.deconv2d(h2, [self.hparas['BATCH_SIZE'], s2, s2, gf_dim * 1], name='g_h3') h3 = tf.nn.relu(ops.batch_norm(h3)) h4 = ops.deconv2d(h3, [self.hparas['BATCH_SIZE'], s, s, 3], name='g_h4') self.generator_net = tf.tanh(h4) / 2.0 + 0.5 self.outputs = tf.tanh(h4) / 2.0 + 0.5
def generator(z, is_training): # Firstly let's reshape input vector into 3-d tensor. z_ = ops.linear(z, GENERATOR_DENSE_SIZE*4*4, 'g_h0_lin') h_in = tf.reshape(z_, [-1, 4, 4, GENERATOR_DENSE_SIZE]) g_batch_norm_in=ops.batch_norm(name='g_batch_norm_in') h_in_bn = g_batch_norm_in(h_in,is_training) h_in_z=ops.lrelu(x=h_in_bn,name='g_lr_1') h_1=ops.deconv2d(h_in_z,output_shape=[BATCH_SIZE,8,8,512],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_1") g_batch_norm_1=ops.batch_norm(name='g_batch_norm_1') h_1_bn = g_batch_norm_1(h_1,is_training) h_1_z=ops.lrelu(x=h_1_bn,name='g_lr_2') h_1_z_dr=tf.nn.dropout(h_1_z,0.3) h_2=ops.deconv2d(h_1_z_dr,output_shape=[BATCH_SIZE,16,16,256],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_2") g_batch_norm_2=ops.batch_norm(name='g_batch_norm_2') h_2_bn = g_batch_norm_2(h_2,is_training) h_2_z=ops.lrelu(x=h_2_bn,name='g_lr_3') h_2_z_dr=tf.nn.dropout(h_2_z,0.3) h_3=ops.deconv2d(h_2_z_dr,output_shape=[BATCH_SIZE,32,32,128],k_h=5,k_w=5,d_h=2, d_w=2,name="g_deconv_3") g_batch_norm_3=ops.batch_norm(name='g_batch_norm_3') h_3_bn = g_batch_norm_3(h_3,is_training) h_3_z=ops.lrelu(x=h_3_bn,name='g_lr_4') h_out = ops.deconv2d(h_3_z, [BATCH_SIZE, IMAGE_SIZE, IMAGE_SIZE, N_CHANNELS], name='g_out') return tf.nn.tanh(h_out)
def discriminator(self, image, is_training, reuse=False): with tf.variable_scope("discriminator"): if reuse: tf.get_variable_scope().reuse_variables() h0 = lrelu(conv2d(image, self.discriminator_dim, scope="d_h0_conv")) h1 = lrelu( batch_norm(conv2d(h0, self.discriminator_dim * 2, scope="d_h1_conv"), is_training, scope="d_bn_1")) h2 = lrelu( batch_norm(conv2d(h1, self.discriminator_dim * 4, scope="d_h2_conv"), is_training, scope="d_bn_2")) h3 = lrelu( batch_norm(conv2d(h2, self.discriminator_dim * 8, sh=1, sw=1, scope="d_h3_conv"), is_training, scope="d_bn_3")) # real or fake binary loss fc1 = fc(tf.reshape(h3, [self.batch_size, -1]), 1, scope="d_fc1") # category loss fc2 = fc(tf.reshape(h3, [self.batch_size, -1]), self.embedding_num, scope="d_fc2") return tf.nn.sigmoid(fc1), fc1, fc2
def generator(inputs): """ Defines the graph for the generator. Takes input (random samples from uniform distribution) and returns a Spectrogram. :param inputs: A vector with random entries (either from a uniform or a gaussian distribution). :return: A generated audio sample, that hopefully fools the discriminator. """ with tf.variable_scope("g_") as scope: net = fc(inputs, 28672, 'fc1') net = tf.reshape(net, [-1, 4, 7, 1024]) net = tf.nn.relu(net) net = deconv(net, [FLAGS.kernel_size,FLAGS.kernel_size,1024, 1024], 'deconv1') net = batch_norm(True, net, 'bn2') net = tf.nn.relu(net) net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 512, 1024], 'deconv2') net = batch_norm(True, net, 'bn3') net = tf.nn.relu(net) net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 256, 512], 'deconv3') net = batch_norm(True, net, 'bn4') net = tf.nn.relu(net) net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 128, 256], 'deconv4') net = batch_norm(True, net, 'bn5') net = tf.nn.relu(net) net = deconv(net, [FLAGS.kernel_size, FLAGS.kernel_size, 2, 128], 'deconv5') net = net[:, :98, :201, :] # Map logmag and phase to [-1,1] logmag, phase = tf.unstack(net, axis=3) phase = tf.nn.tanh(phase) logmag = tf.nn.tanh(logmag) net = tf.stack([logmag, phase], axis=3) return net
def g_block(self, inputs, is_training): h = tf.nn.relu( batch_norm(conv2d(inputs, self.g_depth, 3, 1, use_bias=False), is_training)) h = batch_norm(conv2d(h, self.g_depth, 3, 1, use_bias=False), is_training) h = tf.add(inputs, h) return h
def _initialize_params(self): all_weights = {} batch_norms = {} gen_vars = [] disc_vars = [] # init generator weights prev_layer_dim = self.z_dim for layer_i in xrange(len(self.generator_params['dim'])): name = 'gen_w' + str(layer_i) all_weights[name] = ops.variable(name, [self.generator_params['ksize'][layer_i], self.generator_params['ksize'][layer_i], self.generator_params['dim'][layer_i], prev_layer_dim], self.initializer) gen_vars.append(all_weights[name]) if layer_i+1==len(self.generator_params['dim']): name = 'gen_b' + str(layer_i) all_weights[name] = ops.variable(name, [self.generator_params['dim'][layer_i]], ) gen_vars.append(all_weights[name]) else: name = 'gen_bn' + str(layer_i) batch_norms[name] = ops.batch_norm(self.generator_params['dim'][layer_i], name=name) prev_layer_dim = self.generator_params['dim'][layer_i] # init discriminator weights for disc_i in xrange(len(self.discriminators_params)): prev_layer_dim = self.image_dim cur_params = self.discriminators_params[disc_i] for layer_i in xrange(len(cur_params['dim'])): name = 'disc' + str(disc_i) + '_w' + str(layer_i) all_weights[name] = ops.variable(name, [cur_params['ksize'][layer_i], cur_params['ksize'][layer_i], prev_layer_dim, cur_params['dim'][layer_i]], self.initializer) disc_vars.append(all_weights[name]) if layer_i+1==len(cur_params['dim']): name = 'disc' + str(disc_i) + '_b' + str(layer_i) all_weights[name] = ops.variable(name, [cur_params['dim'][layer_i]], tf.constant_initializer(0.0)) disc_vars.append(all_weights[name]) else: name = 'disc_' + str(disc_i) + '_bn' + str(layer_i) batch_norms[name] = ops.batch_norm(cur_params['dim'][layer_i], name=name) prev_layer_dim = cur_params['dim'][layer_i] return all_weights, batch_norms, gen_vars, disc_vars
def GeneratorCNN(z, config, reuse=None): ''' maps z to a 64x64 images with values in [-1,1] uses batch normalization internally ''' #trying to get around batch_size like this: batch_size = tf.shape(z)[0] #batch_size=tf.placeholder_with_default(64,[],'bs') with tf.variable_scope("generator", reuse=reuse) as vs: g_bn0 = batch_norm(name='g_bn0') g_bn1 = batch_norm(name='g_bn1') g_bn2 = batch_norm(name='g_bn2') g_bn3 = batch_norm(name='g_bn3') s_h, s_w = config.gf_dim, config.gf_dim #64,64 s_h2, s_w2 = conv_out_size_same(s_h, 2), conv_out_size_same(s_w, 2) s_h4, s_w4 = conv_out_size_same(s_h2, 2), conv_out_size_same(s_w2, 2) s_h8, s_w8 = conv_out_size_same(s_h4, 2), conv_out_size_same(s_w4, 2) s_h16, s_w16 = conv_out_size_same(s_h8, 2), conv_out_size_same(s_w8, 2) # project `z` and reshape z_, self_h0_w, self_h0_b = linear(z, config.gf_dim * 8 * s_h16 * s_w16, 'g_h0_lin', with_w=True) self_h0 = tf.reshape(z_, [-1, s_h16, s_w16, config.gf_dim * 8]) h0 = tf.nn.relu(g_bn0(self_h0)) h1, h1_w, h1_b = deconv2d(h0, [batch_size, s_h8, s_w8, config.gf_dim * 4], name='g_h1', with_w=True) h1 = tf.nn.relu(g_bn1(h1)) h2, h2_w, h2_b = deconv2d(h1, [batch_size, s_h4, s_w4, config.gf_dim * 2], name='g_h2', with_w=True) h2 = tf.nn.relu(g_bn2(h2)) h3, h3_w, h3_b = deconv2d(h2, [batch_size, s_h2, s_w2, config.gf_dim * 1], name='g_h3', with_w=True) h3 = tf.nn.relu(g_bn3(h3)) h4, h4_w, h4_b = deconv2d(h3, [batch_size, s_h, s_w, config.c_dim], name='g_h4', with_w=True) out = tf.nn.tanh(h4) variables = tf.contrib.framework.get_variables(vs) return out, variables
def testReuseUpdateOps(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) ops.batch_norm(images, scope='bn') self.assertEquals( len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 2) ops.batch_norm(images, scope='bn', reuse=True) self.assertEquals( len(tf.get_collection(ops.UPDATE_OPS_COLLECTION)), 4)
def testMovingAverageVariables(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) ops.batch_norm(images, scale=True) moving_mean = tf.moving_average_variables()[0] moving_variance = tf.moving_average_variables()[1] self.assertEquals(moving_mean.op.name, 'BatchNorm/moving_mean') self.assertEquals(moving_variance.op.name, 'BatchNorm/moving_variance')
def res_build_block_v1(X, increase=False, init=1.0, stddev=1.0, training=False, projection=None, name=None): print('input shape for {} :: {}'.format(name, X.get_shape().as_list())) shortcut_projection = X stride = 1 in_channels = X.get_shape().as_list()[-1] out_channels = in_channels if increase: out_channels = 2 * in_channels stride = 2 if projection: name_ = name + 'projection_batch_norm' shortcut_projection = projection(shortcut_projection, out_channels=out_channels, training=training, name=name_) else: shortcut_projection = maxpool(shortcut_projection, filter_size=[1, 2, 2, 1], stride_size=[1, 2, 2, 1], padding='VALID') pad = (out_channels - in_channels) shortcut_projection = tf.pad(shortcut_projection, [[0, 0], [0, 0], [0, 0], [0, pad]]) with tf.name_scope('conv1layer'): name_ = name + 'conv1layer_batch_norm' X = conv(X, filter_size=3, out_channels=out_channels, stride_size=stride, padding='SAME', init_bias=init, stddev=stddev) X = batch_norm(X, training, name=name_) X = tf.nn.relu(X) with tf.name_scope('conv2layer'): name_ = name + 'conv2layer_batch_norm' X = conv(X, filter_size=3, out_channels=out_channels, stride_size=1, padding='SAME', init_bias=init, stddev=stddev) X = batch_norm(X, training, name=name_) with tf.name_scope('residual'): X += shortcut_projection X = tf.nn.relu(X) print('output shape for {} :: {}'.format(name, X.get_shape().as_list())) return X
def __call__(self, x, is_reuse=False, is_train=True, is_intermediate=False): with tf.variable_scope('generator') as scope: if is_reuse: scope.reuse_variables() unit_size = self.img_size[0] // (2**self.layer_n) unit_n = self.smallest_hidden_unit_n * (2**(self.layer_n - 1)) batch_size = int(x.shape[0]) if is_intermediate: intermediate_xs = list() with tf.variable_scope('pre'): x = linear(x, unit_size * unit_size * unit_n) x = tf.reshape(x, (batch_size, unit_size, unit_size, unit_n)) if self.is_bn: x = batch_norm(x, is_train) x = tf.nn.relu(x) if is_intermediate: y = avgpool2d(x, unit_size, 1, 'VALID') y = tf.reshape(y, (batch_size, -1)) intermediate_xs.append(y) for i in range(self.layer_n): with tf.variable_scope('layer{}'.format(i)): if i == self.layer_n - 1: unit_n = self.img_dim else: unit_n = self.smallest_hidden_unit_n * (2**( self.layer_n - i - 2)) x_shape = x.get_shape().as_list() if self.is_transpose: x = deconv2d(x, [ x_shape[0], x_shape[1] * 2, x_shape[1] * 2, unit_n ], self.k_size, 2, 'SAME') else: x = tf.image.resize_bilinear( x, (x_shape[1] * 2, x_shape[2] * 2)) x = conv2d(x, unit_n, self.k_size, 1, 'SAME') if i != self.layer_n - 1: if self.is_bn: x = batch_norm(x, is_train) x = tf.nn.relu(x) if is_intermediate: y = tf.reshape(x, (batch_size, -1)) intermediate_xs.append(y) x = tf.nn.tanh(x) if is_intermediate: return x, intermediate_xs return x
def testReuseVariables(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) ops.batch_norm(images, scale=True, scope='bn') ops.batch_norm(images, scale=True, scope='bn', reuse=True) beta = variables.get_variables_by_name('beta') gamma = variables.get_variables_by_name('gamma') self.assertEquals(len(beta), 1) self.assertEquals(len(gamma), 1) moving_vars = tf.get_collection('moving_vars') self.assertEquals(len(moving_vars), 2)
def testUpdateOps(self): height, width = 3, 3 with self.test_session(): images = tf.random_uniform((5, height, width, 3), seed=1) ops.batch_norm(images) update_ops = tf.get_collection(ops.UPDATE_OPS_COLLECTION) update_moving_mean = update_ops[0] update_moving_variance = update_ops[1] self.assertEquals(update_moving_mean.op.name, 'BatchNorm/AssignMovingAvg') self.assertEquals(update_moving_variance.op.name, 'BatchNorm/AssignMovingAvg_1')
def discriminator(image, reuse=False): d_bn1 = ops.batch_norm(FLAGS.batch_size, name='d_bn1') d_bn2 = ops.batch_norm(FLAGS.batch_size, name='d_bn2') d_bn3 = ops.batch_norm(FLAGS.batch_size, name='d_bn3') image = tf.reshape(image, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 1]) if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu(ops.conv2d(image, FLAGS.df_dim, name='d_h0_conv')) h1 = ops.lrelu(d_bn1(ops.conv2d(h0, FLAGS.df_dim * 2, name='d_h1_conv'))) h2 = ops.lrelu(d_bn2(ops.conv2d(h1, FLAGS.df_dim * 4, name='d_h2_conv'))) h3 = ops.lrelu(d_bn3(ops.conv2d(h2, FLAGS.df_dim * 8, name='d_h3_conv'))) h4 = ops.linear(tf.reshape(h3, [FLAGS.batch_size, -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4)
def __init__(self, sess, image_size, batch_size=64, sample_size=64, lowres=8, z_dim=100, gf_dim=64, df_dim=64, gfc_dim=1024, dfc_dim=1024, c_dim=3, checkpoint_dir=None, lamda=0.01, center_scale=0.25): # Currently, image size must be a (power of 2) and (8 or higher). assert(image_size & (image_size - 1) == 0 and image_size >= 8) print("========INITIALIZING THE WENGER DCGAN============") # INIT from train-dcgan! self.sess = sess self.is_crop = False self.batch_size = batch_size self.image_size = image_size self.sample_size = sample_size self.image_shape = [image_size, image_size, c_dim] self.lowres = lowres self.lowres_size = image_size // lowres self.lowres_shape = [self.lowres_size, self.lowres_size, c_dim] self.z_dim = z_dim self.gf_dim = gf_dim self.df_dim = df_dim self.gfc_dim = gfc_dim self.dfc_dim = dfc_dim self.lamda = lamda self.c_dim = c_dim self.center_scale = center_scale # batch normalization : deals with poor initialization helps gradient flow self.d_bns = [ batch_norm(name='d_bn{}'.format(i,)) for i in range(4)] log_size = int(math.log(image_size) / math.log(2)) self.g_bns = [ batch_norm(name='g_bn{}'.format(i,)) for i in range(log_size)] self.checkpoint_dir = checkpoint_dir print("========CALLING SETUP TO BUILD MODEL!============") self.setup() self.model_name = "DCGAN.model"
def block_without_condition(x, out_channels, is_training, name): with tf.variable_scope(name): bn0 = ops.batch_norm(name='bn_0') bn1 = ops.batch_norm(name='bn_1') # bn = ops.batch_norm() x = bn0(x, is_training) x = tf.nn.relu(x) x = usample(x) x = ops.snconv2d(x, out_channels, 3, 3, 1, 1, name='snconv1') x = bn1(x, is_training) x = tf.nn.relu(x) x = ops.snconv2d(x, out_channels, 3, 3, 1, 1, name='snconv2') return x
def discriminator(image, reuse=False): d_bn1 = ops.batch_norm(FLAGS.batch_size, name='d_bn1') d_bn2 = ops.batch_norm(FLAGS.batch_size, name='d_bn2') d_bn3 = ops.batch_norm(FLAGS.batch_size, name='d_bn3') image = tf.reshape( image, [FLAGS.batch_size, FLAGS.image_size, FLAGS.image_size, 1]) if reuse: tf.get_variable_scope().reuse_variables() h0 = ops.lrelu(ops.conv2d(image, FLAGS.df_dim, name='d_h0_conv')) h1 = ops.lrelu(d_bn1(ops.conv2d(h0, FLAGS.df_dim * 2, name='d_h1_conv'))) h2 = ops.lrelu(d_bn2(ops.conv2d(h1, FLAGS.df_dim * 4, name='d_h2_conv'))) h3 = ops.lrelu(d_bn3(ops.conv2d(h2, FLAGS.df_dim * 8, name='d_h3_conv'))) h4 = ops.linear(tf.reshape(h3, [FLAGS.batch_size, -1]), 1, 'd_h3_lin') return tf.nn.sigmoid(h4)
def __init__(self, sess, batch_size=32, image_size=256, lam=0.8, checkpoint_dir=None): self.sess = sess self.batch_size = batch_size self.image_size = image_size self.image_shape = [image_size, image_size, 3] self.lam = lam self.checkpoint_dir = checkpoint_dir self.global_step = tf.Variable(0, trainable=False) self.images = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='images') self.d_bns = [batch_norm(name='d_bn{}'.format(i)) for i in range(5)] self.g_bns = [batch_norm(name='g_bn{}'.format(i, )) for i in range(15)] self.D_real = self.discriminator(self.images, self.image_size) self.masks = tf.placeholder(tf.float32, [batch_size] + self.image_shape, name='masks') self.MG = tf.multiply(self.images, self.masks) self.G = self.generator(self.MG) self.D_fake = self.discriminator(self.G, self.image_size, reuse=True) self.D_loss = -tf.reduce_mean(self.D_real) + tf.reduce_mean(self.D_fake) self.G_loss_d = -tf.reduce_mean(self.D_fake) self.G_loss_l = tf.reduce_mean(tf.contrib.layers.flatten( tf.multiply(self.G - self.images, self.G - self.images))) self.G_loss = (1 - self.lam) * self.G_loss_d + self.lam * self.G_loss_l self.images_summary = tf.summary.image("image", self.images) self.D_loss_summary = tf.summary.scalar("d_loss", self.D_loss) self.G_loss_d_summary = tf.summary.scalar("g_loss_d", self.G_loss_d) self.G_loss_l_summary = tf.summary.scalar("g_loss_l", self.G_loss_l) self.G_loss_summary = tf.summary.scalar("g_loss", self.G_loss) self.MG_summary = tf.summary.image("mg", self.MG) self.G_summary = tf.summary.image("g", self.G) t_vars = tf.trainable_variables() self.d_vars = [var for var in t_vars if 'd_' in var.name] self.g_vars = [var for var in t_vars if 'g_' in var.name] self.clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in self.d_vars] self.saver = tf.train.Saver(max_to_keep=10) self.g_summary = tf.summary.merge([ self.G_summary, self.MG_summary, self.G_loss_summary, self.G_loss_l_summary, self.G_loss_d_summary]) self.d_summary = tf.summary.merge([ self.images_summary, self.G_summary, self.MG_summary, self.D_loss_summary]) self.writer = tf.summary.FileWriter(os.path.join(self.checkpoint_dir, "logs"), self.sess.graph)
def prepare_model(self): with tf.variable_scope("LSTMTDNN"): self.char_inputs = [] self.word_inputs = [] self.cnn_outputs = [] if self.use_char: char_W = tf.get_variable("char_embed", [self.char_vocab_size, self.char_embed_dim]) if self.use_word: word_W = tf.get_variable("word_embed", [self.word_vocab_size, self.word_embed_dim]) with tf.variable_scope("CNN") as scope: self.char_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length, self.max_word_length]) self.word_inputs = tf.placeholder(tf.int32, [self.batch_size, self.seq_length]) char_indices = tf.split(1, self.seq_length, self.char_inputs) word_indices = tf.split(1, self.seq_length, tf.expand_dims(self.word_inputs, -1)) for idx in xrange(self.seq_length): char_index = tf.reshape(char_indices[idx], [-1, self.max_word_length]) word_index = tf.reshape(word_indices[idx], [-1, 1]) if idx != 0: scope.reuse_variables() if self.use_char: # [batch_size x word_max_length, char_embed] char_embed = tf.nn.embedding_lookup(char_W, char_index) char_cnn = TDNN(char_embed, self.char_embed_dim, self.feature_maps, self.kernels) if self.use_word: word_embed = tf.nn.embedding_lookup(word_W, word_index) cnn_output = tf.concat(1, [char_cnn.output, tf.squeeze(word_embed, [1])]) else: cnn_output = char_cnn.output else: cnn_output = tf.squeeze(tf.nn.embedding_lookup(word_W, word_index)) if self.use_batch_norm: bn = batch_norm() norm_output = bn(tf.expand_dims(tf.expand_dims(cnn_output, 1), 1)) cnn_output = tf.squeeze(norm_output) if highway: #cnn_output = highway(input_, input_dim_length, self.highway_layers, 0) cnn_output = highway(cnn_output, cnn_output.get_shape()[1], self.highway_layers, 0) self.cnn_outputs.append(cnn_output) with tf.variable_scope("LSTM") as scope: self.cell = tf.nn.rnn_cell.BasicLSTMCell(self.rnn_size) self.stacked_cell = tf.nn.rnn_cell.MultiRNNCell([self.cell] * self.layer_depth) outputs, _ = tf.nn.rnn(self.stacked_cell, self.cnn_outputs, dtype=tf.float32) self.lstm_outputs = [] self.true_outputs = tf.placeholder(tf.int64, [self.batch_size, self.seq_length]) loss = 0 true_outputs = tf.split(1, self.seq_length, self.true_outputs) for idx, (top_h, true_output) in enumerate(zip(outputs, true_outputs)): if self.dropout_prob > 0: top_h = tf.nn.dropout(top_h, self.dropout_prob) if self.hsm > 0: self.lstm_outputs.append(top_h) else: if idx != 0: scope.reuse_variables() proj = tf.nn.rnn_cell._linear(top_h, self.word_vocab_size, 0) self.lstm_outputs.append(proj) loss += tf.nn.sparse_softmax_cross_entropy_with_logits(self.lstm_outputs[idx], tf.squeeze(true_output)) self.loss = tf.reduce_mean(loss) / self.seq_length tf.scalar_summary("loss", self.loss) tf.scalar_summary("perplexity", tf.exp(self.loss))