def __get_discriminator_loss(self, D1, D2): """ Loss for the discriminator network :param D1:logits computed with a discriminator networks from real images :param D2:logits computed with a discriminator networks from generated images :return:Cross entropy loss, postive samples have implicit labels 1, negative 0ss """ return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) + losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __get_discrinator_loss(self, D1, D2): ''' Loss for discriminator network Args: D1 compute discriminator network from real images D2 compute discriminator network from generated images ''' return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) + losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __get_generator_loss1(self, D2,D1,stddev): '''Loss for the genetor. Maximize probability of generating images that discrimator cannot differentiate. Returns: see the paper ''' # rec_loss = tf.reduce_mean(0.5 * (tf.square(mean) + tf.square(stddev) - 2.0 * tf.log(stddev + 0.01) - 1.0)) # rec_lossH1 = tf.reduce_sum(self.flag * tf.square(tf.reduce_sum(tf.square(mean1 - mean), 1) - self.dis)) return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2))))
def __get_discrinator_loss(self, D1, D2): '''Loss for the discriminator network Args: D1: logits computed with a discriminator networks from real images D2: logits computed with a discriminator networks from generated images Returns: Cross entropy loss, positive samples have implicit labels 1, negative 0s ''' return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) + losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def model_function(features, targets, mode): # don't need one-hot encoding since target is already in one-hot format # sigmoid also will work although the interpretability is difficult; # The output with the max. value corresponds to the 'class' - whether sigmoid or softmax outputs = layers.fully_connected( inputs=features, num_outputs=10, # 10 perceptrons for 10 numbers (0 to 9) activation_fn=None ) # Use "None" as activation function specified in "sigmoid_cross_entropy" loss # layer gives direct/plain outputs - linear activation. To compute losses, we use softmax on top of plain outputs # Calculate loss using cross-entropy error; also use the 'sigmoid' activation function # sigmoid and cross-entropy combined together to handle log(0) and other border-case issues loss = losses.sigmoid_cross_entropy(outputs, targets) optimizer = layers.optimize_loss( loss=loss, # step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int' global_step=tf.contrib.framework.get_global_step(), learning_rate=0.5, optimizer="SGD") # Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value # Returning both fractional values and corresponding labels probs = tf.sigmoid(outputs) return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def _sigmoid_entropy(probabilities, targets, weights=None): return metrics.mean( losses.sigmoid_cross_entropy(probabilities, _squeeze_and_onehot( targets, array_ops.shape(probabilities)[1])), weights=weights)
def _sigmoid_entropy(probabilities, targets, weights=None): return metric_ops.streaming_mean( losses.sigmoid_cross_entropy(probabilities, _squeeze_and_onehot( targets, array_ops.shape(probabilities)[1])), weights=weights)
def __get_generator_loss(self, D2): """ Loss for generator.Maximize probability of generating images that discrimator cannot differentiate. :param D2: :return: """ return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
def __get_generator_loss(self, D2): '''Loss for the genetor. Maximize probability of generating images that discrimator cannot differentiate. Returns: see the paper ''' return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
def vae_loss(x_reconstructed, x_true): # Reconstruction loss encode_decode_loss = image_dim * image_dim * image_channels * sigmoid_cross_entropy( x_reconstructed, x_true) # KL Divergence loss kl_div_loss = 1 + z_std - tf.square(z_mean) - tf.exp(z_std) kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1) return tf.reduce_mean(encode_decode_loss + kl_div_loss)
def model_function(features, targets, mode): # Configure the single layer perceptron model outputs = layers.fully_connected(inputs=features, num_outputs=10, activation_fn=None) # Calculate loss using mean squared error loss = losses.sigmoid_cross_entropy(outputs, targets) # Create an optimizer for minimizing the loss function optimizer = layers.optimize_loss( loss=loss, global_step=tf.contrib.framework.get_global_step(), learning_rate=0.5, optimizer="SGD") probs = tf.sigmoid(outputs) return {'probs':probs, 'labels':tf.arg_max(probs,1)}, loss, optimizer
def __get_generator_loss(self, D2): return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)) * 0.9)
def __get_discrinator_loss(self, D1, D2): return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1)) * 0.9) + losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __init__(self, params): # This defines the generator network - it takes samples from a noise # distribution as input, and passes them through an MLP. with tf.variable_scope('G'): self.z = tf.placeholder(tf.float32, shape=(params.batch_size, 100)) self.G = generator(self.z, params.hidden_size) # The discriminator tries to tell the difference between samples from # the true data distribution (self.x) and the generated samples # (self.z). # # Here we create two copies of the discriminator network # that share parameters, as you cannot use the same network with # different inputs in TensorFlow. self.x = tf.placeholder(tf.float32, shape=(params.batch_size, 28, 28, 1)) with tf.variable_scope('D'): self.D1 = discriminator(self.x, params.hidden_size, params.minibatch) with tf.variable_scope('D', reuse=True): self.D2 = discriminator(self.G, params.hidden_size, params.minibatch) # Define the loss for discriminator and generator networks # (see the original paper for details), and create optimizers for both self.D1 = tf.reshape(self.D1, (-1, 1)) self.D2 = tf.reshape(self.D2, (-1, 1)) print "D1: ", self.D1 print "D2: ", self.D2 # self.loss_d = tf.reduce_mean(-log(self.D1) - log(1 - self.D2)) # self.loss_g = tf.reduce_mean(-log(self.D2)) real_loss = losses.sigmoid_cross_entropy( self.D1, tf.ones([params.batch_size, 1])) fake_loss = losses.sigmoid_cross_entropy( self.D2, tf.fill([params.batch_size, 1], -1)) self.loss_d = real_loss + fake_loss self.loss_g = losses.sigmoid_cross_entropy( self.D2, tf.ones([params.batch_size, 1])) vars = tf.trainable_variables() self.d_params = [v for v in vars if v.name.startswith('D/')] self.g_params = [v for v in vars if v.name.startswith('G/')] ## Clip Ops self.clip_discriminator = [] for var in self.d_params: self.clip_discriminator.append( tf.assign( var, tf.clip_by_value(var, -params.clip_value, params.clip_value))) # print "REmove attemp" # print self.g_params # print [v for v in vars if v.name.startswith('G/gshared/')][0] # print [v for v in vars if v.name.startswith('G/dshared/')] self.g_params.remove( [v for v in vars if v.name.startswith('G/gshared/w')][0]) # self.g_params.remove([v for v in vars if v.name.startswith('G/gshared/b')][0]) # self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/w')][0]) # self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/b')][0]) print self.loss_d print self.loss_g self.opt_d = optimizer(self.loss_d, self.d_params, params.d_learning_rate, params.beta1) self.opt_g = optimizer(self.loss_g, self.g_params, params.g_learning_rate, params.beta1) d_loss_summary = tf.summary.scalar('d_loss', self.loss_d) g_loss_summary = tf.summary.scalar('g_loss', self.loss_g) g_out_summary = tf.summary.histogram('g_out', self.G) d_real_out_summary = tf.summary.histogram('d_real_out', self.D1) d_fake_out_summary = tf.summary.histogram('d_fake_out', self.D2) self.d_summary = tf.summary.merge([d_loss_summary]) self.g_summary = tf.summary.merge([g_loss_summary]) self.test_summary = tf.summary.merge([d_real_out_summary]) # copy_gen_shared() shared_weights_g = [ v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES) if v.name.endswith('gshared') ] shared_weights_g_w = [ v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) if v.name.endswith('gshared/w:0') ] shared_weights_g_b = [ v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) if v.name.endswith('gshared/b:0') ] shared_weights_d_w = [ v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) if v.name.endswith('dshared/w:0') ] shared_weights_d_b = [ v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) if v.name.endswith('dshared/b:0') ] self.copy_d_w_g = shared_weights_g_w[0].assign(shared_weights_d_w[0]) self.copy_d_b_g = shared_weights_g_b[0].assign(shared_weights_d_b[0])
def get_generator_loss(self, d2): return losses.sigmoid_cross_entropy(d2, tf.ones( tf.shape(d2))) # I think shape 1 is too small
def get_discrinator_loss(self, d1, d2): return (losses.sigmoid_cross_entropy(d1, tf.ones(tf.shape(d1))) + losses.sigmoid_cross_entropy(d2, tf.zero(tf.shape(d1))))
def _sigmoid_entropy(probabilities, targets): return metric_ops.streaming_mean(losses.sigmoid_cross_entropy( probabilities, targets))
def discriminator_loss(discriminator_real, discriminator_generated): return ( losses.sigmoid_cross_entropy(discriminator_real, tf.ones(tf.shape(discriminator_real))) + losses.sigmoid_cross_entropy(discriminator_generated, tf.zeros(tf.shape(discriminator_real))))
def generator_loss(discriminator_generated): return losses.sigmoid_cross_entropy( discriminator_generated, tf.ones(tf.shape(discriminator_generated)))
def _sigmoid_entropy(probabilities, targets): return metric_ops.streaming_mean( losses.sigmoid_cross_entropy(probabilities, targets))
def main(): images = YOURDATAHERE # Feed your data here! The program expects batches of 128x128x3 float32 (normalized to be between 0 and 1) images by default tf.image_summary("real", images, max_images=1) z = tf.placeholder(tf.float32, [batch_size, z_dim], name='z') with tf.variable_scope("generator") as scope: gen = generator(z) tf.image_summary("fake", gen, max_images=1) with tf.variable_scope("discriminator") as scope: disc_real = discriminator(images) scope.reuse_variables() disc_fake = discriminator(gen) # Define Losses disc_real_loss = losses.sigmoid_cross_entropy(disc_real, tf.ones([batch_size, 1])) disc_fake_loss = losses.sigmoid_cross_entropy( disc_fake, tf.fill([batch_size, 1], -1.0)) d_loss = disc_real_loss + disc_fake_loss g_loss = losses.sigmoid_cross_entropy(disc_fake, tf.ones([batch_size, 1])) tf.scalar_summary("Discriminator_loss_real", disc_real_loss) tf.scalar_summary("Discrimintator_loss_fake", disc_fake_loss) tf.scalar_summary("Discriminator_loss", d_loss) tf.scalar_summary("Generator_loss", g_loss) # The paper found RMSProp to work better than Adam or other momentum based methods d_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate) g_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate) d_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "discriminator") g_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator") # Create training ops d_train_op = slim.learning.create_train_op(d_loss, d_optimizer, variables_to_train=d_vars) g_train_op = slim.learning.create_train_op(g_loss, g_optimizer, variables_to_train=g_vars) # Create clipping ops, thanks to PatrykChrabaszcz for this! clip_discriminator = [] for var in d_vars: clip_discriminator.append(tf.assign(var, tf.clip_by_value(var, -c, c))) with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) summary_op = tf.merge_all_summaries() summary_writer = tf.train.SummaryWriter(log_dir, sess.graph) saver = tf.train.Saver() init_op = tf.global_variables_initializer() sess.run(init_op) start = 0 # If a checkpoint is found, restore what you can. If not, continue ckpt = tf.train.get_checkpoint_state(log_dir) if ckpt and ckpt.model_checkpoint_path: print("Checkpoint found! Restoring...") saver.restore(sess, ckpt.model_checkpoint_path) # Hackey way to determine what step we're starting on. It feels like there should be some in built function in TensorFlow to do this but I can't find any... start = int(ckpt.model_checkpoint_path.split("-")[-1]) + 1 print("Restored!") else: print("No checkpoint found!") try: current_step = start print("Starting training!") for itr in xrange(start, max_iterations): # As per the reference implementation, the discriminator gets a lot of training early on if current_step < 25 or current_step % 500 == 0: diters = 100 else: diters = d_iters # Train discriminator several times for i in xrange(diters): # Clip all discriminator weights to be between -c and c if i % clip_per == 0: sess.run(clip_discriminator) batch_z = np.random.uniform( -1, 1, [batch_size, z_dim]).astype(np.float32) sess.run(d_train_op, feed_dict={z: batch_z}) # Train generator once batch_z = np.random.uniform(-1, 1, [batch_size, z_dim]).astype( np.float32) sess.run(g_train_op, feed_dict={z: batch_z}) # Give the user some feedback if itr % sum_per == 0: g_loss_val, d_loss_val, summary_str = sess.run( [g_loss, d_loss, summary_op], feed_dict={z: batch_z}) print( "Step: %d, Generator Loss: %g, Discriminator Loss: %g" % (itr, g_loss_val, d_loss_val)) summary_writer.add_summary(summary_str, itr) # Every so often save if itr % save_per == 0: saver.save(sess, os.path.join(log_dir, "model.ckpt"), global_step=itr) current_step = itr except tf.errors.OutOfRangeError: print('Done training -- epoch limit reached!') except KeyboardInterrupt: print("Ending training...") # User terminated with Ctrl-C, save current state saver.save(sess, os.path.join(log_dir, "model.ckpt"), global_step=current_step) finally: coord.request_stop() # Done! coord.join(threads)