Exemple #1
0
 def __get_discriminator_loss(self, D1, D2):
     """
     Loss for the discriminator network
     :param D1:logits computed with a discriminator networks from real images
     :param D2:logits computed with a discriminator networks from generated images
     :return:Cross entropy loss, postive samples have implicit labels 1, negative 0ss
     """
     return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
             losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
Exemple #2
0
    def __get_discrinator_loss(self, D1, D2):
        ''' Loss for discriminator network

            Args:
                D1 compute discriminator network from real images
                D2 compute discriminator network from generated images
        '''
        return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
                losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
Exemple #3
0
    def __get_generator_loss1(self, D2,D1,stddev):
        '''Loss for the genetor. Maximize probability of generating images that
        discrimator cannot differentiate.

        Returns:
            see the paper
        '''
        # rec_loss = tf.reduce_mean(0.5 * (tf.square(mean) + tf.square(stddev) - 2.0 * tf.log(stddev + 0.01) - 1.0))
        # rec_lossH1 = tf.reduce_sum(self.flag * tf.square(tf.reduce_sum(tf.square(mean1 - mean), 1) - self.dis))
        return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2))))
    def __get_discrinator_loss(self, D1, D2):
        '''Loss for the discriminator network

        Args:
            D1: logits computed with a discriminator networks from real images
            D2: logits computed with a discriminator networks from generated images

        Returns:
            Cross entropy loss, positive samples have implicit labels 1, negative 0s
        '''
        return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
                losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def model_function(features, targets, mode):
    # don't need one-hot encoding since target is already in one-hot format

    # sigmoid also will work although the interpretability is difficult;
    # The output with the max. value corresponds to the 'class' - whether sigmoid or softmax
    outputs = layers.fully_connected(
        inputs=features,
        num_outputs=10,  # 10 perceptrons for 10 numbers (0 to 9)
        activation_fn=None
    )  # Use "None" as activation function specified in "sigmoid_cross_entropy" loss
    # layer gives direct/plain outputs - linear activation. To compute losses, we use softmax on top of plain outputs

    # Calculate loss using cross-entropy error; also use the 'sigmoid' activation function
    # sigmoid and cross-entropy combined together to handle log(0) and other border-case issues
    loss = losses.sigmoid_cross_entropy(outputs, targets)

    optimizer = layers.optimize_loss(
        loss=loss,
        # step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int'
        global_step=tf.contrib.framework.get_global_step(),
        learning_rate=0.5,
        optimizer="SGD")

    # Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
    # Returning both fractional values and corresponding labels
    probs = tf.sigmoid(outputs)
    return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def _sigmoid_entropy(probabilities, targets, weights=None):
  return metrics.mean(
      losses.sigmoid_cross_entropy(probabilities,
                                   _squeeze_and_onehot(
                                       targets,
                                       array_ops.shape(probabilities)[1])),
      weights=weights)
Exemple #7
0
def _sigmoid_entropy(probabilities, targets, weights=None):
  return metric_ops.streaming_mean(
      losses.sigmoid_cross_entropy(probabilities,
                                   _squeeze_and_onehot(
                                       targets,
                                       array_ops.shape(probabilities)[1])),
      weights=weights)
Exemple #8
0
 def __get_generator_loss(self, D2):
     """
     Loss for generator.Maximize probability of generating images that
         discrimator cannot differentiate.
     :param D2:
     :return:
     """
     return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
    def __get_generator_loss(self, D2):
        '''Loss for the genetor. Maximize probability of generating images that
        discrimator cannot differentiate.

        Returns:
            see the paper
        '''
        return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
def vae_loss(x_reconstructed, x_true):
    # Reconstruction loss
    encode_decode_loss = image_dim * image_dim * image_channels * sigmoid_cross_entropy(
        x_reconstructed, x_true)
    # KL Divergence loss
    kl_div_loss = 1 + z_std - tf.square(z_mean) - tf.exp(z_std)
    kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1)
    return tf.reduce_mean(encode_decode_loss + kl_div_loss)
Exemple #11
0
def model_function(features, targets, mode):    

  # Configure the single layer perceptron model
  outputs = layers.fully_connected(inputs=features,
                                                 num_outputs=10,
                                                 activation_fn=None)
   # Calculate loss using mean squared error
  loss = losses.sigmoid_cross_entropy(outputs, targets)

  # Create an optimizer for minimizing the loss function
  optimizer = layers.optimize_loss(
      loss=loss,
      global_step=tf.contrib.framework.get_global_step(),
      learning_rate=0.5,
      optimizer="SGD")

  probs = tf.sigmoid(outputs)
  return {'probs':probs, 'labels':tf.arg_max(probs,1)}, loss, optimizer
Exemple #12
0
 def __get_generator_loss(self, D2):
     return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)) * 0.9)
Exemple #13
0
 def __get_discrinator_loss(self, D1, D2):
     return (losses.sigmoid_cross_entropy(D1,
                                          tf.ones(tf.shape(D1)) * 0.9) +
             losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
Exemple #14
0
    def __init__(self, params):
        # This defines the generator network - it takes samples from a noise
        # distribution as input, and passes them through an MLP.

        with tf.variable_scope('G'):
            self.z = tf.placeholder(tf.float32, shape=(params.batch_size, 100))
            self.G = generator(self.z, params.hidden_size)

        # The discriminator tries to tell the difference between samples from
        # the true data distribution (self.x) and the generated samples
        # (self.z).
        #
        # Here we create two copies of the discriminator network
        # that share parameters, as you cannot use the same network with
        # different inputs in TensorFlow.
        self.x = tf.placeholder(tf.float32,
                                shape=(params.batch_size, 28, 28, 1))

        with tf.variable_scope('D'):
            self.D1 = discriminator(self.x, params.hidden_size,
                                    params.minibatch)
        with tf.variable_scope('D', reuse=True):
            self.D2 = discriminator(self.G, params.hidden_size,
                                    params.minibatch)

        # Define the loss for discriminator and generator networks
        # (see the original paper for details), and create optimizers for both

        self.D1 = tf.reshape(self.D1, (-1, 1))
        self.D2 = tf.reshape(self.D2, (-1, 1))

        print "D1: ", self.D1
        print "D2: ", self.D2

        # self.loss_d = tf.reduce_mean(-log(self.D1) - log(1 - self.D2))
        # self.loss_g = tf.reduce_mean(-log(self.D2))

        real_loss = losses.sigmoid_cross_entropy(
            self.D1, tf.ones([params.batch_size, 1]))
        fake_loss = losses.sigmoid_cross_entropy(
            self.D2, tf.fill([params.batch_size, 1], -1))
        self.loss_d = real_loss + fake_loss
        self.loss_g = losses.sigmoid_cross_entropy(
            self.D2, tf.ones([params.batch_size, 1]))

        vars = tf.trainable_variables()
        self.d_params = [v for v in vars if v.name.startswith('D/')]
        self.g_params = [v for v in vars if v.name.startswith('G/')]

        ## Clip Ops

        self.clip_discriminator = []

        for var in self.d_params:
            self.clip_discriminator.append(
                tf.assign(
                    var,
                    tf.clip_by_value(var, -params.clip_value,
                                     params.clip_value)))

        # print "REmove attemp"
        # print self.g_params
        # print [v for v in vars if v.name.startswith('G/gshared/')][0]
        # print [v for v in vars if v.name.startswith('G/dshared/')]

        self.g_params.remove(
            [v for v in vars if v.name.startswith('G/gshared/w')][0])
        # self.g_params.remove([v for v in vars if v.name.startswith('G/gshared/b')][0])

        # self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/w')][0])
        # self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/b')][0])

        print self.loss_d
        print self.loss_g

        self.opt_d = optimizer(self.loss_d, self.d_params,
                               params.d_learning_rate, params.beta1)
        self.opt_g = optimizer(self.loss_g, self.g_params,
                               params.g_learning_rate, params.beta1)

        d_loss_summary = tf.summary.scalar('d_loss', self.loss_d)
        g_loss_summary = tf.summary.scalar('g_loss', self.loss_g)

        g_out_summary = tf.summary.histogram('g_out', self.G)
        d_real_out_summary = tf.summary.histogram('d_real_out', self.D1)
        d_fake_out_summary = tf.summary.histogram('d_fake_out', self.D2)

        self.d_summary = tf.summary.merge([d_loss_summary])
        self.g_summary = tf.summary.merge([g_loss_summary])

        self.test_summary = tf.summary.merge([d_real_out_summary])

        # copy_gen_shared()

        shared_weights_g = [
            v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
            if v.name.endswith('gshared')
        ]

        shared_weights_g_w = [
            v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
            if v.name.endswith('gshared/w:0')
        ]
        shared_weights_g_b = [
            v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
            if v.name.endswith('gshared/b:0')
        ]

        shared_weights_d_w = [
            v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
            if v.name.endswith('dshared/w:0')
        ]
        shared_weights_d_b = [
            v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
            if v.name.endswith('dshared/b:0')
        ]

        self.copy_d_w_g = shared_weights_g_w[0].assign(shared_weights_d_w[0])
        self.copy_d_b_g = shared_weights_g_b[0].assign(shared_weights_d_b[0])
Exemple #15
0
 def get_generator_loss(self, d2):
     return losses.sigmoid_cross_entropy(d2, tf.ones(
         tf.shape(d2)))  # I think shape 1 is too small
Exemple #16
0
 def get_discrinator_loss(self, d1, d2):
     return (losses.sigmoid_cross_entropy(d1, tf.ones(tf.shape(d1))) +
             losses.sigmoid_cross_entropy(d2, tf.zero(tf.shape(d1))))
Exemple #17
0
def _sigmoid_entropy(probabilities, targets):
  return metric_ops.streaming_mean(losses.sigmoid_cross_entropy(
      probabilities, targets))
Exemple #18
0
def discriminator_loss(discriminator_real, discriminator_generated):
    return (
        losses.sigmoid_cross_entropy(discriminator_real,
                                     tf.ones(tf.shape(discriminator_real))) +
        losses.sigmoid_cross_entropy(discriminator_generated,
                                     tf.zeros(tf.shape(discriminator_real))))
Exemple #19
0
def generator_loss(discriminator_generated):
    return losses.sigmoid_cross_entropy(
        discriminator_generated, tf.ones(tf.shape(discriminator_generated)))
Exemple #20
0
def _sigmoid_entropy(probabilities, targets):
    return metric_ops.streaming_mean(
        losses.sigmoid_cross_entropy(probabilities, targets))
def main():
    images = YOURDATAHERE  # Feed your data here! The program expects batches of 128x128x3 float32 (normalized to be between 0 and 1) images by default
    tf.image_summary("real", images, max_images=1)

    z = tf.placeholder(tf.float32, [batch_size, z_dim], name='z')

    with tf.variable_scope("generator") as scope:
        gen = generator(z)
        tf.image_summary("fake", gen, max_images=1)

    with tf.variable_scope("discriminator") as scope:
        disc_real = discriminator(images)
        scope.reuse_variables()
        disc_fake = discriminator(gen)

    # Define Losses
    disc_real_loss = losses.sigmoid_cross_entropy(disc_real,
                                                  tf.ones([batch_size, 1]))
    disc_fake_loss = losses.sigmoid_cross_entropy(
        disc_fake, tf.fill([batch_size, 1], -1.0))

    d_loss = disc_real_loss + disc_fake_loss
    g_loss = losses.sigmoid_cross_entropy(disc_fake, tf.ones([batch_size, 1]))

    tf.scalar_summary("Discriminator_loss_real", disc_real_loss)
    tf.scalar_summary("Discrimintator_loss_fake", disc_fake_loss)
    tf.scalar_summary("Discriminator_loss", d_loss)
    tf.scalar_summary("Generator_loss", g_loss)

    # The paper found RMSProp to work better than Adam or other momentum based methods
    d_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
    g_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)

    d_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
                               "discriminator")
    g_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator")
    # Create training ops
    d_train_op = slim.learning.create_train_op(d_loss,
                                               d_optimizer,
                                               variables_to_train=d_vars)
    g_train_op = slim.learning.create_train_op(g_loss,
                                               g_optimizer,
                                               variables_to_train=g_vars)

    # Create clipping ops, thanks to PatrykChrabaszcz for this!
    clip_discriminator = []
    for var in d_vars:
        clip_discriminator.append(tf.assign(var, tf.clip_by_value(var, -c, c)))

    with tf.Session() as sess:
        coord = tf.train.Coordinator()
        threads = tf.train.start_queue_runners(sess, coord)
        summary_op = tf.merge_all_summaries()
        summary_writer = tf.train.SummaryWriter(log_dir, sess.graph)
        saver = tf.train.Saver()

        init_op = tf.global_variables_initializer()
        sess.run(init_op)

        start = 0
        # If a checkpoint is found, restore what you can. If not, continue
        ckpt = tf.train.get_checkpoint_state(log_dir)
        if ckpt and ckpt.model_checkpoint_path:
            print("Checkpoint found! Restoring...")
            saver.restore(sess, ckpt.model_checkpoint_path)
            # Hackey way to determine what step we're starting on. It feels like there should be some in built function in TensorFlow to do this but I can't find any...
            start = int(ckpt.model_checkpoint_path.split("-")[-1]) + 1
            print("Restored!")
        else:
            print("No checkpoint found!")

        try:
            current_step = start
            print("Starting training!")
            for itr in xrange(start, max_iterations):

                # As per the reference implementation, the discriminator gets a lot of training early on
                if current_step < 25 or current_step % 500 == 0:
                    diters = 100
                else:
                    diters = d_iters

                # Train discriminator several times
                for i in xrange(diters):
                    # Clip all discriminator weights to be between -c and c
                    if i % clip_per == 0:
                        sess.run(clip_discriminator)
                    batch_z = np.random.uniform(
                        -1, 1, [batch_size, z_dim]).astype(np.float32)
                    sess.run(d_train_op, feed_dict={z: batch_z})

                # Train generator once
                batch_z = np.random.uniform(-1, 1, [batch_size, z_dim]).astype(
                    np.float32)
                sess.run(g_train_op, feed_dict={z: batch_z})

                # Give the user some feedback
                if itr % sum_per == 0:
                    g_loss_val, d_loss_val, summary_str = sess.run(
                        [g_loss, d_loss, summary_op], feed_dict={z: batch_z})
                    print(
                        "Step: %d, Generator Loss: %g, Discriminator Loss: %g"
                        % (itr, g_loss_val, d_loss_val))
                    summary_writer.add_summary(summary_str, itr)

                # Every so often save
                if itr % save_per == 0:
                    saver.save(sess,
                               os.path.join(log_dir, "model.ckpt"),
                               global_step=itr)
                current_step = itr

        except tf.errors.OutOfRangeError:
            print('Done training -- epoch limit reached!')

        except KeyboardInterrupt:
            print("Ending training...")
            # User terminated with Ctrl-C, save current state
            saver.save(sess,
                       os.path.join(log_dir, "model.ckpt"),
                       global_step=current_step)

        finally:
            coord.request_stop()

        # Done!
        coord.join(threads)