def __get_discriminator_loss(self, D1, D2):
"""
Loss for the discriminator network
:param D1:logits computed with a discriminator networks from real images
:param D2:logits computed with a discriminator networks from generated images
:return:Cross entropy loss, postive samples have implicit labels 1, negative 0ss
"""
return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __get_discrinator_loss(self, D1, D2):
''' Loss for discriminator network
Args:
D1 compute discriminator network from real images
D2 compute discriminator network from generated images
'''
return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __get_generator_loss1(self, D2,D1,stddev):
'''Loss for the genetor. Maximize probability of generating images that
discrimator cannot differentiate.
Returns:
see the paper
'''
# rec_loss = tf.reduce_mean(0.5 * (tf.square(mean) + tf.square(stddev) - 2.0 * tf.log(stddev + 0.01) - 1.0))
# rec_lossH1 = tf.reduce_sum(self.flag * tf.square(tf.reduce_sum(tf.square(mean1 - mean), 1) - self.dis))
return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2))))
def __get_discrinator_loss(self, D1, D2):
'''Loss for the discriminator network
Args:
D1: logits computed with a discriminator networks from real images
D2: logits computed with a discriminator networks from generated images
Returns:
Cross entropy loss, positive samples have implicit labels 1, negative 0s
'''
return (losses.sigmoid_cross_entropy(D1, tf.ones(tf.shape(D1))) +
losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def model_function(features, targets, mode):
# don't need one-hot encoding since target is already in one-hot format
# sigmoid also will work although the interpretability is difficult;
# The output with the max. value corresponds to the 'class' - whether sigmoid or softmax
outputs = layers.fully_connected(
inputs=features,
num_outputs=10, # 10 perceptrons for 10 numbers (0 to 9)
activation_fn=None
) # Use "None" as activation function specified in "sigmoid_cross_entropy" loss
# layer gives direct/plain outputs - linear activation. To compute losses, we use softmax on top of plain outputs
# Calculate loss using cross-entropy error; also use the 'sigmoid' activation function
# sigmoid and cross-entropy combined together to handle log(0) and other border-case issues
loss = losses.sigmoid_cross_entropy(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
# step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int'
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.5,
optimizer="SGD")
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
probs = tf.sigmoid(outputs)
return {'probs': probs, 'labels': tf.argmax(probs, 1)}, loss, optimizer
def _sigmoid_entropy(probabilities, targets, weights=None):
return metrics.mean(
losses.sigmoid_cross_entropy(probabilities,
_squeeze_and_onehot(
targets,
array_ops.shape(probabilities)[1])),
weights=weights)
def _sigmoid_entropy(probabilities, targets, weights=None):
return metric_ops.streaming_mean(
losses.sigmoid_cross_entropy(probabilities,
_squeeze_and_onehot(
targets,
array_ops.shape(probabilities)[1])),
weights=weights)
def __get_generator_loss(self, D2):
"""
Loss for generator.Maximize probability of generating images that
discrimator cannot differentiate.
:param D2:
:return:
"""
return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
def __get_generator_loss(self, D2):
'''Loss for the genetor. Maximize probability of generating images that
discrimator cannot differentiate.
Returns:
see the paper
'''
return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)))
def vae_loss(x_reconstructed, x_true):
# Reconstruction loss
encode_decode_loss = image_dim * image_dim * image_channels * sigmoid_cross_entropy(
x_reconstructed, x_true)
# KL Divergence loss
kl_div_loss = 1 + z_std - tf.square(z_mean) - tf.exp(z_std)
kl_div_loss = -0.5 * tf.reduce_sum(kl_div_loss, 1)
return tf.reduce_mean(encode_decode_loss + kl_div_loss)
def model_function(features, targets, mode):
# Configure the single layer perceptron model
outputs = layers.fully_connected(inputs=features,
num_outputs=10,
activation_fn=None)
# Calculate loss using mean squared error
loss = losses.sigmoid_cross_entropy(outputs, targets)
# Create an optimizer for minimizing the loss function
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.5,
optimizer="SGD")
probs = tf.sigmoid(outputs)
return {'probs':probs, 'labels':tf.arg_max(probs,1)}, loss, optimizer
def __get_generator_loss(self, D2):
return losses.sigmoid_cross_entropy(D2, tf.ones(tf.shape(D2)) * 0.9)
def __get_discrinator_loss(self, D1, D2):
return (losses.sigmoid_cross_entropy(D1,
tf.ones(tf.shape(D1)) * 0.9) +
losses.sigmoid_cross_entropy(D2, tf.zeros(tf.shape(D1))))
def __init__(self, params):
# This defines the generator network - it takes samples from a noise
# distribution as input, and passes them through an MLP.
with tf.variable_scope('G'):
self.z = tf.placeholder(tf.float32, shape=(params.batch_size, 100))
self.G = generator(self.z, params.hidden_size)
# The discriminator tries to tell the difference between samples from
# the true data distribution (self.x) and the generated samples
# (self.z).
#
# Here we create two copies of the discriminator network
# that share parameters, as you cannot use the same network with
# different inputs in TensorFlow.
self.x = tf.placeholder(tf.float32,
shape=(params.batch_size, 28, 28, 1))
with tf.variable_scope('D'):
self.D1 = discriminator(self.x, params.hidden_size,
params.minibatch)
with tf.variable_scope('D', reuse=True):
self.D2 = discriminator(self.G, params.hidden_size,
params.minibatch)
# Define the loss for discriminator and generator networks
# (see the original paper for details), and create optimizers for both
self.D1 = tf.reshape(self.D1, (-1, 1))
self.D2 = tf.reshape(self.D2, (-1, 1))
print "D1: ", self.D1
print "D2: ", self.D2
# self.loss_d = tf.reduce_mean(-log(self.D1) - log(1 - self.D2))
# self.loss_g = tf.reduce_mean(-log(self.D2))
real_loss = losses.sigmoid_cross_entropy(
self.D1, tf.ones([params.batch_size, 1]))
fake_loss = losses.sigmoid_cross_entropy(
self.D2, tf.fill([params.batch_size, 1], -1))
self.loss_d = real_loss + fake_loss
self.loss_g = losses.sigmoid_cross_entropy(
self.D2, tf.ones([params.batch_size, 1]))
vars = tf.trainable_variables()
self.d_params = [v for v in vars if v.name.startswith('D/')]
self.g_params = [v for v in vars if v.name.startswith('G/')]
## Clip Ops
self.clip_discriminator = []
for var in self.d_params:
self.clip_discriminator.append(
tf.assign(
var,
tf.clip_by_value(var, -params.clip_value,
params.clip_value)))
# print "REmove attemp"
# print self.g_params
# print [v for v in vars if v.name.startswith('G/gshared/')][0]
# print [v for v in vars if v.name.startswith('G/dshared/')]
self.g_params.remove(
[v for v in vars if v.name.startswith('G/gshared/w')][0])
# self.g_params.remove([v for v in vars if v.name.startswith('G/gshared/b')][0])
# self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/w')][0])
# self.d_params.remove([v for v in vars if v.name.startswith('D/dshared/b')][0])
print self.loss_d
print self.loss_g
self.opt_d = optimizer(self.loss_d, self.d_params,
params.d_learning_rate, params.beta1)
self.opt_g = optimizer(self.loss_g, self.g_params,
params.g_learning_rate, params.beta1)
d_loss_summary = tf.summary.scalar('d_loss', self.loss_d)
g_loss_summary = tf.summary.scalar('g_loss', self.loss_g)
g_out_summary = tf.summary.histogram('g_out', self.G)
d_real_out_summary = tf.summary.histogram('d_real_out', self.D1)
d_fake_out_summary = tf.summary.histogram('d_fake_out', self.D2)
self.d_summary = tf.summary.merge([d_loss_summary])
self.g_summary = tf.summary.merge([g_loss_summary])
self.test_summary = tf.summary.merge([d_real_out_summary])
# copy_gen_shared()
shared_weights_g = [
v for v in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
if v.name.endswith('gshared')
]
shared_weights_g_w = [
v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
if v.name.endswith('gshared/w:0')
]
shared_weights_g_b = [
v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
if v.name.endswith('gshared/b:0')
]
shared_weights_d_w = [
v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
if v.name.endswith('dshared/w:0')
]
shared_weights_d_b = [
v for v in tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
if v.name.endswith('dshared/b:0')
]
self.copy_d_w_g = shared_weights_g_w[0].assign(shared_weights_d_w[0])
self.copy_d_b_g = shared_weights_g_b[0].assign(shared_weights_d_b[0])
def get_generator_loss(self, d2):
return losses.sigmoid_cross_entropy(d2, tf.ones(
tf.shape(d2))) # I think shape 1 is too small
def get_discrinator_loss(self, d1, d2):
return (losses.sigmoid_cross_entropy(d1, tf.ones(tf.shape(d1))) +
losses.sigmoid_cross_entropy(d2, tf.zero(tf.shape(d1))))
def _sigmoid_entropy(probabilities, targets):
return metric_ops.streaming_mean(losses.sigmoid_cross_entropy(
probabilities, targets))
def discriminator_loss(discriminator_real, discriminator_generated):
return (
losses.sigmoid_cross_entropy(discriminator_real,
tf.ones(tf.shape(discriminator_real))) +
losses.sigmoid_cross_entropy(discriminator_generated,
tf.zeros(tf.shape(discriminator_real))))
def generator_loss(discriminator_generated):
return losses.sigmoid_cross_entropy(
discriminator_generated, tf.ones(tf.shape(discriminator_generated)))
def _sigmoid_entropy(probabilities, targets):
return metric_ops.streaming_mean(
losses.sigmoid_cross_entropy(probabilities, targets))
def main():
images = YOURDATAHERE # Feed your data here! The program expects batches of 128x128x3 float32 (normalized to be between 0 and 1) images by default
tf.image_summary("real", images, max_images=1)
z = tf.placeholder(tf.float32, [batch_size, z_dim], name='z')
with tf.variable_scope("generator") as scope:
gen = generator(z)
tf.image_summary("fake", gen, max_images=1)
with tf.variable_scope("discriminator") as scope:
disc_real = discriminator(images)
scope.reuse_variables()
disc_fake = discriminator(gen)
# Define Losses
disc_real_loss = losses.sigmoid_cross_entropy(disc_real,
tf.ones([batch_size, 1]))
disc_fake_loss = losses.sigmoid_cross_entropy(
disc_fake, tf.fill([batch_size, 1], -1.0))
d_loss = disc_real_loss + disc_fake_loss
g_loss = losses.sigmoid_cross_entropy(disc_fake, tf.ones([batch_size, 1]))
tf.scalar_summary("Discriminator_loss_real", disc_real_loss)
tf.scalar_summary("Discrimintator_loss_fake", disc_fake_loss)
tf.scalar_summary("Discriminator_loss", d_loss)
tf.scalar_summary("Generator_loss", g_loss)
# The paper found RMSProp to work better than Adam or other momentum based methods
d_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
g_optimizer = tf.train.RMSPropOptimizer(learning_rate=learning_rate)
d_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
"discriminator")
g_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "generator")
# Create training ops
d_train_op = slim.learning.create_train_op(d_loss,
d_optimizer,
variables_to_train=d_vars)
g_train_op = slim.learning.create_train_op(g_loss,
g_optimizer,
variables_to_train=g_vars)
# Create clipping ops, thanks to PatrykChrabaszcz for this!
clip_discriminator = []
for var in d_vars:
clip_discriminator.append(tf.assign(var, tf.clip_by_value(var, -c, c)))
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess, coord)
summary_op = tf.merge_all_summaries()
summary_writer = tf.train.SummaryWriter(log_dir, sess.graph)
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
sess.run(init_op)
start = 0
# If a checkpoint is found, restore what you can. If not, continue
ckpt = tf.train.get_checkpoint_state(log_dir)
if ckpt and ckpt.model_checkpoint_path:
print("Checkpoint found! Restoring...")
saver.restore(sess, ckpt.model_checkpoint_path)
# Hackey way to determine what step we're starting on. It feels like there should be some in built function in TensorFlow to do this but I can't find any...
start = int(ckpt.model_checkpoint_path.split("-")[-1]) + 1
print("Restored!")
else:
print("No checkpoint found!")
try:
current_step = start
print("Starting training!")
for itr in xrange(start, max_iterations):
# As per the reference implementation, the discriminator gets a lot of training early on
if current_step < 25 or current_step % 500 == 0:
diters = 100
else:
diters = d_iters
# Train discriminator several times
for i in xrange(diters):
# Clip all discriminator weights to be between -c and c
if i % clip_per == 0:
sess.run(clip_discriminator)
batch_z = np.random.uniform(
-1, 1, [batch_size, z_dim]).astype(np.float32)
sess.run(d_train_op, feed_dict={z: batch_z})
# Train generator once
batch_z = np.random.uniform(-1, 1, [batch_size, z_dim]).astype(
np.float32)
sess.run(g_train_op, feed_dict={z: batch_z})
# Give the user some feedback
if itr % sum_per == 0:
g_loss_val, d_loss_val, summary_str = sess.run(
[g_loss, d_loss, summary_op], feed_dict={z: batch_z})
print(
"Step: %d, Generator Loss: %g, Discriminator Loss: %g"
% (itr, g_loss_val, d_loss_val))
summary_writer.add_summary(summary_str, itr)
# Every so often save
if itr % save_per == 0:
saver.save(sess,
os.path.join(log_dir, "model.ckpt"),
global_step=itr)
current_step = itr
except tf.errors.OutOfRangeError:
print('Done training -- epoch limit reached!')
except KeyboardInterrupt:
print("Ending training...")
# User terminated with Ctrl-C, save current state
saver.save(sess,
os.path.join(log_dir, "model.ckpt"),
global_step=current_step)
finally:
coord.request_stop()
# Done!
coord.join(threads)