def __init__(self, X_sampled, latent_dim, global_step):
flat_data_dim = int(np.prod(X_sampled.get_shape().as_list()[1:]))
X_sampled_flat = flatten(X_sampled)
with tf.variable_scope("Encoder"):
h = mlp(X_sampled_flat,
layer_sizes=(128, 128),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=tf.nn.relu)
# Computed parameters for encoding distribution.
Z_mu = tf.layers.dense(h, latent_dim, activation=None)
Z_log_var = tf.layers.dense(h, latent_dim, activation=None)
# Sampling through reparametrization z = mu + sigma * epsilon
# L = 1, i.e. one sampling of z is enough if the batch size is big enough according to VAE paper.
epsilon = tf.random_normal(tf.shape(Z_log_var),
mean=0.0,
stddev=1.0)
Z = Z_mu + tf.exp(0.5 * Z_log_var) * epsilon
with tf.variable_scope("Decoder"):
logits = mlp(Z,
layer_sizes=(128, 128, flat_data_dim),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=None)
# Computed parameters for multiple Bernoullis.
bernoulli_probs = tf.nn.sigmoid(logits)
with tf.name_scope("Training"):
with tf.name_scope("Loss"):
with tf.name_scope("KL_loss"):
# KL loss for gaussian encoding distribution, q(z|x).
# Negative since we want to maximize the lower bound estimator.
kl_loss = -0.5 * tf.reduce_sum(
1.0 + Z_log_var - tf.square(Z_mu) - tf.exp(Z_log_var),
axis=1)
with tf.name_scope("Reconstruction_loss"):
# Expected negative log-likelihood as loss.
reconstruction_loss = \
tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits,
labels=X_sampled_flat), axis=1)
# TODO: Maybe try KL cost annealing.
loss = tf.reduce_mean(kl_loss + reconstruction_loss)
self.optimization_step = (tf.train.AdamOptimizer(
learning_rate=1e-4).minimize(loss,
global_step=global_step))
self.global_step = global_step
self.Z = Z
if bernoulli_probs.get_shape().ndims > 2:
self.X_generated = bernoulli_probs
else:
self.X_generated = tf.reshape(bernoulli_probs, [-1] +
X_sampled.get_shape().as_list()[1:])
def critic(X):
return mlp(X, layer_sizes=(128, 128, 128, 1),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=None) # Note: No activation in final layer for Wasserstein GAN.
def generator(Z):
return mlp(Z, layer_sizes=(128, 128, 128, flat_data_dim),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=generator_final_activation)
def discriminator(X):
return mlp(X,
layer_sizes=(128, 128, 1),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=tf.nn.sigmoid)
def discriminator(X):
# Note: Discriminator is an autoencoder in BEGAN paper.
X_reconstructed = mlp(X, layer_sizes=(128, autoencoder_hidden_dim, 128, flat_data_dim),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=tf.nn.sigmoid)
return tf.reduce_sum((X - X_reconstructed) ** 2, axis=1)
def discriminator(X):
return mlp(X, layer_sizes=(128, 128, 128, 1),
intermediate_activation_fn=leaky_relu,
final_activation_fn=None) # Note: No activation in final layer for Least Squares GAN.
def discriminator_q_shared(X):
return mlp(X,
layer_sizes=(128, 128, 128),
intermediate_activation_fn=tf.nn.relu,
final_activation_fn=tf.nn.relu)