def __init__(self, X_sampled, latent_dim, global_step): flat_data_dim = int(np.prod(X_sampled.get_shape().as_list()[1:])) X_sampled_flat = flatten(X_sampled) with tf.variable_scope("Encoder"): h = mlp(X_sampled_flat, layer_sizes=(128, 128), intermediate_activation_fn=tf.nn.relu, final_activation_fn=tf.nn.relu) # Computed parameters for encoding distribution. Z_mu = tf.layers.dense(h, latent_dim, activation=None) Z_log_var = tf.layers.dense(h, latent_dim, activation=None) # Sampling through reparametrization z = mu + sigma * epsilon # L = 1, i.e. one sampling of z is enough if the batch size is big enough according to VAE paper. epsilon = tf.random_normal(tf.shape(Z_log_var), mean=0.0, stddev=1.0) Z = Z_mu + tf.exp(0.5 * Z_log_var) * epsilon with tf.variable_scope("Decoder"): logits = mlp(Z, layer_sizes=(128, 128, flat_data_dim), intermediate_activation_fn=tf.nn.relu, final_activation_fn=None) # Computed parameters for multiple Bernoullis. bernoulli_probs = tf.nn.sigmoid(logits) with tf.name_scope("Training"): with tf.name_scope("Loss"): with tf.name_scope("KL_loss"): # KL loss for gaussian encoding distribution, q(z|x). # Negative since we want to maximize the lower bound estimator. kl_loss = -0.5 * tf.reduce_sum( 1.0 + Z_log_var - tf.square(Z_mu) - tf.exp(Z_log_var), axis=1) with tf.name_scope("Reconstruction_loss"): # Expected negative log-likelihood as loss. reconstruction_loss = \ tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X_sampled_flat), axis=1) # TODO: Maybe try KL cost annealing. loss = tf.reduce_mean(kl_loss + reconstruction_loss) self.optimization_step = (tf.train.AdamOptimizer( learning_rate=1e-4).minimize(loss, global_step=global_step)) self.global_step = global_step self.Z = Z if bernoulli_probs.get_shape().ndims > 2: self.X_generated = bernoulli_probs else: self.X_generated = tf.reshape(bernoulli_probs, [-1] + X_sampled.get_shape().as_list()[1:])
def critic(X): return mlp(X, layer_sizes=(128, 128, 128, 1), intermediate_activation_fn=tf.nn.relu, final_activation_fn=None) # Note: No activation in final layer for Wasserstein GAN.
def generator(Z): return mlp(Z, layer_sizes=(128, 128, 128, flat_data_dim), intermediate_activation_fn=tf.nn.relu, final_activation_fn=generator_final_activation)
def discriminator(X): return mlp(X, layer_sizes=(128, 128, 1), intermediate_activation_fn=tf.nn.relu, final_activation_fn=tf.nn.sigmoid)
def discriminator(X): # Note: Discriminator is an autoencoder in BEGAN paper. X_reconstructed = mlp(X, layer_sizes=(128, autoencoder_hidden_dim, 128, flat_data_dim), intermediate_activation_fn=tf.nn.relu, final_activation_fn=tf.nn.sigmoid) return tf.reduce_sum((X - X_reconstructed) ** 2, axis=1)
def discriminator(X): return mlp(X, layer_sizes=(128, 128, 128, 1), intermediate_activation_fn=leaky_relu, final_activation_fn=None) # Note: No activation in final layer for Least Squares GAN.
def discriminator_q_shared(X): return mlp(X, layer_sizes=(128, 128, 128), intermediate_activation_fn=tf.nn.relu, final_activation_fn=tf.nn.relu)