def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('prior_recons'): self.prior_recons = losses.get_reconst_loss(self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss) self.prior_recons_m = tf.reduce_mean(self.prior_recons) with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss(self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv if 'post_' in v.name]) with tf.variable_scope('encoder_loss', reuse=self.config.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2*self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.config.reuse): self.divergence_cost = losses.get_self_divergence(self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('annvae_loss', reuse=self.config.reuse): c = self.anneal(self.config.c_max, self.global_step_tensor, self.config.itr_thd) self.anneal_reg = self.config.ann_gamma * tf.math.abs(self.div_cost_m - c) self.annvae_loss = tf.add(self.ae_loss, self.anneal_reg) with tf.variable_scope('bayae_loss', reuse=self.config.reuse): if self.config.isConv: self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \ tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), \ tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), self.config.prior_div_cost) else: self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \ self.prior_mean, self.prior_var, self.config.prior_div_cost) self.bayae_loss = tf.add(tf.cast(self.config.num_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss') self.bayvae_loss = tf.add(tf.cast(self.config.num_batches, 'float32') * self.vae_loss, self.bay_div, name='bayvae_loss') self.annbayvae_loss = tf.add(tf.cast(self.config.num_batches, 'float32') * self.annvae_loss, self.bay_div, name='bayvae_loss') with tf.variable_scope('optimizer', reuse=self.config.reuse): self.optimizer = tf.train.AdamOptimizer(self.lr) self.train_step = self.optimizer.minimize(self.annbayvae_loss, global_step=self.global_step_tensor) self.losses = ['ELBO_AnnBayVAE', 'BayVAE', 'BayAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Div_{}'.format(self.config.div_cost), 'Regul_anneal_reg', 'Regul_L2', 'prior_recons_{}'.format(self.config.prior_reconst_loss), 'bayesian_div_{}'.format(self.config.prior_div_cost)]
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.name_scope('prior_recons'): self.prior_recons = losses.get_reconst_loss( self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss) self.prior_recons_m = tf.reduce_mean(self.prior_recons) with tf.variable_scope("L2_loss", reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum( [tf.nn.l2_loss(v) for v in tv if 'post_' in v.name]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') + self.prior_recons_m with tf.variable_scope('bayae_loss', reuse=self.config.reuse): if self.config.isConv: self.bay_div = -1 * losses.get_divergence(self.post_mean, self.post_var, \ tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), self.config.prior_div_cost) else: self.bay_div = -1 * losses.get_divergence(self.post_mean, self.post_var, \ self.prior_mean, self.prior_var, self.config.prior_div_cost) self.bayae_loss = tf.add( tf.cast(self.config.num_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss') with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = tf.train.AdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.bayae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_BayAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Regul_L2', 'prior_recons_{}'.format(self.config.prior_reconst_loss), 'bayesian_div_{}'.format(self.config.prior_div_cost) ]
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('dipae_loss', reuse=self.config.reuse): self.covar_reg = self.regularizer(self.encoder_mean, self.encoder_var) self.dipae_loss = tf.add(self.ae_loss, self.covar_reg) with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.dipae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_DIPcovAE', 'Regul_Covariance_Prior', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Regul_L2' ]
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('optimizer', reuse=self.config.reuse): self.optimizer = tf.train.AdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.vae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Div_{}'.format(self.config.div_cost), 'Regul_L2' ]
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('bvae_loss', reuse=self.config.reuse): self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m) self.bvae_loss = tf.add(self.ae_loss, self.beta_reg) with tf.variable_scope('btcvae_loss', reuse=self.config.reuse): """ Based on Equation 4 with alpha = gamma = 1 of "Isolating Sources of Disentanglement in Variational Autoencoders" (https: // arxiv.org / pdf / 1802.04942). If alpha = gamma = 1, Eq 4 can be written as ELBO + (1 - beta) * TC. """ tc = tf.multiply(1-self.config.beta, self.total_correlation(self.latent_batch, self.encoder_mean, \ self.encoder_logvar)) self.tc_beta_reg = tf.add(self.div_cost_m, tc) self.btcvae_loss = tf.add(self.ae_loss, self.tc_beta_reg) with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.btcvae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_Beta-TC-VAE', 'Beta-VAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Regul_tc_beta_reg', 'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), 'Regul_L2' ]
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss(self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv ]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2*self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence(self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('bvae_loss', reuse=self.config.reuse): self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m) self.bvae_loss = tf.add(self.ae_loss, self.beta_reg) with tf.variable_scope('annvae_loss', reuse=self.config.reuse): c = self.anneal(self.config.c_max, self.global_step_tensor, self.config.itr_thd) self.anneal_reg = self.config.ann_gamma * tf.math.abs(self.div_cost_m - c) self.annvae_loss = tf.add(self.ae_loss, self.anneal_reg) self.annbvae_loss = tf.add(self.bvae_loss, self.anneal_reg) with tf.variable_scope("optimizer" ,reuse=self.config.reuse): self.optimizer = tf.train.AdamOptimizer(self.lr) self.train_step = self.optimizer.minimize(self.annbvae_loss, global_step=self.global_step_tensor) self.losses = ['ELBO_AnnBeta-VAE', 'Beta-VAE', 'annVAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Div_{}'.format(self.config.div_cost), \ 'Regul_beta_reg', 'Regul_anneal_reg', 'Div_{}'.format(self.config.div_cost), 'Regul_L2']
def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('prior_recons'): self.prior_recons = losses.get_reconst_loss( self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss) self.prior_recons_m = tf.reduce_mean(self.prior_recons) with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('bvae_loss', reuse=self.config.reuse): self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m) self.bvae_loss = tf.add(self.ae_loss, self.beta_reg) with tf.variable_scope('bayae_loss', reuse=self.config.reuse): if self.config.isConv: self.bay_div = -1 * losses.get_QP_kl(self.encoder_mean, self.encoder_var, \ tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), \ tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim])) else: self.bay_div = -1 * losses.get_QP_kl(self.encoder_mean, self.encoder_var, \ self.prior_mean, self.prior_var) self.bayae_loss = tf.add( tf.cast(self.config.ntrain_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss') self.bayvae_loss = tf.add( tf.cast(self.config.ntrain_batches, 'float32') * self.vae_loss, self.bay_div, name='bayvae_loss') self.baybvae_loss = tf.add( tf.cast(self.config.ntrain_batches, 'float32') * self.bvae_loss, self.bay_div, name='baybvae_loss') with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.baybvae_loss, global_step=self.global_step_tensor) self.losses = ['ELBO_BayBeta-VAE', \ 'BayVAE', 'BayAE', 'prior_recons_{}'.format(self.config.prior_reconst_loss), \ 'Beta-VAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), \ 'bayesian_div_{}'.format(self.config.prior_div_cost), \ 'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), \ 'Regul_L2']