def test_model(test_trees, labels, embeddings, embedding_lookup, opt): logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter num_feats = len(embeddings[0]) random.shuffle(test_trees) # build the inputs and outputs of the network nodes_node, children_node, codecaps_node = network.init_net_treecaps( num_feats, len(labels)) out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_step = optimizer.minimize(loss_node) sess = tf.Session() sess.run(tf.global_variables_initializer()) with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') correct_labels = [] predictions = [] print('Computing training accuracy...') for batch in sampling.batch_samples( sampling.gen_samples(test_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children, }) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) target_names = list(labels) print( classification_report(correct_labels, predictions, target_names=target_names)) print(confusion_matrix(correct_labels, predictions)) print('*' * 50) print('Accuracy:', accuracy_score(correct_labels, predictions)) print('*' * 50)
class AnnBayBVAEGraph(BaseGraph): def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [self.config.batch_size, self.config.width, self.config.height, self.config.num_channels], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch , [-1,self.x_flat_dim]) self.latent_batch = tf.placeholder(tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') self.sample_batch = tf.random_normal((self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32) ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining _sampling_reconst from X...') self.sample_flat = tf.multiply(tf.ones([self.config.MC_samples, self.config.batch_size, self.x_flat_dim]), self.x_batch_flat) sample_flat_shape = self.sample_flat.get_shape().as_list() if self.config.isConv: sample_input = tf.reshape(self.sample_flat , [-1, self.config.width, self.config.height, self.config.num_channels]) print('\n[*] _sampling_reconst shape {}'.format(sample_input.shape)) else: print('\n[*] _sampling_reconst shape {}'.format(sample_flat_shape)) #################################################################################### print('\n[*] Defining prior ...') print('\n[*] Defining prior encoder...') with tf.variable_scope('prior_encoder', reuse=self.config.reuse): Qlatent_sample = self.create_encoder(input_=sample_input if self.config.isConv else self.sample_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='prior_en_', isConv=self.config.isConv) self.prior_mean = Qlatent_sample.output self.prior_var = tf.nn.sigmoid(self.prior_mean) self.latent_sample = self.prior_mean print('\n[*] Defining prior decoder...') with tf.variable_scope('prior_decoder', reuse=self.config.reuse): Psample_latent = self.create_decoder(input_=self.latent_sample, hidden_dim=self.config.hidden_dim, output_dim=sample_flat_shape[-1], num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='prior_de_', isConv=self.config.isConv) self.sample_recons_flat = Psample_latent.output if self.config.isConv: self.sample_recons_flat = tf.reshape(self.sample_recons_flat, sample_flat_shape) print('\n[*] _sampling_reconst tsne_cost shape {}'.format(self.sample_recons_flat.shape)) #################################################################################### print('\n[*] Defining posterior ...') print('\n[*] Defining posterior encoders...') with tf.variable_scope('encoder_mean', reuse=self.config.reuse): Qlatent_x_mean = self.create_encoder(input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='post_enmean_', isConv=self.config.isConv) self.encoder_mean = Qlatent_x_mean.output with tf.variable_scope('encoder_var', reuse=self.config.reuse): Qlatent_x_var = self.create_encoder(input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.softplus, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='post_envar_', isConv=self.config.isConv) self.encoder_var = Qlatent_x_var.output print('\n[*] Reparameterization trick...') self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon) eps = tf.random_normal((self.config.batch_size, self.config.latent_dim), 0, 1, dtype=tf.float32) self.latent = tf.add(self.encoder_mean, tf.multiply(tf.sqrt(self.encoder_var), eps)) self.latent_batch = self.latent print('\n[*] Defining decoder...') with tf.variable_scope('decoder_mean', reuse=self.config.reuse): Px_latent_mean = self.create_decoder(input_=self.latent_batch, hidden_dim=self.config.hidden_dim, output_dim=self.x_flat_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='post_de_', isConv=self.config.isConv) self.x_recons_flat = Px_latent_mean.output self.x_recons = tf.reshape(self.x_recons_flat , [-1,self.config.width, self.config.height, self.config.num_channels]) ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('prior_recons'): self.prior_recons = losses.get_reconst_loss(self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss) self.prior_recons_m = tf.reduce_mean(self.prior_recons) with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss(self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv ]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2*self.L2_loss, name='encoder_loss') with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence(self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('bvae_loss', reuse=self.config.reuse): self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m) self.bvae_loss = tf.add(self.ae_loss, self.beta_reg) with tf.variable_scope('annvae_loss', reuse=self.config.reuse): c = self.anneal(self.config.c_max, self.global_step_tensor, self.config.itr_thd) self.anneal_reg = self.config.ann_gamma * tf.math.abs(self.div_cost_m - c) self.annvae_loss = tf.add(self.ae_loss, self.anneal_reg) with tf.variable_scope('bayae_loss', reuse=self.config.reuse): if self.config.isConv: self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \ tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), \ tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), self.config.prior_div_cost) else: self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \ self.prior_mean, self.prior_var, self.config.prior_div_cost) self.bayae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss') self.bayvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.vae_loss, self.bay_div, name='bayvae_loss') self.baybvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.bvae_loss, self.bay_div, name='baybvae_loss') self.annbayvae_loss = tf.add(self.baybvae_loss, self.anneal_reg) self.annbvae_loss = tf.add(self.bvae_loss, self.anneal_reg) self.annbaybvae_loss = self.baybvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.bvae_loss, self.bay_div, name='annbaybvae_loss') with tf.variable_scope("optimizer" ,reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize(self.annbaybvae_loss, global_step=self.global_step_tensor) self.losses = ['ELBO_AnnBayBeta-VAE', 'AnnBeta-VAE', 'BayBeta-VAE',\ 'BayVAE', 'BayAE', 'prior_recons_{}'.format(self.config.prior_reconst_loss), \ 'Beta-VAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), \ 'bayesian_div_{}'.format(self.config.prior_div_cost), \ 'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), \ 'Regul_L2'] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x): tensors = [self.train_step, self.annbaybvae_loss, self.annbvae_loss, self.baybvae_loss, self.bayvae_loss, self.bayae_loss, self.prior_recons_m, self.bvae_loss, self.bae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.bay_div, self.beta_reg, self.div_cost_m, self.L2_loss] feed_dict = {self.x_batch: x} _, annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \ bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_loss = session.run(tensors, feed_dict=feed_dict) return annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \ bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los def test_epoch(self, session, x): tensors = [self.annbaybvae_loss, self.annbvae_loss, self.baybvae_loss, self.bayvae_loss, self.bayae_loss, self.prior_recons_m, self.bvae_loss, self.bae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.bay_div, self.beta_reg, self.div_cost_m, self.L2_loss] feed_dict = {self.x_batch: x} annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \ bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los = session.run(tensors, feed_dict=feed_dict) return annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \ bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los
class DIPcovVAEGraph(BaseGraph): def extra_sittings(self): self.d = self.config.d_factor * self.config.lambda_d def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [ self.config.batch_size, self.config.width, self.config.height, self.config.num_channels ], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim]) self.latent_batch = tf.placeholder( tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') self.sample_batch = tf.random_normal( (self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32) ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining encoders...') with tf.variable_scope('encoder_mean', reuse=self.config.reuse): Qlatent_x_mean = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='enmean_', isConv=self.config.isConv) self.encoder_mean = Qlatent_x_mean.output with tf.variable_scope('encoder_var', reuse=self.config.reuse): Qlatent_x_var = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.softplus, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='envar_', isConv=self.config.isConv) self.encoder_var = Qlatent_x_var.output print('\n[*] Reparameterization trick...') self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon) eps = tf.random_normal( (self.config.batch_size, self.config.latent_dim), 0, 1, dtype=tf.float32) self.latent = tf.add(self.encoder_mean, tf.multiply(tf.sqrt(self.encoder_var), eps)) self.latent_batch = self.latent print('\n[*] Defining decoder...') with tf.variable_scope('decoder_mean', reuse=self.config.reuse): Px_latent_mean = self.create_decoder( input_=self.latent_batch, hidden_dim=self.config.hidden_dim, output_dim=self.x_flat_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='de_', isConv=self.config.isConv) self.x_recons_flat = Px_latent_mean.output self.x_recons = tf.reshape(self.x_recons_flat, [ -1, self.config.width, self.config.height, self.config.num_channels ]) ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('dipae_loss', reuse=self.config.reuse): self.covar_reg = tf.add( self.regularizer(self.encoder_mean, self.encoder_logvar), self.div_cost_m) self.dipvae_loss = tf.add(self.ae_loss, self.covar_reg) with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.dipvae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_DIPcovVAE', 'Regul_Covariance_Prior', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Div_{}'.format(self.config.div_cost), 'Regul_L2' ] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x): tensors = [ self.train_step, self.dipvae_loss, self.covar_reg, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} _, dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss = session.run( tensors, feed_dict=feed_dict) return dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss def test_epoch(self, session, x): tensors = [ self.dipvae_loss, self.covar_reg, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss = session.run( tensors, feed_dict=feed_dict) return dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss ''' ------------------------------------------------------------------------------ DIP OPERATIONS ------------------------------------------------------------------------------ ''' def compute_covariance_latent_mean(self, latent_mean): """ :param latent_mean: :return: Computes the covariance of latent_mean. Uses cov(latent_mean) = E[latent_mean*latent_mean^T] - E[latent_mean]E[latent_mean]^T. Args: latent_mean: Encoder mean, tensor of size [batch_size, num_latent]. Returns: cov_latent_mean: Covariance of encoder mean, tensor of size [latent_dim, latent_dim]. """ exp_latent_mean_latent_mean_t = tf.reduce_mean( tf.expand_dims(latent_mean, 2) * tf.expand_dims(latent_mean, 1), axis=0) expectation_latent_mean = tf.reduce_mean(latent_mean, axis=0) cov_latent_mean = tf.subtract( exp_latent_mean_latent_mean_t, tf.expand_dims(expectation_latent_mean, 1) * tf.expand_dims(expectation_latent_mean, 0)) return cov_latent_mean def regularize_diag_off_diag_dip(self, covariance_matrix, lambda_od, lambda_d): """ Compute on and off diagonal regularizers for DIP-VAE models. Penalize deviations of covariance_matrix from the identity matrix. Uses different weights for the deviations of the diagonal and off diagonal entries. Args: covariance_matrix: Tensor of size [num_latent, num_latent] to covar_reg. lambda_od: Weight of penalty for off diagonal elements. lambda_d: Weight of penalty for diagonal elements. Returns: dip_regularizer: Regularized deviation from diagonal of covariance_matrix. """ covariance_matrix_diagonal = tf.diag_part(covariance_matrix) covariance_matrix_off_diagonal = covariance_matrix - tf.diag( covariance_matrix_diagonal) dip_regularizer = tf.add( lambda_od * tf.reduce_sum(covariance_matrix_off_diagonal**2), lambda_d * tf.reduce_sum((covariance_matrix_diagonal - 1)**2)) return dip_regularizer def regularizer(self, latent_mean, latent_logvar): cov_latent_mean = self.compute_covariance_latent_mean(latent_mean) # Eq 6 page 4 # mu = z_mean is [batch_size, num_latent] # Compute cov_p(x) [mu(x)] = E[mu*mu^T] - E[mu]E[mu]^T] cov_dip_regularizer = self.regularize_diag_off_diag_dip( cov_latent_mean, self.config.lambda_d, self.d) return cov_dip_regularizer
class VAEGraph(BaseGraph): def build_graph(self): self.create_inputs() self.create_graph() self.create_loss_optimizer() def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [ self.config.batch_size, self.config.width, self.config.height, self.config.num_channels ], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim]) self.latent_batch = tf.placeholder( tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') self.sample_batch = tf.random_normal( (self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32) ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining encoders...') with tf.variable_scope('encoder_mean', reuse=self.config.reuse): Qlatent_x_mean = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='enmean_', isConv=self.config.isConv) self.encoder_mean = Qlatent_x_mean.output with tf.variable_scope('encoder_var', reuse=self.config.reuse): Qlatent_x_var = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.softplus, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='envar_', isConv=self.config.isConv) self.encoder_var = Qlatent_x_var.output print('\n[*] Reparameterization trick...') self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon) eps = tf.random_normal( (self.config.batch_size, self.config.latent_dim), 0, 1, dtype=tf.float32) self.latent = tf.add(self.encoder_mean, tf.multiply(tf.sqrt(self.encoder_var), eps)) self.latent_batch = self.latent print('\n[*] Defining decoder...') with tf.variable_scope('decoder_mean', reuse=self.config.reuse): Px_latent_mean = self.create_decoder( input_=self.latent_batch, hidden_dim=self.config.hidden_dim, output_dim=self.x_flat_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='de_', isConv=self.config.isConv) self.x_recons_flat = Px_latent_mean.output self.x_recons = tf.reshape(self.x_recons_flat, [ -1, self.config.width, self.config.height, self.config.num_channels ]) ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('optimizer', reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.vae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Div_{}'.format(self.config.div_cost), 'Regul_L2' ] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x): tensors = [ self.train_step, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} _, loss, aeloss, recons, div, L2_loss = session.run( tensors, feed_dict=feed_dict) return loss, aeloss, recons, div, L2_loss def test_epoch(self, session, x): tensors = [ self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} loss, aeloss, recons, div, L2_loss = session.run(tensors, feed_dict=feed_dict) return loss, aeloss, recons, div, L2_loss
class BayAEGraph(BaseGraph): def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [ self.config.batch_size, self.config.width, self.config.height, self.config.num_channels ], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim]) self.latent_batch = tf.placeholder( tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') self.sample_batch = tf.random_normal( (self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32) ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining _sampling_reconst from X...') self.sample_flat = tf.multiply( tf.ones([ self.config.MC_samples, self.config.batch_size, self.x_flat_dim ]), self.x_batch_flat) sample_flat_shape = self.sample_flat.get_shape().as_list() if self.config.isConv: sample_input = tf.reshape(self.sample_flat, [ -1, self.config.width, self.config.height, self.config.num_channels ]) print('\n[*] _sampling_reconst shape {}'.format( sample_input.shape)) else: print('\n[*] _sampling_reconst shape {}'.format(sample_flat_shape)) #################################################################################### print('\n[*] Defining prior ...') print('\n[*] Defining prior encoder...') with tf.variable_scope('prior_encoder', reuse=self.config.reuse): Qlatent_sample = self.create_encoder( input_=sample_input if self.config.isConv else self.sample_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='prior_en_', isConv=self.config.isConv) self.prior_mean = Qlatent_sample.output self.prior_var = tf.nn.sigmoid(self.prior_mean) self.latent_sample = self.prior_mean print('\n[*] Defining prior decoder...') with tf.variable_scope('prior_decoder', reuse=self.config.reuse): Psample_latent = self.create_decoder( input_=self.latent_sample, hidden_dim=self.config.hidden_dim, output_dim=sample_flat_shape[-1], num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='prior_de_', isConv=self.config.isConv) self.sample_recons_flat = Psample_latent.output if self.config.isConv: self.sample_recons_flat = tf.reshape(self.sample_recons_flat, sample_flat_shape) print('\n[*] _sampling_reconst tsne_cost shape {}'.format( self.sample_recons_flat.shape)) #################################################################################### print('\n[*] Defining posterior ...') print('\n[*] Defining posterior encoder...') with tf.variable_scope('post_encoder', reuse=self.config.reuse): Qlatent_x = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='post_en_', isConv=self.config.isConv) self.post_mean = Qlatent_x.output self.post_var = tf.nn.sigmoid(self.post_mean) self.latent = self.post_mean self.latent_batch = self.latent print('\n[*] Defining posterior decoder...') with tf.variable_scope('post_decoder', reuse=self.config.reuse): Px_latent = self.create_decoder( input_=self.latent_batch, hidden_dim=self.config.hidden_dim, output_dim=self.x_flat_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='post_de_', isConv=self.config.isConv) self.x_recons_flat = Px_latent.output self.x_recons = tf.reshape(self.x_recons_flat, [ -1, self.config.width, self.config.height, self.config.num_channels ]) ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.name_scope('prior_recons'): self.prior_recons = losses.get_reconst_loss( self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss) self.prior_recons_m = tf.reduce_mean(self.prior_recons) with tf.variable_scope("L2_loss", reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum( [tf.nn.l2_loss(v) for v in tv if 'post_' in v.name]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') + self.prior_recons_m with tf.variable_scope('bayae_loss', reuse=self.config.reuse): if self.config.isConv: self.bay_div = -1 * losses.get_QP_kl(self.post_mean, self.post_var, \ tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim])) else: self.bay_div = -1 * losses.get_QP_kl(self.post_mean, self.post_var, \ self.prior_mean, self.prior_var) self.bayae_loss = tf.add( tf.cast(self.config.ntrain_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss') with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.bayae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_BayAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Regul_L2', 'prior_recons_{}'.format(self.config.prior_reconst_loss), 'bayesian_div_{}'.format(self.config.prior_div_cost) ] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x): tensors = [ self.train_step, self.bayae_loss, self.ae_loss, self.loss_reconstruction_m, self.L2_loss, self.prior_recons_m, self.bay_div ] feed_dict = {self.x_batch: x} _, bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div = session.run( tensors, feed_dict=feed_dict) return bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div def test_epoch(self, session, x): tensors = [ self.bayae_loss, self.ae_loss, self.loss_reconstruction_m, self.L2_loss, self.prior_recons_m, self.bay_div ] feed_dict = {self.x_batch: x} bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div = session.run( tensors, feed_dict=feed_dict) return bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div
def main(train_opt, test_opt): train_opt.model_path = os.path.join( train_opt.model_path, util_functions.form_tbcnn_model_path(train_opt)) checkfile = os.path.join(train_opt.model_path, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(train_opt.model_path) print("The model path : " + str(checkfile)) if ckpt and ckpt.model_checkpoint_path: print("-------Continue training with old model-------- : " + str(checkfile)) tbcnn_model = TBCNN(train_opt) tbcnn_model.feed_forward() train_data_loader = BaseDataLoader(train_opt.batch_size, train_opt.label_size, train_opt.tree_size_threshold_upper, train_opt.tree_size_threshold_lower, train_opt.train_path, True) test_data_loader = BaseDataLoader(test_opt.batch_size, test_opt.label_size, test_opt.tree_size_threshold_upper, test_opt.tree_size_threshold_lower, test_opt.test_path, False) optimizer = RAdamOptimizer(train_opt.lr) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_point = optimizer.minimize(tbcnn_model.loss) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) init = tf.global_variables_initializer() best_f1 = test_opt.best_f1 with tf.Session() as sess: sess.run(init) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) for epoch in range(1, train_opt.epochs + 1): train_batch_iterator = ThreadedIterator( train_data_loader.make_minibatch_iterator(), max_queue_size=train_opt.worker) for train_step, train_batch_data in enumerate( train_batch_iterator): print("***************") # print(train_batch_data["batch_node_index"].shape) # print(train_batch_data["batch_node_type_id"].shape) # print(train_batch_data["batch_node_sub_tokens_id"].shape) # print(train_batch_data["batch_children_index"].shape) # print(train_batch_data["batch_children_node_type_id"].shape) # print(train_batch_data["batch_children_node_sub_tokens_id"].shape) # print(train_batch_data["batch_labels_one_hot"]) # print("Labels : " + str(train_batch_data["batch_labels"])) # print("Tree sizes : " + str(train_batch_data["batch_size"])) # for children_index in train_batch_data["batch_children_index"]: # print("Children_index : " + str(len(children_index))) _, err = sess.run( [training_point, tbcnn_model.loss], feed_dict={ tbcnn_model.placeholders["node_type"]: train_batch_data["batch_node_type_id"], tbcnn_model.placeholders["node_token"]: train_batch_data["batch_node_sub_tokens_id"], tbcnn_model.placeholders["children_index"]: train_batch_data["batch_children_index"], tbcnn_model.placeholders["children_node_type"]: train_batch_data["batch_children_node_type_id"], tbcnn_model.placeholders["children_node_token"]: train_batch_data["batch_children_node_sub_tokens_id"], tbcnn_model.placeholders["labels"]: train_batch_data["batch_labels_one_hot"], tbcnn_model.placeholders["dropout_rate"]: 0.3 }) print("Epoch:", epoch, "Step:", train_step, "Loss:", err, "Best F1:", best_f1) if train_step % train_opt.checkpoint_every == 0 and train_step > 0: #Perform Validation print("Perform validation.....") correct_labels = [] predictions = [] test_batch_iterator = ThreadedIterator( test_data_loader.make_minibatch_iterator(), max_queue_size=test_opt.worker) for test_step, test_batch_data in enumerate( test_batch_iterator): print("***************") print(test_batch_data["batch_size"]) scores = sess.run( [tbcnn_model.softmax], feed_dict={ tbcnn_model.placeholders["node_type"]: test_batch_data["batch_node_type_id"], tbcnn_model.placeholders["node_token"]: test_batch_data["batch_node_sub_tokens_id"], tbcnn_model.placeholders["children_index"]: test_batch_data["batch_children_index"], tbcnn_model.placeholders["children_node_type"]: test_batch_data["batch_children_node_type_id"], tbcnn_model.placeholders["children_node_token"]: test_batch_data[ "batch_children_node_sub_tokens_id"], tbcnn_model.placeholders["labels"]: test_batch_data["batch_labels_one_hot"], tbcnn_model.placeholders["dropout_rate"]: 0.0 }) batch_correct_labels = list( np.argmax(test_batch_data["batch_labels_one_hot"], axis=1)) batch_predictions = list(np.argmax(scores[0], axis=1)) print(batch_correct_labels) print(batch_predictions) correct_labels.extend( np.argmax(test_batch_data["batch_labels_one_hot"], axis=1)) predictions.extend(np.argmax(scores[0], axis=1)) print(correct_labels) print(predictions) f1 = float( f1_score(correct_labels, predictions, average="micro")) print(classification_report(correct_labels, predictions)) print('F1:', f1) print('Best F1:', best_f1) # print(confusion_matrix(correct_labels, predictions)) if f1 > best_f1: best_f1 = f1 saver.save(sess, checkfile) print('Checkpoint saved, epoch:' + str(epoch) + ', step: ' + str(train_step) + ', loss: ' + str(err) + '.')
def main(opt): opt.model_path = os.path.join(opt.model_path, form_model_path(opt)) checkfile = os.path.join(opt.model_path, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(opt.model_path) print("The model path : " + str(checkfile)) print("Loss : " + str(opt.loss)) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model : " + str(checkfile)) print("Loading vocabs.........") node_type_lookup, node_token_lookup, subtree_lookup = load_vocabs(opt) opt.node_type_lookup = node_type_lookup opt.node_token_lookup = node_token_lookup opt.subtree_lookup = subtree_lookup if opt.task == 1: train_dataset = CodeClassificationData(opt, True, False, False) if opt.task == 0: val_opt = copy.deepcopy(opt) val_opt.node_token_lookup = node_token_lookup validation_dataset = CodeClassificationData(val_opt, False, False, True) print("Initializing tree caps model...........") corder = CorderModel(opt) print("Finished initializing corder model...........") loss_node = corder.loss optimizer = RAdamOptimizer(opt.lr) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_point = optimizer.minimize(loss_node) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) init = tf.global_variables_initializer() # best_f1_score = get_best_f1_score(opt) # print("Best f1 score : " + str(best_f1_score)) with tf.Session() as sess: sess.run(init) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) if opt.task == 1: for epoch in range(1, opt.epochs + 1): train_batch_iterator = ThreadedIterator( train_dataset.make_minibatch_iterator(), max_queue_size=opt.worker) train_accs = [] for train_step, train_batch_data in enumerate( train_batch_iterator): print("--------------------------") # print(train_batch_data["batch_subtrees_ids"]) logging.info(str(train_batch_data["batch_subtree_id"])) _, err = sess.run( [training_point, corder.loss], feed_dict={ corder.placeholders["node_types"]: train_batch_data["batch_node_types"], corder.placeholders["node_tokens"]: train_batch_data["batch_node_tokens"], corder.placeholders["children_indices"]: train_batch_data["batch_children_indices"], corder.placeholders["children_node_types"]: train_batch_data["batch_children_node_types"], corder.placeholders["children_node_tokens"]: train_batch_data["batch_children_node_tokens"], corder.placeholders["labels"]: train_batch_data["batch_subtree_id"], corder.placeholders["dropout_rate"]: 0.3 }) logging.info("Training at epoch " + str(epoch) + " and step " + str(train_step) + " with loss " + str(err)) print("Epoch:", epoch, "Step:", train_step, "Training loss:", err) if train_step % opt.checkpoint_every == 0 and train_step > 0: saver.save(sess, checkfile) print('Checkpoint saved, epoch:' + str(epoch) + ', step: ' + str(train_step) + ', loss: ' + str(err) + '.') if opt.task == 0: validation_batch_iterator = ThreadedIterator( validation_dataset.make_minibatch_iterator(), max_queue_size=opt.worker) for val_step, val_batch_data in enumerate( validation_batch_iterator): scores = sess.run( [corder.code_vector], feed_dict={ corder.placeholders["node_types"]: val_batch_data["batch_node_types"], corder.placeholders["node_tokens"]: val_batch_data["batch_node_tokens"], corder.placeholders["children_indices"]: val_batch_data["batch_children_indices"], corder.placeholders["children_node_types"]: val_batch_data["batch_children_node_types"], corder.placeholders["children_node_tokens"]: val_batch_data["batch_children_node_tokens"], corder.placeholders["dropout_rate"]: 0.0 }) for i, vector in enumerate(scores[0]): file_name = "analysis/rosetta_sampled_softmax_train.csv" with open(file_name, "a") as f: vector_score = [] for score in vector: vector_score.append(str(score)) # print(val_batch_data["batch_file_path"]) line = str(val_batch_data["batch_file_path"] [i]) + "," + " ".join(vector_score) f.write(line) f.write("\n")
class EmbeddingGraph(BaseGraph): def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [ self.config.batch_size, self.config.width, self.config.height, self.config.num_channels ], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim]) self.joint_probabilities_batch = tf.placeholder( tf.float32, [self.config.batch_size, self.config.batch_size], name='joint_probabilities_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining encoder...') with tf.variable_scope('encoder', reuse=self.config.reuse): Qlatent_x = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.n_components, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='en_', isConv=self.config.isConv) self.latent = Qlatent_x.output ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.variable_scope('e_divergence_cost', reuse=self.config.reuse): kerneled_embedding = kernels.get_kernel( X=self.latent, batch_size=self.config.batch_size, param=self.config.df, epsilon=self.config.epsilon, kernel=self.config.kernel_mode) self.e_divergence_cost = losses.get_distributions_div_cost( self.joint_probabilities_batch, kerneled_embedding, self.config.e_div_cost) self.e_div_cost_m = tf.reduce_mean(self.e_divergence_cost) with tf.variable_scope("L2_loss", reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('embedding_loss', reuse=self.config.reuse): self.embedding_loss = tf.add(tf.reduce_mean( self.e_divergence_cost), self.config.l2 * self.L2_loss, name='embedding_loss') with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.embedding_loss, global_step=self.global_step_tensor) self.losses = [ 'Embedding_loss', 'E_diverg_{}'.format(self.config.e_div_cost), 'Regul_L2' ] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x, p): tensors = [ self.train_step, self.embedding_loss, self.e_divergence_cost, self.L2_loss ] feed_dict = {self.x_batch: x, self.joint_probabilities_batch: p} _, loss, e_div, L2_loss = session.run(tensors, feed_dict=feed_dict) return loss, e_div, L2_loss def test_epoch(self, session, x, p): tensors = [self.embedding_loss, self.e_divergence_cost, self.L2_loss] feed_dict = {self.x_batch: x, self.joint_probabilities_batch: p} loss, e_div, L2_loss = session.run(tensors, feed_dict=feed_dict) return loss, e_div, L2_loss
def main(opt): opt.model_path = os.path.join(opt.model_path, form_model_path(opt)) checkfile = os.path.join(opt.model_path, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(opt.model_path) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model : " + str(checkfile)) print("Loading vocabs.........") train_label_lookup, node_type_lookup, node_token_lookup, val_label_lookup = load_vocabs(opt) opt.label_lookup = train_label_lookup opt.label_size = len(train_label_lookup.keys()) opt.node_type_lookup = node_type_lookup opt.node_token_lookup = node_token_lookup if opt.task == 1: train_dataset = MethodNamePredictionData(opt, opt.train_path, True, False, False) val_opt = copy.deepcopy(opt) val_opt.label_lookup = val_label_lookup val_opt.num_labels = len(val_label_lookup.keys()) val_opt.node_token_lookup = node_token_lookup validation_dataset = MethodNamePredictionData(val_opt, opt.val_path, False, False, True) print("Initializing tree caps model...........") treecaps = TreeCapsModel(opt) # network.init_net_treecaps(30,30) print("Finished initializing tree caps model...........") code_caps = treecaps.code_caps loss_node = treecaps.loss softmax_values = treecaps.softmax_values logits = treecaps.logits optimizer = RAdamOptimizer(opt.lr) # optimizer = tf.compat.v1.train.AdamOptimizer(opt.lr) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_point = optimizer.minimize(loss_node) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) init = tf.global_variables_initializer() best_f1_score = get_best_f1_score(opt) print("Best f1 score : " + str(best_f1_score)) num_caps_top_a = int(opt.num_conv*opt.output_size/opt.num_channel)*opt.top_a with tf.Session() as sess: sess.run(init) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) validation_batch_iterator = ThreadedIterator(validation_dataset.make_minibatch_iterator(), max_queue_size=5) # f1_scores_of_val_data = [] all_predicted_labels = [] all_ground_truth_labels = [] for val_step, val_batch_data in enumerate(validation_batch_iterator): alpha_IJ_shape = (opt.batch_size, int(num_caps_top_a/opt.top_a*val_batch_data["batch_node_types"].shape[1]), num_caps_top_a) alpha_IJ = np.zeros(alpha_IJ_shape) scores, alpha_IJ_scores= sess.run( [logits, treecaps.alpha_IJ], feed_dict={ treecaps.placeholders["node_types"]: val_batch_data["batch_node_types"], treecaps.placeholders["node_tokens"]: val_batch_data["batch_node_tokens"], treecaps.placeholders["children_indices"]: val_batch_data["batch_children_indices"], treecaps.placeholders["children_node_types"]: val_batch_data["batch_children_node_types"], treecaps.placeholders["children_node_tokens"]: val_batch_data["batch_children_node_tokens"], treecaps.placeholders["labels"]: val_batch_data["batch_labels"], treecaps.placeholders["alpha_IJ"]: alpha_IJ, treecaps.placeholders["is_training"]: False } ) alpha_IJ_scores = np.reshape(alpha_IJ_scores, (opt.batch_size, val_batch_data["batch_node_types"].shape[1], 8, opt.top_a, 8)) alpha_IJ_scores = np.sum(alpha_IJ_scores, axis=2) alpha_IJ_scores = np.sum(alpha_IJ_scores, axis=3) alpha_IJ_scores = np.squeeze(alpha_IJ_scores, axis=0) alpha_IJ_scores = np.transpose(alpha_IJ_scores) predictions = np.argmax(scores, axis=1) ground_truths = np.argmax(val_batch_data['batch_labels'], axis=1) predicted_labels = [] for prediction in predictions: predicted_labels.append(train_label_lookup.inverse[prediction]) ground_truth_labels = [] for ground_truth in ground_truths: ground_truth_labels.append( val_label_lookup.inverse[ground_truth]) f1_score = evaluation.calculate_f1_scores(predicted_labels, ground_truth_labels) print(ground_truth_labels) print(predicted_labels) print("F1:", f1_score, "Step:", val_step) if f1_score > 0: node_types = val_batch_data["batch_node_types"][0] node_tokens_text = val_batch_data["batch_node_tokens_text"][0] node_indexes = val_batch_data["batch_node_indexes"][0] file_path = val_batch_data["batch_file_path"][0] file_path_splits = file_path.split("/") file_path_splits[1] = "java-small" file_path_splits[len(file_path_splits) - 1] = file_path_splits[len(file_path_splits) - 1].replace(".pkl",".java") file_path = "/".join(file_path_splits) analysis_folder = os.path.join("analysis", "_".join(file_path_splits[-2:]).replace(".java","")) try: from pathlib import Path Path(analysis_folder).mkdir(parents=True, exist_ok=True) except Exception as e: print(e) count = 0 for capsule in alpha_IJ_scores: # print(val_batch_data["batch_node_indexes"]) connection_strength = capsule all_tuples = [] for i, node_index in enumerate(node_indexes): tuple_of_info = [] tuple_of_info.append(str(node_indexes[i])) tuple_of_info.append(str(node_types[i])) tuple_of_info.append(str(connection_strength[i])) tuple_of_info.append(node_tokens_text[i]) tuple_of_info = tuple(tuple_of_info) # print(tuple_of_info) all_tuples.append(tuple_of_info) all_tuples = sorted(all_tuples, key=lambda x: x[2]) all_tuples.reverse() with open(os.path.join(analysis_folder, "Group_" + str(count) + ".txt"), "w") as f: for t in all_tuples: line = ";".join(list(t)) f.write(line) f.write("\n") with open(os.path.join(analysis_folder, "result.txt"), "w") as f1: f1.write("Predicted : " + str(predicted_labels[0])) f1.write("\n") f1.write("Ground truth : " + str(ground_truth_labels[0])) f1.write("\n") import shutil try: print("Trying to copy original source file....") shutil.copy(file_path, analysis_folder) except Exception as e: print(e) count += 1
def main(train_opt, test_opt): train_opt.model_path = os.path.join( train_opt.model_path, util_functions.form_tbcnn_model_path(train_opt)) checkfile = os.path.join(train_opt.model_path, 'cnn_tree.ckpt') ckpt = tf.train.get_checkpoint_state(train_opt.model_path) print("The model path : " + str(checkfile)) if ckpt and ckpt.model_checkpoint_path: print("-------Continue training with old model-------- : " + str(checkfile)) tbcnn_model = InferCodeModel(train_opt) tbcnn_model.feed_forward() train_data_loader = BaseDataLoader(train_opt.batch_size, train_opt.tree_size_threshold_upper, train_opt.tree_size_threshold_lower, train_opt.train_tree_path, train_opt.train_bucket_path, True) # test_data_loader = BaseDataLoader(test_opt.batch_size, test_opt.tree_size_threshold_upper, test_opt.tree_size_threshold_lower, test_opt.test_path, False) optimizer = RAdamOptimizer(train_opt.lr) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): training_point = optimizer.minimize(tbcnn_model.loss) saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5) init = tf.global_variables_initializer() best_f1 = test_opt.best_f1 with tf.Session() as sess: sess.run(init) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") print("Checkpoint path : " + str(ckpt.model_checkpoint_path)) saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) for epoch in range(1, train_opt.epochs + 1): train_batch_iterator = ThreadedIterator( train_data_loader.make_minibatch_iterator(), max_queue_size=train_opt.worker) for train_step, train_batch_data in enumerate( train_batch_iterator): print("***************") print(train_batch_data["batch_subtree_id"]) _, err = sess.run( [training_point, tbcnn_model.loss], feed_dict={ tbcnn_model.placeholders["node_type"]: train_batch_data["batch_node_type_id"], tbcnn_model.placeholders["node_token"]: train_batch_data["batch_node_sub_tokens_id"], tbcnn_model.placeholders["children_index"]: train_batch_data["batch_children_index"], tbcnn_model.placeholders["children_node_type"]: train_batch_data["batch_children_node_type_id"], tbcnn_model.placeholders["children_node_token"]: train_batch_data["batch_children_node_sub_tokens_id"], tbcnn_model.placeholders["subtree_labels"]: train_batch_data["batch_subtree_id"], tbcnn_model.placeholders["dropout_rate"]: 0.3 }) print("Epoch:", epoch, "Step:", train_step, "Loss:", err, "Best F1:", best_f1)
class BTCVAEGraph(BaseGraph): def build_graph(self): self.create_inputs() self.create_graph() self.create_loss_optimizer() def create_inputs(self): with tf.variable_scope('inputs', reuse=self.config.reuse): self.x_batch = tf.placeholder(tf.float32, [ self.config.batch_size, self.config.width, self.config.height, self.config.num_channels ], name='x_batch') self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim]) self.latent_batch = tf.placeholder( tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch') self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr') self.sample_batch = tf.random_normal( (self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32) ''' ------------------------------------------------------------------------------ GRAPH FUNCTIONS ------------------------------------------------------------------------------ ''' def create_graph(self): print('\n[*] Defining encoders...') with tf.variable_scope('encoder_mean', reuse=self.config.reuse): Qlatent_x_mean = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=None, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='enmean_', isConv=self.config.isConv) self.encoder_mean = Qlatent_x_mean.output with tf.variable_scope('encoder_var', reuse=self.config.reuse): Qlatent_x_var = self.create_encoder( input_=self.x_batch if self.config.isConv else self.x_batch_flat, hidden_dim=self.config.hidden_dim, output_dim=self.config.latent_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.softplus, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='envar_', isConv=self.config.isConv) self.encoder_var = Qlatent_x_var.output print('\n[*] Reparameterization trick...') self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon) eps = tf.random_normal( (self.config.batch_size, self.config.latent_dim), 0, 1, dtype=tf.float32) self.latent = tf.add(self.encoder_mean, tf.multiply(tf.sqrt(self.encoder_var), eps)) self.latent_batch = self.latent print('\n[*] Defining decoder...') with tf.variable_scope('decoder_mean', reuse=self.config.reuse): Px_latent_mean = self.create_decoder( input_=self.latent_batch, hidden_dim=self.config.hidden_dim, output_dim=self.x_flat_dim, num_layers=self.config.num_layers, transfer_fct=self.config.transfer_fct, act_out=tf.nn.sigmoid, reuse=self.config.reuse, kinit=self.config.kinit, bias_init=self.config.bias_init, drop_rate=self.config.dropout, prefix='de_', isConv=self.config.isConv) self.x_recons_flat = Px_latent_mean.output self.x_recons = tf.reshape(self.x_recons_flat, [ -1, self.config.width, self.config.height, self.config.num_channels ]) ''' ------------------------------------------------------------------------------ LOSSES ------------------------------------------------------------------------------ ''' def create_loss_optimizer(self): print('[*] Defining Loss Functions and Optimizer...') with tf.name_scope('reconstruct'): self.reconstruction = losses.get_reconst_loss( self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss) self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction) with tf.variable_scope('L2_loss', reuse=self.config.reuse): tv = tf.trainable_variables() self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv]) with tf.variable_scope('encoder_loss', reuse=self.config.reuse): self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2 * self.L2_loss, name='encoder_loss') with tf.variable_scope('divergence_cost', reuse=self.config.reuse): self.divergence_cost = losses.get_self_divergence( self.encoder_mean, self.encoder_logvar, self.config.div_cost) self.div_cost_m = tf.reduce_mean(self.divergence_cost) with tf.variable_scope('vae_loss', reuse=self.config.reuse): self.vae_loss = tf.add(self.ae_loss, self.div_cost_m) with tf.variable_scope('bvae_loss', reuse=self.config.reuse): self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m) self.bvae_loss = tf.add(self.ae_loss, self.beta_reg) with tf.variable_scope('btcvae_loss', reuse=self.config.reuse): """ Based on Equation 4 with alpha = gamma = 1 of "Isolating Sources of Disentanglement in Variational Autoencoders" (https: // arxiv.org / pdf / 1802.04942). If alpha = gamma = 1, Eq 4 can be written as ELBO + (1 - beta) * TC. """ tc = tf.multiply(1-self.config.beta, self.total_correlation(self.latent_batch, self.encoder_mean, \ self.encoder_logvar)) self.tc_beta_reg = tf.add(self.div_cost_m, tc) self.btcvae_loss = tf.add(self.ae_loss, self.tc_beta_reg) with tf.variable_scope("optimizer", reuse=self.config.reuse): self.optimizer = RAdamOptimizer(self.lr) self.train_step = self.optimizer.minimize( self.btcvae_loss, global_step=self.global_step_tensor) self.losses = [ 'ELBO_Beta-TC-VAE', 'Beta-VAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), 'Regul_tc_beta_reg', 'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), 'Regul_L2' ] ''' ------------------------------------------------------------------------------ FIT & EVALUATE TENSORS ------------------------------------------------------------------------------ ''' def train_epoch(self, session, x): tensors = [ self.train_step, self.btcvae_loss, self.bvae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.tc_beta_reg, self.beta_reg, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} _, loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss = session.run( tensors, feed_dict=feed_dict) return loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss def test_epoch(self, session, x): tensors = [ self.btcvae_loss, self.bvae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.tc_beta_reg, self.beta_reg, self.div_cost_m, self.L2_loss ] feed_dict = {self.x_batch: x} loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss = session.run( tensors, feed_dict=feed_dict) return loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss ''' ------------------------------------------------------------------------------ GRAPH OPERATIONS ------------------------------------------------------------------------------ ''' def gaussian_log_density(self, samples, mean, log_var): pi = tf.constant(math.pi) normalization = tf.log(2. * pi) inv_sigma = tf.exp(-log_var) tmp = (samples - mean) return -0.5 * (tmp * tmp * inv_sigma + log_var + normalization) def total_correlation(self, latent, latent_mean, latent_logvar): """Estimate of total correlation on a batch. We need to compute the expectation over a batch of: E_j [log(q(latent(x_j))) - log(prod_l q(latent(x_j)_l))]. We ignore the constants as they do not matter for the minimization. The constant should be equal to (num_latents - 1) * log(batch_size * dataset_size) Args: latent: [batch_size, num_latents]-tensor with sampled representation. z_mean: [batch_size, num_latents]-tensor with mean of the encoder. z_logvar: [batch_size, num_latents]-tensor with log variance of the encoder. Returns: Total correlation estimated on a batch. """ # Compute log(q(latent(x_j)|x_i)) for every _sampling_reconst in the batch, which is a # tensor of size [batch_size, batch_size, num_latents]. In the following # comments, [batch_size, batch_size, num_latents] are indexed by [j, i, l]. log_qlatent_prob = self.gaussian_log_density( tf.expand_dims(latent, 1), tf.expand_dims(latent_mean, 0), tf.expand_dims(latent_logvar, 0)) # Compute log prod_l p(latent(x_j)_l) = sum_l(log(sum_i(q(latent(z_j)_l|x_i))) # + constant) for each _sampling_reconst in the batch, which is a vector of size # [batch_size,]. log_qlatent_product = tf.reduce_sum(tf.reduce_logsumexp( log_qlatent_prob, axis=1, keepdims=False), axis=1, keepdims=False) # Compute log(q(Qx(x_j))) as log(sum_i(q(Qx(x_j)|x_i))) + constant = # log(sum_i(prod_l q(Qx(x_j)_l|x_i))) + constant. log_qlatent = tf.reduce_logsumexp(tf.reduce_sum(log_qlatent_prob, axis=2, keepdims=False), axis=1, keepdims=False) return tf.reduce_mean(log_qlatent - log_qlatent_product)
def __init__(self, config, word_embedding, sess): self.name = config.name self.batch_size = config.batch_size_train # bach size self.num_nodes = config.num_nodes # number of user for diffusion network self.n_sequence = config.n_sequence # user number in diffusion path self.total_nodes = config.total_nodes # total number of user # content feature self.word_embedding = word_embedding # word embedding matrix self.word_num = config.word_num # words number in tweet # user feature self.d_user_fea = config.d_user_fea # dimension of the user feature self.ratio = config.ratio self.residual = True # diffusion structure self.insTancenorm = True self.feat_in = config.feat_in # number of feature self.z_size_1 = config.z_size_1 self.z_size_2 = config.z_size_2 # CTLSTM continuous-time LSTM self.embeding_size = config.embedding_size self.emb_learning_rate = config.emb_learning_rate self.num_ctlstm = config.num_ctlstm # view size self.equal = False self.view_size = config.view_size # content self.kernel_sizes = [3, 4, 5] # attention capsule self.capsule_size = config.capsule_size # iteration num for routing self.iter_routing = config.iter_routing # prediction self.num_label = 4 self.h1 = config.h1 self.sess = sess self.max_grad_norm = config.max_grad_norm self.learning_rate = config.learning_rate self.n_time_interval = config.n_time_interval # 时间间隔 6 self.scale1 = config.l1 # 正则化方法 self.scale2 = config.l2 self.stddev = config.stddev self.initializer = tf.random_normal_initializer(stddev=self.stddev) self._build_placeholders() self._build_var() self._build_model() self.global_step = tf.Variable(0, name='global_step', trainable=False) learning_rate = tf.compat.v1.train.exponential_decay( self.learning_rate, self.global_step, 20000, 0.9, staircase=True) # linear decay over time var_list_all = [ var for var in tf.compat.v1.trainable_variables() if not 'embedding' in var.name ] var_list_dy = [ var for var in tf.compat.v1.trainable_variables() if 'embedding' in var.name ] # user dynamic embedding lossL2 = tf.add_n([tf.nn.l2_loss(v) for v in var_list_all]) * self.scale2 opt1 = RAdamOptimizer(learning_rate=learning_rate) # all opt2 = RAdamOptimizer( learning_rate=self.emb_learning_rate) # for user embedding train_op1 = opt1.minimize(self.loss + lossL2, var_list=var_list_all, global_step=self.global_step) train_op2 = opt2.minimize(self.loss, var_list=var_list_dy) self.train_op = tf.group(train_op1, train_op2) init_op = tf.global_variables_initializer() self.sess.run(init_op)
def train_model(train_trees, val_trees, labels, embeddings, embedding_lookup, opt): max_acc = 0.0 logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter num_feats = len(embeddings[0]) random.shuffle(train_trees) nodes_node, children_node, codecaps_node = network.init_net_treecaps( num_feats, len(labels)) codecaps_node = tf.identity(codecaps_node, name="codecaps_node") out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_step = optimizer.minimize(loss_node) ### init the graph sess = tf.Session() sess.run(tf.global_variables_initializer()) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') print("Begin training..........") num_batches = len(train_trees) // batch_size + ( 1 if len(train_trees) % batch_size != 0 else 0) for epoch in range(1, epochs + 1): bar = progressbar.ProgressBar(maxval=len(train_trees), widgets=[ progressbar.Bar('=', '[', ']'), ' ', progressbar.Percentage() ]) bar.start() for i, batch in enumerate( sampling.batch_samples( sampling.gen_samples(train_trees, labels, embeddings, embedding_lookup), batch_size)): nodes, children, batch_labels = batch step = (epoch - 1) * num_batches + i * batch_size if not nodes: continue _, err, out = sess.run( [train_step, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels }) bar.update(i + 1) bar.finish() correct_labels = [] predictions = [] logits = [] for batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels, embeddings, embedding_lookup), 1): nodes, children, batch_labels = batch output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children }) correct_labels.append(np.argmax(batch_labels)) predictions.append(np.argmax(output)) logits.append(output) target_names = list(labels) acc = accuracy_score(correct_labels, predictions) if (acc > max_acc): max_acc = acc saver.save(sess, checkfile) np.save(opt.model_path + '/logits', np.array(logits)) np.save(opt.model_path + '/correct', np.array(correct_labels)) print('Epoch', str(epoch), 'Accuracy:', acc, 'Max Acc: ', max_acc) csv_log.write(str(epoch) + ',' + str(acc) + ',' + str(max_acc) + '\n') print("Finish all iters, storring the whole model..........")
def train_model(train_trees, val_trees, labels, embedding_lookup, opt): max_acc = 0.0 logdir = opt.model_path batch_size = opt.train_batch_size epochs = opt.niter random.shuffle(train_trees) nodes_node, children_node, codecaps_node = network.init_net_treecaps(50, embedding_lookup, len(labels)) codecaps_node = tf.identity(codecaps_node, name="codecaps_node") out_node = network.out_layer(codecaps_node) labels_node, loss_node = network.loss_layer(codecaps_node, len(labels)) optimizer = RAdamOptimizer(opt.lr) train_point = optimizer.minimize(loss_node) ### init the graph sess = tf.Session() sess.run(tf.global_variables_initializer()) # Initialize the variables (i.e. assign their default value) init = tf.global_variables_initializer() with tf.name_scope('saver'): saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(logdir) if ckpt and ckpt.model_checkpoint_path: print("Continue training with old model") saver.restore(sess, ckpt.model_checkpoint_path) for i, var in enumerate(saver._var_list): print('Var {}: {}'.format(i, var)) checkfile = os.path.join(logdir, 'tree_network.ckpt') print("Begin training..........") num_batches = len(train_trees) // batch_size + (1 if len(train_trees) % batch_size != 0 else 0) max_acc = 0.0 for epoch in range(1, epochs+1): for train_step, train_batch in enumerate(sampling.batch_samples( sampling.gen_samples(train_trees, labels), batch_size )): nodes, children, batch_labels = train_batch # step = (epoch - 1) * num_batches + train_step * batch_size if not nodes: continue _, err, out = sess.run( [train_point, loss_node, out_node], feed_dict={ nodes_node: nodes, children_node: children, labels_node: batch_labels } ) print("Epoch : ", str(epoch), "Step : ", train_step, "Loss : ", err, "Max Acc: ",max_acc) if train_step % 1000 == 0 and train_step > 0: correct_labels = [] predictions = [] # logits = [] for test_batch in sampling.batch_samples( sampling.gen_samples(val_trees, labels), batch_size ): print("---------------") nodes, children, batch_labels = test_batch print(batch_labels) output = sess.run([out_node], feed_dict={ nodes_node: nodes, children_node: children } ) batch_correct_labels = np.argmax(batch_labels, axis=1) batch_predictions = np.argmax(output[0], axis=1) correct_labels.extend(batch_correct_labels) predictions.extend(batch_predictions) # logits.append(output) print(batch_correct_labels) print(batch_predictions) acc = accuracy_score(correct_labels, predictions) if (acc>max_acc): max_acc = acc saver.save(sess, checkfile) print("Saved checkpoint....") print('Epoch',str(epoch),'Accuracy:', acc, 'Max Acc: ',max_acc) csv_log.write(str(epoch)+','+str(acc)+','+str(max_acc)+'\n') print("Finish all iters, storring the whole model..........")