def test_model(test_trees, labels, embeddings, embedding_lookup, opt):
logdir = opt.model_path
batch_size = opt.train_batch_size
epochs = opt.niter
num_feats = len(embeddings[0])
random.shuffle(test_trees)
# build the inputs and outputs of the network
nodes_node, children_node, codecaps_node = network.init_net_treecaps(
num_feats, len(labels))
out_node = network.out_layer(codecaps_node)
labels_node, loss_node = network.loss_layer(codecaps_node, len(labels))
optimizer = RAdamOptimizer(opt.lr)
train_step = optimizer.minimize(loss_node)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
with tf.name_scope('saver'):
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(logdir)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
checkfile = os.path.join(logdir, 'tree_network.ckpt')
correct_labels = []
predictions = []
print('Computing training accuracy...')
for batch in sampling.batch_samples(
sampling.gen_samples(test_trees, labels, embeddings,
embedding_lookup), 1):
nodes, children, batch_labels = batch
output = sess.run([out_node],
feed_dict={
nodes_node: nodes,
children_node: children,
})
correct_labels.append(np.argmax(batch_labels))
predictions.append(np.argmax(output))
target_names = list(labels)
print(
classification_report(correct_labels,
predictions,
target_names=target_names))
print(confusion_matrix(correct_labels, predictions))
print('*' * 50)
print('Accuracy:', accuracy_score(correct_labels, predictions))
print('*' * 50)
class AnnBayBVAEGraph(BaseGraph):
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [self.config.batch_size, self.config.width, self.config.height, self.config.num_channels], name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch , [-1,self.x_flat_dim])
self.latent_batch = tf.placeholder(tf.float32, [self.config.batch_size, self.config.latent_dim], name='px_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate, shape=None, name='lr')
self.sample_batch = tf.random_normal((self.config.batch_size, self.config.latent_dim), -1, 1, dtype=tf.float32)
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining _sampling_reconst from X...')
self.sample_flat = tf.multiply(tf.ones([self.config.MC_samples, self.config.batch_size, self.x_flat_dim]), self.x_batch_flat)
sample_flat_shape = self.sample_flat.get_shape().as_list()
if self.config.isConv:
sample_input = tf.reshape(self.sample_flat , [-1, self.config.width, self.config.height, self.config.num_channels])
print('\n[*] _sampling_reconst shape {}'.format(sample_input.shape))
else:
print('\n[*] _sampling_reconst shape {}'.format(sample_flat_shape))
####################################################################################
print('\n[*] Defining prior ...')
print('\n[*] Defining prior encoder...')
with tf.variable_scope('prior_encoder', reuse=self.config.reuse):
Qlatent_sample = self.create_encoder(input_=sample_input if self.config.isConv else self.sample_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='prior_en_',
isConv=self.config.isConv)
self.prior_mean = Qlatent_sample.output
self.prior_var = tf.nn.sigmoid(self.prior_mean)
self.latent_sample = self.prior_mean
print('\n[*] Defining prior decoder...')
with tf.variable_scope('prior_decoder', reuse=self.config.reuse):
Psample_latent = self.create_decoder(input_=self.latent_sample,
hidden_dim=self.config.hidden_dim,
output_dim=sample_flat_shape[-1],
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='prior_de_',
isConv=self.config.isConv)
self.sample_recons_flat = Psample_latent.output
if self.config.isConv:
self.sample_recons_flat = tf.reshape(self.sample_recons_flat, sample_flat_shape)
print('\n[*] _sampling_reconst tsne_cost shape {}'.format(self.sample_recons_flat.shape))
####################################################################################
print('\n[*] Defining posterior ...')
print('\n[*] Defining posterior encoders...')
with tf.variable_scope('encoder_mean', reuse=self.config.reuse):
Qlatent_x_mean = self.create_encoder(input_=self.x_batch if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='post_enmean_',
isConv=self.config.isConv)
self.encoder_mean = Qlatent_x_mean.output
with tf.variable_scope('encoder_var', reuse=self.config.reuse):
Qlatent_x_var = self.create_encoder(input_=self.x_batch if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.softplus,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='post_envar_',
isConv=self.config.isConv)
self.encoder_var = Qlatent_x_var.output
print('\n[*] Reparameterization trick...')
self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon)
eps = tf.random_normal((self.config.batch_size, self.config.latent_dim), 0, 1, dtype=tf.float32)
self.latent = tf.add(self.encoder_mean, tf.multiply(tf.sqrt(self.encoder_var), eps))
self.latent_batch = self.latent
print('\n[*] Defining decoder...')
with tf.variable_scope('decoder_mean', reuse=self.config.reuse):
Px_latent_mean = self.create_decoder(input_=self.latent_batch,
hidden_dim=self.config.hidden_dim,
output_dim=self.x_flat_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='post_de_',
isConv=self.config.isConv)
self.x_recons_flat = Px_latent_mean.output
self.x_recons = tf.reshape(self.x_recons_flat , [-1,self.config.width, self.config.height, self.config.num_channels])
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.name_scope('prior_recons'):
self.prior_recons = losses.get_reconst_loss(self.sample_flat, self.sample_recons_flat, self.config.prior_reconst_loss)
self.prior_recons_m = tf.reduce_mean(self.prior_recons)
with tf.name_scope('reconstruct'):
self.reconstruction = losses.get_reconst_loss(self.x_batch_flat, self.x_recons_flat, self.config.reconst_loss)
self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction)
with tf.variable_scope('L2_loss', reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum([ tf.nn.l2_loss(v) for v in tv ])
with tf.variable_scope('encoder_loss', reuse=self.config.reuse):
self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction), self.config.l2*self.L2_loss, name='encoder_loss')
with tf.variable_scope('vae_loss', reuse=self.config.reuse):
self.vae_loss = tf.add(self.ae_loss, self.div_cost_m)
with tf.variable_scope('divergence_cost', reuse=self.config.reuse):
self.divergence_cost = losses.get_self_divergence(self.encoder_mean, self.encoder_logvar, self.config.div_cost)
self.div_cost_m = tf.reduce_mean(self.divergence_cost)
with tf.variable_scope('vae_loss', reuse=self.config.reuse):
self.vae_loss = tf.add(self.ae_loss, self.div_cost_m)
with tf.variable_scope('bvae_loss', reuse=self.config.reuse):
self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m)
self.bvae_loss = tf.add(self.ae_loss, self.beta_reg)
with tf.variable_scope('annvae_loss', reuse=self.config.reuse):
c = self.anneal(self.config.c_max, self.global_step_tensor, self.config.itr_thd)
self.anneal_reg = self.config.ann_gamma * tf.math.abs(self.div_cost_m - c)
self.annvae_loss = tf.add(self.ae_loss, self.anneal_reg)
with tf.variable_scope('bayae_loss', reuse=self.config.reuse):
if self.config.isConv:
self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \
tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), \
tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]),
self.config.prior_div_cost)
else:
self.bay_div = -1 * losses.get_divergence(self.encoder_mean, self.encoder_var, \
self.prior_mean, self.prior_var,
self.config.prior_div_cost)
self.bayae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.ae_loss, self.bay_div, name='bayae_loss')
self.bayvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.vae_loss, self.bay_div, name='bayvae_loss')
self.baybvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.bvae_loss, self.bay_div,
name='baybvae_loss')
self.annbayvae_loss = tf.add(self.baybvae_loss, self.anneal_reg)
self.annbvae_loss = tf.add(self.bvae_loss, self.anneal_reg)
self.annbaybvae_loss = self.baybvae_loss = tf.add(tf.cast(self.config.ntrain_batches, 'float32') * self.bvae_loss, self.bay_div,
name='annbaybvae_loss')
with tf.variable_scope("optimizer" ,reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(self.annbaybvae_loss, global_step=self.global_step_tensor)
self.losses = ['ELBO_AnnBayBeta-VAE', 'AnnBeta-VAE', 'BayBeta-VAE',\
'BayVAE', 'BayAE', 'prior_recons_{}'.format(self.config.prior_reconst_loss), \
'Beta-VAE', 'VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss), \
'bayesian_div_{}'.format(self.config.prior_div_cost), \
'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), \
'Regul_L2']
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x):
tensors = [self.train_step, self.annbaybvae_loss, self.annbvae_loss, self.baybvae_loss,
self.bayvae_loss, self.bayae_loss, self.prior_recons_m, self.bvae_loss,
self.bae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.bay_div,
self.beta_reg, self.div_cost_m,
self.L2_loss]
feed_dict = {self.x_batch: x}
_, annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \
bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_loss = session.run(tensors, feed_dict=feed_dict)
return annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \
bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los
def test_epoch(self, session, x):
tensors = [self.annbaybvae_loss, self.annbvae_loss, self.baybvae_loss,
self.bayvae_loss, self.bayae_loss, self.prior_recons_m, self.bvae_loss,
self.bae_loss, self.vae_loss, self.ae_loss, self.loss_reconstruction_m, self.bay_div,
self.beta_reg, self.div_cost_m,
self.L2_loss]
feed_dict = {self.x_batch: x}
annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \
bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los = session.run(tensors, feed_dict=feed_dict)
return annbaybvae_loss, annbvae_loss, baybvae_loss, bayvae_loss, bayae_loss, prior_recons, \
bvae_loss, bae_loss, vae_loss, ae_loss, loss_recon, bay_div, beta_reg, div_cost, L2_los
class DIPcovVAEGraph(BaseGraph):
def extra_sittings(self):
self.d = self.config.d_factor * self.config.lambda_d
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [
self.config.batch_size, self.config.width, self.config.height,
self.config.num_channels
],
name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim])
self.latent_batch = tf.placeholder(
tf.float32, [self.config.batch_size, self.config.latent_dim],
name='px_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate,
shape=None,
name='lr')
self.sample_batch = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
-1,
1,
dtype=tf.float32)
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining encoders...')
with tf.variable_scope('encoder_mean', reuse=self.config.reuse):
Qlatent_x_mean = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='enmean_',
isConv=self.config.isConv)
self.encoder_mean = Qlatent_x_mean.output
with tf.variable_scope('encoder_var', reuse=self.config.reuse):
Qlatent_x_var = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.softplus,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='envar_',
isConv=self.config.isConv)
self.encoder_var = Qlatent_x_var.output
print('\n[*] Reparameterization trick...')
self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon)
eps = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
0,
1,
dtype=tf.float32)
self.latent = tf.add(self.encoder_mean,
tf.multiply(tf.sqrt(self.encoder_var), eps))
self.latent_batch = self.latent
print('\n[*] Defining decoder...')
with tf.variable_scope('decoder_mean', reuse=self.config.reuse):
Px_latent_mean = self.create_decoder(
input_=self.latent_batch,
hidden_dim=self.config.hidden_dim,
output_dim=self.x_flat_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='de_',
isConv=self.config.isConv)
self.x_recons_flat = Px_latent_mean.output
self.x_recons = tf.reshape(self.x_recons_flat, [
-1, self.config.width, self.config.height, self.config.num_channels
])
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.name_scope('reconstruct'):
self.reconstruction = losses.get_reconst_loss(
self.x_batch_flat, self.x_recons_flat,
self.config.reconst_loss)
self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction)
with tf.variable_scope('L2_loss', reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv])
with tf.variable_scope('encoder_loss', reuse=self.config.reuse):
self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction),
self.config.l2 * self.L2_loss,
name='encoder_loss')
with tf.variable_scope('divergence_cost', reuse=self.config.reuse):
self.divergence_cost = losses.get_self_divergence(
self.encoder_mean, self.encoder_logvar, self.config.div_cost)
self.div_cost_m = tf.reduce_mean(self.divergence_cost)
with tf.variable_scope('vae_loss', reuse=self.config.reuse):
self.vae_loss = tf.add(self.ae_loss, self.div_cost_m)
with tf.variable_scope('dipae_loss', reuse=self.config.reuse):
self.covar_reg = tf.add(
self.regularizer(self.encoder_mean, self.encoder_logvar),
self.div_cost_m)
self.dipvae_loss = tf.add(self.ae_loss, self.covar_reg)
with tf.variable_scope("optimizer", reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(
self.dipvae_loss, global_step=self.global_step_tensor)
self.losses = [
'ELBO_DIPcovVAE', 'Regul_Covariance_Prior', 'VAE', 'AE',
'Recons_{}'.format(self.config.reconst_loss),
'Div_{}'.format(self.config.div_cost), 'Regul_L2'
]
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x):
tensors = [
self.train_step, self.dipvae_loss, self.covar_reg, self.vae_loss,
self.ae_loss, self.loss_reconstruction_m, self.div_cost_m,
self.L2_loss
]
feed_dict = {self.x_batch: x}
_, dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss = session.run(
tensors, feed_dict=feed_dict)
return dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss
def test_epoch(self, session, x):
tensors = [
self.dipvae_loss, self.covar_reg, self.vae_loss, self.ae_loss,
self.loss_reconstruction_m, self.div_cost_m, self.L2_loss
]
feed_dict = {self.x_batch: x}
dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss = session.run(
tensors, feed_dict=feed_dict)
return dipvae_loss, covar_reg, vae_loss, ae_loss, loss_recons, div, L2_loss
'''
------------------------------------------------------------------------------
DIP OPERATIONS
------------------------------------------------------------------------------
'''
def compute_covariance_latent_mean(self, latent_mean):
"""
:param latent_mean:
:return:
Computes the covariance of latent_mean.
Uses cov(latent_mean) = E[latent_mean*latent_mean^T] - E[latent_mean]E[latent_mean]^T.
Args:
latent_mean: Encoder mean, tensor of size [batch_size, num_latent].
Returns:
cov_latent_mean: Covariance of encoder mean, tensor of size [latent_dim, latent_dim].
"""
exp_latent_mean_latent_mean_t = tf.reduce_mean(
tf.expand_dims(latent_mean, 2) * tf.expand_dims(latent_mean, 1),
axis=0)
expectation_latent_mean = tf.reduce_mean(latent_mean, axis=0)
cov_latent_mean = tf.subtract(
exp_latent_mean_latent_mean_t,
tf.expand_dims(expectation_latent_mean, 1) *
tf.expand_dims(expectation_latent_mean, 0))
return cov_latent_mean
def regularize_diag_off_diag_dip(self, covariance_matrix, lambda_od,
lambda_d):
"""
Compute on and off diagonal regularizers for DIP-VAE models.
Penalize deviations of covariance_matrix from the identity matrix. Uses
different weights for the deviations of the diagonal and off diagonal entries.
Args:
covariance_matrix: Tensor of size [num_latent, num_latent] to covar_reg.
lambda_od: Weight of penalty for off diagonal elements.
lambda_d: Weight of penalty for diagonal elements.
Returns:
dip_regularizer: Regularized deviation from diagonal of covariance_matrix.
"""
covariance_matrix_diagonal = tf.diag_part(covariance_matrix)
covariance_matrix_off_diagonal = covariance_matrix - tf.diag(
covariance_matrix_diagonal)
dip_regularizer = tf.add(
lambda_od * tf.reduce_sum(covariance_matrix_off_diagonal**2),
lambda_d * tf.reduce_sum((covariance_matrix_diagonal - 1)**2))
return dip_regularizer
def regularizer(self, latent_mean, latent_logvar):
cov_latent_mean = self.compute_covariance_latent_mean(latent_mean)
# Eq 6 page 4
# mu = z_mean is [batch_size, num_latent]
# Compute cov_p(x) [mu(x)] = E[mu*mu^T] - E[mu]E[mu]^T]
cov_dip_regularizer = self.regularize_diag_off_diag_dip(
cov_latent_mean, self.config.lambda_d, self.d)
return cov_dip_regularizer
class VAEGraph(BaseGraph):
def build_graph(self):
self.create_inputs()
self.create_graph()
self.create_loss_optimizer()
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [
self.config.batch_size, self.config.width, self.config.height,
self.config.num_channels
],
name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim])
self.latent_batch = tf.placeholder(
tf.float32, [self.config.batch_size, self.config.latent_dim],
name='px_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate,
shape=None,
name='lr')
self.sample_batch = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
-1,
1,
dtype=tf.float32)
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining encoders...')
with tf.variable_scope('encoder_mean', reuse=self.config.reuse):
Qlatent_x_mean = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='enmean_',
isConv=self.config.isConv)
self.encoder_mean = Qlatent_x_mean.output
with tf.variable_scope('encoder_var', reuse=self.config.reuse):
Qlatent_x_var = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.softplus,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='envar_',
isConv=self.config.isConv)
self.encoder_var = Qlatent_x_var.output
print('\n[*] Reparameterization trick...')
self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon)
eps = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
0,
1,
dtype=tf.float32)
self.latent = tf.add(self.encoder_mean,
tf.multiply(tf.sqrt(self.encoder_var), eps))
self.latent_batch = self.latent
print('\n[*] Defining decoder...')
with tf.variable_scope('decoder_mean', reuse=self.config.reuse):
Px_latent_mean = self.create_decoder(
input_=self.latent_batch,
hidden_dim=self.config.hidden_dim,
output_dim=self.x_flat_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='de_',
isConv=self.config.isConv)
self.x_recons_flat = Px_latent_mean.output
self.x_recons = tf.reshape(self.x_recons_flat, [
-1, self.config.width, self.config.height, self.config.num_channels
])
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.name_scope('reconstruct'):
self.reconstruction = losses.get_reconst_loss(
self.x_batch_flat, self.x_recons_flat,
self.config.reconst_loss)
self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction)
with tf.variable_scope('L2_loss', reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv])
with tf.variable_scope('encoder_loss', reuse=self.config.reuse):
self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction),
self.config.l2 * self.L2_loss,
name='encoder_loss')
with tf.variable_scope('divergence_cost', reuse=self.config.reuse):
self.divergence_cost = losses.get_self_divergence(
self.encoder_mean, self.encoder_logvar, self.config.div_cost)
self.div_cost_m = tf.reduce_mean(self.divergence_cost)
with tf.variable_scope('vae_loss', reuse=self.config.reuse):
self.vae_loss = tf.add(self.ae_loss, self.div_cost_m)
with tf.variable_scope('optimizer', reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(
self.vae_loss, global_step=self.global_step_tensor)
self.losses = [
'ELBO_VAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss),
'Div_{}'.format(self.config.div_cost), 'Regul_L2'
]
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x):
tensors = [
self.train_step, self.vae_loss, self.ae_loss,
self.loss_reconstruction_m, self.div_cost_m, self.L2_loss
]
feed_dict = {self.x_batch: x}
_, loss, aeloss, recons, div, L2_loss = session.run(
tensors, feed_dict=feed_dict)
return loss, aeloss, recons, div, L2_loss
def test_epoch(self, session, x):
tensors = [
self.vae_loss, self.ae_loss, self.loss_reconstruction_m,
self.div_cost_m, self.L2_loss
]
feed_dict = {self.x_batch: x}
loss, aeloss, recons, div, L2_loss = session.run(tensors,
feed_dict=feed_dict)
return loss, aeloss, recons, div, L2_loss
class BayAEGraph(BaseGraph):
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [
self.config.batch_size, self.config.width, self.config.height,
self.config.num_channels
],
name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim])
self.latent_batch = tf.placeholder(
tf.float32, [self.config.batch_size, self.config.latent_dim],
name='px_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate,
shape=None,
name='lr')
self.sample_batch = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
-1,
1,
dtype=tf.float32)
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining _sampling_reconst from X...')
self.sample_flat = tf.multiply(
tf.ones([
self.config.MC_samples, self.config.batch_size, self.x_flat_dim
]), self.x_batch_flat)
sample_flat_shape = self.sample_flat.get_shape().as_list()
if self.config.isConv:
sample_input = tf.reshape(self.sample_flat, [
-1, self.config.width, self.config.height,
self.config.num_channels
])
print('\n[*] _sampling_reconst shape {}'.format(
sample_input.shape))
else:
print('\n[*] _sampling_reconst shape {}'.format(sample_flat_shape))
####################################################################################
print('\n[*] Defining prior ...')
print('\n[*] Defining prior encoder...')
with tf.variable_scope('prior_encoder', reuse=self.config.reuse):
Qlatent_sample = self.create_encoder(
input_=sample_input
if self.config.isConv else self.sample_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='prior_en_',
isConv=self.config.isConv)
self.prior_mean = Qlatent_sample.output
self.prior_var = tf.nn.sigmoid(self.prior_mean)
self.latent_sample = self.prior_mean
print('\n[*] Defining prior decoder...')
with tf.variable_scope('prior_decoder', reuse=self.config.reuse):
Psample_latent = self.create_decoder(
input_=self.latent_sample,
hidden_dim=self.config.hidden_dim,
output_dim=sample_flat_shape[-1],
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='prior_de_',
isConv=self.config.isConv)
self.sample_recons_flat = Psample_latent.output
if self.config.isConv:
self.sample_recons_flat = tf.reshape(self.sample_recons_flat,
sample_flat_shape)
print('\n[*] _sampling_reconst tsne_cost shape {}'.format(
self.sample_recons_flat.shape))
####################################################################################
print('\n[*] Defining posterior ...')
print('\n[*] Defining posterior encoder...')
with tf.variable_scope('post_encoder', reuse=self.config.reuse):
Qlatent_x = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='post_en_',
isConv=self.config.isConv)
self.post_mean = Qlatent_x.output
self.post_var = tf.nn.sigmoid(self.post_mean)
self.latent = self.post_mean
self.latent_batch = self.latent
print('\n[*] Defining posterior decoder...')
with tf.variable_scope('post_decoder', reuse=self.config.reuse):
Px_latent = self.create_decoder(
input_=self.latent_batch,
hidden_dim=self.config.hidden_dim,
output_dim=self.x_flat_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='post_de_',
isConv=self.config.isConv)
self.x_recons_flat = Px_latent.output
self.x_recons = tf.reshape(self.x_recons_flat, [
-1, self.config.width, self.config.height, self.config.num_channels
])
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.name_scope('reconstruct'):
self.reconstruction = losses.get_reconst_loss(
self.x_batch_flat, self.x_recons_flat,
self.config.reconst_loss)
self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction)
with tf.name_scope('prior_recons'):
self.prior_recons = losses.get_reconst_loss(
self.sample_flat, self.sample_recons_flat,
self.config.prior_reconst_loss)
self.prior_recons_m = tf.reduce_mean(self.prior_recons)
with tf.variable_scope("L2_loss", reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in tv if 'post_' in v.name])
with tf.variable_scope('encoder_loss', reuse=self.config.reuse):
self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction),
self.config.l2 * self.L2_loss,
name='encoder_loss') + self.prior_recons_m
with tf.variable_scope('bayae_loss', reuse=self.config.reuse):
if self.config.isConv:
self.bay_div = -1 * losses.get_QP_kl(self.post_mean, self.post_var, \
tf.reshape(self.prior_mean, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]), tf.reshape(self.prior_var, [self.config.MC_samples, self.config.batch_size, self.config.latent_dim]))
else:
self.bay_div = -1 * losses.get_QP_kl(self.post_mean, self.post_var, \
self.prior_mean, self.prior_var)
self.bayae_loss = tf.add(
tf.cast(self.config.ntrain_batches, 'float32') * self.ae_loss,
self.bay_div,
name='bayae_loss')
with tf.variable_scope("optimizer", reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(
self.bayae_loss, global_step=self.global_step_tensor)
self.losses = [
'ELBO_BayAE', 'AE', 'Recons_{}'.format(self.config.reconst_loss),
'Regul_L2',
'prior_recons_{}'.format(self.config.prior_reconst_loss),
'bayesian_div_{}'.format(self.config.prior_div_cost)
]
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x):
tensors = [
self.train_step, self.bayae_loss, self.ae_loss,
self.loss_reconstruction_m, self.L2_loss, self.prior_recons_m,
self.bay_div
]
feed_dict = {self.x_batch: x}
_, bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div = session.run(
tensors, feed_dict=feed_dict)
return bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div
def test_epoch(self, session, x):
tensors = [
self.bayae_loss, self.ae_loss, self.loss_reconstruction_m,
self.L2_loss, self.prior_recons_m, self.bay_div
]
feed_dict = {self.x_batch: x}
bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div = session.run(
tensors, feed_dict=feed_dict)
return bayae_loss, aeloss, recons, L2_loss, prior_recons, bay_div
def main(train_opt, test_opt):
train_opt.model_path = os.path.join(
train_opt.model_path, util_functions.form_tbcnn_model_path(train_opt))
checkfile = os.path.join(train_opt.model_path, 'cnn_tree.ckpt')
ckpt = tf.train.get_checkpoint_state(train_opt.model_path)
print("The model path : " + str(checkfile))
if ckpt and ckpt.model_checkpoint_path:
print("-------Continue training with old model-------- : " +
str(checkfile))
tbcnn_model = TBCNN(train_opt)
tbcnn_model.feed_forward()
train_data_loader = BaseDataLoader(train_opt.batch_size,
train_opt.label_size,
train_opt.tree_size_threshold_upper,
train_opt.tree_size_threshold_lower,
train_opt.train_path, True)
test_data_loader = BaseDataLoader(test_opt.batch_size, test_opt.label_size,
test_opt.tree_size_threshold_upper,
test_opt.tree_size_threshold_lower,
test_opt.test_path, False)
optimizer = RAdamOptimizer(train_opt.lr)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
training_point = optimizer.minimize(tbcnn_model.loss)
saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5)
init = tf.global_variables_initializer()
best_f1 = test_opt.best_f1
with tf.Session() as sess:
sess.run(init)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
print("Checkpoint path : " + str(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
for epoch in range(1, train_opt.epochs + 1):
train_batch_iterator = ThreadedIterator(
train_data_loader.make_minibatch_iterator(),
max_queue_size=train_opt.worker)
for train_step, train_batch_data in enumerate(
train_batch_iterator):
print("***************")
# print(train_batch_data["batch_node_index"].shape)
# print(train_batch_data["batch_node_type_id"].shape)
# print(train_batch_data["batch_node_sub_tokens_id"].shape)
# print(train_batch_data["batch_children_index"].shape)
# print(train_batch_data["batch_children_node_type_id"].shape)
# print(train_batch_data["batch_children_node_sub_tokens_id"].shape)
# print(train_batch_data["batch_labels_one_hot"])
# print("Labels : " + str(train_batch_data["batch_labels"]))
# print("Tree sizes : " + str(train_batch_data["batch_size"]))
# for children_index in train_batch_data["batch_children_index"]:
# print("Children_index : " + str(len(children_index)))
_, err = sess.run(
[training_point, tbcnn_model.loss],
feed_dict={
tbcnn_model.placeholders["node_type"]:
train_batch_data["batch_node_type_id"],
tbcnn_model.placeholders["node_token"]:
train_batch_data["batch_node_sub_tokens_id"],
tbcnn_model.placeholders["children_index"]:
train_batch_data["batch_children_index"],
tbcnn_model.placeholders["children_node_type"]:
train_batch_data["batch_children_node_type_id"],
tbcnn_model.placeholders["children_node_token"]:
train_batch_data["batch_children_node_sub_tokens_id"],
tbcnn_model.placeholders["labels"]:
train_batch_data["batch_labels_one_hot"],
tbcnn_model.placeholders["dropout_rate"]:
0.3
})
print("Epoch:", epoch, "Step:", train_step, "Loss:", err,
"Best F1:", best_f1)
if train_step % train_opt.checkpoint_every == 0 and train_step > 0:
#Perform Validation
print("Perform validation.....")
correct_labels = []
predictions = []
test_batch_iterator = ThreadedIterator(
test_data_loader.make_minibatch_iterator(),
max_queue_size=test_opt.worker)
for test_step, test_batch_data in enumerate(
test_batch_iterator):
print("***************")
print(test_batch_data["batch_size"])
scores = sess.run(
[tbcnn_model.softmax],
feed_dict={
tbcnn_model.placeholders["node_type"]:
test_batch_data["batch_node_type_id"],
tbcnn_model.placeholders["node_token"]:
test_batch_data["batch_node_sub_tokens_id"],
tbcnn_model.placeholders["children_index"]:
test_batch_data["batch_children_index"],
tbcnn_model.placeholders["children_node_type"]:
test_batch_data["batch_children_node_type_id"],
tbcnn_model.placeholders["children_node_token"]:
test_batch_data[
"batch_children_node_sub_tokens_id"],
tbcnn_model.placeholders["labels"]:
test_batch_data["batch_labels_one_hot"],
tbcnn_model.placeholders["dropout_rate"]:
0.0
})
batch_correct_labels = list(
np.argmax(test_batch_data["batch_labels_one_hot"],
axis=1))
batch_predictions = list(np.argmax(scores[0], axis=1))
print(batch_correct_labels)
print(batch_predictions)
correct_labels.extend(
np.argmax(test_batch_data["batch_labels_one_hot"],
axis=1))
predictions.extend(np.argmax(scores[0], axis=1))
print(correct_labels)
print(predictions)
f1 = float(
f1_score(correct_labels, predictions, average="micro"))
print(classification_report(correct_labels, predictions))
print('F1:', f1)
print('Best F1:', best_f1)
# print(confusion_matrix(correct_labels, predictions))
if f1 > best_f1:
best_f1 = f1
saver.save(sess, checkfile)
print('Checkpoint saved, epoch:' + str(epoch) +
', step: ' + str(train_step) + ', loss: ' +
str(err) + '.')
def main(opt):
opt.model_path = os.path.join(opt.model_path, form_model_path(opt))
checkfile = os.path.join(opt.model_path, 'cnn_tree.ckpt')
ckpt = tf.train.get_checkpoint_state(opt.model_path)
print("The model path : " + str(checkfile))
print("Loss : " + str(opt.loss))
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model : " + str(checkfile))
print("Loading vocabs.........")
node_type_lookup, node_token_lookup, subtree_lookup = load_vocabs(opt)
opt.node_type_lookup = node_type_lookup
opt.node_token_lookup = node_token_lookup
opt.subtree_lookup = subtree_lookup
if opt.task == 1:
train_dataset = CodeClassificationData(opt, True, False, False)
if opt.task == 0:
val_opt = copy.deepcopy(opt)
val_opt.node_token_lookup = node_token_lookup
validation_dataset = CodeClassificationData(val_opt, False, False,
True)
print("Initializing tree caps model...........")
corder = CorderModel(opt)
print("Finished initializing corder model...........")
loss_node = corder.loss
optimizer = RAdamOptimizer(opt.lr)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
training_point = optimizer.minimize(loss_node)
saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5)
init = tf.global_variables_initializer()
# best_f1_score = get_best_f1_score(opt)
# print("Best f1 score : " + str(best_f1_score))
with tf.Session() as sess:
sess.run(init)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
print("Checkpoint path : " + str(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
if opt.task == 1:
for epoch in range(1, opt.epochs + 1):
train_batch_iterator = ThreadedIterator(
train_dataset.make_minibatch_iterator(),
max_queue_size=opt.worker)
train_accs = []
for train_step, train_batch_data in enumerate(
train_batch_iterator):
print("--------------------------")
# print(train_batch_data["batch_subtrees_ids"])
logging.info(str(train_batch_data["batch_subtree_id"]))
_, err = sess.run(
[training_point, corder.loss],
feed_dict={
corder.placeholders["node_types"]:
train_batch_data["batch_node_types"],
corder.placeholders["node_tokens"]:
train_batch_data["batch_node_tokens"],
corder.placeholders["children_indices"]:
train_batch_data["batch_children_indices"],
corder.placeholders["children_node_types"]:
train_batch_data["batch_children_node_types"],
corder.placeholders["children_node_tokens"]:
train_batch_data["batch_children_node_tokens"],
corder.placeholders["labels"]:
train_batch_data["batch_subtree_id"],
corder.placeholders["dropout_rate"]:
0.3
})
logging.info("Training at epoch " + str(epoch) +
" and step " + str(train_step) +
" with loss " + str(err))
print("Epoch:", epoch, "Step:", train_step,
"Training loss:", err)
if train_step % opt.checkpoint_every == 0 and train_step > 0:
saver.save(sess, checkfile)
print('Checkpoint saved, epoch:' + str(epoch) +
', step: ' + str(train_step) + ', loss: ' +
str(err) + '.')
if opt.task == 0:
validation_batch_iterator = ThreadedIterator(
validation_dataset.make_minibatch_iterator(),
max_queue_size=opt.worker)
for val_step, val_batch_data in enumerate(
validation_batch_iterator):
scores = sess.run(
[corder.code_vector],
feed_dict={
corder.placeholders["node_types"]:
val_batch_data["batch_node_types"],
corder.placeholders["node_tokens"]:
val_batch_data["batch_node_tokens"],
corder.placeholders["children_indices"]:
val_batch_data["batch_children_indices"],
corder.placeholders["children_node_types"]:
val_batch_data["batch_children_node_types"],
corder.placeholders["children_node_tokens"]:
val_batch_data["batch_children_node_tokens"],
corder.placeholders["dropout_rate"]:
0.0
})
for i, vector in enumerate(scores[0]):
file_name = "analysis/rosetta_sampled_softmax_train.csv"
with open(file_name, "a") as f:
vector_score = []
for score in vector:
vector_score.append(str(score))
# print(val_batch_data["batch_file_path"])
line = str(val_batch_data["batch_file_path"]
[i]) + "," + " ".join(vector_score)
f.write(line)
f.write("\n")
class EmbeddingGraph(BaseGraph):
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [
self.config.batch_size, self.config.width, self.config.height,
self.config.num_channels
],
name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim])
self.joint_probabilities_batch = tf.placeholder(
tf.float32, [self.config.batch_size, self.config.batch_size],
name='joint_probabilities_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate,
shape=None,
name='lr')
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining encoder...')
with tf.variable_scope('encoder', reuse=self.config.reuse):
Qlatent_x = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.n_components,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='en_',
isConv=self.config.isConv)
self.latent = Qlatent_x.output
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.variable_scope('e_divergence_cost', reuse=self.config.reuse):
kerneled_embedding = kernels.get_kernel(
X=self.latent,
batch_size=self.config.batch_size,
param=self.config.df,
epsilon=self.config.epsilon,
kernel=self.config.kernel_mode)
self.e_divergence_cost = losses.get_distributions_div_cost(
self.joint_probabilities_batch, kerneled_embedding,
self.config.e_div_cost)
self.e_div_cost_m = tf.reduce_mean(self.e_divergence_cost)
with tf.variable_scope("L2_loss", reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv])
with tf.variable_scope('embedding_loss', reuse=self.config.reuse):
self.embedding_loss = tf.add(tf.reduce_mean(
self.e_divergence_cost),
self.config.l2 * self.L2_loss,
name='embedding_loss')
with tf.variable_scope("optimizer", reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(
self.embedding_loss, global_step=self.global_step_tensor)
self.losses = [
'Embedding_loss', 'E_diverg_{}'.format(self.config.e_div_cost),
'Regul_L2'
]
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x, p):
tensors = [
self.train_step, self.embedding_loss, self.e_divergence_cost,
self.L2_loss
]
feed_dict = {self.x_batch: x, self.joint_probabilities_batch: p}
_, loss, e_div, L2_loss = session.run(tensors, feed_dict=feed_dict)
return loss, e_div, L2_loss
def test_epoch(self, session, x, p):
tensors = [self.embedding_loss, self.e_divergence_cost, self.L2_loss]
feed_dict = {self.x_batch: x, self.joint_probabilities_batch: p}
loss, e_div, L2_loss = session.run(tensors, feed_dict=feed_dict)
return loss, e_div, L2_loss
def main(opt):
opt.model_path = os.path.join(opt.model_path, form_model_path(opt))
checkfile = os.path.join(opt.model_path, 'cnn_tree.ckpt')
ckpt = tf.train.get_checkpoint_state(opt.model_path)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model : " + str(checkfile))
print("Loading vocabs.........")
train_label_lookup, node_type_lookup, node_token_lookup, val_label_lookup = load_vocabs(opt)
opt.label_lookup = train_label_lookup
opt.label_size = len(train_label_lookup.keys())
opt.node_type_lookup = node_type_lookup
opt.node_token_lookup = node_token_lookup
if opt.task == 1:
train_dataset = MethodNamePredictionData(opt, opt.train_path, True, False, False)
val_opt = copy.deepcopy(opt)
val_opt.label_lookup = val_label_lookup
val_opt.num_labels = len(val_label_lookup.keys())
val_opt.node_token_lookup = node_token_lookup
validation_dataset = MethodNamePredictionData(val_opt, opt.val_path, False, False, True)
print("Initializing tree caps model...........")
treecaps = TreeCapsModel(opt)
# network.init_net_treecaps(30,30)
print("Finished initializing tree caps model...........")
code_caps = treecaps.code_caps
loss_node = treecaps.loss
softmax_values = treecaps.softmax_values
logits = treecaps.logits
optimizer = RAdamOptimizer(opt.lr)
# optimizer = tf.compat.v1.train.AdamOptimizer(opt.lr)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
training_point = optimizer.minimize(loss_node)
saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5)
init = tf.global_variables_initializer()
best_f1_score = get_best_f1_score(opt)
print("Best f1 score : " + str(best_f1_score))
num_caps_top_a = int(opt.num_conv*opt.output_size/opt.num_channel)*opt.top_a
with tf.Session() as sess:
sess.run(init)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
print("Checkpoint path : " + str(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
validation_batch_iterator = ThreadedIterator(validation_dataset.make_minibatch_iterator(), max_queue_size=5)
# f1_scores_of_val_data = []
all_predicted_labels = []
all_ground_truth_labels = []
for val_step, val_batch_data in enumerate(validation_batch_iterator):
alpha_IJ_shape = (opt.batch_size, int(num_caps_top_a/opt.top_a*val_batch_data["batch_node_types"].shape[1]), num_caps_top_a)
alpha_IJ = np.zeros(alpha_IJ_shape)
scores, alpha_IJ_scores= sess.run(
[logits, treecaps.alpha_IJ],
feed_dict={
treecaps.placeholders["node_types"]: val_batch_data["batch_node_types"],
treecaps.placeholders["node_tokens"]: val_batch_data["batch_node_tokens"],
treecaps.placeholders["children_indices"]: val_batch_data["batch_children_indices"],
treecaps.placeholders["children_node_types"]: val_batch_data["batch_children_node_types"],
treecaps.placeholders["children_node_tokens"]: val_batch_data["batch_children_node_tokens"],
treecaps.placeholders["labels"]: val_batch_data["batch_labels"],
treecaps.placeholders["alpha_IJ"]: alpha_IJ,
treecaps.placeholders["is_training"]: False
}
)
alpha_IJ_scores = np.reshape(alpha_IJ_scores, (opt.batch_size, val_batch_data["batch_node_types"].shape[1], 8, opt.top_a, 8))
alpha_IJ_scores = np.sum(alpha_IJ_scores, axis=2)
alpha_IJ_scores = np.sum(alpha_IJ_scores, axis=3)
alpha_IJ_scores = np.squeeze(alpha_IJ_scores, axis=0)
alpha_IJ_scores = np.transpose(alpha_IJ_scores)
predictions = np.argmax(scores, axis=1)
ground_truths = np.argmax(val_batch_data['batch_labels'], axis=1)
predicted_labels = []
for prediction in predictions:
predicted_labels.append(train_label_lookup.inverse[prediction])
ground_truth_labels = []
for ground_truth in ground_truths:
ground_truth_labels.append(
val_label_lookup.inverse[ground_truth])
f1_score = evaluation.calculate_f1_scores(predicted_labels, ground_truth_labels)
print(ground_truth_labels)
print(predicted_labels)
print("F1:", f1_score, "Step:", val_step)
if f1_score > 0:
node_types = val_batch_data["batch_node_types"][0]
node_tokens_text = val_batch_data["batch_node_tokens_text"][0]
node_indexes = val_batch_data["batch_node_indexes"][0]
file_path = val_batch_data["batch_file_path"][0]
file_path_splits = file_path.split("/")
file_path_splits[1] = "java-small"
file_path_splits[len(file_path_splits) - 1] = file_path_splits[len(file_path_splits) - 1].replace(".pkl",".java")
file_path = "/".join(file_path_splits)
analysis_folder = os.path.join("analysis", "_".join(file_path_splits[-2:]).replace(".java",""))
try:
from pathlib import Path
Path(analysis_folder).mkdir(parents=True, exist_ok=True)
except Exception as e:
print(e)
count = 0
for capsule in alpha_IJ_scores:
# print(val_batch_data["batch_node_indexes"])
connection_strength = capsule
all_tuples = []
for i, node_index in enumerate(node_indexes):
tuple_of_info = []
tuple_of_info.append(str(node_indexes[i]))
tuple_of_info.append(str(node_types[i]))
tuple_of_info.append(str(connection_strength[i]))
tuple_of_info.append(node_tokens_text[i])
tuple_of_info = tuple(tuple_of_info)
# print(tuple_of_info)
all_tuples.append(tuple_of_info)
all_tuples = sorted(all_tuples, key=lambda x: x[2])
all_tuples.reverse()
with open(os.path.join(analysis_folder, "Group_" + str(count) + ".txt"), "w") as f:
for t in all_tuples:
line = ";".join(list(t))
f.write(line)
f.write("\n")
with open(os.path.join(analysis_folder, "result.txt"), "w") as f1:
f1.write("Predicted : " + str(predicted_labels[0]))
f1.write("\n")
f1.write("Ground truth : " + str(ground_truth_labels[0]))
f1.write("\n")
import shutil
try:
print("Trying to copy original source file....")
shutil.copy(file_path, analysis_folder)
except Exception as e:
print(e)
count += 1
def main(train_opt, test_opt):
train_opt.model_path = os.path.join(
train_opt.model_path, util_functions.form_tbcnn_model_path(train_opt))
checkfile = os.path.join(train_opt.model_path, 'cnn_tree.ckpt')
ckpt = tf.train.get_checkpoint_state(train_opt.model_path)
print("The model path : " + str(checkfile))
if ckpt and ckpt.model_checkpoint_path:
print("-------Continue training with old model-------- : " +
str(checkfile))
tbcnn_model = InferCodeModel(train_opt)
tbcnn_model.feed_forward()
train_data_loader = BaseDataLoader(train_opt.batch_size,
train_opt.tree_size_threshold_upper,
train_opt.tree_size_threshold_lower,
train_opt.train_tree_path,
train_opt.train_bucket_path, True)
# test_data_loader = BaseDataLoader(test_opt.batch_size, test_opt.tree_size_threshold_upper, test_opt.tree_size_threshold_lower, test_opt.test_path, False)
optimizer = RAdamOptimizer(train_opt.lr)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
training_point = optimizer.minimize(tbcnn_model.loss)
saver = tf.train.Saver(save_relative_paths=True, max_to_keep=5)
init = tf.global_variables_initializer()
best_f1 = test_opt.best_f1
with tf.Session() as sess:
sess.run(init)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
print("Checkpoint path : " + str(ckpt.model_checkpoint_path))
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
for epoch in range(1, train_opt.epochs + 1):
train_batch_iterator = ThreadedIterator(
train_data_loader.make_minibatch_iterator(),
max_queue_size=train_opt.worker)
for train_step, train_batch_data in enumerate(
train_batch_iterator):
print("***************")
print(train_batch_data["batch_subtree_id"])
_, err = sess.run(
[training_point, tbcnn_model.loss],
feed_dict={
tbcnn_model.placeholders["node_type"]:
train_batch_data["batch_node_type_id"],
tbcnn_model.placeholders["node_token"]:
train_batch_data["batch_node_sub_tokens_id"],
tbcnn_model.placeholders["children_index"]:
train_batch_data["batch_children_index"],
tbcnn_model.placeholders["children_node_type"]:
train_batch_data["batch_children_node_type_id"],
tbcnn_model.placeholders["children_node_token"]:
train_batch_data["batch_children_node_sub_tokens_id"],
tbcnn_model.placeholders["subtree_labels"]:
train_batch_data["batch_subtree_id"],
tbcnn_model.placeholders["dropout_rate"]:
0.3
})
print("Epoch:", epoch, "Step:", train_step, "Loss:", err,
"Best F1:", best_f1)
class BTCVAEGraph(BaseGraph):
def build_graph(self):
self.create_inputs()
self.create_graph()
self.create_loss_optimizer()
def create_inputs(self):
with tf.variable_scope('inputs', reuse=self.config.reuse):
self.x_batch = tf.placeholder(tf.float32, [
self.config.batch_size, self.config.width, self.config.height,
self.config.num_channels
],
name='x_batch')
self.x_batch_flat = tf.reshape(self.x_batch, [-1, self.x_flat_dim])
self.latent_batch = tf.placeholder(
tf.float32, [self.config.batch_size, self.config.latent_dim],
name='px_batch')
self.lr = tf.placeholder_with_default(self.config.learning_rate,
shape=None,
name='lr')
self.sample_batch = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
-1,
1,
dtype=tf.float32)
'''
------------------------------------------------------------------------------
GRAPH FUNCTIONS
------------------------------------------------------------------------------
'''
def create_graph(self):
print('\n[*] Defining encoders...')
with tf.variable_scope('encoder_mean', reuse=self.config.reuse):
Qlatent_x_mean = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=None,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='enmean_',
isConv=self.config.isConv)
self.encoder_mean = Qlatent_x_mean.output
with tf.variable_scope('encoder_var', reuse=self.config.reuse):
Qlatent_x_var = self.create_encoder(
input_=self.x_batch
if self.config.isConv else self.x_batch_flat,
hidden_dim=self.config.hidden_dim,
output_dim=self.config.latent_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.softplus,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='envar_',
isConv=self.config.isConv)
self.encoder_var = Qlatent_x_var.output
print('\n[*] Reparameterization trick...')
self.encoder_logvar = tf.log(self.encoder_var + self.config.epsilon)
eps = tf.random_normal(
(self.config.batch_size, self.config.latent_dim),
0,
1,
dtype=tf.float32)
self.latent = tf.add(self.encoder_mean,
tf.multiply(tf.sqrt(self.encoder_var), eps))
self.latent_batch = self.latent
print('\n[*] Defining decoder...')
with tf.variable_scope('decoder_mean', reuse=self.config.reuse):
Px_latent_mean = self.create_decoder(
input_=self.latent_batch,
hidden_dim=self.config.hidden_dim,
output_dim=self.x_flat_dim,
num_layers=self.config.num_layers,
transfer_fct=self.config.transfer_fct,
act_out=tf.nn.sigmoid,
reuse=self.config.reuse,
kinit=self.config.kinit,
bias_init=self.config.bias_init,
drop_rate=self.config.dropout,
prefix='de_',
isConv=self.config.isConv)
self.x_recons_flat = Px_latent_mean.output
self.x_recons = tf.reshape(self.x_recons_flat, [
-1, self.config.width, self.config.height, self.config.num_channels
])
'''
------------------------------------------------------------------------------
LOSSES
------------------------------------------------------------------------------
'''
def create_loss_optimizer(self):
print('[*] Defining Loss Functions and Optimizer...')
with tf.name_scope('reconstruct'):
self.reconstruction = losses.get_reconst_loss(
self.x_batch_flat, self.x_recons_flat,
self.config.reconst_loss)
self.loss_reconstruction_m = tf.reduce_mean(self.reconstruction)
with tf.variable_scope('L2_loss', reuse=self.config.reuse):
tv = tf.trainable_variables()
self.L2_loss = tf.reduce_sum([tf.nn.l2_loss(v) for v in tv])
with tf.variable_scope('encoder_loss', reuse=self.config.reuse):
self.ae_loss = tf.add(tf.reduce_mean(self.reconstruction),
self.config.l2 * self.L2_loss,
name='encoder_loss')
with tf.variable_scope('divergence_cost', reuse=self.config.reuse):
self.divergence_cost = losses.get_self_divergence(
self.encoder_mean, self.encoder_logvar, self.config.div_cost)
self.div_cost_m = tf.reduce_mean(self.divergence_cost)
with tf.variable_scope('vae_loss', reuse=self.config.reuse):
self.vae_loss = tf.add(self.ae_loss, self.div_cost_m)
with tf.variable_scope('bvae_loss', reuse=self.config.reuse):
self.beta_reg = tf.multiply(self.config.beta, self.div_cost_m)
self.bvae_loss = tf.add(self.ae_loss, self.beta_reg)
with tf.variable_scope('btcvae_loss', reuse=self.config.reuse):
"""
Based on Equation 4 with alpha = gamma = 1 of "Isolating Sources of Disentanglement in Variational
Autoencoders"
(https: // arxiv.org / pdf / 1802.04942).
If alpha = gamma = 1, Eq 4 can be
written as ELBO + (1 - beta) * TC.
"""
tc = tf.multiply(1-self.config.beta, self.total_correlation(self.latent_batch, self.encoder_mean, \
self.encoder_logvar))
self.tc_beta_reg = tf.add(self.div_cost_m, tc)
self.btcvae_loss = tf.add(self.ae_loss, self.tc_beta_reg)
with tf.variable_scope("optimizer", reuse=self.config.reuse):
self.optimizer = RAdamOptimizer(self.lr)
self.train_step = self.optimizer.minimize(
self.btcvae_loss, global_step=self.global_step_tensor)
self.losses = [
'ELBO_Beta-TC-VAE', 'Beta-VAE', 'VAE', 'AE',
'Recons_{}'.format(self.config.reconst_loss), 'Regul_tc_beta_reg',
'Regul_beta_reg', 'Div_{}'.format(self.config.div_cost), 'Regul_L2'
]
'''
------------------------------------------------------------------------------
FIT & EVALUATE TENSORS
------------------------------------------------------------------------------
'''
def train_epoch(self, session, x):
tensors = [
self.train_step, self.btcvae_loss, self.bvae_loss, self.vae_loss,
self.ae_loss, self.loss_reconstruction_m, self.tc_beta_reg,
self.beta_reg, self.div_cost_m, self.L2_loss
]
feed_dict = {self.x_batch: x}
_, loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss = session.run(
tensors, feed_dict=feed_dict)
return loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss
def test_epoch(self, session, x):
tensors = [
self.btcvae_loss, self.bvae_loss, self.vae_loss, self.ae_loss,
self.loss_reconstruction_m, self.tc_beta_reg, self.beta_reg,
self.div_cost_m, self.L2_loss
]
feed_dict = {self.x_batch: x}
loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss = session.run(
tensors, feed_dict=feed_dict)
return loss, bvaeloss, vaeloss, aeloss, recons, tc_beta_reg, beta_reg, div_cost, L2_loss
'''
------------------------------------------------------------------------------
GRAPH OPERATIONS
------------------------------------------------------------------------------
'''
def gaussian_log_density(self, samples, mean, log_var):
pi = tf.constant(math.pi)
normalization = tf.log(2. * pi)
inv_sigma = tf.exp(-log_var)
tmp = (samples - mean)
return -0.5 * (tmp * tmp * inv_sigma + log_var + normalization)
def total_correlation(self, latent, latent_mean, latent_logvar):
"""Estimate of total correlation on a batch.
We need to compute the expectation over a batch of: E_j [log(q(latent(x_j))) -
log(prod_l q(latent(x_j)_l))]. We ignore the constants as they do not matter
for the minimization. The constant should be equal to (num_latents - 1) *
log(batch_size * dataset_size)
Args:
latent: [batch_size, num_latents]-tensor with sampled representation.
z_mean: [batch_size, num_latents]-tensor with mean of the encoder.
z_logvar: [batch_size, num_latents]-tensor with log variance of the encoder.
Returns:
Total correlation estimated on a batch.
"""
# Compute log(q(latent(x_j)|x_i)) for every _sampling_reconst in the batch, which is a
# tensor of size [batch_size, batch_size, num_latents]. In the following
# comments, [batch_size, batch_size, num_latents] are indexed by [j, i, l].
log_qlatent_prob = self.gaussian_log_density(
tf.expand_dims(latent, 1), tf.expand_dims(latent_mean, 0),
tf.expand_dims(latent_logvar, 0))
# Compute log prod_l p(latent(x_j)_l) = sum_l(log(sum_i(q(latent(z_j)_l|x_i)))
# + constant) for each _sampling_reconst in the batch, which is a vector of size
# [batch_size,].
log_qlatent_product = tf.reduce_sum(tf.reduce_logsumexp(
log_qlatent_prob, axis=1, keepdims=False),
axis=1,
keepdims=False)
# Compute log(q(Qx(x_j))) as log(sum_i(q(Qx(x_j)|x_i))) + constant =
# log(sum_i(prod_l q(Qx(x_j)_l|x_i))) + constant.
log_qlatent = tf.reduce_logsumexp(tf.reduce_sum(log_qlatent_prob,
axis=2,
keepdims=False),
axis=1,
keepdims=False)
return tf.reduce_mean(log_qlatent - log_qlatent_product)
def __init__(self, config, word_embedding, sess):
self.name = config.name
self.batch_size = config.batch_size_train # bach size
self.num_nodes = config.num_nodes # number of user for diffusion network
self.n_sequence = config.n_sequence # user number in diffusion path
self.total_nodes = config.total_nodes # total number of user
# content feature
self.word_embedding = word_embedding # word embedding matrix
self.word_num = config.word_num # words number in tweet
# user feature
self.d_user_fea = config.d_user_fea # dimension of the user feature
self.ratio = config.ratio
self.residual = True
# diffusion structure
self.insTancenorm = True
self.feat_in = config.feat_in # number of feature
self.z_size_1 = config.z_size_1
self.z_size_2 = config.z_size_2
# CTLSTM continuous-time LSTM
self.embeding_size = config.embedding_size
self.emb_learning_rate = config.emb_learning_rate
self.num_ctlstm = config.num_ctlstm
# view size
self.equal = False
self.view_size = config.view_size
# content
self.kernel_sizes = [3, 4, 5]
# attention capsule
self.capsule_size = config.capsule_size
# iteration num for routing
self.iter_routing = config.iter_routing
# prediction
self.num_label = 4
self.h1 = config.h1
self.sess = sess
self.max_grad_norm = config.max_grad_norm
self.learning_rate = config.learning_rate
self.n_time_interval = config.n_time_interval # 时间间隔 6
self.scale1 = config.l1 # 正则化方法
self.scale2 = config.l2
self.stddev = config.stddev
self.initializer = tf.random_normal_initializer(stddev=self.stddev)
self._build_placeholders()
self._build_var()
self._build_model()
self.global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.compat.v1.train.exponential_decay(
self.learning_rate, self.global_step, 20000, 0.9,
staircase=True) # linear decay over time
var_list_all = [
var for var in tf.compat.v1.trainable_variables()
if not 'embedding' in var.name
]
var_list_dy = [
var for var in tf.compat.v1.trainable_variables()
if 'embedding' in var.name
] # user dynamic embedding
lossL2 = tf.add_n([tf.nn.l2_loss(v)
for v in var_list_all]) * self.scale2
opt1 = RAdamOptimizer(learning_rate=learning_rate) # all
opt2 = RAdamOptimizer(
learning_rate=self.emb_learning_rate) # for user embedding
train_op1 = opt1.minimize(self.loss + lossL2,
var_list=var_list_all,
global_step=self.global_step)
train_op2 = opt2.minimize(self.loss, var_list=var_list_dy)
self.train_op = tf.group(train_op1, train_op2)
init_op = tf.global_variables_initializer()
self.sess.run(init_op)
def train_model(train_trees, val_trees, labels, embeddings, embedding_lookup,
opt):
max_acc = 0.0
logdir = opt.model_path
batch_size = opt.train_batch_size
epochs = opt.niter
num_feats = len(embeddings[0])
random.shuffle(train_trees)
nodes_node, children_node, codecaps_node = network.init_net_treecaps(
num_feats, len(labels))
codecaps_node = tf.identity(codecaps_node, name="codecaps_node")
out_node = network.out_layer(codecaps_node)
labels_node, loss_node = network.loss_layer(codecaps_node, len(labels))
optimizer = RAdamOptimizer(opt.lr)
train_step = optimizer.minimize(loss_node)
### init the graph
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
with tf.name_scope('saver'):
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(logdir)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
checkfile = os.path.join(logdir, 'tree_network.ckpt')
print("Begin training..........")
num_batches = len(train_trees) // batch_size + (
1 if len(train_trees) % batch_size != 0 else 0)
for epoch in range(1, epochs + 1):
bar = progressbar.ProgressBar(maxval=len(train_trees),
widgets=[
progressbar.Bar('=', '[', ']'), ' ',
progressbar.Percentage()
])
bar.start()
for i, batch in enumerate(
sampling.batch_samples(
sampling.gen_samples(train_trees, labels, embeddings,
embedding_lookup), batch_size)):
nodes, children, batch_labels = batch
step = (epoch - 1) * num_batches + i * batch_size
if not nodes:
continue
_, err, out = sess.run(
[train_step, loss_node, out_node],
feed_dict={
nodes_node: nodes,
children_node: children,
labels_node: batch_labels
})
bar.update(i + 1)
bar.finish()
correct_labels = []
predictions = []
logits = []
for batch in sampling.batch_samples(
sampling.gen_samples(val_trees, labels, embeddings,
embedding_lookup), 1):
nodes, children, batch_labels = batch
output = sess.run([out_node],
feed_dict={
nodes_node: nodes,
children_node: children
})
correct_labels.append(np.argmax(batch_labels))
predictions.append(np.argmax(output))
logits.append(output)
target_names = list(labels)
acc = accuracy_score(correct_labels, predictions)
if (acc > max_acc):
max_acc = acc
saver.save(sess, checkfile)
np.save(opt.model_path + '/logits', np.array(logits))
np.save(opt.model_path + '/correct', np.array(correct_labels))
print('Epoch', str(epoch), 'Accuracy:', acc, 'Max Acc: ', max_acc)
csv_log.write(str(epoch) + ',' + str(acc) + ',' + str(max_acc) + '\n')
print("Finish all iters, storring the whole model..........")
def train_model(train_trees, val_trees, labels, embedding_lookup, opt):
max_acc = 0.0
logdir = opt.model_path
batch_size = opt.train_batch_size
epochs = opt.niter
random.shuffle(train_trees)
nodes_node, children_node, codecaps_node = network.init_net_treecaps(50, embedding_lookup, len(labels))
codecaps_node = tf.identity(codecaps_node, name="codecaps_node")
out_node = network.out_layer(codecaps_node)
labels_node, loss_node = network.loss_layer(codecaps_node, len(labels))
optimizer = RAdamOptimizer(opt.lr)
train_point = optimizer.minimize(loss_node)
### init the graph
sess = tf.Session()
sess.run(tf.global_variables_initializer())
# Initialize the variables (i.e. assign their default value)
init = tf.global_variables_initializer()
with tf.name_scope('saver'):
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(logdir)
if ckpt and ckpt.model_checkpoint_path:
print("Continue training with old model")
saver.restore(sess, ckpt.model_checkpoint_path)
for i, var in enumerate(saver._var_list):
print('Var {}: {}'.format(i, var))
checkfile = os.path.join(logdir, 'tree_network.ckpt')
print("Begin training..........")
num_batches = len(train_trees) // batch_size + (1 if len(train_trees) % batch_size != 0 else 0)
max_acc = 0.0
for epoch in range(1, epochs+1):
for train_step, train_batch in enumerate(sampling.batch_samples(
sampling.gen_samples(train_trees, labels), batch_size
)):
nodes, children, batch_labels = train_batch
# step = (epoch - 1) * num_batches + train_step * batch_size
if not nodes:
continue
_, err, out = sess.run(
[train_point, loss_node, out_node],
feed_dict={
nodes_node: nodes,
children_node: children,
labels_node: batch_labels
}
)
print("Epoch : ", str(epoch), "Step : ", train_step, "Loss : ", err, "Max Acc: ",max_acc)
if train_step % 1000 == 0 and train_step > 0:
correct_labels = []
predictions = []
# logits = []
for test_batch in sampling.batch_samples(
sampling.gen_samples(val_trees, labels), batch_size
):
print("---------------")
nodes, children, batch_labels = test_batch
print(batch_labels)
output = sess.run([out_node],
feed_dict={
nodes_node: nodes,
children_node: children
}
)
batch_correct_labels = np.argmax(batch_labels, axis=1)
batch_predictions = np.argmax(output[0], axis=1)
correct_labels.extend(batch_correct_labels)
predictions.extend(batch_predictions)
# logits.append(output)
print(batch_correct_labels)
print(batch_predictions)
acc = accuracy_score(correct_labels, predictions)
if (acc>max_acc):
max_acc = acc
saver.save(sess, checkfile)
print("Saved checkpoint....")
print('Epoch',str(epoch),'Accuracy:', acc, 'Max Acc: ',max_acc)
csv_log.write(str(epoch)+','+str(acc)+','+str(max_acc)+'\n')
print("Finish all iters, storring the whole model..........")