def test_epoch(self):
self.logger.warn("Testing evaluation...")
scores_im1 = []
scores_im2 = []
scores_z1 = []
scores_z2 = []
inference_time = []
true_labels = []
summaries = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0,
scale=1.0,
size=[self.config.data_loader.test_batch, self.noise_dim])
feed_dict = {
self.model.image_input: test_batch,
self.model.noise_tensor: noise,
self.model.is_training: False,
}
scores_im1 += self.sess.run(self.model.img_score_l1,
feed_dict=feed_dict).tolist()
scores_im2 += self.sess.run(self.model.img_score_l2,
feed_dict=feed_dict).tolist()
scores_z1 += self.sess.run(self.model.z_score_l1,
feed_dict=feed_dict).tolist()
scores_z2 += self.sess.run(self.model.z_score_l2,
feed_dict=feed_dict).tolist()
summaries += self.sess.run([self.model.sum_op_im_test],
feed_dict=feed_dict)
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=summaries,
summarizer="test")
scores_im1 = np.asarray(scores_im1)
scores_im2 = np.asarray(scores_im2)
scores_z1 = np.asarray(scores_z1)
scores_z2 = np.asarray(scores_z2)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
save_results(
self.config.log.result_dir,
scores_im1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_im2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_z1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"z1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_z2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"z2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
def train_and_test(dataset, nb_epochs, weight, degree, random_seed, label,
enable_sm, score_method, enable_early_stop):
""" Runs the AnoGAN on the specified dataset
Note:
Saves summaries on tensorboard. To display them, please use cmd line
tensorboard --logdir=model.training_logdir() --port=number
Args:
dataset (str): name of the dataset
nb_epochs (int): number of epochs
weight (float): weight in the inverting loss function
degree (int): degree of the norm in the feature matching
random_seed (int): trying different seeds for averaging the results
label (int): label which is normal for image experiments
enable_sm (bool): allow TF summaries for monitoring the training
score_method (str): which metric to use for the ablation study
enable_early_stop (bool): allow early stopping for determining the number of epochs
"""
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
logger = logging.getLogger("AnoGAN.run.{}.{}".format(dataset, label))
# Import model and data
network = importlib.import_module('anogan.{}_utilities'.format(dataset))
data = importlib.import_module("data.{}".format(dataset))
# Parameters
starting_lr = network.learning_rate
batch_size = network.batch_size
latent_dim = network.latent_dim
ema_decay = 0.999
# Data
logger.info('Data loading...')
trainx, trainy = data.get_train(label)
if enable_early_stop:
validx, validy = data.get_valid(label)
nr_batches_valid = int(validx.shape[0] / batch_size)
trainx_copy = trainx.copy()
testx, testy = data.get_test(label)
global_step = tf.Variable(0, name='global_step', trainable=False)
# Placeholders
x_pl = tf.placeholder(tf.float32,
shape=(None, trainx.shape[1]),
name="input")
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl")
rng = np.random.RandomState(random_seed)
nr_batches_train = int(trainx.shape[0] / batch_size)
nr_batches_test = int(testx.shape[0] / batch_size)
logger.info('Building graph...')
logger.warn("The GAN is training with the following parameters:")
display_parameters(batch_size, starting_lr, ema_decay, weight, degree,
label, trainx.shape[1])
gen = network.generator
dis = network.discriminator
# Sample noise from random normal distribution
random_z = tf.random_normal([batch_size, latent_dim],
mean=0.0,
stddev=1.0,
name='random_z')
# Generate images with generator
x_gen = gen(x_pl, random_z, is_training=is_training_pl)
real_d, inter_layer_real = dis(x_pl, is_training=is_training_pl)
fake_d, inter_layer_fake = dis(x_gen,
is_training=is_training_pl,
reuse=True)
with tf.name_scope('loss_functions'):
# Calculate seperate losses for discriminator with real and fake images
real_discriminator_loss = tf.losses.sigmoid_cross_entropy(
tf.ones_like(real_d), real_d, scope='real_discriminator_loss')
fake_discriminator_loss = tf.losses.sigmoid_cross_entropy(
tf.zeros_like(fake_d), fake_d, scope='fake_discriminator_loss')
# Add discriminator losses
loss_discriminator = real_discriminator_loss + fake_discriminator_loss
# Calculate loss for generator by flipping label on discriminator output
loss_generator = tf.losses.sigmoid_cross_entropy(
tf.ones_like(fake_d), fake_d, scope='generator_loss')
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
dvars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='discriminator')
gvars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='generator')
update_ops_gen = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='generator')
update_ops_dis = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='discriminator')
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5)
with tf.control_dependencies(update_ops_gen):
gen_op = optimizer.minimize(loss_generator,
var_list=gvars,
global_step=global_step)
with tf.control_dependencies(update_ops_dis):
dis_op = optimizer.minimize(loss_discriminator, var_list=dvars)
# Exponential Moving Average for inference
def train_op_with_ema_dependency(vars, op):
ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
maintain_averages_op = ema.apply(vars)
with tf.control_dependencies([op]):
train_op = tf.group(maintain_averages_op)
return train_op, ema
train_gen_op, gen_ema = train_op_with_ema_dependency(gvars, gen_op)
train_dis_op, dis_ema = train_op_with_ema_dependency(dvars, dis_op)
### Testing ###
with tf.variable_scope("latent_variable"):
z_optim = tf.get_variable(
name='z_optim',
shape=[batch_size, latent_dim],
initializer=tf.truncated_normal_initializer())
reinit_z = z_optim.initializer
# EMA
x_gen_ema = gen(x_pl,
random_z,
is_training=is_training_pl,
getter=get_getter(gen_ema),
reuse=True)
rec_x_ema = gen(x_pl,
z_optim,
is_training=is_training_pl,
getter=get_getter(gen_ema),
reuse=True)
# Pass real and fake images into discriminator separately
real_d_ema, inter_layer_real_ema = dis(x_pl,
is_training=is_training_pl,
getter=get_getter(gen_ema),
reuse=True)
fake_d_ema, inter_layer_fake_ema = dis(rec_x_ema,
is_training=is_training_pl,
getter=get_getter(gen_ema),
reuse=True)
with tf.name_scope('Testing'):
with tf.variable_scope('Reconstruction_loss'):
delta = x_pl - rec_x_ema
delta_flat = tf.contrib.layers.flatten(delta)
reconstruction_score = tf.norm(delta_flat,
ord=degree,
axis=1,
keep_dims=False,
name='epsilon')
with tf.variable_scope('Discriminator_scores'):
if score_method == 'cross-e':
dis_score = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(fake_d_ema), logits=fake_d_ema)
else:
fm = inter_layer_real_ema - inter_layer_fake_ema
fm = tf.contrib.layers.flatten(fm)
dis_score = tf.norm(fm,
ord=degree,
axis=1,
keep_dims=False,
name='d_loss')
dis_score = tf.squeeze(dis_score)
with tf.variable_scope('Score'):
loss_invert = weight * reconstruction_score \
+ (1 - weight) * dis_score
rec_error_valid = tf.reduce_mean(loss_invert)
with tf.variable_scope("Test_learning_rate"):
step_lr = tf.Variable(0, trainable=False)
learning_rate_invert = 0.001
reinit_lr = tf.variables_initializer(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope="Test_learning_rate"))
with tf.name_scope('Test_optimizer'):
invert_op = tf.train.AdamOptimizer(learning_rate_invert).\
minimize(loss_invert,global_step=step_lr, var_list=[z_optim],
name='optimizer')
reinit_optim = tf.variables_initializer(
tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='Test_optimizer'))
reinit_test_graph_op = [reinit_z, reinit_lr, reinit_optim]
with tf.name_scope("Scores"):
list_scores = loss_invert
if enable_sm:
with tf.name_scope('training_summary'):
with tf.name_scope('dis_summary'):
tf.summary.scalar('real_discriminator_loss',
real_discriminator_loss, ['dis'])
tf.summary.scalar('fake_discriminator_loss',
fake_discriminator_loss, ['dis'])
tf.summary.scalar('discriminator_loss', loss_discriminator,
['dis'])
with tf.name_scope('gen_summary'):
tf.summary.scalar('loss_generator', loss_generator, ['gen'])
with tf.name_scope('img_summary'):
heatmap_pl_latent = tf.placeholder(tf.float32,
shape=(1, 480, 640, 3),
name="heatmap_pl_latent")
sum_op_latent = tf.summary.image('heatmap_latent',
heatmap_pl_latent)
with tf.name_scope('validation_summary'):
tf.summary.scalar('valid', rec_error_valid, ['v'])
if dataset in IMAGES_DATASETS:
with tf.name_scope('image_summary'):
tf.summary.image('reconstruct', x_gen, 8, ['image'])
tf.summary.image('input_images', x_pl, 8, ['image'])
else:
heatmap_pl_rec = tf.placeholder(tf.float32,
shape=(1, 480, 640, 3),
name="heatmap_pl_rec")
with tf.name_scope('image_summary'):
tf.summary.image('heatmap_rec', heatmap_pl_rec, 1,
['image'])
sum_op_dis = tf.summary.merge_all('dis')
sum_op_gen = tf.summary.merge_all('gen')
sum_op = tf.summary.merge([sum_op_dis, sum_op_gen])
sum_op_im = tf.summary.merge_all('image')
sum_op_valid = tf.summary.merge_all('v')
logdir = create_logdir(dataset, weight, label, random_seed)
sv = tf.train.Supervisor(logdir=logdir,
save_summaries_secs=None,
save_model_secs=None)
logger.info('Start training...')
with sv.managed_session(config=config) as sess:
logger.info('Initialization done')
writer = tf.summary.FileWriter(logdir, sess.graph)
train_batch = 0
epoch = 0
best_valid_loss = 0
while not sv.should_stop() and epoch < nb_epochs:
lr = starting_lr
begin = time.time()
trainx = trainx[rng.permutation(
trainx.shape[0])] # shuffling unl dataset
trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])]
train_loss_dis, train_loss_gen = [0, 0]
# training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
# train discriminator
feed_dict = {
x_pl: trainx[ran_from:ran_to],
is_training_pl: True,
learning_rate: lr
}
_, ld, step = sess.run(
[train_dis_op, loss_discriminator, global_step],
feed_dict=feed_dict)
train_loss_dis += ld
# train generator
feed_dict = {
x_pl: trainx_copy[ran_from:ran_to],
is_training_pl: True,
learning_rate: lr
}
_, lg = sess.run([train_gen_op, loss_generator],
feed_dict=feed_dict)
train_loss_gen += lg
if enable_sm:
sm = sess.run(sum_op, feed_dict=feed_dict)
writer.add_summary(sm, step)
if t % FREQ_PRINT == 0: # inspect reconstruction
# t = np.random.randint(0,400)
# ran_from = t
# ran_to = t + batch_size
# sm = sess.run(sum_op_im, feed_dict={x_pl: trainx[ran_from:ran_to],is_training_pl: False})
# writer.add_summary(sm, train_batch)
# data = sess.run(z_gen, feed_dict={
# x_pl: trainx[ran_from:ran_to],
# is_training_pl: False})
# data = np.expand_dims(heatmap(data), axis=0)
# sml = sess.run(sum_op_latent, feed_dict={
# heatmap_pl_latent: data,
# is_training_pl: False})
#
# writer.add_summary(sml, train_batch)
if dataset in IMAGES_DATASETS:
sm = sess.run(sum_op_im,
feed_dict={
x_pl: trainx[ran_from:ran_to],
is_training_pl: False
})
#
# else:
# data = sess.run(z_gen, feed_dict={
# x_pl: trainx[ran_from:ran_to],
# z_pl: np.random.normal(
# size=[batch_size, latent_dim]),
# is_training_pl: False})
# data = np.expand_dims(heatmap(data), axis=0)
# sm = sess.run(sum_op_im, feed_dict={
# heatmap_pl_rec: data,
# is_training_pl: False})
writer.add_summary(sm, step) #train_batch)
train_batch += 1
train_loss_gen /= nr_batches_train
train_loss_dis /= nr_batches_train
# logger.info('Epoch terminated')
# print("Epoch %d | time = %ds | loss gen = %.4f | loss dis = %.4f "
# % (epoch, time.time() - begin, train_loss_gen, train_loss_dis))
##EARLY STOPPING
if (epoch + 1) % FREQ_EV == 0 and enable_early_stop:
logger.info('Validation...')
inds = rng.permutation(validx.shape[0])
validx = validx[inds] # shuffling dataset
validy = validy[inds] # shuffling dataset
valid_loss = 0
# Create scores
for t in range(nr_batches_valid):
# construct randomly permuted minibatches
display_progression_epoch(t, nr_batches_valid)
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
feed_dict = {
x_pl: validx[ran_from:ran_to],
is_training_pl: False
}
for _ in range(STEPS_NUMBER):
_ = sess.run(invert_op, feed_dict=feed_dict)
vl = sess.run(rec_error_valid, feed_dict=feed_dict)
valid_loss += vl
sess.run(reinit_test_graph_op)
valid_loss /= nr_batches_valid
sess.run(reinit_test_graph_op)
if enable_sm:
sm = sess.run(sum_op_valid, feed_dict=feed_dict)
writer.add_summary(sm, step) # train_batch)
logger.info('Validation: valid loss {:.4f}'.format(valid_loss))
if valid_loss < best_valid_loss or epoch == FREQ_EV - 1:
best_valid_loss = valid_loss
logger.info(
"Best model - valid loss = {:.4f} - saving...".format(
best_valid_loss))
sv.saver.save(sess,
logdir + '/model.ckpt',
global_step=step)
nb_without_improvements = 0
else:
nb_without_improvements += FREQ_EV
if nb_without_improvements > PATIENCE:
sv.request_stop()
logger.warning(
"Early stopping at epoch {} with weights from epoch {}"
.format(epoch, epoch - nb_without_improvements))
epoch += 1
logger.warn('Testing evaluation...')
step = sess.run(global_step)
sv.saver.save(sess, logdir + '/model.ckpt', global_step=step)
rect_x, rec_error, latent, scores = [], [], [], []
inference_time = []
# Create scores
for t in range(nr_batches_test):
# construct randomly permuted minibatches
display_progression_epoch(t, nr_batches_test)
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
begin_val_batch = time.time()
feed_dict = {x_pl: testx[ran_from:ran_to], is_training_pl: False}
for _ in range(STEPS_NUMBER):
_ = sess.run(invert_op, feed_dict=feed_dict)
brect_x, brec_error, bscores, blatent = sess.run(
[rec_x_ema, reconstruction_score, loss_invert, z_optim],
feed_dict=feed_dict)
rect_x.append(brect_x)
rec_error.append(brec_error)
scores.append(bscores)
latent.append(blatent)
sess.run(reinit_test_graph_op)
inference_time.append(time.time() - begin_val_batch)
logger.info('Testing : mean inference time is %.4f' %
(np.mean(inference_time)))
if testx.shape[0] % batch_size != 0:
batch, size = batch_fill(testx, batch_size)
feed_dict = {x_pl: batch, is_training_pl: False}
for _ in range(STEPS_NUMBER):
_ = sess.run(invert_op, feed_dict=feed_dict)
brect_x, brec_error, bscores, blatent = sess.run(
[rec_x_ema, reconstruction_score, loss_invert, z_optim],
feed_dict=feed_dict)
rect_x.append(brect_x[:size])
rec_error.append(brec_error[:size])
scores.append(bscores[:size])
latent.append(blatent[:size])
sess.run(reinit_test_graph_op)
rect_x = np.concatenate(rect_x, axis=0)
rec_error = np.concatenate(rec_error, axis=0)
scores = np.concatenate(scores, axis=0)
latent = np.concatenate(latent, axis=0)
save_results(scores, testy, 'anogan', dataset, score_method, weight,
label, random_seed)
def test_epoch(self):
self.logger.warn("Testing evaluation...")
scores_im1 = []
scores_im2 = []
scores_comb_im = []
scores_comb_z = []
scores_final_1 = []
scores_final_2 = []
scores_final_3 = []
summaries = []
if self.config.trainer.enable_disc_xx:
scores_final_4 = []
if self.config.trainer.enable_disc_zz:
scores_final_5 = []
scores_final_6 = []
inference_time = []
true_labels = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
factor_list = self.config.trainer.feature_match_weight_2
for f in factor_list:
scores_im1 = []
scores_im2 = []
scores_comb_im = []
scores_comb_z = []
scores_final_1 = []
scores_final_2 = []
scores_final_3 = []
inference_time = []
true_labels = []
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run([self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0, scale=1.0, size=[self.config.data_loader.test_batch, self.noise_dim]
)
feature_match1 = f
feature_match2 = self.config.trainer.feature_match_weight
feed_dict = {
self.model.image_input: test_batch,
self.model.noise_tensor: noise,
self.model.is_training_gen: False,
self.model.is_training_dis: False,
self.model.is_training_enc_g: False,
self.model.is_training_enc_r: False,
self.model.feature_match1 : feature_match1,
self.model.feature_match2 : feature_match2,
}
scores_im1 += self.sess.run(self.model.img_score_l1, feed_dict=feed_dict).tolist()
scores_im2 += self.sess.run(self.model.img_score_l2, feed_dict=feed_dict).tolist()
scores_comb_im += self.sess.run(self.model.score_comb_im, feed_dict=feed_dict).tolist()
scores_comb_z += self.sess.run(self.model.score_comb_z, feed_dict=feed_dict).tolist()
scores_final_1 += self.sess.run(self.model.final_score_1, feed_dict=feed_dict).tolist()
scores_final_2 += self.sess.run(self.model.final_score_2, feed_dict=feed_dict).tolist()
scores_final_3 += self.sess.run(self.model.final_score_3, feed_dict=feed_dict).tolist()
summaries += self.sess.run([self.model.sum_op_im_test], feed_dict=feed_dict)
if self.config.trainer.enable_disc_xx:
scores_final_4 += self.sess.run(
self.model.final_score_4, feed_dict=feed_dict
).tolist()
if self.config.trainer.enable_disc_zz:
# scores_final_5 += self.sess.run(
# self.model.final_score_5, feed_dict=feed_dict
# ).tolist()
scores_final_6 += self.sess.run(
self.model.final_score_6, feed_dict=feed_dict
).tolist()
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
self.summarizer.add_tensorboard(step=cur_epoch, summaries=summaries, summarizer="test")
scores_im1 = np.asarray(scores_im1)
scores_im2 = np.asarray(scores_im2)
scores_comb_im = np.asarray(scores_comb_im)
scores_comb_z = np.asarray(scores_comb_z)
scores_final_1 = np.asarray(scores_final_1)
scores_final_2 = np.asarray(scores_final_2)
scores_final_3 = np.asarray(scores_final_3)
if self.config.trainer.enable_disc_xx:
scores_final_4 = np.asarray(scores_final_4)
if self.config.trainer.enable_disc_zz:
# scores_final_5 = np.asarray(scores_final_5)
scores_final_6 = np.asarray(scores_final_6)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.logger.info("Testing: Mean inference time is {:4f}".format(inference_time))
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
postfix = "_2_{}".format(str(f))
save_results(
self.config.log.result_dir,
scores_im1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im1{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_im2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im2{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_comb_im,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"comb_im{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_comb_z,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"comb_z{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_final_1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"final_1{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_final_2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"final_2{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
save_results(
self.config.log.result_dir,
scores_final_3,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"final_3{}".format(postfix),
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
postfix=postfix
)
if self.config.trainer.enable_disc_xx:
save_results(
self.config.log.result_dir,
scores_final_4,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"final_4",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
if self.config.trainer.enable_disc_zz:
# save_results(
# self.config.log.result_dir,
# scores_final_5,
# true_labels,
# self.config.model.name,
# self.config.data_loader.dataset_name,
# "final_5",
# "paper",
# self.config.trainer.label,
# self.config.data_loader.random_seed,
# self.logger,
# step,
# percentile=percentiles,
# )
save_results(
self.config.log.result_dir,
scores_final_6,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"final_6",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
def test_epoch(self):
self.logger.warn("Testing evaluation...")
scores_1 = []
scores_2 = []
inference_time = []
true_labels = []
summaries = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0,
scale=1.0,
size=[self.config.data_loader.test_batch, self.noise_dim])
feed_dict = {
self.model.image_input: test_batch,
self.model.noise_tensor: noise,
self.model.is_training: False,
}
scores_1 += self.sess.run(self.model.list_scores_1,
feed_dict=feed_dict).tolist()
scores_2 += self.sess.run(self.model.list_scores_2,
feed_dict=feed_dict).tolist()
summaries += self.sess.run([self.model.sum_op_im_test],
feed_dict=feed_dict)
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
# Since the higher anomaly score indicates the anomalous one, and we inverted the labels to show that
# normal images are 0 meaning that contains no anomaly and anomalous images are 1 meaning that it contains
# an anomalous region, we first scale the scores and then invert them to match the scores
scores_1 = np.asarray(scores_1)
scores_2 = np.asarray(scores_2)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=summaries,
summarizer="test")
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
save_results(
self.config.log.result_dir,
scores_1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"fm_1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"fm_2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
def train_and_test(nb_epochs, weight, method, degree, random_seed):
""" Runs the Bigan on the KDD dataset
Note:
Saves summaries on tensorboard. To display them, please use cmd line
tensorboard --logdir=model.training_logdir() --port=number
Args:
nb_epochs (int): number of epochs
weight (float, optional): weight for the anomaly score composition
method (str, optional): 'fm' for ``Feature Matching`` or "cross-e"
for ``cross entropy``, "efm" etc.
anomalous_label (int): int in range 0 to 10, is the class/digit
which is considered outlier
"""
logger = logging.getLogger("BiGAN.train.kdd.{}".format(method))
# Placeholders
input_pl = tf.placeholder(tf.float32,
shape=data.get_shape_input(),
name="input")
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl")
# Data
trainx, trainy = data.get_train()
trainx_copy = trainx.copy()
testx, testy = data.get_test()
trainx_org, trainy_org = data.get_train_org()
print('samples:', trainx.shape, testx.shape, trainx_org.shape)
# Parameters
starting_lr = network.learning_rate
batch_size = network.batch_size
latent_dim = network.latent_dim
ema_decay = 0.9999
rng = np.random.RandomState(RANDOM_SEED)
nr_batches_train = int(trainx.shape[0] / batch_size)
nr_batches_test = int(testx.shape[0] / batch_size)
nr_batches_train_org = int(trainx_org.shape[0] / batch_size)
logger.info('Building training graph...')
logger.warn("The BiGAN is training with the following parameters:")
display_parameters(batch_size, starting_lr, ema_decay, weight, method,
degree)
gen = network.decoder
enc = network.encoder
dis = network.discriminator
with tf.variable_scope('encoder_model'):
z_gen = enc(input_pl, is_training=is_training_pl)
with tf.variable_scope('generator_model'):
z = tf.random_normal([batch_size, latent_dim])
x_gen = gen(z, is_training=is_training_pl)
with tf.variable_scope('discriminator_model'):
l_encoder, inter_layer_inp = dis(z_gen,
input_pl,
is_training=is_training_pl)
l_generator, inter_layer_rct = dis(z,
x_gen,
is_training=is_training_pl,
reuse=True)
with tf.name_scope('loss_functions'):
# discriminator
loss_dis_enc = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_encoder), logits=l_encoder))
loss_dis_gen = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.zeros_like(l_generator), logits=l_generator))
loss_discriminator = loss_dis_gen + loss_dis_enc
# generator
loss_generator = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_generator), logits=l_generator))
# encoder
loss_encoder = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.zeros_like(l_encoder), logits=l_encoder))
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
tvars = tf.trainable_variables()
dvars = [var for var in tvars if 'discriminator_model' in var.name]
gvars = [var for var in tvars if 'generator_model' in var.name]
evars = [var for var in tvars if 'encoder_model' in var.name]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops_gen = [
x for x in update_ops if ('generator_model' in x.name)
]
update_ops_enc = [x for x in update_ops if ('encoder_model' in x.name)]
update_ops_dis = [
x for x in update_ops if ('discriminator_model' in x.name)
]
optimizer_dis = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5,
name='dis_optimizer')
optimizer_gen = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5,
name='gen_optimizer')
optimizer_enc = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5,
name='enc_optimizer')
with tf.control_dependencies(update_ops_gen):
gen_op = optimizer_gen.minimize(loss_generator, var_list=gvars)
with tf.control_dependencies(update_ops_enc):
enc_op = optimizer_enc.minimize(loss_encoder, var_list=evars)
with tf.control_dependencies(update_ops_dis):
dis_op = optimizer_dis.minimize(loss_discriminator, var_list=dvars)
# Exponential Moving Average for estimation
dis_ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
maintain_averages_op_dis = dis_ema.apply(dvars)
with tf.control_dependencies([dis_op]):
train_dis_op = tf.group(maintain_averages_op_dis)
gen_ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
maintain_averages_op_gen = gen_ema.apply(gvars)
with tf.control_dependencies([gen_op]):
train_gen_op = tf.group(maintain_averages_op_gen)
enc_ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
maintain_averages_op_enc = enc_ema.apply(evars)
with tf.control_dependencies([enc_op]):
train_enc_op = tf.group(maintain_averages_op_enc)
with tf.name_scope('summary'):
with tf.name_scope('dis_summary'):
tf.summary.scalar('loss_discriminator', loss_discriminator,
['dis'])
tf.summary.scalar('loss_dis_encoder', loss_dis_enc, ['dis'])
tf.summary.scalar('loss_dis_gen', loss_dis_gen, ['dis'])
with tf.name_scope('gen_summary'):
tf.summary.scalar('loss_generator', loss_generator, ['gen'])
tf.summary.scalar('loss_encoder', loss_encoder, ['gen'])
sum_op_dis = tf.summary.merge_all('dis')
sum_op_gen = tf.summary.merge_all('gen')
logger.info('Building testing graph...')
with tf.variable_scope('encoder_model'):
z_gen_ema = enc(input_pl,
is_training=is_training_pl,
getter=get_getter(enc_ema),
reuse=True)
with tf.variable_scope('generator_model'):
reconstruct_ema = gen(z_gen_ema,
is_training=is_training_pl,
getter=get_getter(gen_ema),
reuse=True)
with tf.variable_scope('discriminator_model'):
l_encoder_ema, inter_layer_inp_ema = dis(z_gen_ema,
input_pl,
is_training=is_training_pl,
getter=get_getter(dis_ema),
reuse=True)
l_generator_ema, inter_layer_rct_ema = dis(z_gen_ema,
reconstruct_ema,
is_training=is_training_pl,
getter=get_getter(dis_ema),
reuse=True)
with tf.name_scope('Testing'):
with tf.variable_scope('Reconstruction_loss'):
delta = input_pl - reconstruct_ema
delta_flat = tf.contrib.layers.flatten(delta)
gen_score = tf.norm(delta_flat,
ord=degree,
axis=1,
keep_dims=False,
name='epsilon')
with tf.variable_scope('Discriminator_loss'):
if method == "cross-e":
dis_score = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_generator_ema),
logits=l_generator_ema)
elif method == "fm":
fm = inter_layer_inp_ema - inter_layer_rct_ema
fm = tf.contrib.layers.flatten(fm)
dis_score = tf.norm(fm,
ord=degree,
axis=1,
keep_dims=False,
name='d_loss')
dis_score = tf.squeeze(dis_score)
with tf.variable_scope('Score'):
list_scores = (1 - weight) * gen_score + weight * dis_score
logdir = create_logdir(weight, method, random_seed)
sv = tf.train.Supervisor(logdir=logdir,
save_summaries_secs=None,
save_model_secs=120)
logger.info('Start training...')
with sv.managed_session() as sess:
logger.info('Initialization done')
writer = tf.summary.FileWriter(logdir, sess.graph)
train_batch = 0
epoch = 0
while not sv.should_stop() and epoch < nb_epochs:
lr = starting_lr
begin = time.time()
# construct randomly permuted minibatches
trainx = trainx[rng.permutation(
trainx.shape[0])] # shuffling dataset
trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])]
train_loss_dis, train_loss_gen, train_loss_enc = [0, 0, 0]
# training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
# train discriminator
feed_dict = {
input_pl: trainx[ran_from:ran_to],
is_training_pl: True,
learning_rate: lr
}
_, ld, sm = sess.run(
[train_dis_op, loss_discriminator, sum_op_dis],
feed_dict=feed_dict)
train_loss_dis += ld
writer.add_summary(sm, train_batch)
# train generator and encoder
feed_dict = {
input_pl: trainx_copy[ran_from:ran_to],
is_training_pl: True,
learning_rate: lr
}
_, _, le, lg, sm = sess.run([
train_gen_op, train_enc_op, loss_encoder, loss_generator,
sum_op_gen
],
feed_dict=feed_dict)
train_loss_gen += lg
train_loss_enc += le
writer.add_summary(sm, train_batch)
train_batch += 1
train_loss_gen /= nr_batches_train
train_loss_enc /= nr_batches_train
train_loss_dis /= nr_batches_train
logger.info('Epoch terminated')
print(
"Epoch %d | time = %ds | loss gen = %.4f | loss enc = %.4f | loss dis = %.4f "
% (epoch, time.time() - begin, train_loss_gen, train_loss_enc,
train_loss_dis))
epoch += 1
logger.warn('Training evaluation...')
inds = rng.permutation(trainx_org.shape[0])
trainx_org = trainx_org[inds] # shuffling dataset
trainy_org = trainy_org[inds] # shuffling dataset
scores = []
inference_time = []
# Create scores
for t in range(nr_batches_train_org):
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
begin_val_batch = time.time()
feed_dict = {
input_pl: trainx_org[ran_from:ran_to],
is_training_pl: False
}
scores += sess.run(list_scores, feed_dict=feed_dict).tolist()
inference_time.append(time.time() - begin_val_batch)
logger.info('Testing : mean inference time is %.4f' %
(np.mean(inference_time)))
ran_from = nr_batches_train_org * batch_size
ran_to = (nr_batches_train_org + 1) * batch_size
size = trainx_org[ran_from:ran_to].shape[0]
fill = np.ones([batch_size - size, 70])
batch = np.concatenate([trainx_org[ran_from:ran_to], fill], axis=0)
feed_dict = {input_pl: batch, is_training_pl: False}
batch_score = sess.run(list_scores, feed_dict=feed_dict).tolist()
scores += batch_score[:size]
# Highest 80% are anomalous
per = np.percentile(scores, 80)
y_pred_org = scores.copy()
y_pred_org = np.array(y_pred_org)
inds = (y_pred_org < per)
inds_comp = (y_pred_org >= per)
y_pred_org[inds] = 0
y_pred_org[inds_comp] = 1
roc_auc = do_roc(scores, trainy_org, "train_", "Results/", True)
prc_auc = do_prc(scores, trainy_org, "train_", "Results/", True)
#prg_auc = do_prg(scores, trainy_org, "train_", "Results/", True)
precision, recall, f1, _ = precision_recall_fscore_support(
trainy_org, y_pred_org, average='binary')
print("Testing : Prec = %.4f | Rec = %.4f | F1 = %.4f " %
(precision, recall, f1))
save_results(scores,
trainy_org,
'bigan',
'sia_train_air_4',
'fm',
'0.5',
'1',
2018,
'outlier',
0.1,
step=-1)
logger.warn('Testing evaluation...')
inds = rng.permutation(testx.shape[0])
testx = testx[inds] # shuffling dataset
testy = testy[inds] # shuffling dataset
scores = []
inference_time = []
# Create scores
for t in range(nr_batches_test):
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
begin_val_batch = time.time()
feed_dict = {
input_pl: testx[ran_from:ran_to],
is_training_pl: False
}
scores += sess.run(list_scores, feed_dict=feed_dict).tolist()
inference_time.append(time.time() - begin_val_batch)
logger.info('Testing : mean inference time is %.4f' %
(np.mean(inference_time)))
ran_from = nr_batches_test * batch_size
ran_to = (nr_batches_test + 1) * batch_size
size = testx[ran_from:ran_to].shape[0]
fill = np.ones([batch_size - size, 70])
batch = np.concatenate([testx[ran_from:ran_to], fill], axis=0)
feed_dict = {input_pl: batch, is_training_pl: False}
batch_score = sess.run(list_scores, feed_dict=feed_dict).tolist()
scores += batch_score[:size]
# Highest 80% are anomalous
per = np.percentile(scores, 80)
y_pred = scores.copy()
y_pred = np.array(y_pred)
inds = (y_pred < per)
inds_comp = (y_pred >= per)
y_pred[inds] = 0
y_pred[inds_comp] = 1
roc_auc = do_roc(scores, testy, "test_", "Results/", True)
prc_auc = do_prc(scores, testy, "test_", "Results/", True)
#prg_auc = do_prg(scores, testy, "test_", "Results/", True)
precision, recall, f1, _ = precision_recall_fscore_support(
testy, y_pred, average='binary')
print("Testing : Prec = %.4f | Rec = %.4f | F1 = %.4f " %
(precision, recall, f1))
# SIA metrics calculation
TN_train, FN_train, FP_train, TP_train = my_confusion_matrix(
trainy_org, y_pred_org)
TN_test, FN_test, FP_test, TP_test = my_confusion_matrix(testy, y_pred)
overall_train, average_train, sens_train, spec_train, ppr_train = derive_metric(
TN_train, FN_train, FP_train, TP_train)
overall_test, average_test, sens_test, spec_test, ppr_test = derive_metric(
TN_test, FN_test, FP_test, TP_test)
print('Train Metrics:', overall_train, average_train, sens_train,
spec_train, ppr_train)
print('Test Metrics:', overall_test, average_test, sens_test,
spec_test, ppr_test)
save_results(scores,
testy,
'bigan',
'sia_test_air_4',
'fm',
'0.5',
'1',
2018,
'outlier',
0.1,
step=-1)
def train_and_test(dataset, nb_epochs, degree, random_seed, label,
allow_zz, enable_sm, score_method,
enable_early_stop, do_spectral_norm):
""" Runs the AliCE on the specified dataset
Note:
Saves summaries on tensorboard. To display them, please use cmd line
tensorboard --logdir=model.training_logdir() --port=number
Args:
dataset (str): name of the dataset
nb_epochs (int): number of epochs
degree (int): degree of the norm in the feature matching
random_seed (int): trying different seeds for averaging the results
label (int): label which is normal for image experiments
allow_zz (bool): allow the d_zz discriminator or not for ablation study
enable_sm (bool): allow TF summaries for monitoring the training
score_method (str): which metric to use for the ablation study
enable_early_stop (bool): allow early stopping for determining the number of epochs
do_spectral_norm (bool): allow spectral norm or not for ablation study
"""
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
logger = logging.getLogger("ALAD.run.{}.{}".format(
dataset, label))
# Import model and data
network = importlib.import_module('alad.{}_utilities'.format(dataset))
data = importlib.import_module("data.{}".format(dataset))
# Parameters
starting_lr = network.learning_rate
batch_size = network.batch_size
latent_dim = network.latent_dim
ema_decay = 0.999
global_step = tf.Variable(0, name='global_step', trainable=False)
# Placeholders
x_pl = tf.placeholder(tf.float32, shape=data.get_shape_input(),
name="input_x")
z_pl = tf.placeholder(tf.float32, shape=[None, latent_dim],
name="input_z")
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl")
# Data
# label 部分都已经转换为 0 normal数据 / 1 abnormal数据
logger.info('Data loading...')
trainx, trainy = data.get_train(label)
if enable_early_stop: validx, validy = data.get_valid(label)
trainx_copy = trainx.copy()
testx, testy = data.get_test(label)
rng = np.random.RandomState(random_seed)
nr_batches_train = int(trainx.shape[0] / batch_size)
nr_batches_test = int(testx.shape[0] / batch_size)
logger.info('Building graph...')
logger.warn("ALAD is training with the following parameters:")
display_parameters(batch_size, starting_lr, ema_decay, degree, label,
allow_zz, score_method, do_spectral_norm)
gen = network.decoder
enc = network.encoder
dis_xz = network.discriminator_xz
dis_xx = network.discriminator_xx
dis_zz = network.discriminator_zz
with tf.variable_scope('encoder_model'):
z_gen = enc(x_pl, is_training=is_training_pl,
do_spectral_norm=do_spectral_norm)
with tf.variable_scope('generator_model'):
x_gen = gen(z_pl, is_training=is_training_pl)
rec_x = gen(z_gen, is_training=is_training_pl, reuse=True)
with tf.variable_scope('encoder_model'):
rec_z = enc(x_gen, is_training=is_training_pl, reuse=True,
do_spectral_norm=do_spectral_norm)
with tf.variable_scope('discriminator_model_xz'):
l_encoder, inter_layer_inp_xz = dis_xz(x_pl, z_gen,
is_training=is_training_pl,
do_spectral_norm=do_spectral_norm)
l_generator, inter_layer_rct_xz = dis_xz(x_gen, z_pl,
is_training=is_training_pl,
reuse=True,
do_spectral_norm=do_spectral_norm)
with tf.variable_scope('discriminator_model_xx'):
x_logit_real, inter_layer_inp_xx = dis_xx(x_pl, x_pl,
is_training=is_training_pl,
do_spectral_norm=do_spectral_norm)
x_logit_fake, inter_layer_rct_xx = dis_xx(x_pl, rec_x, is_training=is_training_pl,
reuse=True, do_spectral_norm=do_spectral_norm)
with tf.variable_scope('discriminator_model_zz'):
z_logit_real, _ = dis_zz(z_pl, z_pl, is_training=is_training_pl,
do_spectral_norm=do_spectral_norm)
z_logit_fake, _ = dis_zz(z_pl, rec_z, is_training=is_training_pl,
reuse=True, do_spectral_norm=do_spectral_norm)
with tf.name_scope('loss_functions'):
# discriminator xz
loss_dis_enc = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_encoder),logits=l_encoder))
loss_dis_gen = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.zeros_like(l_generator),logits=l_generator))
dis_loss_xz = loss_dis_gen + loss_dis_enc
# discriminator xx
x_real_dis = tf.nn.sigmoid_cross_entropy_with_logits(
logits=x_logit_real, labels=tf.ones_like(x_logit_real))
x_fake_dis = tf.nn.sigmoid_cross_entropy_with_logits(
logits=x_logit_fake, labels=tf.zeros_like(x_logit_fake))
dis_loss_xx = tf.reduce_mean(x_real_dis + x_fake_dis)
# discriminator zz
z_real_dis = tf.nn.sigmoid_cross_entropy_with_logits(
logits=z_logit_real, labels=tf.ones_like(z_logit_real))
z_fake_dis = tf.nn.sigmoid_cross_entropy_with_logits(
logits=z_logit_fake, labels=tf.zeros_like(z_logit_fake))
dis_loss_zz = tf.reduce_mean(z_real_dis + z_fake_dis)
loss_discriminator = dis_loss_xz + dis_loss_xx + dis_loss_zz if \
allow_zz else dis_loss_xz + dis_loss_xx
# generator and encoder
gen_loss_xz = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_generator),logits=l_generator))
enc_loss_xz = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.zeros_like(l_encoder), logits=l_encoder))
x_real_gen = tf.nn.sigmoid_cross_entropy_with_logits(
logits=x_logit_real, labels=tf.zeros_like(x_logit_real))
x_fake_gen = tf.nn.sigmoid_cross_entropy_with_logits(
logits=x_logit_fake, labels=tf.ones_like(x_logit_fake))
z_real_gen = tf.nn.sigmoid_cross_entropy_with_logits(
logits=z_logit_real, labels=tf.zeros_like(z_logit_real))
z_fake_gen = tf.nn.sigmoid_cross_entropy_with_logits(
logits=z_logit_fake, labels=tf.ones_like(z_logit_fake))
cost_x = tf.reduce_mean(x_real_gen + x_fake_gen)
cost_z = tf.reduce_mean(z_real_gen + z_fake_gen)
cycle_consistency_loss = cost_x + cost_z if allow_zz else cost_x
loss_generator = gen_loss_xz + cycle_consistency_loss
loss_encoder = enc_loss_xz + cycle_consistency_loss
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
tvars = tf.trainable_variables()
dxzvars = [var for var in tvars if 'discriminator_model_xz' in var.name]
dxxvars = [var for var in tvars if 'discriminator_model_xx' in var.name]
dzzvars = [var for var in tvars if 'discriminator_model_zz' in var.name]
gvars = [var for var in tvars if 'generator_model' in var.name]
evars = [var for var in tvars if 'encoder_model' in var.name]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops_gen = [x for x in update_ops if ('generator_model' in x.name)]
update_ops_enc = [x for x in update_ops if ('encoder_model' in x.name)]
update_ops_dis_xz = [x for x in update_ops if
('discriminator_model_xz' in x.name)]
update_ops_dis_xx = [x for x in update_ops if
('discriminator_model_xx' in x.name)]
update_ops_dis_zz = [x for x in update_ops if
('discriminator_model_zz' in x.name)]
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5)
with tf.control_dependencies(update_ops_gen):
gen_op = optimizer.minimize(loss_generator, var_list=gvars,
global_step=global_step)
with tf.control_dependencies(update_ops_enc):
enc_op = optimizer.minimize(loss_encoder, var_list=evars)
with tf.control_dependencies(update_ops_dis_xz):
dis_op_xz = optimizer.minimize(dis_loss_xz, var_list=dxzvars)
with tf.control_dependencies(update_ops_dis_xx):
dis_op_xx = optimizer.minimize(dis_loss_xx, var_list=dxxvars)
with tf.control_dependencies(update_ops_dis_zz):
dis_op_zz = optimizer.minimize(dis_loss_zz, var_list=dzzvars)
# Exponential Moving Average for inference
def train_op_with_ema_dependency(vars, op):
ema = tf.train.ExponentialMovingAverage(decay=ema_decay)
maintain_averages_op = ema.apply(vars)
with tf.control_dependencies([op]):
train_op = tf.group(maintain_averages_op)
return train_op, ema
train_gen_op, gen_ema = train_op_with_ema_dependency(gvars, gen_op)
train_enc_op, enc_ema = train_op_with_ema_dependency(evars, enc_op)
train_dis_op_xz, xz_ema = train_op_with_ema_dependency(dxzvars,
dis_op_xz)
train_dis_op_xx, xx_ema = train_op_with_ema_dependency(dxxvars,
dis_op_xx)
train_dis_op_zz, zz_ema = train_op_with_ema_dependency(dzzvars,
dis_op_zz)
with tf.variable_scope('encoder_model'):
z_gen_ema = enc(x_pl, is_training=is_training_pl,
getter=get_getter(enc_ema), reuse=True,
do_spectral_norm=do_spectral_norm)
with tf.variable_scope('generator_model'):
rec_x_ema = gen(z_gen_ema, is_training=is_training_pl,
getter=get_getter(gen_ema), reuse=True)
x_gen_ema = gen(z_pl, is_training=is_training_pl,
getter=get_getter(gen_ema), reuse=True)
with tf.variable_scope('discriminator_model_xx'):
l_encoder_emaxx, inter_layer_inp_emaxx = dis_xx(x_pl, x_pl,
is_training=is_training_pl,
getter=get_getter(xx_ema),
reuse=True,
do_spectral_norm=do_spectral_norm)
l_generator_emaxx, inter_layer_rct_emaxx = dis_xx(x_pl, rec_x_ema,
is_training=is_training_pl,
getter=get_getter(
xx_ema),
reuse=True,
do_spectral_norm=do_spectral_norm)
with tf.name_scope('Testing'):
with tf.variable_scope('Scores'):
score_ch = tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(l_generator_emaxx),
logits=l_generator_emaxx)
score_ch = tf.squeeze(score_ch)
rec = x_pl - rec_x_ema
rec = tf.contrib.layers.flatten(rec)
score_l1 = tf.norm(rec, ord=1, axis=1,
keep_dims=False, name='d_loss')
score_l1 = tf.squeeze(score_l1)
rec = x_pl - rec_x_ema
rec = tf.contrib.layers.flatten(rec)
score_l2 = tf.norm(rec, ord=2, axis=1,
keep_dims=False, name='d_loss')
score_l2 = tf.squeeze(score_l2)
inter_layer_inp, inter_layer_rct = inter_layer_inp_emaxx, \
inter_layer_rct_emaxx
fm = inter_layer_inp - inter_layer_rct
fm = tf.contrib.layers.flatten(fm)
score_fm = tf.norm(fm, ord=degree, axis=1,
keep_dims=False, name='d_loss')
score_fm = tf.squeeze(score_fm)
if enable_early_stop:
rec_error_valid = tf.reduce_mean(score_fm)
if enable_sm:
with tf.name_scope('summary'):
with tf.name_scope('dis_summary'):
tf.summary.scalar('loss_discriminator', loss_discriminator, ['dis'])
tf.summary.scalar('loss_dis_encoder', loss_dis_enc, ['dis'])
tf.summary.scalar('loss_dis_gen', loss_dis_gen, ['dis'])
tf.summary.scalar('loss_dis_xz', dis_loss_xz, ['dis'])
tf.summary.scalar('loss_dis_xx', dis_loss_xx, ['dis'])
if allow_zz:
tf.summary.scalar('loss_dis_zz', dis_loss_zz, ['dis'])
with tf.name_scope('gen_summary'):
tf.summary.scalar('loss_generator', loss_generator, ['gen'])
tf.summary.scalar('loss_encoder', loss_encoder, ['gen'])
tf.summary.scalar('loss_encgen_dxx', cost_x, ['gen'])
if allow_zz:
tf.summary.scalar('loss_encgen_dzz', cost_z, ['gen'])
if enable_early_stop:
with tf.name_scope('validation_summary'):
tf.summary.scalar('valid', rec_error_valid, ['v'])
with tf.name_scope('img_summary'):
heatmap_pl_latent = tf.placeholder(tf.float32,
shape=(1, 480, 640, 3),
name="heatmap_pl_latent")
sum_op_latent = tf.summary.image('heatmap_latent', heatmap_pl_latent)
if dataset in IMAGES_DATASETS:
with tf.name_scope('image_summary'):
tf.summary.image('reconstruct', rec_x, 8, ['image'])
tf.summary.image('input_images', x_pl, 8, ['image'])
else:
heatmap_pl_rec = tf.placeholder(tf.float32, shape=(1, 480, 640, 3),
name="heatmap_pl_rec")
with tf.name_scope('image_summary'):
tf.summary.image('heatmap_rec', heatmap_pl_rec, 1, ['image'])
sum_op_dis = tf.summary.merge_all('dis')
sum_op_gen = tf.summary.merge_all('gen')
sum_op = tf.summary.merge([sum_op_dis, sum_op_gen])
sum_op_im = tf.summary.merge_all('image')
sum_op_valid = tf.summary.merge_all('v')
logdir = create_logdir(dataset, label, random_seed, allow_zz, score_method,
do_spectral_norm)
saver = tf.train.Saver(max_to_keep=2)
save_model_secs = None if enable_early_stop else 20
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=None, saver=saver, save_model_secs=save_model_secs)
logger.info('Start training...')
with sv.managed_session(config=config) as sess:
step = sess.run(global_step)
logger.info('Initialization done at step {}'.format(step/nr_batches_train))
writer = tf.summary.FileWriter(logdir, sess.graph)
train_batch = 0
epoch = 0
best_valid_loss = 0
request_stop = False
while not sv.should_stop() and epoch < nb_epochs:
lr = starting_lr
begin = time.time()
# construct randomly permuted minibatches
trainx = trainx[rng.permutation(trainx.shape[0])] # shuffling dataset
trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])]
train_loss_dis_xz, train_loss_dis_xx, train_loss_dis_zz, \
train_loss_dis, train_loss_gen, train_loss_enc = [0, 0, 0, 0, 0, 0]
# Training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
# train discriminator
feed_dict = {x_pl: trainx[ran_from:ran_to],
z_pl: np.random.normal(size=[batch_size, latent_dim]),
is_training_pl: True,
learning_rate:lr}
_, _, _, ld, ldxz, ldxx, ldzz, step = sess.run([train_dis_op_xz,
train_dis_op_xx,
train_dis_op_zz,
loss_discriminator,
dis_loss_xz,
dis_loss_xx,
dis_loss_zz,
global_step],
feed_dict=feed_dict)
train_loss_dis += ld
train_loss_dis_xz += ldxz
train_loss_dis_xx += ldxx
train_loss_dis_zz += ldzz
# train generator and encoder
feed_dict = {x_pl: trainx_copy[ran_from:ran_to],
z_pl: np.random.normal(size=[batch_size, latent_dim]),
is_training_pl: True,
learning_rate:lr}
_,_, le, lg = sess.run([train_gen_op,
train_enc_op,
loss_encoder,
loss_generator],
feed_dict=feed_dict)
train_loss_gen += lg
train_loss_enc += le
if enable_sm:
sm = sess.run(sum_op, feed_dict=feed_dict)
writer.add_summary(sm, step)
if t % FREQ_PRINT == 0 and dataset in IMAGES_DATASETS: # inspect reconstruction
t = np.random.randint(0, trainx.shape[0]-batch_size)
ran_from = t
ran_to = t + batch_size
feed_dict = {x_pl: trainx[ran_from:ran_to],
z_pl: np.random.normal(
size=[batch_size, latent_dim]),
is_training_pl: False}
sm = sess.run(sum_op_im, feed_dict=feed_dict)
writer.add_summary(sm, step)#train_batch)
train_batch += 1
train_loss_gen /= nr_batches_train
train_loss_enc /= nr_batches_train
train_loss_dis /= nr_batches_train
train_loss_dis_xz /= nr_batches_train
train_loss_dis_xx /= nr_batches_train
train_loss_dis_zz /= nr_batches_train
logger.info('Epoch terminated')
if allow_zz:
print("Epoch %d | time = %ds | loss gen = %.4f | loss enc = %.4f | "
"loss dis = %.4f | loss dis xz = %.4f | loss dis xx = %.4f | "
"loss dis zz = %.4f"
% (epoch, time.time() - begin, train_loss_gen,
train_loss_enc, train_loss_dis, train_loss_dis_xz,
train_loss_dis_xx, train_loss_dis_zz))
else:
print("Epoch %d | time = %ds | loss gen = %.4f | loss enc = %.4f | "
"loss dis = %.4f | loss dis xz = %.4f | loss dis xx = %.4f | "
% (epoch, time.time() - begin, train_loss_gen,
train_loss_enc, train_loss_dis, train_loss_dis_xz,
train_loss_dis_xx))
##EARLY STOPPING
if (epoch + 1) % FREQ_EV == 0 and enable_early_stop:
valid_loss = 0
feed_dict = {x_pl: validx,
z_pl: np.random.normal(size=[validx.shape[0], latent_dim]),
is_training_pl: False}
vl, lat = sess.run([rec_error_valid, rec_z], feed_dict=feed_dict)
valid_loss += vl
if enable_sm:
sm = sess.run(sum_op_valid, feed_dict=feed_dict)
writer.add_summary(sm, step) # train_batch)
logger.info('Validation: valid loss {:.4f}'.format(valid_loss))
if (valid_loss < best_valid_loss or epoch == FREQ_EV-1):
best_valid_loss = valid_loss
logger.info("Best model - valid loss = {:.4f} - saving...".format(best_valid_loss))
sv.saver.save(sess, logdir+'/model.ckpt', global_step=step)
nb_without_improvements = 0
else:
nb_without_improvements += FREQ_EV
if nb_without_improvements > PATIENCE:
sv.request_stop()
logger.warning(
"Early stopping at epoch {} with weights from epoch {}".format(
epoch, epoch - nb_without_improvements))
epoch += 1
sv.saver.save(sess, logdir+'/model.ckpt', global_step=step)
logger.warn('Testing evaluation...')
scores_ch = []
scores_l1 = []
scores_l2 = []
scores_fm = []
inference_time = []
# Create scores
for t in range(nr_batches_test):
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
begin_test_time_batch = time.time()
feed_dict = {x_pl: testx[ran_from:ran_to],
z_pl: np.random.normal(size=[batch_size, latent_dim]),
is_training_pl:False}
scores_ch += sess.run(score_ch, feed_dict=feed_dict).tolist()
scores_l1 += sess.run(score_l1, feed_dict=feed_dict).tolist()
scores_l2 += sess.run(score_l2, feed_dict=feed_dict).tolist()
scores_fm += sess.run(score_fm, feed_dict=feed_dict).tolist()
inference_time.append(time.time() - begin_test_time_batch)
inference_time = np.mean(inference_time)
logger.info('Testing : mean inference time is %.4f' % (inference_time))
if testx.shape[0] % batch_size != 0:
batch, size = batch_fill(testx, batch_size)
feed_dict = {x_pl: batch,
z_pl: np.random.normal(size=[batch_size, latent_dim]),
is_training_pl: False}
bscores_ch = sess.run(score_ch,feed_dict=feed_dict).tolist()
bscores_l1 = sess.run(score_l1,feed_dict=feed_dict).tolist()
bscores_l2 = sess.run(score_l2,feed_dict=feed_dict).tolist()
bscores_fm = sess.run(score_fm,feed_dict=feed_dict).tolist()
scores_ch += bscores_ch[:size]
scores_l1 += bscores_l1[:size]
scores_l2 += bscores_l2[:size]
scores_fm += bscores_fm[:size]
model = 'alad_sn{}_dzz{}'.format(do_spectral_norm, allow_zz)
save_results(scores_ch, testy, model, dataset, 'ch',
'dzzenabled{}'.format(allow_zz), label, random_seed, step)
save_results(scores_l1, testy, model, dataset, 'l1',
'dzzenabled{}'.format(allow_zz), label, random_seed, step)
save_results(scores_l2, testy, model, dataset, 'l2',
'dzzenabled{}'.format(allow_zz), label, random_seed, step)
save_results(scores_fm, testy, model, dataset, 'fm',
'dzzenabled{}'.format(allow_zz), label, random_seed, step)
def test_epoch(self):
# Evaluation for the testing
self.logger.info("Testing evaluation...")
scores_ch = []
scores_l1 = []
scores_l2 = []
scores_fm = []
inference_time = []
true_labels = []
summaries = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0,
scale=1.0,
size=[self.config.data_loader.test_batch, self.noise_dim])
feed_dict = {
self.model.image_tensor: test_batch,
self.model.noise_tensor: noise,
self.model.is_training: False,
}
scores_ch += self.sess.run(self.model.score_ch,
feed_dict=feed_dict).tolist()
scores_l1 += self.sess.run(self.model.score_l1,
feed_dict=feed_dict).tolist()
scores_l2 += self.sess.run(self.model.score_l2,
feed_dict=feed_dict).tolist()
scores_fm += self.sess.run(self.model.score_fm,
feed_dict=feed_dict).tolist()
summaries += self.sess.run([self.model.sum_op_im_test],
feed_dict=feed_dict)
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
scores_ch = np.asarray(scores_ch)
scores_l1 = np.asarray(scores_l1)
scores_l2 = np.asarray(scores_l2)
scores_fm = np.asarray(scores_fm)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=summaries,
summarizer="test")
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
# TODO BATCH FILL ?
model = "alad_sn{}_dzz{}".format(self.config.trainer.do_spectral_norm,
self.config.trainer.allow_zz)
random_seed = 42
label = 0
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
save_results(
self.config.log.result_dir,
scores_ch,
true_labels,
model,
self.config.data_loader.dataset_name,
"ch",
"dzzenabled{}".format(self.config.trainer.allow_zz),
label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_l1,
true_labels,
model,
self.config.data_loader.dataset_name,
"l1",
"dzzenabled{}".format(self.config.trainer.allow_zz),
label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_l2,
true_labels,
model,
self.config.data_loader.dataset_name,
"l2",
"dzzenabled{}".format(self.config.trainer.allow_zz),
label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_fm,
true_labels,
model,
self.config.data_loader.dataset_name,
"fm",
"dzzenabled{}".format(self.config.trainer.allow_zz),
label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
def train_and_test(dataset, nb_epochs, random_seed, label):
""" Runs DSEBM on available datasets
Note:
Saves summaries on tensorboard. To display them, please use cmd line
tensorboard --logdir=model.training_logdir() --port=number
Args:
dataset (string): dataset to run the model on
nb_epochs (int): number of epochs
random_seed (int): trying different seeds for averaging the results
label (int): label which is normal for image experiments
anomaly_type (string): "novelty" for 100% normal samples in the training set
"outlier" for a contamined training set
anomaly_proportion (float): if "outlier", anomaly proportion in the training set
"""
logger = logging.getLogger("DSEBM.run.{}.{}".format(dataset, label))
# Import model and data
network = importlib.import_module('dsebm.{}_utilities'.format(dataset))
data = importlib.import_module("data.{}".format(dataset))
# Parameters
starting_lr = network.learning_rate
batch_size = network.batch_size
# Placeholders
x_pl = tf.placeholder(tf.float32,
shape=data.get_shape_input(),
name="input")
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl")
#test
y_true = tf.placeholder(tf.int32, shape=[None], name="y_true")
logger.info('Building training graph...')
logger.warn("The DSEBM is training with the following parameters:")
display_parameters(batch_size, starting_lr, label)
net = network.network
global_step = tf.train.get_or_create_global_step()
noise = tf.random_normal(shape=tf.shape(x_pl),
mean=0.0,
stddev=1.,
dtype=tf.float32)
x_noise = x_pl + noise
with tf.variable_scope('network'):
b_prime_shape = list(data.get_shape_input())
b_prime_shape[0] = batch_size
b_prime = tf.get_variable(name='b_prime',
shape=b_prime_shape) #tf.shape(x_pl))
net_out = net(x_pl, is_training=is_training_pl)
net_out_noise = net(x_noise, is_training=is_training_pl, reuse=True)
with tf.name_scope('energies'):
energy = 0.5 * tf.reduce_sum(tf.square(x_pl - b_prime)) \
- tf.reduce_sum(net_out)
energy_noise = 0.5 * tf.reduce_sum(tf.square(x_noise - b_prime)) \
- tf.reduce_sum(net_out_noise)
with tf.name_scope('reconstructions'):
# reconstruction
grad = tf.gradients(energy, x_pl)
fx = x_pl - tf.gradients(energy, x_pl)
fx = tf.squeeze(fx, axis=0)
fx_noise = x_noise - tf.gradients(energy_noise, x_noise)
with tf.name_scope("loss_function"):
# DSEBM for images
if len(data.get_shape_input()) == 4:
loss = tf.reduce_mean(
tf.reduce_sum(tf.square(x_pl - fx_noise), axis=[1, 2, 3]))
# DSEBM for tabular data
else:
loss = tf.reduce_mean(tf.square(x_pl - fx_noise))
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
tvars = tf.trainable_variables()
netvars = [var for var in tvars if 'network' in var.name]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops_net = [x for x in update_ops if ('network' in x.name)]
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
name='optimizer')
with tf.control_dependencies(update_ops_net):
train_op = optimizer.minimize(loss,
var_list=netvars,
global_step=global_step)
with tf.variable_scope('Scores'):
with tf.name_scope('Energy_score'):
flat = tf.layers.flatten(x_pl - b_prime)
if len(data.get_shape_input()) == 4:
list_scores_energy = 0.5 * tf.reduce_sum(tf.square(flat), axis=1) \
- tf.reduce_sum(net_out, axis=[1, 2, 3])
else:
list_scores_energy = 0.5 * tf.reduce_sum(tf.square(flat), axis=1) \
- tf.reduce_sum(net_out, axis=1)
with tf.name_scope('Reconstruction_score'):
delta = x_pl - fx
delta_flat = tf.layers.flatten(delta)
list_scores_reconstruction = tf.norm(delta_flat,
ord=2,
axis=1,
keepdims=False,
name='reconstruction')
# with tf.name_scope('predictions'):
# # Highest 20% are anomalous
# if dataset=="kdd":
# per = tf.contrib.distributions.percentile(list_scores_energy, 80)
# else:
# per = tf.contrib.distributions.percentile(list_scores_energy, 95)
# y_pred = tf.greater_equal(list_scores_energy, per)
#
# #y_test_true = tf.cast(y_test_true, tf.float32)
# cm = tf.confusion_matrix(y_true, y_pred, num_classes=2)
# recall = cm[1,1]/(cm[1,0]+cm[1,1])
# precision = cm[1,1]/(cm[0,1]+cm[1,1])
# f1 = 2*precision*recall/(precision + recall)
with tf.name_scope('training_summary'):
tf.summary.scalar('score_matching_loss', loss, ['net'])
tf.summary.scalar('energy', energy, ['net'])
if dataset in IMAGES_DATASETS:
with tf.name_scope('image_summary'):
tf.summary.image('reconstruct', fx, 6, ['image'])
tf.summary.image('input_images', x_pl, 6, ['image'])
sum_op_im = tf.summary.merge_all('image')
sum_op_net = tf.summary.merge_all('net')
logdir = create_logdir(dataset, label, random_seed)
sv = tf.train.Supervisor(logdir=logdir + "/train",
save_summaries_secs=None,
save_model_secs=None)
# Data
logger.info('Data loading...')
trainx, trainy = data.get_train(label)
trainx_copy = trainx.copy()
if dataset in IMAGES_DATASETS: validx, validy = data.get_valid(label)
testx, testy = data.get_test(label)
rng = np.random.RandomState(RANDOM_SEED)
nr_batches_train = int(trainx.shape[0] / batch_size)
if dataset in IMAGES_DATASETS:
nr_batches_valid = int(validx.shape[0] / batch_size)
nr_batches_test = int(testx.shape[0] / batch_size)
logger.info("Train: {} samples in {} batches".format(
trainx.shape[0], nr_batches_train))
if dataset in IMAGES_DATASETS:
logger.info("Valid: {} samples in {} batches".format(
validx.shape[0], nr_batches_valid))
logger.info("Test: {} samples in {} batches".format(
testx.shape[0], nr_batches_test))
logger.info('Start training...')
with sv.managed_session() as sess:
logger.info('Initialization done')
train_writer = tf.summary.FileWriter(logdir + "/train", sess.graph)
valid_writer = tf.summary.FileWriter(logdir + "/valid", sess.graph)
train_batch = 0
epoch = 0
best_valid_loss = 0
train_losses = [0] * STRIP_EV
while not sv.should_stop() and epoch < nb_epochs:
lr = starting_lr
begin = time.time()
trainx = trainx[rng.permutation(
trainx.shape[0])] # shuffling unl dataset
trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])]
losses, energies = [0, 0]
# training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
# train the net
feed_dict = {
x_pl: trainx[ran_from:ran_to],
is_training_pl: True,
learning_rate: lr
}
_, ld, en, sm, step = sess.run(
[train_op, loss, energy, sum_op_net, global_step],
feed_dict=feed_dict)
losses += ld
energies += en
train_writer.add_summary(sm, step)
if t % FREQ_PRINT == 0 and dataset in IMAGES_DATASETS: # inspect reconstruction
t = np.random.randint(0, 40)
ran_from = t
ran_to = t + batch_size
sm = sess.run(sum_op_im,
feed_dict={
x_pl: trainx[ran_from:ran_to],
is_training_pl: False
})
train_writer.add_summary(sm, step)
train_batch += 1
losses /= nr_batches_train
energies /= nr_batches_train
# Remembering loss for early stopping
train_losses[epoch % STRIP_EV] = losses
logger.info('Epoch terminated')
print("Epoch %d | time = %ds | loss = %.4f | energy = %.4f " %
(epoch, time.time() - begin, losses, energies))
if (epoch + 1) % FREQ_SNAP == 0 and dataset in IMAGES_DATASETS:
print("Take a snap of the reconstructions...")
x = trainx[:batch_size]
feed_dict = {x_pl: x, is_training_pl: False}
rect_x = sess.run(fx, feed_dict=feed_dict)
nama_e_wa = "dsebm/reconstructions/{}/{}/" \
"{}_epoch{}".format(dataset,
label,
random_seed, epoch)
nb_imgs = 50
save_grid_plot(x[:nb_imgs], rect_x[:nb_imgs], nama_e_wa,
nb_imgs)
if (epoch + 1) % FREQ_EV == 0 and dataset in IMAGES_DATASETS:
logger.info("Validation")
inds = rng.permutation(validx.shape[0])
validx = validx[inds] # shuffling dataset
validy = validy[inds] # shuffling dataset
valid_loss = 0
for t in range(nr_batches_valid):
display_progression_epoch(t, nr_batches_valid)
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
# train the net
feed_dict = {
x_pl: validx[ran_from:ran_to],
y_true: validy[ran_from:ran_to],
is_training_pl: False
}
vl, sm, step = sess.run([loss, sum_op_net, global_step],
feed_dict=feed_dict)
valid_writer.add_summary(sm, step + t) #train_batch)
valid_loss += vl
valid_loss /= nr_batches_valid
# train the net
logger.info("Validation loss at step " + str(step) + ":" +
str(valid_loss))
##EARLY STOPPING
#UPDATE WEIGHTS
if valid_loss < best_valid_loss or epoch == FREQ_EV - 1:
best_valid_loss = valid_loss
logger.info("Best model - loss={} - saving...".format(
best_valid_loss))
sv.saver.save(sess,
logdir + '/train/model.ckpt',
global_step=step)
nb_without_improvements = 0
else:
nb_without_improvements += FREQ_EV
if nb_without_improvements > PATIENCE:
sv.request_stop()
logger.warning(
"Early stopping at epoch {} with weights from epoch {}"
.format(epoch, epoch - nb_without_improvements))
epoch += 1
logger.warn('Testing evaluation...')
step = sess.run(global_step)
scores_e = []
scores_r = []
inference_time = []
# Create scores
for t in range(nr_batches_test):
# construct randomly permuted minibatches
ran_from = t * batch_size
ran_to = (t + 1) * batch_size
begin_val_batch = time.time()
feed_dict = {x_pl: testx[ran_from:ran_to], is_training_pl: False}
scores_e += sess.run(list_scores_energy,
feed_dict=feed_dict).tolist()
scores_r += sess.run(list_scores_reconstruction,
feed_dict=feed_dict).tolist()
inference_time.append(time.time() - begin_val_batch)
logger.info('Testing : mean inference time is %.4f' %
(np.mean(inference_time)))
if testx.shape[0] % batch_size != 0:
batch, size = batch_fill(testx, batch_size)
feed_dict = {x_pl: batch, is_training_pl: False}
batch_score_e = sess.run(list_scores_energy,
feed_dict=feed_dict).tolist()
batch_score_r = sess.run(list_scores_reconstruction,
feed_dict=feed_dict).tolist()
scores_e += batch_score_e[:size]
scores_r += batch_score_r[:size]
save_results(scores_e, testy, 'dsebm', dataset, 'energy', "test",
label, random_seed, step)
save_results(scores_r, testy, 'dsebm', dataset, 'reconstruction',
"test", label, random_seed, step)
def test_epoch(self):
self.logger.warn("Testing evaluation...")
scores_im1 = []
scores_im2 = []
scores_comb = []
scores_mask1 = []
scores_mask2 = []
scores_pipe = []
scores_pipe_2 = []
inference_time = []
true_labels = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0,
scale=1.0,
size=[self.config.data_loader.test_batch, self.noise_dim])
feed_dict = {
self.model.image_input: test_batch,
self.model.noise_tensor: noise,
self.model.is_training_gen: False,
self.model.is_training_dis: False,
self.model.is_training_enc_g: False,
self.model.is_training_enc_r: False,
}
scores_im1 += self.sess.run(self.model.img_score_l1,
feed_dict=feed_dict).tolist()
scores_im2 += self.sess.run(self.model.img_score_l2,
feed_dict=feed_dict).tolist()
scores_comb += self.sess.run(self.model.score_comb,
feed_dict=feed_dict).tolist()
scores_mask1 += self.sess.run(self.model.mask_score_1,
feed_dict=feed_dict).tolist()
scores_mask2 += self.sess.run(self.model.mask_score_2,
feed_dict=feed_dict).tolist()
scores_pipe += self.sess.run(self.model.pipe_score,
feed_dict=feed_dict).tolist()
scores_pipe_2 += self.sess.run(self.model.pipe_score_2,
feed_dict=feed_dict).tolist()
if self.config.trainer.enable_disc_xx:
# scores_final_3 += self.sess.run(
# self.model.final_score_3, feed_dict=feed_dict
# ).tolist()
scores_final_4 += self.sess.run(self.model.final_score_4,
feed_dict=feed_dict).tolist()
if self.config.trainer.enable_disc_zz:
# scores_final_5 += self.sess.run(
# self.model.final_score_5, feed_dict=feed_dict
# ).tolist()
scores_final_6 += self.sess.run(self.model.final_score_6,
feed_dict=feed_dict).tolist()
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
scores_im1 = np.asarray(scores_im1)
scores_im2 = np.asarray(scores_im2)
scores_comb = np.asarray(scores_comb)
scores_pipe = np.asarray(scores_pipe)
scores_pipe_2 = np.asarray(scores_pipe_2)
scores_mask1 = np.asarray(scores_mask1)
scores_mask2 = np.asarray(scores_mask2)
if self.config.trainer.enable_disc_xx:
#scores_final_3 = np.asarray(scores_final_3)
scores_final_4 = np.asarray(scores_final_4)
if self.config.trainer.enable_disc_zz:
#scores_final_5 = np.asarray(scores_final_5)
scores_final_6 = np.asarray(scores_final_6)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
save_results(
self.config.log.result_dir,
scores_im1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_im2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"im2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_comb,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"comb",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_mask1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"mask_1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_mask2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"mask_2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_pipe,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"scores_pipe_1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_pipe_2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"scores_pipe_2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
def train_and_test(dataset, nb_epochs, K, l1, l2, label,
random_seed):
""" Runs the DAGMM on the specified dataset
Note:
Saves summaries on tensorboard. To display them, please use cmd line
tensorboard --logdir=model.training_logdir() --port=number
Args:
nb_epochs (int): number of epochs
weight (float, optional): weight for the anomaly score composition
anomalous_label (int): int in range 0 to 10, is the class/digit
which is considered outlier
"""
logger = logging.getLogger("DAGMM.train.{}.{}".format(dataset,label))
# Import model and data
model = importlib.import_module('dagmm.{}_utilities'.format(dataset))
data = importlib.import_module("data.{}".format(dataset))
# Parameters
starting_lr = model.params["learning_rate"]
batch_size = model.params["batch_size"]
if l1==-1: l1 = model.params["l1"]
if l2==-1: l2 = model.params["l2"]
if K==-1: K = model.params["K"]
# Placeholders
x_pl = tf.placeholder(tf.float32, data.get_shape_input())
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
learning_rate = tf.placeholder(tf.float32, shape=(), name="lr_pl")
logger.info('Building training graph...')
logger.warning("The DAGMM is training with the following parameters:")
display_parameters(batch_size, starting_lr, l1, l2,
label)
global_step = tf.Variable(0, name='global_step', trainable=False)
enc = model.encoder
dec = model.decoder
feat_ex = model.feature_extractor
est = model.estimator
#feature extraction for images
if model.params["is_image"] and not METHOD=="pca":
x_features = image_features.extract_features(x_pl)
else:
x_features = x_pl
n_features = x_features.shape[1]
with tf.variable_scope('encoder_model'):
z_c = enc(x_features, is_training=is_training_pl)
with tf.variable_scope('decoder_model'):
x_rec = dec(z_c, n_features, is_training=is_training_pl)
with tf.variable_scope('feature_extractor_model'):
x_flat = tf.layers.flatten(x_features)
x_rec_flat = tf.layers.flatten(x_rec)
z_r = feat_ex(x_flat, x_rec_flat)
z = tf.concat([z_c, z_r], axis=1)
with tf.variable_scope('estimator_model'):
gamma = est(z, K, is_training=is_training_pl)
with tf.variable_scope('gmm'):
energy, penalty = gmm.compute_energy_and_penalty(z, gamma, is_training_pl)
with tf.name_scope('loss_functions'):
# reconstruction error
rec_error = reconstruction_error(x_flat, x_rec_flat)
loss_rec = tf.reduce_mean(rec_error)
# probabilities to observe
loss_energy = tf.reduce_mean(energy)
# full loss
full_loss = loss_rec + l1*loss_energy + l2*penalty
with tf.name_scope('optimizer'):
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
beta1=0.5, name='dis_optimizer')
train_op = optimizer.minimize(full_loss, global_step=global_step)
with tf.name_scope('predictions'):
# Highest 20% are anomalous
if dataset=="kdd":
per = tf.contrib.distributions.percentile(energy, 80)
else:
per = tf.contrib.distributions.percentile(energy, 80)
y_pred = tf.greater_equal(energy, per)
with tf.name_scope('summary'):
with tf.name_scope('loss_summary'):
tf.summary.scalar('loss_rec', loss_rec, ['loss'])
tf.summary.scalar('mean_energy', loss_energy, ['loss'])
tf.summary.scalar('penalty', penalty, ['loss'])
tf.summary.scalar('full_loss', full_loss, ['loss'])
sum_op_loss = tf.summary.merge_all('loss')
# Data
logger.info('Data loading...')
trainx, trainy = data.get_train(label)
trainx_copy = trainx.copy()
testx, testy = data.get_test(label)
if model.params["is_image"] and METHOD=="pca":
logger.info('PCA...')
trainx = trainx.reshape([trainx.shape[0], -1])
testx = testx.reshape([testx.shape[0], -1])
trainx, testx = image_features.pca(trainx, testx, 20)
logger.info('Done')
rng = np.random.RandomState(RANDOM_SEED)
nr_batches_train = int(trainx.shape[0] / batch_size)
nr_batches_test = int(testx.shape[0] / batch_size)
logdir = create_logdir(dataset, K, l1, l2, label, random_seed)
sv = tf.train.Supervisor(logdir=logdir, save_summaries_secs=None,
save_model_secs=10)
logger.info('Start training...')
with sv.managed_session() as sess:
logger.info('Initialization done')
writer = tf.summary.FileWriter(logdir, sess.graph)
train_batch = 0
epoch = 0
while not sv.should_stop() and epoch < nb_epochs:
lr = starting_lr
begin = time.time()
# construct randomly permuted minibatches
trainx = trainx[rng.permutation(trainx.shape[0])] # shuffling dataset
trainx_copy = trainx_copy[rng.permutation(trainx.shape[0])]
train_loss_rec, train_loss = [0, 0]
# training
for t in range(nr_batches_train):
display_progression_epoch(t, nr_batches_train)
ran_from = t * batch_size
ran_to = ran_from + batch_size
feed_dict = {x_pl: trainx[ran_from:ran_to],
is_training_pl: True,
learning_rate:lr}
_, lrec, loss, sm, step = sess.run([train_op,
loss_rec,
full_loss,
sum_op_loss,
global_step],
feed_dict=feed_dict)
train_loss_rec += lrec
train_loss += loss
writer.add_summary(sm, step)#train_batch)
if np.isnan(loss):
logger.info("Loss is nan - Stopping")
break
train_batch += 1
if np.isnan(loss):
logger.info("Loss is nan - Stopping")
break
train_loss_rec /= nr_batches_train
train_loss /= nr_batches_train
logger.info('Epoch terminated')
print("Epoch %d | time = %ds | loss rec = %.4f "
"| loss = %.4f"
% (epoch, time.time() - begin, train_loss_rec,
train_loss))
epoch += 1
logger.warning('Testing evaluation...')
inds = rng.permutation(testx.shape[0])
##TESTING PER BATCHS
inference_time = []
scores = []
for t in range(nr_batches_test+1):
ran_from = t * batch_size
ran_to = min(ran_from + batch_size, testx.shape[0])
feed_dict = {x_pl: testx[ran_from:ran_to],
is_training_pl: False}
begin_val = time.time()
if l1>0:
scoresb, step = sess.run([energy, global_step], feed_dict=feed_dict)
else:
scoresb, step = sess.run([rec_error, global_step], feed_dict=feed_dict)
scores.append(scoresb)
inference_time.append(time.time() - begin_val)
scores = np.concatenate(scores, axis = 0)
logger.warning('Testing : inference time is %.4f' % (
np.mean(inference_time)))
#scores = np.array(scores)
save_results(scores, testy, 'dagmm/K{}'.format(K), dataset, None, str(l1)+"_"+str(l2), label,
random_seed, step)
def train_epoch(self):
begin = time()
# Attach the epoch loop to a variable
loop = tqdm(range(self.config.data_loader.num_iter_per_epoch))
# Define the lists for summaries and losses
gen_losses = []
disc_losses = []
disc_xz_losses = []
disc_xx_losses = []
disc_zz_losses = []
summaries = []
# Get the current epoch counter
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
image = self.data.image
for _ in loop:
loop.set_description("Epoch:{}".format(cur_epoch + 1))
loop.refresh() # to show immediately the update
sleep(0.01)
lg, ld, ldxz, ldxx, ldzz, sum_g, sum_d = self.train_step(
image, cur_epoch)
gen_losses.append(lg)
disc_losses.append(ld)
disc_xz_losses.append(ldxz)
disc_xx_losses.append(ldxx)
disc_zz_losses.append(ldzz)
summaries.append(sum_g)
summaries.append(sum_d)
self.logger.info("Epoch {} terminated".format(cur_epoch))
self.summarizer.add_tensorboard(step=cur_epoch, summaries=summaries)
# Check for reconstruction
if cur_epoch % self.config.log.frequency_test == 0:
image_eval = self.sess.run(image)
feed_dict = {
self.model.image_input: image_eval,
self.model.is_training: False
}
reconstruction = self.sess.run(self.model.sum_op_im,
feed_dict=feed_dict)
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=[reconstruction])
# Get the means of the loss values to display
gl_m = np.mean(gen_losses)
dl_m = np.mean(disc_losses)
dlxz_m = np.mean(disc_xz_losses)
dlxx_m = np.mean(disc_xx_losses)
dlzz_m = np.mean(disc_zz_losses)
if self.config.trainer.allow_zz:
self.logger.info(
"Epoch {} | time = {} | loss gen = {:4f} |"
"loss dis = {:4f} | loss dis xz = {:4f} | loss dis xx = {:4f} | "
"loss dis zz = {:4f}".format(cur_epoch,
time() - begin, gl_m, dl_m,
dlxz_m, dlxx_m, dlzz_m))
else:
self.logger.info(
"Epoch {} | time = {} | loss gen = {:4f} | "
"loss dis = {:4f} | loss dis xz = {:4f} | loss dis xx = {:4f} | "
.format(cur_epoch,
time() - begin, gl_m, dl_m, dlxz_m, dlxx_m))
# Save the model state
# self.model.save(self.sess)
if (
cur_epoch + 1
) % self.config.trainer.frequency_eval == 0 and self.config.trainer.enable_early_stop:
valid_loss = 0
image_valid = self.sess.run(self.data.valid_image)
feed_dict = {
self.model.image_input: image_valid,
self.model.is_training: False
}
vl = self.sess.run([self.model.rec_error_valid],
feed_dict=feed_dict)
valid_loss += vl[0]
if self.config.log.enable_summary:
sm = self.sess.run(self.model.sum_op_valid,
feed_dict=feed_dict)
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=[sm],
summarizer="valid")
self.logger.info(
"Validation: valid loss {:.4f}".format(valid_loss))
if (valid_loss < self.best_valid_loss
or cur_epoch == self.config.trainer.frequency_eval - 1):
self.best_valid_loss = valid_loss
self.logger.info(
"Best model - valid loss = {:.4f} - saving...".format(
self.best_valid_loss))
# Save the model state
self.model.save(self.sess)
self.nb_without_improvements = 0
else:
self.nb_without_improvements += self.config.trainer.frequency_eval
if self.nb_without_improvements > self.config.trainer.patience:
self.patience_lost = True
self.logger.warning(
"Early stopping at epoch {} with weights from epoch {}".
format(cur_epoch,
cur_epoch - self.nb_without_improvements))
self.logger.warn("Testing evaluation...")
scores = []
inference_time = []
true_labels = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
for _ in test_loop:
test_batch_begin = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
feed_dict = {
self.model.image_input: test_batch,
self.model.is_training: False
}
scores += self.sess.run(self.model.score,
feed_dict=feed_dict).tolist()
inference_time.append(time() - test_batch_begin)
true_labels += test_labels.tolist()
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
scores = np.asarray(scores)
scores_scaled = (scores - min(scores)) / (max(scores) - min(scores))
step = self.sess.run(self.model.global_step_tensor)
save_results(
self.config.log.result_dir,
scores_scaled,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"fm",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
)
def test_epoch(self):
# Evaluation for the testing
self.logger.info("Testing evaluation...")
rect_x, rec_error,rec_error2, latent, scores_1, scores_2 = [],[], [], [], [], []
inference_time = []
true_labels = []
summaries = []
# Create the scores
test_loop = tqdm(range(self.config.data_loader.num_iter_per_test))
cur_epoch = self.model.cur_epoch_tensor.eval(self.sess)
for _ in test_loop:
begin_val_batch = time()
test_batch, test_labels = self.sess.run(
[self.data.test_image, self.data.test_label])
test_loop.refresh() # to show immediately the update
sleep(0.01)
noise = np.random.normal(
loc=0.0,
scale=1.0,
size=[self.config.data_loader.test_batch, self.noise_dim])
feed_dict = {
self.model.image_input: test_batch,
self.model.noise_tensor: noise,
self.model.is_training: False,
}
for _ in range(self.config.trainer.steps_number):
_ = self.sess.run(self.model.invert_op, feed_dict=feed_dict)
brect_x, brec_error, brec_error2, bscores_1, bscores_2, blatent = self.sess.run(
[
self.model.rec_gen_ema,
self.model.reconstruction_score_1,
self.model.reconstruction_score_2,
self.model.loss_invert_1,
self.model.loss_invert_2,
self.model.z_optim,
],
feed_dict=feed_dict,
)
rect_x.append(brect_x)
rec_error.append(brec_error)
rec_error2.append(brec_error2)
scores_1.append(bscores_1)
scores_2.append(bscores_2)
latent.append(blatent)
self.sess.run(self.model.reinit_test_graph_op)
inference_time.append(time() - begin_val_batch)
true_labels += test_labels.tolist()
summaries += self.sess.run([self.model.sum_op_im_test],
feed_dict=feed_dict)
true_labels = np.asarray(true_labels)
inference_time = np.mean(inference_time)
self.summarizer.add_tensorboard(step=cur_epoch,
summaries=summaries,
summarizer="test")
self.logger.info(
"Testing: Mean inference time is {:4f}".format(inference_time))
rect_x = np.concatenate(rect_x, axis=0)
rec_error = np.concatenate(rec_error, axis=0)
rec_error2 = np.concatenate(rec_error2, axis=0)
scores_1 = np.concatenate(scores_1, axis=0)
scores_2 = np.concatenate(scores_2, axis=0)
latent = np.concatenate(latent, axis=0)
step = self.sess.run(self.model.global_step_tensor)
percentiles = np.asarray(self.config.trainer.percentiles)
save_results(
self.config.log.result_dir,
scores_1,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"fm_1",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
save_results(
self.config.log.result_dir,
scores_2,
true_labels,
self.config.model.name,
self.config.data_loader.dataset_name,
"fm_2",
"paper",
self.config.trainer.label,
self.config.data_loader.random_seed,
self.logger,
step,
percentile=percentiles,
)
