def initialize_variables(self, dataset):
self.all_layers, self.trainable_layers = (), ()
self.n_conv_layers = 0
self.n_dense_layers = 0
self.n_relu_layers = 0
self.n_leaky_relu_layers = 0
self.n_bn_layers = 0
self.n_norm_layers = 0
self.n_abs_layers = 0
self.n_maxpool_layers = 0
self.n_upscale_layers = 0
self.n_dropout_layers = 0
self.n_dimshuffle_layers = 0
self.n_reshape_layers = 0
self.R_init = 0
self.learning_rate_init = Cfg.learning_rate.get_value()
self.it = 0
self.clock = 0
self.pretrained = False # set to True after pretraining such that dictionary initialization mustn't be repeated
self.diag = {} # init an empty dictionary to hold diagnostics
self.log = Log(dataset_name=dataset)
self.ad_log = AD_Log()
self.dense_layers, self.conv_layers, = [], []
def __init__(self, loss, dataset, kernel, **kwargs):
# initialize
self.svm = None
self.cv_svm = None
self.loss = loss
self.kernel = kernel
self.K_train = None
self.K_val = None
self.K_test = None
self.nu = None
self.gamma = None
self.initialize_svm(loss, **kwargs)
# load dataset
load_dataset(self, dataset)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.val_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
self.rho = None
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
def __init__(self,
dataset,
n_estimators=100,
max_samples='auto',
contamination=0.1,
**kwargs):
# load dataset
load_dataset(self, dataset)
# initialize
self.isoForest = None
self.n_estimators = n_estimators
self.max_samples = max_samples
self.contamination = contamination
self.initialize_isoForest(seed=self.data.seed, **kwargs)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
def __init__(self, dataset, kernel, **kwargs):
# initialize
self.kde = None
self.kernel = kernel
self.bandwidth = None
self.initialize_kde(**kwargs)
# load dataset
load_dataset(self, dataset)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
class NeuralNet:
def __init__(self,
dataset,
use_weights=None,
pretrain=False,
profile=False):
"""
initialize instance
"""
# whether to enable profiling in Theano functions
self.profile = profile
self.initialize_variables(dataset)
# load dataset
load_dataset(self, dataset.lower(), pretrain)
if use_weights and not pretrain:
self.load_weights(use_weights)
def initialize_variables(self, dataset):
self.all_layers, self.trainable_layers = (), ()
self.n_conv_layers = 0
self.n_dense_layers = 0
self.n_relu_layers = 0
self.n_leaky_relu_layers = 0
self.n_bn_layers = 0
self.n_norm_layers = 0
self.n_abs_layers = 0
self.n_maxpool_layers = 0
self.n_upscale_layers = 0
self.n_dropout_layers = 0
self.n_dimshuffle_layers = 0
self.n_reshape_layers = 0
self.R_init = 0
self.learning_rate_init = Cfg.learning_rate.get_value()
self.it = 0
self.clock = 0
self.pretrained = False # set to True after pretraining such that dictionary initialization mustn't be repeated
self.diag = {} # init an empty dictionary to hold diagnostics
self.log = Log(dataset_name=dataset)
self.ad_log = AD_Log()
self.dense_layers, self.conv_layers, = [], []
def compile_updates(self):
""" create network from architecture given in modules (determined by dataset)
create Theano compiled functions
"""
opt.sgd.updates.create_update(self)
def compile_autoencoder(self):
"""
create network from autoencoder architecture (determined by dataset)
and compile Theano update functions.
"""
print("Compiling autoencoder...")
opt.sgd.updates.create_autoencoder(self)
print("Autoencoder compiled.")
def load_data(self, data_loader=None, pretrain=False):
self.data = data_loader()
if pretrain:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
elif Cfg.reconstruction_loss:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
elif Cfg.svdd_loss and Cfg.reconstruction_penalty:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
else:
self.data.build_architecture(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
def flush_data(self):
self.data._X_train = None
self.data._y_train = None
self.data._X_val = None
self.data._y_val = None
self.data._X_test = None
self.data._y_test = None
print("Data flushed from network.")
def next_layers(self, layer):
flag = False
for current_layer in self.all_layers:
if flag:
yield current_layer
if current_layer is layer:
flag = True
def previous_layers(self, layer):
flag = False
for current_layer in reversed(self.all_layers):
if flag:
yield current_layer
if current_layer is layer:
flag = True
def start_clock(self):
self.clock = time.time()
def stop_clock(self):
self.clocked = time.time() - self.clock
print("Total elapsed time: %g" % self.clocked)
def pretrain(self, solver, lr, n_epochs):
"""
pre-train weights with an autoencoder
"""
self.ae_solver = solver.lower()
self.ae_learning_rate = lr
self.ae_n_epochs = n_epochs
# set learning rate
lr_tmp = Cfg.learning_rate.get_value()
Cfg.learning_rate.set_value(Cfg.floatX(lr))
self.compile_autoencoder()
from opt.sgd.train import train_autoencoder
train_autoencoder(self)
# remove layer attributes, re-initialize network and reset learning rate
for layer in self.all_layers:
delattr(self, layer.name + "_layer")
self.initialize_variables(self.data.dataset_name)
Cfg.learning_rate.set_value(Cfg.floatX(lr_tmp))
self.pretrained = True # set to True that dictionary initialization mustn't be repeated
# load network architecture
if Cfg.svdd_loss and Cfg.reconstruction_penalty:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
else:
self.data.build_architecture(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
# load weights learned by autoencoder
self.load_weights(Cfg.xp_path + "/ae_pretrained_weights.p")
def train(self, solver, n_epochs=10, save_at=0, save_to=''):
self.solver = solver.lower()
self.ae_solver = solver.lower()
self.n_epochs = n_epochs
self.save_at = save_at
self.save_to = save_to
self.log['solver'] = self.solver
self.log['save_at'] = self.save_at
self.compile_updates()
from opt.sgd.train import train_network
self.start_clock()
train_network(self)
self.stop_clock()
# self.log.save_to_file()
def evaluate(self, solver):
# this could be simplified to only compiling the forwardpropagation...
self.solver = solver.lower() # needed for compiling backprop
self.compile_updates()
print("Evaluating network with current weights...")
self.initialize_diagnostics(1)
self.copy_parameters()
# perform forward passes on training, val, and test set
_, _ = performance(self, which_set='train', epoch=0, print_=True)
_, _ = performance(self, which_set='val', epoch=0, print_=True)
_, _ = performance(self, which_set='test', epoch=0, print_=True)
print("Evaluation on train, val, and test set completed.")
def log_results(self, filename=None):
"""
log the results relevant for anomaly detection
"""
self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
self.ad_log['train_time'] = self.train_time
self.ad_log['val_auc'] = self.diag['val']['auc'][-1]
self.ad_log['val_accuracy'] = self.diag['val']['acc'][-1]
self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
self.ad_log['test_time'] = self.test_time
self.ad_log.save_to_file(filename=filename)
def addInputLayer(self, **kwargs):
self.input_layer = InputLayer(name="input", **kwargs)
self.input_layer.inp_ndim = len(kwargs["shape"])
def addConvLayer(self, use_batch_norm=False, **kwargs):
"""
Add convolutional layer.
If batch norm flag is True, the convolutional layer
will be followed by a batch-normalization layer
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_conv_layers += 1
name = "conv%i" % self.n_conv_layers
new_layer = ConvLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
self.trainable_layers += (new_layer, )
if use_batch_norm:
self.n_bn_layers += 1
name = "bn%i" % self.n_bn_layers
self.all_layers += (BatchNorm(new_layer, name=name), )
def addDenseLayer(self, use_batch_norm=False, **kwargs):
"""
Add dense layer.
If batch norm flag is True, the dense layer
will be followed by a batch-normalization layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dense_layers += 1
name = "dense%i" % self.n_dense_layers
new_layer = DenseLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
self.trainable_layers += (new_layer, )
if use_batch_norm:
self.n_bn_layers += 1
name = "bn%i" % self.n_bn_layers
self.all_layers += (BatchNorm(new_layer, name=name), )
def addSigmoidLayer(self, **kwargs):
"""
Add sigmoid classification layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
new_layer = Sigmoid(input_layer, **kwargs)
self.all_layers += (new_layer, )
self.n_layers = len(self.all_layers)
def addSoftmaxLayer(self, **kwargs):
"""
Add softmax multi-class classification layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
new_layer = Softmax(input_layer, **kwargs)
self.all_layers += (new_layer, )
self.n_layers = len(self.all_layers)
def addNormLayer(self, **kwargs):
"""
Add layer which normalizes its input to length 1.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_norm_layers += 1
name = "norm%i" % self.n_norm_layers
new_layer = Norm(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addAbsLayer(self, **kwargs):
"""
Add layer which returns the absolute value of its input.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_abs_layers += 1
name = "abs%i" % self.n_abs_layers
new_layer = Abs(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addReLU(self, **kwargs):
"""
Add ReLU activation layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_relu_layers += 1
name = "relu%i" % self.n_relu_layers
new_layer = ReLU(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addLeakyReLU(self, **kwargs):
"""
Add leaky ReLU activation layer. (with leakiness=0.01)
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_leaky_relu_layers += 1
name = "leaky_relu%i" % self.n_leaky_relu_layers
new_layer = LeakyReLU(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addMaxPool(self, **kwargs):
"""
Add MaxPooling activation layer.
"""
input_layer = self.input_layer if not self.all_layers\
else self.all_layers[-1]
self.n_maxpool_layers += 1
name = "maxpool%i" % self.n_maxpool_layers
new_layer = MaxPool(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addUpscale(self, **kwargs):
"""
Add Upscaling activation layer.
"""
input_layer = self.input_layer if not self.all_layers\
else self.all_layers[-1]
self.n_upscale_layers += 1
name = "upscale%i" % self.n_upscale_layers
new_layer = Upscale(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addDropoutLayer(self, **kwargs):
"""
Add Dropout layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dropout_layers += 1
name = "dropout%i" % self.n_dropout_layers
new_layer = DropoutLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addDimshuffleLayer(self, **kwargs):
"""
Add Dimshuffle layer to reorder dimensions
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dimshuffle_layers += 1
name = "dimshuffle%i" % self.n_dimshuffle_layers
new_layer = Dimshuffle(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addReshapeLayer(self, **kwargs):
"""
Add reshape layer to reshape dimensions
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_reshape_layers += 1
name = "reshape%i" % self.n_reshape_layers
new_layer = Reshape(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def setFeatureLayer(self):
"""
sets the currently highest layer of the current network to be the code layer (compression layer)
"""
setattr(self, "feature_layer", self.all_layers[-1])
def dump_weights(self, filename=None, pretrain=False):
dump_weights(self, filename, pretrain=pretrain)
def load_weights(self, filename=None):
assert filename and os.path.exists(filename)
load_weights(self, filename)
def update_R(self):
"""
method to update R while leaving the network parameters and center c fixed in a block coordinate optimization
"""
print("Updating radius R...")
# Get updates
R = update_R(self.diag['train']['rep'],
self.cvar.get_value(),
solver=Cfg.R_update_solver,
scalar_method=Cfg.R_update_scalar_method,
lp_obj=Cfg.R_update_lp_obj)
# Update R
self.Rvar.set_value(Cfg.floatX(R))
print("Radius R updated.")
def update_R_c(self):
"""
method to update R and c while leaving the network parameters fixed in a block coordinate optimization
"""
print("Updating radius R and center c...")
# Get updates
R, c = update_R_c(self.diag['train']['rep'],
np.sum(self.diag['train']['rep']**2, axis=1),
solver=Cfg.QP_solver)
# Update values
self.Rvar.set_value(Cfg.floatX(R))
self.cvar.set_value(Cfg.floatX(c))
print("Radius R and center c updated.")
def initialize_diagnostics(self, n_epochs):
"""
initialize diagnostics for the neural network
"""
# data-dependent diagnostics for train, validation, and test set
self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_train)
self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_val)
self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_test)
# network parameter diagnostics
self.diag['network'] = NNetParamDiag(self, n_epochs=n_epochs)
# Best results (highest AUC on test set)
self.auc_best = 0
self.auc_best_epoch = 0 # determined by highest AUC on test set
self.best_weight_dict = None
def save_objective_and_accuracy(self, epoch, which_set, objective,
accuracy):
"""
save objective and accuracy of epoch
"""
self.diag[which_set]['objective'][epoch] = objective
self.diag[which_set]['acc'][epoch] = accuracy
def save_initial_parameters(self):
"""
save a copy of the initial network parameters for diagnostics.
"""
self.W_init = []
self.b_init = []
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.W_init.append(None)
self.b_init.append(None)
i = 0
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.W_init[i] = layer.W.get_value()
if layer.b is not None:
self.b_init[i] = layer.b.get_value()
i += 1
def copy_parameters(self):
"""
save a copy of the current network parameters in order to monitor the difference between epochs.
"""
i = 0
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.diag['network']['W_copy'][i] = layer.W.get_value()
if layer.b is not None:
self.diag['network']['b_copy'][i] = layer.b.get_value()
i += 1
def copy_initial_parameters_to_cache(self):
"""
Save a copy of the initial parameters in cache
"""
self.diag['network']['W_copy'] = list(self.W_init)
self.diag['network']['b_copy'] = list(self.b_init)
def save_network_diagnostics(self, epoch, l2, R):
"""
save diagnostics of the network
"""
self.diag['network']['l2_penalty'][epoch] = l2
self.log['l2_penalty'].append(float(l2))
i = 0
j = 0
for layer in self.trainable_layers:
if layer.isdense:
self.diag['network']['W_norms'][i][:, epoch] = np.sum(
layer.W.get_value()**2, axis=0)
if layer.b is not None:
self.diag['network']['b_norms'][
i][:, epoch] = layer.b.get_value()**2
i += 1
if layer.isdense | layer.isconv:
dW = np.sqrt(
np.sum((layer.W.get_value() -
self.diag['network']['W_copy'][j])**2))
self.diag['network']['dW_norms'][j][epoch] = dW
if layer.b is not None:
db = np.sqrt(
np.sum((layer.b.get_value() -
self.diag['network']['b_copy'][j])**2))
self.diag['network']['db_norms'][j][epoch] = db
j += 1
# diagnostics only relevant for the SVDD loss
if Cfg.svdd_loss:
self.diag['network']['R'][epoch] = R
self.diag['network']['c_norm'][epoch] = np.sqrt(
np.sum(self.cvar.get_value()**2))
def track_best_results(self, epoch):
"""
Save network parameters where AUC on the test set was highest.
"""
if self.diag['test']['auc'][epoch] > self.auc_best:
self.auc_best = self.diag['test']['auc'][epoch]
self.auc_best_epoch = epoch
self.best_weight_dict = dict()
for layer in self.trainable_layers:
self.best_weight_dict[layer.name + "_w"] = layer.W.get_value()
if layer.b is not None:
self.best_weight_dict[layer.name +
"_b"] = layer.b.get_value()
if Cfg.svdd_loss:
self.best_weight_dict["R"] = self.Rvar.get_value()
def dump_best_weights(self, filename):
"""
pickle the network parameters, where AUC on the test set was highest.
"""
with open(filename, 'wb') as f:
pickle.dump(self.best_weight_dict, f)
print("Parameters of best epoch saved in %s" % filename)
def save_diagnostics(self, which_set, epoch, scores, rep_norm, rep,
emp_loss, reconstruction_penalty):
"""
save diagnostics for which_set of epoch
"""
if self.data.n_classes == 2:
if which_set == 'train':
y = self.data._y_train
if which_set == 'val':
y = self.data._y_val
if which_set == 'test':
y = self.data._y_test
self.diag[which_set]['scores'][:, epoch] = scores
if sum(y) > 0:
AUC = roc_auc_score(y, scores)
self.diag[which_set]['auc'][epoch] = AUC
self.log[which_set + '_auc'].append(float(AUC))
print("{:32} {:.2f}%".format(which_set.title() + ' AUC:',
100. * AUC))
scores_normal = scores[y == 0]
scores_outlier = scores[y == 1]
normal_summary = get_five_number_summary(scores_normal)
outlier_summary = get_five_number_summary(scores_outlier)
self.log[which_set +
'_normal_scores_summary'].append(normal_summary)
self.log[which_set +
'_outlier_scores_summary'].append(outlier_summary)
self.diag[which_set]['rep'] = rep
self.diag[which_set]['rep_norm'][:, epoch] = rep_norm
rep_norm_normal = rep_norm[y == 0]
rep_norm_outlier = rep_norm[y == 1]
normal_summary = get_five_number_summary(rep_norm_normal)
outlier_summary = get_five_number_summary(rep_norm_outlier)
self.log[which_set +
'_normal_rep_norm_summary'].append(normal_summary)
self.log[which_set +
'_outlier_rep_norm_summary'].append(outlier_summary)
if Cfg.svdd_loss:
rep_mean = np.mean(rep, axis=0)
self.diag[which_set]['output_mean_norm'][epoch] = np.sqrt(
np.sum(rep_mean**2))
self.diag[which_set]['c_mean_diff'][epoch] = np.sqrt(
np.sum((rep_mean - self.cvar.get_value())**2))
self.diag[which_set]['reconstruction_penalty'][
epoch] = reconstruction_penalty
self.diag[which_set]['emp_loss'][epoch] = float(emp_loss)
self.log[which_set + '_emp_loss'].append(float(emp_loss))
def initialize_ae_diagnostics(self, n_epochs):
"""
initialize diagnostic variables for autoencoder network.
"""
self.train_time = 0
self.test_time = 0
self.best_weight_dict = None
# data-dependent diagnostics for train, validation, and test set
self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_train)
self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_val)
self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_test)
# network parameters
self.diag['network'] = {}
self.diag['network']['l2_penalty'] = np.zeros(n_epochs,
dtype=Cfg.floatX)
def save_ae_diagnostics(self, which_set, epoch, error, scores, l2):
"""
save autoencoder diagnostics for which_set
"""
self.diag[which_set]['objective'][epoch] = error + l2
self.diag[which_set]['emp_loss'][epoch] = error
self.diag[which_set]['scores'][:, epoch] = scores
self.diag['network']['l2_penalty'][epoch] = l2
if self.data.n_classes == 2:
if which_set == 'train':
y = self.data._y_train
if which_set == 'val':
y = self.data._y_val
if which_set == 'test':
y = self.data._y_test
if sum(y) > 0:
AUC = roc_auc_score(y, scores)
self.diag[which_set]['auc'][epoch] = AUC
class IsoForest(object):
def __init__(self,
dataset,
n_estimators=100,
max_samples='auto',
contamination=0.1,
**kwargs):
# load dataset
load_dataset(self, dataset)
# initialize
self.isoForest = None
self.n_estimators = n_estimators
self.max_samples = max_samples
self.contamination = contamination
self.initialize_isoForest(seed=self.data.seed, **kwargs)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
def initialize_isoForest(self, seed=0, **kwargs):
self.isoForest = IsolationForest(n_estimators=self.n_estimators,
max_samples=self.max_samples,
contamination=self.contamination,
n_jobs=-1,
random_state=seed,
**kwargs)
def load_data(self, data_loader=None, pretrain=False):
self.data = data_loader()
def start_clock(self):
self.clock = time.time()
def stop_clock(self):
self.clocked = time.time() - self.clock
print("Total elapsed time: %g" % self.clocked)
def train(self):
if self.data._X_train.ndim > 2:
X_train_shape = self.data._X_train.shape
X_train = self.data._X_train.reshape(X_train_shape[0], -1)
else:
X_train = self.data._X_train
print("Starting training...")
self.start_clock()
self.isoForest.fit(X_train.astype(np.float32))
self.stop_clock()
self.train_time = self.clocked
def predict(self, which_set='train'):
assert which_set in ('train', 'test')
if which_set == 'train':
X = self.data._X_train
y = self.data._y_train
if which_set == 'test':
X = self.data._X_test
y = self.data._y_test
# reshape to 2D if input is tensor
if X.ndim > 2:
X_shape = X.shape
X = X.reshape(X_shape[0], -1)
print("Starting prediction...")
self.start_clock()
scores = (-1.0) * self.isoForest.decision_function(
X.astype(np.float32)) # compute anomaly score
y_pred = (self.isoForest.predict(X.astype(np.float32))
== -1) * 1 # get prediction
self.diag[which_set]['scores'][:, 0] = scores.flatten()
self.diag[which_set]['acc'][0] = 100.0 * sum(y == y_pred) / len(y)
if sum(y) > 0:
auc = roc_auc_score(y, scores.flatten())
self.diag[which_set]['auc'][0] = auc
self.stop_clock()
if which_set == 'test':
self.test_time = self.clocked
def dump_model(self, filename=None):
dump_isoForest(self, filename)
def load_model(self, filename=None):
assert filename and os.path.exists(filename)
load_isoForest(self, filename)
def log_results(self, filename=None):
"""
log the results relevant for anomaly detection
"""
self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
self.ad_log['train_time'] = self.train_time
self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
self.ad_log['test_time'] = self.test_time
self.ad_log.save_to_file(filename=filename)
class SVM(object):
def __init__(self, loss, dataset, kernel, **kwargs):
# initialize
self.svm = None
self.cv_svm = None
self.loss = loss
self.kernel = kernel
self.K_train = None
self.K_val = None
self.K_test = None
self.nu = None
self.gamma = None
self.initialize_svm(loss, **kwargs)
# load dataset
load_dataset(self, dataset)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.val_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
self.rho = None
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
def initialize_svm(self, loss, **kwargs):
assert loss in ('SVC', 'OneClassSVM')
if self.kernel in ('linear', 'poly', 'rbf', 'sigmoid'):
kernel = self.kernel
else:
kernel = 'precomputed'
if loss == 'SVC':
self.svm = svm.SVC(kernel=kernel, C=Cfg.svm_C, **kwargs)
if loss == 'OneClassSVM':
self.svm = svm.OneClassSVM(kernel=kernel, nu=Cfg.svm_nu, **kwargs)
self.cv_svm = svm.OneClassSVM(kernel=kernel,
nu=Cfg.svm_nu,
**kwargs)
def load_data(self, data_loader=None, pretrain=False):
self.data = data_loader()
def flush_data(self):
self.data._X_train = None
self.data._y_train = None
self.data._X_val = None
self.data._y_val = None
self.data._X_test = None
self.data._y_test = None
print("Data flushed from model.")
def start_clock(self):
self.clock = time.time()
def stop_clock(self):
self.clocked = time.time() - self.clock
print("Total elapsed time: %g" % self.clocked)
def train(self, GridSearch=True, **kwargs):
if self.data._X_train.ndim > 2:
X_train_shape = self.data._X_train.shape
X_train = self.data._X_train.reshape(X_train_shape[0],
np.prod(X_train_shape[1:]))
else:
X_train = self.data._X_train
print("Starting training...")
self.start_clock()
if self.loss == 'SVC':
if self.kernel in ('DegreeKernel', 'WeightedDegreeKernel'):
self.get_kernel_matrix(kernel=self.kernel,
which_set='train',
**kwargs)
self.svm.fit(self.K_train, self.data._y_train)
else:
self.svm.fit(X_train, self.data._y_train)
if self.loss == 'OneClassSVM':
if self.kernel in ('DegreeKernel', 'WeightedDegreeKernel'):
self.get_kernel_matrix(kernel=self.kernel,
which_set='train',
**kwargs)
self.svm.fit(self.K_train)
else:
if GridSearch and self.kernel == 'rbf':
# use grid search cross-validation to select gamma
print("Using GridSearchCV for hyperparameter selection...")
# sample small hold-out set from test set for hyperparameter selection. Save as val set.
n_val_set = int(0.1 * self.data.n_test)
n_test_out = 0
n_test_norm = 0
n_val_out = 0
n_val_norm = 0
while (n_test_out == 0) | (n_test_norm == 0) | (
n_val_out == 0) | (n_val_norm == 0):
perm = np.random.permutation(self.data.n_test)
self.data._X_val = self.data._X_test[perm[:n_val_set]]
self.data._y_val = self.data._y_test[perm[:n_val_set]]
# only accept small test set if AUC can be computed on val and test set
n_test_out = np.sum(
self.data._y_test[perm[:n_val_set]])
n_test_norm = np.sum(
self.data._y_test[perm[:n_val_set]] == 0)
n_val_out = np.sum(self.data._y_test[perm[n_val_set:]])
n_val_norm = np.sum(
self.data._y_test[perm[n_val_set:]] == 0)
self.data._X_test = self.data._X_test[perm[n_val_set:]]
self.data._y_test = self.data._y_test[perm[n_val_set:]]
self.data.n_val = len(self.data._y_val)
self.data.n_test = len(self.data._y_test)
self.diag['val']['scores'] = np.zeros(
(len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros(
(len(self.data._y_test), 1))
cv_auc = 0.0
cv_acc = 0
for gamma in np.logspace(-10, -1, num=10, base=2):
# train on selected gamma
self.cv_svm = svm.OneClassSVM(kernel='rbf',
nu=Cfg.svm_nu,
gamma=gamma)
self.cv_svm.fit(X_train)
# predict on small hold-out set
self.predict(which_set='val')
# save model if AUC on hold-out set improved
if self.diag['val']['auc'] > cv_auc:
self.svm = self.cv_svm
self.nu = Cfg.svm_nu
self.gamma = gamma
cv_auc = self.diag['val']['auc']
cv_acc = self.diag['val']['acc']
# save results of best cv run
self.diag['val']['auc'] = cv_auc
self.diag['val']['acc'] = cv_acc
else:
# if rbf-kernel, re-initialize svm with gamma minimizing the
# numerical error
if self.kernel == 'rbf':
gamma = 1 / (np.max(pairwise_distances(X_train))**2)
self.svm = svm.OneClassSVM(kernel='rbf',
nu=Cfg.svm_nu,
gamma=gamma)
self.svm.fit(X_train)
self.nu = Cfg.svm_nu
self.gamma = gamma
self.stop_clock()
self.train_time = self.clocked
def predict(self, which_set='train', **kwargs):
assert which_set in ('train', 'val', 'test')
if which_set == 'train':
X = self.data._X_train
y = self.data._y_train
if which_set == 'val':
X = self.data._X_val
y = self.data._y_val
if which_set == 'test':
X = self.data._X_test
y = self.data._y_test
# reshape to 2D if input is tensor
if X.ndim > 2:
X_shape = X.shape
X = X.reshape(X_shape[0], np.prod(X_shape[1:]))
print("Starting prediction...")
self.start_clock()
if self.loss == 'SVC':
if self.kernel in ('DegreeKernel', 'WeightedDegreeKernel'):
self.get_kernel_matrix(kernel=self.kernel,
which_set=which_set,
**kwargs)
if which_set == 'train':
scores = self.svm.decision_function(self.K_train)
if which_set == 'test':
scores = self.svm.decision_function(self.K_test)
else:
scores = self.svm.decision_function(X)
auc = roc_auc_score(y, scores[:, 0])
self.diag[which_set]['scores'] = scores
self.diag[which_set]['auc'][0] = auc
if self.loss == 'OneClassSVM':
if self.kernel in ('DegreeKernel', 'WeightedDegreeKernel'):
self.get_kernel_matrix(kernel=self.kernel,
which_set=which_set,
**kwargs)
if which_set == 'train':
scores = (-1.0) * self.svm.decision_function(self.K_train)
y_pred = (self.svm.predict(self.K_train) == -1) * 1
if which_set == 'test':
scores = (-1.0) * self.svm.decision_function(self.K_test)
y_pred = (self.svm.predict(self.K_test) == -1) * 1
else:
if which_set == "val":
scores = (-1.0) * self.cv_svm.decision_function(X)
y_pred = (self.cv_svm.predict(X) == -1) * 1
else:
scores = (-1.0) * self.svm.decision_function(X)
y_pred = (self.svm.predict(X) == -1) * 1
self.diag[which_set]['scores'][:, 0] = scores.flatten()
self.diag[which_set]['acc'][0] = 100.0 * sum(y == y_pred) / len(y)
if sum(y) > 0:
auc = roc_auc_score(y, scores.flatten())
self.diag[which_set]['auc'][0] = auc
self.stop_clock()
if which_set == 'test':
self.rho = -self.svm.intercept_[0]
self.test_time = self.clocked
if which_set == 'val':
self.val_time = self.clocked
def dump_model(self, filename=None):
dump_svm(self, filename)
def load_model(self, filename=None):
assert filename and os.path.exists(filename)
load_svm(self, filename)
def log_results(self, filename=None):
"""
log the results relevant for anomaly detection
"""
self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
self.ad_log['train_time'] = self.train_time
self.ad_log['val_auc'] = self.diag['val']['auc'][-1]
self.ad_log['val_accuracy'] = self.diag['val']['acc'][-1]
self.ad_log['val_time'] = self.val_time
self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
self.ad_log['test_time'] = self.test_time
self.ad_log.save_to_file(filename=filename)
def get_kernel_matrix(self, kernel, which_set='train', **kwargs):
assert kernel in ('DegreeKernel', 'WeightedDegreeKernel')
if kernel == 'DegreeKernel':
kernel_function = degree_kernel
if kernel == 'WeightedDegreeKernel':
kernel_function = weighted_degree_kernel
if which_set == 'train':
self.K_train = kernel_function(self.data._X_train,
self.data._X_train, **kwargs)
if which_set == 'val':
self.K_val = kernel_function(self.data._X_val, self.data._X_train,
**kwargs)
if which_set == 'test':
self.K_test = kernel_function(self.data._X_test,
self.data._X_train, **kwargs)
class LeNet:
def __init__(self,
dataset,
use_weights=None,
pretrain=False,
profile=False):
"""
initialize instance
"""
self.initialize_variables(dataset)
# whether to enable profiling in Theano functions
self.profile = profile
W1_init = None
self.addInputLayer(shape=(None, 1, 28, 28))
addConvModule(self,
num_filters=8,
filter_size=(5, 5),
W_init=W1_init,
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
addConvModule(self,
num_filters=4,
filter_size=(5, 5),
bias=Cfg.mnist_bias,
pool_size=(2, 2),
use_batch_norm=Cfg.use_batch_norm)
# Code Layer
if Cfg.mnist_bias:
self.addDenseLayer(num_units=Cfg.mnist_rep_dim)
else:
self.addDenseLayer(num_units=Cfg.mnist_rep_dim, b=None)
self.setFeatureLayer(
) # set the currently highest layer to be the SVDD feature layer
self.addDenseLayer(num_units=10)
self.addSoftmaxLayer()
input_var = T.tensor4('inputs')
target_var = T.ivector('targets')
final_layer = self.all_layers[-1]
prediction = lasagne.layers.get_output(final_layer,
deterministic=False)
loss = lasagne.objectives.categorical_crossentropy(
prediction, target_var)
loss = loss.mean()
params = lasagne.layers.get_all_params(final_layer, trainable=True)
updates = lasagne.updates.nesterov_momentum(loss,
params,
learning_rate=0.0001,
momentum=0.9)
train_fn = theano.function([input_var, target_var],
loss,
updates=updates)
val_fn = theano.function([input_var, target_var], loss)
def train(self):
# train on epoch
i_batch = 0
for batch in self.data.get_epoch_train():
# train
inputs, targets, _ = batch
print "#################"
exit()
self.train_fn(inputs, targets)
# def adversarial(self, images, labels):
# # instantiate model
# keras.backend.set_learning_phase(0)
# preprocessing = (np.array([104]), 1)
#
# fmodel = foolbox.models.TheanoModel(self.input_layer, self.logits, (0, 1), 2, 1, preprocessing)
#
# # apply attack on source image
# # ::-1 reverses the color channels, because Keras ResNet50 expects BGR instead of RGB
# print np.shape(images)
# attack = foolbox.attacks.FGSM(fmodel)
# print 'starting attack'
# for i in range(len(images)):
# image = images[i][0]
# label = labels[i]
# adversarial = attack(np.reshape(image, (28, 28, 1)), label)
# print label
# print np.shape(images)
# # plt.plot(image)
# print np.shape(image)
# print image
# plt.show()
# # adversarial = attack(image[:, :, ::-1], label)
#
# # if the attack fails, adversarial will be None and a warning will be printed
# print adversarial
# time.sleep(2)
# print 'ending attack'
#
# plt.figure()
#
# plt.subplot(1, 3, 1)
# plt.title('Original')
# plt.imshow(image) # division by 255 to convert [0, 255] to [0, 1]
# plt.axis('off')
#
# plt.subplot(1, 3, 2)
# plt.title('Adversarial')
# plt.imshow(adversarial) # ::-1 to convert BGR to RGB
# plt.axis('off')
#
# plt.subplot(1, 3, 3)
# plt.title('Difference')
# difference = adversarial - image
# plt.imshow(difference / abs(difference).max() * 0.2 + 0.5)
# plt.axis('off')
#
# plt.show()
def initialize_variables(self, dataset):
self.all_layers, self.trainable_layers = (), ()
self.n_conv_layers = 0
self.n_dense_layers = 0
self.n_relu_layers = 0
self.n_leaky_relu_layers = 0
self.n_bn_layers = 0
self.n_norm_layers = 0
self.n_abs_layers = 0
self.n_maxpool_layers = 0
self.n_upscale_layers = 0
self.n_dropout_layers = 0
self.n_dimshuffle_layers = 0
self.n_reshape_layers = 0
self.R_init = 0
self.learning_rate_init = Cfg.learning_rate.get_value()
self.it = 0
self.clock = 0
self.pretrained = False # set to True after pretraining such that dictionary initialization mustn't be repeated
self.diag = {} # init an empty dictionary to hold diagnostics
self.log = Log(dataset_name=dataset)
self.ad_log = AD_Log()
self.dense_layers, self.conv_layers, = [], []
def compile_updates(self):
""" create network from architecture given in modules (determined by dataset)
create Theano compiled functions
"""
opt.sgd.updates.create_update(self)
def compile_autoencoder(self):
"""
create network from autoencoder architecture (determined by dataset)
and compile Theano update functions.
"""
print("Compiling autoencoder...")
opt.sgd.updates.create_autoencoder(self)
print("Autoencoder compiled.")
def load_data(self, data_loader=None, pretrain=False):
self.data = data_loader()
if pretrain:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
elif Cfg.reconstruction_loss:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
elif Cfg.svdd_loss and Cfg.reconstruction_penalty:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
else:
self.data.build_architecture(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
def flush_data(self):
self.data._X_train = None
self.data._y_train = None
self.data._X_val = None
self.data._y_val = None
self.data._X_test = None
self.data._y_test = None
print("Data flushed from network.")
def next_layers(self, layer):
flag = False
for current_layer in self.all_layers:
if flag:
yield current_layer
if current_layer is layer:
flag = True
def previous_layers(self, layer):
flag = False
for current_layer in reversed(self.all_layers):
if flag:
yield current_layer
if current_layer is layer:
flag = True
def start_clock(self):
self.clock = time.time()
def stop_clock(self):
self.clocked = time.time() - self.clock
print("Total elapsed time: %g" % self.clocked)
def pretrain(self, solver, lr, n_epochs):
"""
pre-train weights with an autoencoder
"""
self.ae_solver = solver.lower()
self.ae_learning_rate = lr
self.ae_n_epochs = n_epochs
# set learning rate
lr_tmp = Cfg.learning_rate.get_value()
Cfg.learning_rate.set_value(Cfg.floatX(lr))
self.compile_autoencoder()
from opt.sgd.train import train_autoencoder
train_autoencoder(self)
# remove layer attributes, re-initialize network and reset learning rate
for layer in self.all_layers:
delattr(self, layer.name + "_layer")
self.initialize_variables(self.data.dataset_name)
Cfg.learning_rate.set_value(Cfg.floatX(lr_tmp))
self.pretrained = True # set to True that dictionary initialization mustn't be repeated
# load network architecture
if Cfg.svdd_loss and Cfg.reconstruction_penalty:
self.data.build_autoencoder(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
else:
self.data.build_architecture(self)
for layer in self.all_layers:
setattr(self, layer.name + "_layer", layer)
self.log.store_architecture(self)
# load weights learned by autoencoder
self.load_weights(Cfg.xp_path + "/ae_pretrained_weights.p")
def train1(self, solver, n_epochs=10, save_at=0, save_to=''):
self.solver = solver.lower()
self.ae_solver = solver.lower()
self.n_epochs = n_epochs
self.save_at = save_at
self.save_to = save_to
self.log['solver'] = self.solver
self.log['save_at'] = self.save_at
self.compile_updates()
from opt.sgd.train import train_network
self.start_clock()
self.stop_clock()
# self.log.save_to_file()
def evaluate(self, solver):
# this could be simplified to only compiling the forwardpropagation...
self.solver = solver.lower() # needed for compiling backprop
self.compile_updates()
print("Evaluating network with current weights...")
self.initialize_diagnostics(1)
self.copy_parameters()
# perform forward passes on training, val, and test set
_, _ = performance(self, which_set='train', epoch=0, print_=True)
_, _ = performance(self, which_set='val', epoch=0, print_=True)
_, _ = performance(self, which_set='test', epoch=0, print_=True)
print("Evaluation on train, val, and test set completed.")
def log_results(self, filename=None):
"""
log the results relevant for anomaly detection
"""
self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
self.ad_log['train_time'] = self.train_time
self.ad_log['val_auc'] = self.diag['val']['auc'][-1]
self.ad_log['val_accuracy'] = self.diag['val']['acc'][-1]
self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
self.ad_log['test_time'] = self.test_time
self.ad_log.save_to_file(filename=filename)
def addInputLayer(self, **kwargs):
self.input_layer = InputLayer(name="input", **kwargs)
self.input_layer.inp_ndim = len(kwargs["shape"])
def addConvLayer(self, use_batch_norm=False, **kwargs):
"""
Add convolutional layer.
If batch norm flag is True, the convolutional layer
will be followed by a batch-normalization layer
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_conv_layers += 1
name = "conv%i" % self.n_conv_layers
new_layer = ConvLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
self.trainable_layers += (new_layer, )
if use_batch_norm:
self.n_bn_layers += 1
name = "bn%i" % self.n_bn_layers
self.all_layers += (BatchNorm(new_layer, name=name), )
def addDenseLayer(self, use_batch_norm=False, **kwargs):
"""
Add dense layer.
If batch norm flag is True, the dense layer
will be followed by a batch-normalization layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dense_layers += 1
name = "dense%i" % self.n_dense_layers
new_layer = DenseLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
self.trainable_layers += (new_layer, )
if use_batch_norm:
self.n_bn_layers += 1
name = "bn%i" % self.n_bn_layers
self.all_layers += (BatchNorm(new_layer, name=name), )
def addSigmoidLayer(self, **kwargs):
"""
Add sigmoid classification layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
new_layer = Sigmoid(input_layer, **kwargs)
self.all_layers += (new_layer, )
self.n_layers = len(self.all_layers)
def addSoftmaxLayer(self, **kwargs):
"""
Add softmax multi-class classification layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
new_layer = Softmax(input_layer, **kwargs)
self.all_layers += (new_layer, )
self.n_layers = len(self.all_layers)
def addNormLayer(self, **kwargs):
"""
Add layer which normalizes its input to length 1.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_norm_layers += 1
name = "norm%i" % self.n_norm_layers
new_layer = Norm(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addAbsLayer(self, **kwargs):
"""
Add layer which returns the absolute value of its input.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_abs_layers += 1
name = "abs%i" % self.n_abs_layers
new_layer = Abs(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addReLU(self, **kwargs):
"""
Add ReLU activation layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_relu_layers += 1
name = "relu%i" % self.n_relu_layers
new_layer = ReLU(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addLeakyReLU(self, **kwargs):
"""
Add leaky ReLU activation layer. (with leakiness=0.01)
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_leaky_relu_layers += 1
name = "leaky_relu%i" % self.n_leaky_relu_layers
new_layer = LeakyReLU(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addMaxPool(self, **kwargs):
"""
Add MaxPooling activation layer.
"""
input_layer = self.input_layer if not self.all_layers\
else self.all_layers[-1]
self.n_maxpool_layers += 1
name = "maxpool%i" % self.n_maxpool_layers
new_layer = MaxPool(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addUpscale(self, **kwargs):
"""
Add Upscaling activation layer.
"""
input_layer = self.input_layer if not self.all_layers\
else self.all_layers[-1]
self.n_upscale_layers += 1
name = "upscale%i" % self.n_upscale_layers
new_layer = Upscale(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addDropoutLayer(self, **kwargs):
"""
Add Dropout layer.
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dropout_layers += 1
name = "dropout%i" % self.n_dropout_layers
new_layer = DropoutLayer(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addDimshuffleLayer(self, **kwargs):
"""
Add Dimshuffle layer to reorder dimensions
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_dimshuffle_layers += 1
name = "dimshuffle%i" % self.n_dimshuffle_layers
new_layer = Dimshuffle(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def addReshapeLayer(self, **kwargs):
"""
Add reshape layer to reshape dimensions
"""
input_layer = self.input_layer if not self.all_layers \
else self.all_layers[-1]
self.n_reshape_layers += 1
name = "reshape%i" % self.n_reshape_layers
new_layer = Reshape(input_layer, name=name, **kwargs)
self.all_layers += (new_layer, )
def setFeatureLayer(self):
"""
sets the currently highest layer of the current network to be the code layer (compression layer)
"""
setattr(self, "feature_layer", self.all_layers[-1])
def dump_weights(self, filename=None, pretrain=False):
dump_weights(self, filename, pretrain=pretrain)
def load_weights(self, filename=None):
assert filename and os.path.exists(filename)
load_weights(self, filename)
def update_R(self):
"""
method to update R while leaving the network parameters and center c fixed in a block coordinate optimization
"""
print("Updating radius R...")
# Get updates
R = update_R(self.diag['train']['rep'],
self.cvar.get_value(),
solver=Cfg.R_update_solver,
scalar_method=Cfg.R_update_scalar_method,
lp_obj=Cfg.R_update_lp_obj)
# Update R
self.Rvar.set_value(Cfg.floatX(R))
print("Radius R updated.")
def update_R_c(self):
"""
method to update R and c while leaving the network parameters fixed in a block coordinate optimization
"""
print("Updating radius R and center c...")
# Get updates
R, c = update_R_c(self.diag['train']['rep'],
np.sum(self.diag['train']['rep']**2, axis=1),
solver=Cfg.QP_solver)
# Update values
self.Rvar.set_value(Cfg.floatX(R))
self.cvar.set_value(Cfg.floatX(c))
print("Radius R and center c updated.")
def initialize_diagnostics(self, n_epochs):
"""
initialize diagnostics for the neural network
"""
# data-dependent diagnostics for train, validation, and test set
self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_train)
self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_val)
self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_test)
# network parameter diagnostics
self.diag['network'] = NNetParamDiag(self, n_epochs=n_epochs)
# Best results (highest AUC on test set)
self.auc_best = 0
self.auc_best_epoch = 0 # determined by highest AUC on test set
self.best_weight_dict = None
def save_objective_and_accuracy(self, epoch, which_set, objective,
accuracy):
"""
save objective and accuracy of epoch
"""
self.diag[which_set]['objective'][epoch] = objective
self.diag[which_set]['acc'][epoch] = accuracy
def save_initial_parameters(self):
"""
save a copy of the initial network parameters for diagnostics.
"""
self.W_init = []
self.b_init = []
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.W_init.append(None)
self.b_init.append(None)
i = 0
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.W_init[i] = layer.W.get_value()
if layer.b is not None:
self.b_init[i] = layer.b.get_value()
i += 1
def copy_parameters(self):
"""
save a copy of the current network parameters in order to monitor the difference between epochs.
"""
i = 0
for layer in self.trainable_layers:
if layer.isdense | layer.isconv:
self.diag['network']['W_copy'][i] = layer.W.get_value()
if layer.b is not None:
self.diag['network']['b_copy'][i] = layer.b.get_value()
i += 1
def copy_initial_parameters_to_cache(self):
"""
Save a copy of the initial parameters in cache
"""
self.diag['network']['W_copy'] = list(self.W_init)
self.diag['network']['b_copy'] = list(self.b_init)
def save_network_diagnostics(self, epoch, l2, R):
"""
save diagnostics of the network
"""
self.diag['network']['l2_penalty'][epoch] = l2
self.log['l2_penalty'].append(float(l2))
i = 0
j = 0
for layer in self.trainable_layers:
if layer.isdense:
self.diag['network']['W_norms'][i][:, epoch] = np.sum(
layer.W.get_value()**2, axis=0)
if layer.b is not None:
self.diag['network']['b_norms'][
i][:, epoch] = layer.b.get_value()**2
i += 1
if layer.isdense | layer.isconv:
dW = np.sqrt(
np.sum((layer.W.get_value() -
self.diag['network']['W_copy'][j])**2))
self.diag['network']['dW_norms'][j][epoch] = dW
if layer.b is not None:
db = np.sqrt(
np.sum((layer.b.get_value() -
self.diag['network']['b_copy'][j])**2))
self.diag['network']['db_norms'][j][epoch] = db
j += 1
# diagnostics only relevant for the SVDD loss
if Cfg.svdd_loss:
self.diag['network']['R'][epoch] = R
self.diag['network']['c_norm'][epoch] = np.sqrt(
np.sum(self.cvar.get_value()**2))
def track_best_results(self, epoch):
"""
Save network parameters where AUC on the test set was highest.
"""
if self.diag['test']['auc'][epoch] > self.auc_best:
self.auc_best = self.diag['test']['auc'][epoch]
self.auc_best_epoch = epoch
self.best_weight_dict = dict()
for layer in self.trainable_layers:
self.best_weight_dict[layer.name + "_w"] = layer.W.get_value()
if layer.b is not None:
self.best_weight_dict[layer.name +
"_b"] = layer.b.get_value()
if Cfg.svdd_loss:
self.best_weight_dict["R"] = self.Rvar.get_value()
def dump_best_weights(self, filename):
"""
pickle the network parameters, where AUC on the test set was highest.
"""
with open(filename, 'wb') as f:
pickle.dump(self.best_weight_dict, f)
print("Parameters of best epoch saved in %s" % filename)
def save_diagnostics(self, which_set, epoch, scores, rep_norm, rep,
emp_loss, reconstruction_penalty):
"""
save diagnostics for which_set of epoch
"""
if self.data.n_classes == 2:
if which_set == 'train':
y = self.data._y_train
if which_set == 'val':
y = self.data._y_val
if which_set == 'test':
y = self.data._y_test
self.diag[which_set]['scores'][:, epoch] = scores
if sum(y) > 0:
AUC = roc_auc_score(y, scores)
self.diag[which_set]['auc'][epoch] = AUC
self.log[which_set + '_auc'].append(float(AUC))
print("{:32} {:.2f}%".format(which_set.title() + ' AUC:',
100. * AUC))
scores_normal = scores[y == 0]
scores_outlier = scores[y == 1]
normal_summary = get_five_number_summary(scores_normal)
outlier_summary = get_five_number_summary(scores_outlier)
self.log[which_set +
'_normal_scores_summary'].append(normal_summary)
self.log[which_set +
'_outlier_scores_summary'].append(outlier_summary)
self.diag[which_set]['rep'] = rep
self.diag[which_set]['rep_norm'][:, epoch] = rep_norm
rep_norm_normal = rep_norm[y == 0]
rep_norm_outlier = rep_norm[y == 1]
normal_summary = get_five_number_summary(rep_norm_normal)
outlier_summary = get_five_number_summary(rep_norm_outlier)
self.log[which_set +
'_normal_rep_norm_summary'].append(normal_summary)
self.log[which_set +
'_outlier_rep_norm_summary'].append(outlier_summary)
if Cfg.svdd_loss:
rep_mean = np.mean(rep, axis=0)
self.diag[which_set]['output_mean_norm'][epoch] = np.sqrt(
np.sum(rep_mean**2))
self.diag[which_set]['c_mean_diff'][epoch] = np.sqrt(
np.sum((rep_mean - self.cvar.get_value())**2))
self.diag[which_set]['reconstruction_penalty'][
epoch] = reconstruction_penalty
self.diag[which_set]['emp_loss'][epoch] = float(emp_loss)
self.log[which_set + '_emp_loss'].append(float(emp_loss))
def initialize_ae_diagnostics(self, n_epochs):
"""
initialize diagnostic variables for autoencoder network.
"""
self.train_time = 0
self.test_time = 0
self.best_weight_dict = None
# data-dependent diagnostics for train, validation, and test set
self.diag['train'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_train)
self.diag['val'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_val)
self.diag['test'] = NNetDataDiag(n_epochs=n_epochs,
n_samples=self.data.n_test)
# network parameters
self.diag['network'] = {}
self.diag['network']['l2_penalty'] = np.zeros(n_epochs,
dtype=Cfg.floatX)
def save_ae_diagnostics(self, which_set, epoch, error, scores, l2):
"""
save autoencoder diagnostics for which_set
"""
self.diag[which_set]['objective'][epoch] = error + l2
self.diag[which_set]['emp_loss'][epoch] = error
self.diag[which_set]['scores'][:, epoch] = scores
self.diag['network']['l2_penalty'][epoch] = l2
if self.data.n_classes == 2:
if which_set == 'train':
y = self.data._y_train
if which_set == 'val':
y = self.data._y_val
if which_set == 'test':
y = self.data._y_test
if sum(y) > 0:
AUC = roc_auc_score(y, scores)
self.diag[which_set]['auc'][epoch] = AUC
class KDE(object):
def __init__(self, dataset, kernel, **kwargs):
# initialize
self.kde = None
self.kernel = kernel
self.bandwidth = None
self.initialize_kde(**kwargs)
# load dataset
load_dataset(self, dataset)
# train and test time
self.clock = 0
self.clocked = 0
self.train_time = 0
self.test_time = 0
# Scores and AUC
self.diag = {}
self.diag['train'] = {}
self.diag['val'] = {}
self.diag['test'] = {}
self.diag['train']['scores'] = np.zeros((len(self.data._y_train), 1))
self.diag['val']['scores'] = np.zeros((len(self.data._y_val), 1))
self.diag['test']['scores'] = np.zeros((len(self.data._y_test), 1))
self.diag['train']['auc'] = np.zeros(1)
self.diag['val']['auc'] = np.zeros(1)
self.diag['test']['auc'] = np.zeros(1)
self.diag['train']['acc'] = np.zeros(1)
self.diag['val']['acc'] = np.zeros(1)
self.diag['test']['acc'] = np.zeros(1)
# AD results log
self.ad_log = AD_Log()
# diagnostics
self.best_weight_dict = None # attribute to reuse nnet plot-functions
def initialize_kde(self, **kwargs):
self.kde = KernelDensity(kernel=self.kernel, **kwargs)
self.bandwidth = self.kde.bandwidth
def load_data(self, data_loader=None, pretrain=False):
self.data = data_loader()
def start_clock(self):
self.clock = time.time()
def stop_clock(self):
self.clocked = time.time() - self.clock
print("Total elapsed time: %g" % self.clocked)
def train(self, bandwidth_GridSearchCV=True):
if self.data._X_train.ndim > 2:
X_train_shape = self.data._X_train.shape
X_train = self.data._X_train.reshape(X_train_shape[0], -1)
else:
X_train = self.data._X_train
print("Starting training...")
self.start_clock()
if bandwidth_GridSearchCV:
# use grid search cross-validation to select bandwidth
print("Using GridSearchCV for bandwidth selection...")
# params = {'bandwidth': np.logspace(0.5, 5, num=10, base=2)}
params = {'bandwidth': np.logspace(- 4.5, 5, num=20, base=2)}
hyper_kde = GridSearchCV(KernelDensity(kernel=self.kernel), params, n_jobs=-1, cv=5, verbose=0)
hyper_kde.fit(X_train)
self.bandwidth = hyper_kde.best_estimator_.bandwidth
self.kde = hyper_kde.best_estimator_
else:
# if exponential kernel, re-initialize kde with bandwidth minimizing
# the numerical error
if self.kernel == 'exponential':
bandwidth = np.max(pairwise_distances(X_train)) ** 2
self.kde = KernelDensity(kernel=self.kernel,
bandwidth=bandwidth)
self.kde.fit(X_train)
self.stop_clock()
self.train_time = self.clocked
def predict(self, which_set='train'):
assert which_set in ('train', 'test')
if which_set == 'train':
X = self.data._X_train
y = self.data._y_train
if which_set == 'test':
X = self.data._X_test
y = self.data._y_test
# reshape to 2D if input is tensor
if X.ndim > 2:
X_shape = X.shape
X = X.reshape(X_shape[0], -1)
print("Starting prediction...")
self.start_clock()
scores = (-1.0) * self.kde.score_samples(X) # anomaly score
self.diag[which_set]['scores'][:, 0] = scores.flatten()
if sum(y) > 0:
auc = roc_auc_score(y, scores.flatten())
self.diag[which_set]['auc'][0] = auc
self.stop_clock()
if which_set == 'test':
self.test_time = self.clocked
def dump_model(self, filename=None):
dump_kde(self, filename)
def load_model(self, filename=None):
assert filename and os.path.exists(filename)
load_kde(self, filename)
def log_results(self, filename=None):
"""
log the results relevant for anomaly detection
"""
self.ad_log['train_auc'] = self.diag['train']['auc'][-1]
self.ad_log['train_accuracy'] = self.diag['train']['acc'][-1]
self.ad_log['train_time'] = self.train_time
self.ad_log['test_auc'] = self.diag['test']['auc'][-1]
self.ad_log['test_accuracy'] = self.diag['test']['acc'][-1]
self.ad_log['test_time'] = self.test_time
self.ad_log.save_to_file(filename=filename)