def detectors(seed):
if os.environ.get('CIRCLECI', False):
dets = [
AutoEncoder(num_epochs=1, seed=seed),
DAGMM(num_epochs=1, seed=seed),
DAGMM(num_epochs=1,
autoencoder_type=DAGMM.AutoEncoder.LSTM,
seed=seed),
LSTMAD(num_epochs=1, seed=seed),
LSTMED(num_epochs=1, seed=seed),
RecurrentEBM(num_epochs=1, seed=seed)
]
else:
standard_epochs = 40
dets = [
AutoEncoder(num_epochs=standard_epochs, seed=seed),
DAGMM(num_epochs=standard_epochs, seed=seed, lr=1e-4),
DAGMM(num_epochs=standard_epochs,
autoencoder_type=DAGMM.AutoEncoder.LSTM,
seed=seed),
LSTMAD(num_epochs=standard_epochs, seed=seed),
LSTMED(num_epochs=standard_epochs, seed=seed),
RecurrentEBM(num_epochs=standard_epochs, seed=seed)
]
return sorted(dets, key=lambda x: x.framework)
def detectors(seed):
standard_epochs = 40
dets = [AutoEncoder(num_epochs=standard_epochs, seed=seed),
DAGMM(num_epochs=standard_epochs, seed=seed, lr=1e-4),
DAGMM(num_epochs=standard_epochs, autoencoder_type=DAGMM.AutoEncoder.LSTM, seed=seed),
LSTMAD(num_epochs=standard_epochs, seed=seed), LSTMED(num_epochs=standard_epochs, seed=seed),
RecurrentEBM(num_epochs=standard_epochs, seed=seed)]
return sorted(dets, key=lambda x: x.framework)
def different_window_detectors(seed):
standard_epochs = 40
dets = [LSTMAD(num_epochs=standard_epochs)]
for window_size in [13, 25, 50, 100]:
dets.extend([
LSTMED(name='LSTMED Window: ' + str(window_size),
num_epochs=standard_epochs,
seed=seed,
sequence_length=window_size),
AutoEncoder(name='AE Window: ' + str(window_size),
num_epochs=standard_epochs,
seed=seed,
sequence_length=window_size)
])
return dets
OUTLIER_CLASS = 0
mnist = tf.keras.datasets.mnist
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# Label outliers with 1 and normal digits with 0
y_train, y_test = (y_train == OUTLIER_CLASS), (y_test == OUTLIER_CLASS)
x_train = x_train[~y_train] # Remove outliers from the training set
x_train, x_test = x_train / 255, x_test / 255
x_train, x_test = x_train.reshape(-1, 784), x_test.reshape(-1, 784)
self._data = tuple(pd.DataFrame(data=data) for data in [x_train, y_train, x_test, y_test])
x_train, y_train, x_test, y_test = MNIST(seed=0).data()
# Use fewer instances for demonstration purposes
x_train, y_train = x_train[:1000], y_train[:1000]
x_test, y_test = x_test[:100], y_test[:100]
model = AutoEncoder(sequence_length=1, num_epochs=40, hidden_size=10, lr=1e-4)
model.fit(x_train)
error = model.predict(x_test)
print(roc_auc_score(y_test, error)) # e.g. 0.8614
#
"""Borrowed from https://github.com/scikit-learn/scikit-learn/blob/master/examples/manifold/plot_lle_digits.py#L44"""
error = (error - error.min()) / (error.max() - error.min()) # Normalize error
x_test = x_test.values
y_random = np.random.rand(len(x_test)) * 2 - 1
plt.figure(figsize=(20, 10))
ax = plt.subplot(111)
if hasattr(offsetbox, 'AnnotationBbox'):
shown_images = np.array([[1., 1.]])
import glob
import json
import os
import numpy as np
import pandas as pd
import pickle
from src.algorithms import AutoEncoder, DAGMM, RecurrentEBM, LSTMAD, LSTMED
from src.datasets.pandas import PandasDataset
from src.evaluation import Evaluator
seed = np.random.randint(np.iinfo(np.uint32).max, size=1, dtype=np.uint32)[0]
standard_epochs = 40
window_size = 13
dets = [AutoEncoder(num_epochs=standard_epochs, seed=seed)]
def main():
eval_json()
def eval_json():
with open('QueryResult.json') as json_file:
dict_train = json.load(json_file)
df = pd.DataFrame.from_records(dict_train)
ds = PandasDataset('temperatur_sonne_calvin',
df,
ignore=['TYPE', 'LOCATIONID', 'TS'])
evaluator = Evaluator([ds], get_detectors, seed=seed)
evaluator.train()