def l21RDAE(X,
layers,
lamda,
folder,
learning_rate=0.15,
inner=100,
outer=10,
batch_size=133,
inputsize=(28, 28)):
if not os.path.isdir(folder):
os.makedirs(folder)
os.chdir(folder)
with tf.Graph().as_default():
with tf.Session() as sess:
rael21 = l21RDA.RobustL21Autoencoder(sess=sess,
lambda_=lamda * X.shape[0],
layers_sizes=layers)
l21L, l21S = rael21.fit(X=X,
sess=sess,
inner_iteration=inner,
iteration=outer,
batch_size=batch_size,
learning_rate=learning_rate,
verbose=True)
l21R = rael21.getRecon(X=X, sess=sess)
l21H = rael21.transform(X, sess)
l21S.dump("l21S.npk")
os.chdir("../")
def l21RDAE(X, layers, lamda, folder, learning_rate = 0.15, inner = 100, outer = 10, batch_size = 133,inputsize = (28,28)):
if not os.path.isdir(folder):
os.makedirs(folder)
os.chdir(folder)
with tf.Graph().as_default():
with tf.Session() as sess:
rael21 = l21RDA.RobustL21Autoencoder(sess = sess, lambda_= lamda*X.shape[0], layers_sizes=layers)
l21L, l21S = rael21.fit(X = X, sess = sess, inner_iteration = inner, iteration = outer, batch_size = batch_size, learning_rate = learning_rate, verbose = True)
l21R = rael21.getRecon(X = X, sess = sess)
l21H = rael21.transform(X, sess)
Image.fromarray(I.tile_raster_images(X=l21S,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21S.png")
Image.fromarray(I.tile_raster_images(X=l21R,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21R.png")
Image.fromarray(I.tile_raster_images(X=l21L,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21L.png")
l21S.dump("l21S.npk")
os.chdir("../")
def test_compare_with_paper():
"""
Compare this AE with the AE from the paper:
Anomaly Detection with Robusr Deep Autoencoders. Chong Zhou, Randy C. Paffenroth.s
"""
# Take the data from the data forlder of RobustAutoencoder repo ->
data_show_path = os.path.abspath(os.path.join(
'../RobustAutoencoder/data')) # path is relative to notebook path.
if data_show_path not in sys.path:
sys.path.append(data_show_path)
x_data_file = os.path.abspath(
os.path.join('../RobustAutoencoder/data/data.txt'))
y_data_file = os.path.abspath(
os.path.join('../RobustAutoencoder/data/y.txt'))
x_data = np.loadtxt(x_data_file, delimiter=",")
y_data = np.loadtxt(y_data_file, delimiter=",")
# Show the data:
import matplotlib.pyplot as plt
import ImShow as Ii
plt.figure(1)
Xpic = Ii.tile_raster_images(X=x_data,
img_shape=(28, 28),
tile_shape=(10, 10))
plt.imshow(Xpic, cmap='gray')
plt.show()
# Take the data from the data forlder of RobustAutoencoder repo <-
# set AE parameters ->
inner_epochs = 50 # in one outer iteration
outer_iterations = 20
p_re_init = True # whether to reinit the AE on every outer iteration
p_lambda = 0.00095
p_layers = [784, 400, 200]
p_lr = 0.001
p_minibatch_size = 133
p_seed = None
# set AE parameters <-2
# function to evaluate RAE ->
def eval_RAE(sparse_matrix, low_rank_reconstr, caption_text="L21 "):
"""
low_rank_reconstr - only for visualization.
"""
# Calculate statistics
from sklearn.metrics import f1_score
from sklearn.metrics import recall_score as recall
from sklearn.metrics import precision_score as precision
from collections import Counter
l21S = sparse_matrix
l21R = low_rank_reconstr
def binary_error(value):
if value != 0.0:
return "o" # 'majority'
else:
return "m" #'outlier'
def binary_y(value):
if value == 4:
return "m"
else:
return "o"
bi_y = list(map(binary_y, y_data))
print(Counter(bi_y))
#import pdb; pdb.set_trace()
S = l21S
predictions = list(map(binary_error, np.linalg.norm(S, axis=1)))
p = precision(bi_y, predictions, labels=["o", "m"], pos_label="o")
r = recall(bi_y, predictions, labels=["o", "m"], pos_label="o")
f1 = f1_score(bi_y, predictions, labels=["o", "m"], pos_label="o")
print("lambda:", p_lambda)
print("stat:", Counter(predictions))
print("precision", p)
print("recall", r)
print("f1", f1)
# Showing the images
inputsize = (28, 28)
iS = Image.fromarray(
Ii.tile_raster_images(X=l21S,
img_shape=inputsize,
tile_shape=(10, 10),
tile_spacing=(1, 1)))
iR = Image.fromarray(
Ii.tile_raster_images(X=l21R,
img_shape=inputsize,
tile_shape=(10, 10),
tile_spacing=(1, 1)))
iD = Image.fromarray(
Ii.tile_raster_images(X=x_data,
img_shape=inputsize,
tile_shape=(10, 10),
tile_spacing=(1, 1)))
fig, ax = plt.subplots(nrows=1, ncols=3, squeeze=False)
fig.set_size_inches(12, 3)
#import pdb; pdb.set_trace()
ax[0][0].imshow(iR, cmap="gray")
ax[0][1].imshow(iS, cmap="gray")
ax[0][2].imshow(iD, cmap="gray")
ax[0][0].set_title(caption_text + "R")
ax[0][1].set_title(caption_text + "S")
ax[0][2].set_title("X")
ax[0][0].get_xaxis().set_visible(False)
ax[0][0].get_yaxis().set_visible(False)
ax[0][1].get_xaxis().set_visible(False)
ax[0][1].get_yaxis().set_visible(False)
ax[0][2].get_xaxis().set_visible(False)
ax[0][2].get_yaxis().set_visible(False)
plt.show()
# function to evaluate RAE <-
# Train other RAE ->
re_path = os.path.abspath(os.path.join(
'../RobustAutoencoder/')) # path is relative to notebook path.
if re_path not in sys.path:
sys.path.append(re_path)
import PIL.Image as Image
from model import l21RobustDeepAutoencoderOnST as l21RDA
with tf.Graph().as_default():
with tf.Session() as sess:
rael21 = l21RDA.RobustL21Autoencoder(sess=sess,
lambda_=p_lambda *
x_data.shape[0],
layers_sizes=p_layers)
_, l21S = rael21.fit(X=x_data,
sess=sess,
inner_iteration=inner_epochs,
iteration=outer_iterations,
batch_size=p_minibatch_size,
learning_rate=p_lr,
re_init=p_re_init,
verbose=True)
# Attention! there is a bug in the basic autoencoder in the paper, so the learning_rate is not affecting at all.
l21R = rael21.getRecon(X=x_data, sess=sess)
eval_RAE(l21S, l21R)
# Train other RAE <-
# my RAE ->
with tf.Graph().as_default():
sess = tf.Session()
my_rae = RobustAE(sess,
p_layers,
'rows',
p_lambda=p_lambda * x_data.shape[0],
p_seed=p_seed,
p_log_folder='./Log')
_, my_S = my_rae.train(x_data,
p_learning_rate=p_lr,
p_inner_epochs=inner_epochs,
total_iterations=outer_iterations,
p_batch_size=p_minibatch_size,
p_dropout_rate=0,
p_ae_epoch_eval_freq=20,
verbose=True)
my_R = my_rae.getRecon(x_data, p_batch_size=None)
eval_RAE(my_S, my_R)