def l21RDAE(X, layers, lamda, folder, learning_rate=0.15, inner=100, outer=10, batch_size=133, inputsize=(28, 28)): if not os.path.isdir(folder): os.makedirs(folder) os.chdir(folder) with tf.Graph().as_default(): with tf.Session() as sess: rael21 = l21RDA.RobustL21Autoencoder(sess=sess, lambda_=lamda * X.shape[0], layers_sizes=layers) l21L, l21S = rael21.fit(X=X, sess=sess, inner_iteration=inner, iteration=outer, batch_size=batch_size, learning_rate=learning_rate, verbose=True) l21R = rael21.getRecon(X=X, sess=sess) l21H = rael21.transform(X, sess) l21S.dump("l21S.npk") os.chdir("../")
def l21RDAE(X, layers, lamda, folder, learning_rate = 0.15, inner = 100, outer = 10, batch_size = 133,inputsize = (28,28)): if not os.path.isdir(folder): os.makedirs(folder) os.chdir(folder) with tf.Graph().as_default(): with tf.Session() as sess: rael21 = l21RDA.RobustL21Autoencoder(sess = sess, lambda_= lamda*X.shape[0], layers_sizes=layers) l21L, l21S = rael21.fit(X = X, sess = sess, inner_iteration = inner, iteration = outer, batch_size = batch_size, learning_rate = learning_rate, verbose = True) l21R = rael21.getRecon(X = X, sess = sess) l21H = rael21.transform(X, sess) Image.fromarray(I.tile_raster_images(X=l21S,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21S.png") Image.fromarray(I.tile_raster_images(X=l21R,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21R.png") Image.fromarray(I.tile_raster_images(X=l21L,img_shape=inputsize, tile_shape=(10, 10),tile_spacing=(1, 1))).save(r"l21L.png") l21S.dump("l21S.npk") os.chdir("../")
def test_compare_with_paper(): """ Compare this AE with the AE from the paper: Anomaly Detection with Robusr Deep Autoencoders. Chong Zhou, Randy C. Paffenroth.s """ # Take the data from the data forlder of RobustAutoencoder repo -> data_show_path = os.path.abspath(os.path.join( '../RobustAutoencoder/data')) # path is relative to notebook path. if data_show_path not in sys.path: sys.path.append(data_show_path) x_data_file = os.path.abspath( os.path.join('../RobustAutoencoder/data/data.txt')) y_data_file = os.path.abspath( os.path.join('../RobustAutoencoder/data/y.txt')) x_data = np.loadtxt(x_data_file, delimiter=",") y_data = np.loadtxt(y_data_file, delimiter=",") # Show the data: import matplotlib.pyplot as plt import ImShow as Ii plt.figure(1) Xpic = Ii.tile_raster_images(X=x_data, img_shape=(28, 28), tile_shape=(10, 10)) plt.imshow(Xpic, cmap='gray') plt.show() # Take the data from the data forlder of RobustAutoencoder repo <- # set AE parameters -> inner_epochs = 50 # in one outer iteration outer_iterations = 20 p_re_init = True # whether to reinit the AE on every outer iteration p_lambda = 0.00095 p_layers = [784, 400, 200] p_lr = 0.001 p_minibatch_size = 133 p_seed = None # set AE parameters <-2 # function to evaluate RAE -> def eval_RAE(sparse_matrix, low_rank_reconstr, caption_text="L21 "): """ low_rank_reconstr - only for visualization. """ # Calculate statistics from sklearn.metrics import f1_score from sklearn.metrics import recall_score as recall from sklearn.metrics import precision_score as precision from collections import Counter l21S = sparse_matrix l21R = low_rank_reconstr def binary_error(value): if value != 0.0: return "o" # 'majority' else: return "m" #'outlier' def binary_y(value): if value == 4: return "m" else: return "o" bi_y = list(map(binary_y, y_data)) print(Counter(bi_y)) #import pdb; pdb.set_trace() S = l21S predictions = list(map(binary_error, np.linalg.norm(S, axis=1))) p = precision(bi_y, predictions, labels=["o", "m"], pos_label="o") r = recall(bi_y, predictions, labels=["o", "m"], pos_label="o") f1 = f1_score(bi_y, predictions, labels=["o", "m"], pos_label="o") print("lambda:", p_lambda) print("stat:", Counter(predictions)) print("precision", p) print("recall", r) print("f1", f1) # Showing the images inputsize = (28, 28) iS = Image.fromarray( Ii.tile_raster_images(X=l21S, img_shape=inputsize, tile_shape=(10, 10), tile_spacing=(1, 1))) iR = Image.fromarray( Ii.tile_raster_images(X=l21R, img_shape=inputsize, tile_shape=(10, 10), tile_spacing=(1, 1))) iD = Image.fromarray( Ii.tile_raster_images(X=x_data, img_shape=inputsize, tile_shape=(10, 10), tile_spacing=(1, 1))) fig, ax = plt.subplots(nrows=1, ncols=3, squeeze=False) fig.set_size_inches(12, 3) #import pdb; pdb.set_trace() ax[0][0].imshow(iR, cmap="gray") ax[0][1].imshow(iS, cmap="gray") ax[0][2].imshow(iD, cmap="gray") ax[0][0].set_title(caption_text + "R") ax[0][1].set_title(caption_text + "S") ax[0][2].set_title("X") ax[0][0].get_xaxis().set_visible(False) ax[0][0].get_yaxis().set_visible(False) ax[0][1].get_xaxis().set_visible(False) ax[0][1].get_yaxis().set_visible(False) ax[0][2].get_xaxis().set_visible(False) ax[0][2].get_yaxis().set_visible(False) plt.show() # function to evaluate RAE <- # Train other RAE -> re_path = os.path.abspath(os.path.join( '../RobustAutoencoder/')) # path is relative to notebook path. if re_path not in sys.path: sys.path.append(re_path) import PIL.Image as Image from model import l21RobustDeepAutoencoderOnST as l21RDA with tf.Graph().as_default(): with tf.Session() as sess: rael21 = l21RDA.RobustL21Autoencoder(sess=sess, lambda_=p_lambda * x_data.shape[0], layers_sizes=p_layers) _, l21S = rael21.fit(X=x_data, sess=sess, inner_iteration=inner_epochs, iteration=outer_iterations, batch_size=p_minibatch_size, learning_rate=p_lr, re_init=p_re_init, verbose=True) # Attention! there is a bug in the basic autoencoder in the paper, so the learning_rate is not affecting at all. l21R = rael21.getRecon(X=x_data, sess=sess) eval_RAE(l21S, l21R) # Train other RAE <- # my RAE -> with tf.Graph().as_default(): sess = tf.Session() my_rae = RobustAE(sess, p_layers, 'rows', p_lambda=p_lambda * x_data.shape[0], p_seed=p_seed, p_log_folder='./Log') _, my_S = my_rae.train(x_data, p_learning_rate=p_lr, p_inner_epochs=inner_epochs, total_iterations=outer_iterations, p_batch_size=p_minibatch_size, p_dropout_rate=0, p_ae_epoch_eval_freq=20, verbose=True) my_R = my_rae.getRecon(x_data, p_batch_size=None) eval_RAE(my_S, my_R)