def process_nsl(root='/home/naruto/NetLearner'): nslkdd.generate_datasets(binary_label=True, one_hot_encoding=False) raw_X_train = np.load('%s/NSLKDD/train_dataset.npy' % root) y_train = np.load('%s/NSLKDD/train_labels.npy' % root) raw_X_test = np.load('%s/NSLKDD/test_dataset.npy' % root) y_test = np.load('%s/NSLKDD/test_labels.npy' % root) [X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test) permutate_dataset(X_train, y_train) permutate_dataset(X_test, y_test) print('Training set', X_train.shape, y_train.shape) print('Test set', X_test.shape, y_test.shape) return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_unsw(root='/home/naruto/NetLearner'): unsw.generate_dataset(False) raw_X_train = np.load('%s/UNSW/train_dataset.npy' % root) y_train = np.load('%s/UNSW/train_labels.npy' % root) raw_X_test = np.load('%s/UNSW/test_dataset.npy' % root) y_test = np.load('%s/UNSW/test_labels.npy' % root) [X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test) permutate_dataset(X_train, y_train) permutate_dataset(X_test, y_test) print('Training set', X_train.shape, y_train.shape) print('Test set', X_test.shape, y_test.shape) return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_nsl(root='SharedAutoEncoder/'): nslkdd.generate_datasets(True, one_hot_encoding=True, root=root) raw_X_train = np.load(root + 'NSLKDD/train_dataset.npy') y_train = np.load(root + 'NSLKDD/train_labels.npy') raw_X_test = np.load(root + 'NSLKDD/test_dataset.npy') y_test = np.load(root + 'NSLKDD/test_labels.npy') [X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test) permutate_dataset(X_train, y_train) permutate_dataset(X_test, y_test) print('Training set', X_train.shape, y_train.shape) print('Test set', X_test.shape, y_test.shape) return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_nsl(): nslkdd.generate_datasets(binary_label=True) raw_X_train = np.load('NSLKDD/train_dataset.npy') y_train = np.load('NSLKDD/train_labels.npy') raw_X_test = np.load('NSLKDD/test_dataset.npy') y_test = np.load('NSLKDD/test_labels.npy') [X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test) permutate_dataset(X_train, y_train) permutate_dataset(X_test, y_test) print('Training set', X_train.shape, y_train.shape) print('Test set', X_test.shape, y_test.shape) return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_unsw(): unsw.generate_dataset(True) raw_X_train = np.load('UNSW/train_dataset.npy') y_train = np.load('UNSW/train_labels.npy') raw_X_test = np.load('UNSW/test_dataset.npy') y_test = np.load('UNSW/test_labels.npy') [X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test) permutate_dataset(X_train, y_train) permutate_dataset(X_test, y_test) print('Training set', X_train.shape, y_train.shape) print('Test set', X_test.shape, y_test.shape) return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
from netlearner.utils import augment_quantiled, permutate_dataset from netlearner.multilayer_perceptron import MultilayerPerceptron generate_dataset(True) raw_train_dataset = np.load('UNSW/train_dataset.npy') train_labels = np.load('UNSW/train_labels.npy') raw_valid_dataset = np.load('UNSW/valid_dataset.npy') valid_labels = np.load('UNSW/valid_labels.npy') raw_test_dataset = np.load('UNSW/test_dataset.npy') test_labels = np.load('UNSW/test_labels.npy') columns = np.array(range(1, 6) + range(8, 16) + range(17, 19) + range(23, 25) + [26]) [train_dataset, valid_dataset, test_dataset] = augment_quantiled( raw_train_dataset, raw_valid_dataset, raw_test_dataset, columns) permutate_dataset(train_dataset, train_labels) print('Training set', train_dataset.shape, train_labels.shape) print('Validation set', valid_dataset.shape, valid_labels.shape) print('Test set', test_dataset.shape, test_labels.shape) num_samples, feature_size = train_dataset.shape num_labels = train_labels.shape[1] batch_size = 80 keep_prob = 0.80 beta = 0.00008 weights = [1.0, 1.0] num_epochs = [160] init_lrs = [0.001] hidden_layer_sizes = [ [400, 400, 400, 400], # [800, 640], [160, 80], [80, 40],
mlp.get_layer('h1').set_weights(init_weights) mlp.save(pretrained_mlp_path) os.environ['CUDA_VISIBLE_DEVICES'] = '1' model_dir = 'SparseAE/' generate_dataset(True, True, model_dir) data_dir = model_dir + 'UNSW/' pretrained_mlp_path = data_dir + 'sae_mlp.h5' raw_train_dataset = np.load(data_dir + 'train_dataset.npy') raw_test_dataset = np.load(data_dir + 'test_dataset.npy') y = np.load(data_dir + 'train_labels.npy') y_test = np.load(data_dir + 'test_labels.npy') X, _, X_test = min_max_scale(raw_train_dataset, None, raw_test_dataset) X, y = permutate_dataset(X, y) print('Training set', X.shape, y.shape) print('Test set', X_test.shape) num_samples, num_classes = y.shape feature_size = X.shape[1] encoder_size = 800 num_epoch = 160 batch_size = 80 class_weights = None sae_weights = pretrain_model() build_model(sae_weights) fold = 5 skf = StratifiedKFold(n_splits=fold) hist = {'train_loss': [], 'valid_loss': []}
os.environ['CUDA_VISIBLE_DEVICES'] = '3' model_dir = 'KerasMLP/' generate_dataset(False, True, model_dir) data_dir = model_dir + 'NSLKDD/' mlp_path = data_dir + 'mlp.h5' raw_train_dataset = np.load(data_dir + 'train_dataset.npy') raw_valid_dataset = np.load(data_dir + 'valid_dataset.npy') raw_test_dataset = np.load(data_dir + 'test_dataset.npy') train_labels = np.load(data_dir + 'train_labels.npy') valid_labels = np.load(data_dir + 'valid_labels.npy') test_labels = np.load(data_dir + 'test_labels.npy') [train_dataset, valid_dataset, test_dataset] = min_max_scale( raw_train_dataset, raw_valid_dataset, raw_test_dataset) train_dataset, train_labels = permutate_dataset(train_dataset, train_labels) valid_dataset, valid_labels = permutate_dataset(valid_dataset, valid_labels) test_dataset, test_labels = permutate_dataset(test_dataset, test_labels) print('Training set', train_dataset.shape, train_labels.shape) print('Validation set', valid_dataset.shape, valid_labels.shape) print('Test set', test_dataset.shape, test_labels.shape) batch_size = 80 keep_prob = 0.8 num_epoch = 200 tail = 160 incremental = False if incremental is False: num_samples, num_classes = train_labels.shape feature_size = train_dataset.shape[1] hidden_size = [800, 480]
i]) X_train = np.concatenate((X_train, ftr), axis=1) print(X_train.shape) print(X_valid.shape, fv.shape) X_valid = np.concatenate((X_valid, fv), axis=1) X_test = np.concatenate((X_test, fte), axis=1) X_train = np.concatenate((X_train, train_disc), axis=1) X_valid = np.concatenate((X_valid, valid_disc), axis=1) X_test = np.concatenate((X_test, test_disc), axis=1) print("Augmenting discrete & embedding dataset", X_train.shape) [X_train, X_valid, X_test] = min_max_scale(X_train, X_valid, X_test) print("Min-max scaled dataset", X_train.shape, X_test.shape) X_train, y_train = permutate_dataset(X_train, y_train) X_valid, y_valid = permutate_dataset(X_valid, y_valid, 'Valid') X_test, y_test = permutate_dataset(X_test, y_test, 'Test') num_samples, num_features = X_train.shape num_classes = y_train.shape[1] batch_size = 40 keep_prob = 0.8 beta = 0.000 weights = [1.0, 1.0] num_epochs = [160] init_lrs = [0.001] hidden_layer_sizes = [ [400, 200, 100], # [800, 640], [160, 80], [80, 40], # [400, 360, 320],
mp_classifier.exit() np.random.seed(1944) generate_dataset(True) raw_train_dataset = np.load('UNSW/train_dataset.npy') train_labels = np.load('UNSW/train_labels.npy') raw_valid_dataset = np.load('UNSW/valid_dataset.npy') valid_labels = np.load('UNSW/valid_labels.npy') raw_test_dataset = np.load('UNSW/test_dataset.npy') test_labels = np.load('UNSW/test_labels.npy') columns = np.array(range(1, 27)) [train_dataset, valid_dataset, test_dataset] = augment_quantiled(raw_train_dataset, raw_valid_dataset, raw_test_dataset, columns) train_dataset, train_labels = permutate_dataset(train_dataset, train_labels) valid_dataset, valid_labels = permutate_dataset(valid_dataset, valid_labels) test_dataset, test_labels = permutate_dataset(test_dataset, test_labels) # Generate fake attacking data using GAN fake_dataset, fake_labels = generate_fake_data(train_dataset, train_labels) # Mix fake and real data to do supervised learning mixed_dataset = np.concatenate((train_dataset, fake_dataset), axis=0) mixed_labels = np.concatenate((train_labels, fake_labels), axis=0) permutate_dataset(mixed_dataset, mixed_labels, name='Fake') print('Mix trainset with fake set', mixed_dataset.shape, mixed_labels.shape) tf.reset_default_graph() classify(mixed_dataset, mixed_labels, valid_dataset, valid_labels, test_dataset, test_labels)