def process_nsl(root='/home/naruto/NetLearner'):
nslkdd.generate_datasets(binary_label=True, one_hot_encoding=False)
raw_X_train = np.load('%s/NSLKDD/train_dataset.npy' % root)
y_train = np.load('%s/NSLKDD/train_labels.npy' % root)
raw_X_test = np.load('%s/NSLKDD/test_dataset.npy' % root)
y_test = np.load('%s/NSLKDD/test_labels.npy' % root)
[X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test)
permutate_dataset(X_train, y_train)
permutate_dataset(X_test, y_test)
print('Training set', X_train.shape, y_train.shape)
print('Test set', X_test.shape, y_test.shape)
return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_unsw(root='/home/naruto/NetLearner'):
unsw.generate_dataset(False)
raw_X_train = np.load('%s/UNSW/train_dataset.npy' % root)
y_train = np.load('%s/UNSW/train_labels.npy' % root)
raw_X_test = np.load('%s/UNSW/test_dataset.npy' % root)
y_test = np.load('%s/UNSW/test_labels.npy' % root)
[X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test)
permutate_dataset(X_train, y_train)
permutate_dataset(X_test, y_test)
print('Training set', X_train.shape, y_train.shape)
print('Test set', X_test.shape, y_test.shape)
return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_nsl(root='SharedAutoEncoder/'):
nslkdd.generate_datasets(True, one_hot_encoding=True, root=root)
raw_X_train = np.load(root + 'NSLKDD/train_dataset.npy')
y_train = np.load(root + 'NSLKDD/train_labels.npy')
raw_X_test = np.load(root + 'NSLKDD/test_dataset.npy')
y_test = np.load(root + 'NSLKDD/test_labels.npy')
[X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test)
permutate_dataset(X_train, y_train)
permutate_dataset(X_test, y_test)
print('Training set', X_train.shape, y_train.shape)
print('Test set', X_test.shape, y_test.shape)
return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_nsl():
nslkdd.generate_datasets(binary_label=True)
raw_X_train = np.load('NSLKDD/train_dataset.npy')
y_train = np.load('NSLKDD/train_labels.npy')
raw_X_test = np.load('NSLKDD/test_dataset.npy')
y_test = np.load('NSLKDD/test_labels.npy')
[X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test)
permutate_dataset(X_train, y_train)
permutate_dataset(X_test, y_test)
print('Training set', X_train.shape, y_train.shape)
print('Test set', X_test.shape, y_test.shape)
return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
def process_unsw():
unsw.generate_dataset(True)
raw_X_train = np.load('UNSW/train_dataset.npy')
y_train = np.load('UNSW/train_labels.npy')
raw_X_test = np.load('UNSW/test_dataset.npy')
y_test = np.load('UNSW/test_labels.npy')
[X_train, _, X_test] = min_max_scale(raw_X_train, None, raw_X_test)
permutate_dataset(X_train, y_train)
permutate_dataset(X_test, y_test)
print('Training set', X_train.shape, y_train.shape)
print('Test set', X_test.shape, y_test.shape)
return {'X': X_train, 'y': y_train, 'X_test': X_test, 'y_test': y_test}
from netlearner.utils import augment_quantiled, permutate_dataset
from netlearner.multilayer_perceptron import MultilayerPerceptron
generate_dataset(True)
raw_train_dataset = np.load('UNSW/train_dataset.npy')
train_labels = np.load('UNSW/train_labels.npy')
raw_valid_dataset = np.load('UNSW/valid_dataset.npy')
valid_labels = np.load('UNSW/valid_labels.npy')
raw_test_dataset = np.load('UNSW/test_dataset.npy')
test_labels = np.load('UNSW/test_labels.npy')
columns = np.array(range(1, 6) + range(8, 16) + range(17, 19) +
range(23, 25) + [26])
[train_dataset, valid_dataset, test_dataset] = augment_quantiled(
raw_train_dataset, raw_valid_dataset, raw_test_dataset, columns)
permutate_dataset(train_dataset, train_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
num_samples, feature_size = train_dataset.shape
num_labels = train_labels.shape[1]
batch_size = 80
keep_prob = 0.80
beta = 0.00008
weights = [1.0, 1.0]
num_epochs = [160]
init_lrs = [0.001]
hidden_layer_sizes = [
[400, 400, 400, 400],
# [800, 640], [160, 80], [80, 40],
mlp.get_layer('h1').set_weights(init_weights)
mlp.save(pretrained_mlp_path)
os.environ['CUDA_VISIBLE_DEVICES'] = '1'
model_dir = 'SparseAE/'
generate_dataset(True, True, model_dir)
data_dir = model_dir + 'UNSW/'
pretrained_mlp_path = data_dir + 'sae_mlp.h5'
raw_train_dataset = np.load(data_dir + 'train_dataset.npy')
raw_test_dataset = np.load(data_dir + 'test_dataset.npy')
y = np.load(data_dir + 'train_labels.npy')
y_test = np.load(data_dir + 'test_labels.npy')
X, _, X_test = min_max_scale(raw_train_dataset, None, raw_test_dataset)
X, y = permutate_dataset(X, y)
print('Training set', X.shape, y.shape)
print('Test set', X_test.shape)
num_samples, num_classes = y.shape
feature_size = X.shape[1]
encoder_size = 800
num_epoch = 160
batch_size = 80
class_weights = None
sae_weights = pretrain_model()
build_model(sae_weights)
fold = 5
skf = StratifiedKFold(n_splits=fold)
hist = {'train_loss': [], 'valid_loss': []}
os.environ['CUDA_VISIBLE_DEVICES'] = '3'
model_dir = 'KerasMLP/'
generate_dataset(False, True, model_dir)
data_dir = model_dir + 'NSLKDD/'
mlp_path = data_dir + 'mlp.h5'
raw_train_dataset = np.load(data_dir + 'train_dataset.npy')
raw_valid_dataset = np.load(data_dir + 'valid_dataset.npy')
raw_test_dataset = np.load(data_dir + 'test_dataset.npy')
train_labels = np.load(data_dir + 'train_labels.npy')
valid_labels = np.load(data_dir + 'valid_labels.npy')
test_labels = np.load(data_dir + 'test_labels.npy')
[train_dataset, valid_dataset, test_dataset] = min_max_scale(
raw_train_dataset, raw_valid_dataset, raw_test_dataset)
train_dataset, train_labels = permutate_dataset(train_dataset, train_labels)
valid_dataset, valid_labels = permutate_dataset(valid_dataset, valid_labels)
test_dataset, test_labels = permutate_dataset(test_dataset, test_labels)
print('Training set', train_dataset.shape, train_labels.shape)
print('Validation set', valid_dataset.shape, valid_labels.shape)
print('Test set', test_dataset.shape, test_labels.shape)
batch_size = 80
keep_prob = 0.8
num_epoch = 200
tail = 160
incremental = False
if incremental is False:
num_samples, num_classes = train_labels.shape
feature_size = train_dataset.shape[1]
hidden_size = [800, 480]
i])
X_train = np.concatenate((X_train, ftr), axis=1)
print(X_train.shape)
print(X_valid.shape, fv.shape)
X_valid = np.concatenate((X_valid, fv), axis=1)
X_test = np.concatenate((X_test, fte), axis=1)
X_train = np.concatenate((X_train, train_disc), axis=1)
X_valid = np.concatenate((X_valid, valid_disc), axis=1)
X_test = np.concatenate((X_test, test_disc), axis=1)
print("Augmenting discrete & embedding dataset", X_train.shape)
[X_train, X_valid, X_test] = min_max_scale(X_train, X_valid, X_test)
print("Min-max scaled dataset", X_train.shape, X_test.shape)
X_train, y_train = permutate_dataset(X_train, y_train)
X_valid, y_valid = permutate_dataset(X_valid, y_valid, 'Valid')
X_test, y_test = permutate_dataset(X_test, y_test, 'Test')
num_samples, num_features = X_train.shape
num_classes = y_train.shape[1]
batch_size = 40
keep_prob = 0.8
beta = 0.000
weights = [1.0, 1.0]
num_epochs = [160]
init_lrs = [0.001]
hidden_layer_sizes = [
[400, 200, 100],
# [800, 640], [160, 80], [80, 40],
# [400, 360, 320],
mp_classifier.exit()
np.random.seed(1944)
generate_dataset(True)
raw_train_dataset = np.load('UNSW/train_dataset.npy')
train_labels = np.load('UNSW/train_labels.npy')
raw_valid_dataset = np.load('UNSW/valid_dataset.npy')
valid_labels = np.load('UNSW/valid_labels.npy')
raw_test_dataset = np.load('UNSW/test_dataset.npy')
test_labels = np.load('UNSW/test_labels.npy')
columns = np.array(range(1, 27))
[train_dataset, valid_dataset,
test_dataset] = augment_quantiled(raw_train_dataset, raw_valid_dataset,
raw_test_dataset, columns)
train_dataset, train_labels = permutate_dataset(train_dataset, train_labels)
valid_dataset, valid_labels = permutate_dataset(valid_dataset, valid_labels)
test_dataset, test_labels = permutate_dataset(test_dataset, test_labels)
# Generate fake attacking data using GAN
fake_dataset, fake_labels = generate_fake_data(train_dataset, train_labels)
# Mix fake and real data to do supervised learning
mixed_dataset = np.concatenate((train_dataset, fake_dataset), axis=0)
mixed_labels = np.concatenate((train_labels, fake_labels), axis=0)
permutate_dataset(mixed_dataset, mixed_labels, name='Fake')
print('Mix trainset with fake set', mixed_dataset.shape, mixed_labels.shape)
tf.reset_default_graph()
classify(mixed_dataset, mixed_labels, valid_dataset, valid_labels,
test_dataset, test_labels)
