def get_nsl_data():
dataset_names = ['NSLKDD/KDD%s.csv' % x for x in ['Train', 'Test']]
feature_file = 'NSLKDD/feature_names.csv'
headers, _, _, _ = nslkdd.get_feature_names(feature_file)
symbolic_features = nslkdd.discovery_feature_volcabulary(dataset_names)
integer_features = nslkdd.discovery_integer_map(feature_file,
dataset_names)
continuous_features = nslkdd.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'nsl')
test_X, test_y = get_dataset(dataset_names[1], headers, 'nsl')
train_dict = dict()
test_dict = dict()
merged_inputs = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, merged_inputs,
X, test_X, train_dict, test_dict, 'nsl')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, merged_inputs, X,
test_X, train_dict, test_dict, 'nsl')
return train_dict, y, test_dict, test_y
def modality_net_nsl(hidden):
dataset_names = ['NSLKDD/KDD%s.csv' % x for x in ['Train', 'Test']]
feature_file = 'NSLKDD/feature_names.csv'
headers, _, _, _ = nslkdd.get_feature_names(feature_file)
symbolic_features = nslkdd.discovery_feature_volcabulary(dataset_names)
integer_features = nslkdd.discovery_integer_map(feature_file,
dataset_names)
continuous_features = nslkdd.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'nsl')
test_X, test_y = get_dataset(dataset_names[1], headers, 'nsl')
train_dict = dict()
test_dict = dict()
merged_inputs = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, merged_inputs,
X, test_X, train_dict, test_dict, 'nsl')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, merged_inputs, X,
test_X, train_dict, test_dict, 'nsl')
logger.debug('merge input_dim for NSLKDD dataset = %s' % merged_dim)
merge = concatenate(embeddings + large_discrete + [cont_component],
name='concate_features_nsl')
h1 = Dense(hidden[0], activation='relu', name='h1_nsl')(merge)
dropout = Dropout(drop_prob)(h1)
h2 = Dense(hidden[1], activation='relu', name='h2_nsl')(dropout)
bn = BatchNormalization(name='bn_nsl')(h2)
h3 = Dense(hidden[2], activation='sigmoid', name='sigmoid_nsl')(bn)
sm = Dense(2, activation='softmax', name='output_nsl')(h3)
model = Model(inputs=merged_inputs, outputs=sm)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(train_dict, {'output_nsl': y}, batch_size, num_epochs)
modnet['nsl_loss'].append(history.history['loss'])
score = model.evaluate(train_dict, y, y.shape[0])
logger.debug('modnet[nsl] train loss\t%.6f' % score[0])
logger.info('modenet[nsl] train accu\t%.6f' % score[1])
modnet['nsl']['train'].append(score[1])
score = model.evaluate(test_dict, test_y, test_y.shape[0])
logger.debug('modnet[nsl] test loss\t%.6f' % score[0])
logger.info('modenet[nsl] test accu\t%.6f' % score[1])
modnet['nsl']['test'].append(score[1])
model.save_weights('ModalityNets/modnet_nsl.h5')
# np.savez('ModalityNets/nsl_EX.npy', train=EX, test=EX_test)
return merge, merged_inputs, train_dict, test_dict, y, test_y
def get_nsl_data():
dataset_names = ['NSLKDD/KDD%s.csv' % x for x in ['Train', 'Test']]
feature_file = 'NSLKDD/feature_names.csv'
headers, _, _, _ = nslkdd.get_feature_names(feature_file)
symbolic_features = nslkdd.discovery_feature_volcabulary(dataset_names)
integer_features = nslkdd.discovery_integer_map(feature_file,
dataset_names)
continuous_features = nslkdd.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'nsl')
X_test, y_test = get_dataset(dataset_names[1], headers, 'nsl')
train_dict = dict()
test_dict = dict()
input_layer = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, input_layer, X,
X_test, train_dict, test_dict, 'nsl')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, input_layer, X,
X_test, train_dict, test_dict, 'nsl')
pprint('merge input_dim for NSLKDD dataset = %s' % merged_dim)
merged_layer = concatenate(embeddings + large_discrete + [cont_component],
name='concate_features_nsl')
model = Model(inputs=input_layer, outputs=merged_layer)
model.compile('adam', 'mse')
model.summary()
MX = model.predict(train_dict)
MX_test = model.predict(test_dict)
return MX, MX_test, y, y_test
for key in results:
logger.info("%s: %s" % (key, results[key]))
predictions = []
for x in m.predict(test_ib):
predictions.append(x['probabilities'])
conf_table = measure_prediction(np.array(predictions), ohe, model_dir)
history['confusion_table'] = conf_table
return history
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
CSV_COLUMNS, symbolic_names, continuous_names, discrete_names = \
get_feature_names('NSLKDD/feature_names.csv')
print(symbolic_names)
print(continuous_names)
print(discrete_names)
"""
quantile_names = []
for name in continuous_names + discrete_names:
quantile_names.append(name + '_quantile')
"""
# Build wide columns
protocol = tf.feature_column.categorical_column_with_vocabulary_list(
'protocol', get_categorical_values('protocol'))
service = tf.feature_column.categorical_column_with_vocabulary_list(
'service', get_categorical_values('service'))
flag = tf.feature_column.categorical_column_with_vocabulary_list(
'flag', get_categorical_values('flag'))