def get_unsw_data():
dataset_names = [
'UNSW/UNSW_NB15_%s-set.csv' % x for x in ['training', 'testing']
]
feature_file = 'UNSW/feature_names_train_test.csv'
headers, _, _, _ = unsw.get_feature_names(feature_file)
symbolic_features = unsw.discovery_feature_volcabulary(dataset_names)
integer_features = unsw.discovery_integer_map(feature_file, dataset_names)
continuous_features = unsw.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'unsw')
test_X, test_y = get_dataset(dataset_names[1], headers, 'unsw')
train_dict = dict()
test_dict = dict()
merged_inputs = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, merged_inputs,
X, test_X, train_dict, test_dict, 'unsw')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, merged_inputs, X,
test_X, train_dict, test_dict, 'unsw')
return train_dict, y, test_dict, test_y
def modality_net_unsw(hidden):
dataset_names = [
'UNSW/UNSW_NB15_%s-set.csv' % x for x in ['training', 'testing']
]
feature_file = 'UNSW/feature_names_train_test.csv'
headers, _, _, _ = unsw.get_feature_names(feature_file)
symbolic_features = unsw.discovery_feature_volcabulary(dataset_names)
integer_features = unsw.discovery_integer_map(feature_file, dataset_names)
continuous_features = unsw.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'unsw')
test_X, test_y = get_dataset(dataset_names[1], headers, 'unsw')
train_dict = dict()
test_dict = dict()
merged_inputs = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, merged_inputs,
X, test_X, train_dict, test_dict, 'unsw')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, merged_inputs, X,
test_X, train_dict, test_dict, 'unsw')
logger.debug('merge input_dim for UNSW-NB dataset = %s' % merged_dim)
merge = concatenate(embeddings + large_discrete + [cont_component],
name='concate_features_unsw')
h1 = Dense(hidden[0], activation='relu', name='h1_unsw')(merge)
dropout = Dropout(drop_prob)(h1)
h2 = Dense(hidden[1], activation='relu', name='h2_unsw')(dropout)
bn = BatchNormalization(name='bn_unsw')(h2)
h3 = Dense(hidden[2], activation='sigmoid', name='sigmoid_unsw')(bn)
sm = Dense(2, activation='softmax', name='output_unsw')(h3)
model = Model(inputs=merged_inputs, outputs=sm)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(train_dict, {'output_unsw': y}, batch_size, num_epochs)
modnet['unsw_loss'].append(history.history['loss'])
score = model.evaluate(train_dict, y, y.shape[0])
logger.debug('modnet[unsw] train loss\t%.6f' % score[0])
logger.info('modenet[unsw] train accu\t%.6f' % score[1])
modnet['unsw']['train'].append(score[1])
score = model.evaluate(test_dict, test_y, test_y.shape[0])
logger.debug('modnet[unsw] test loss\t%.6f' % score[0])
logger.info('modenet[unsw] test accu\t%.6f' % score[1])
modnet['unsw']['test'].append(score[1])
model.save_weights('ModalityNets/modnet_unsw.h5')
# np.savez('ModalityNets/unsw_EX.npy', train=EX, test=EX_test)
return merge, merged_inputs, train_dict, test_dict, y, test_y
def get_unsw_data():
dataset_names = [
'UNSW/UNSW_NB15_%s-set.csv' % x for x in ['training', 'testing']
]
feature_file = 'UNSW/feature_names_train_test.csv'
headers, _, _, _ = unsw.get_feature_names(feature_file)
symbolic_features = unsw.discovery_feature_volcabulary(dataset_names)
integer_features = unsw.discovery_integer_map(feature_file, dataset_names)
continuous_features = unsw.discovery_continuous_map(
feature_file, dataset_names)
X, y = get_dataset(dataset_names[0], headers, 'unsw')
X_test, y_test = get_dataset(dataset_names[1], headers, 'unsw')
train_dict = dict()
test_dict = dict()
input_layer = []
embeddings = []
large_discrete = []
merged_dim = 0
merged_dim += build_embeddings(symbolic_features, integer_features,
embeddings, large_discrete, input_layer, X,
X_test, train_dict, test_dict, 'unsw')
merged_dim += len(continuous_features)
cont_component = build_continuous(continuous_features, input_layer, X,
X_test, train_dict, test_dict, 'unsw')
pprint('merge input_dim for UNSW-NB dataset = %s' % merged_dim)
merged_layer = concatenate(embeddings + large_discrete + [cont_component],
name='concate_features_unsw')
model = Model(inputs=input_layer, outputs=merged_layer)
model.compile('adam', 'mse')
model.summary()
MX = model.predict(train_dict)
MX_test = model.predict(test_dict)
return MX, MX_test, y, y_test
name='embed_%s' % name)(column)
temp = Flatten(name='flat_%s' % name)(temp)
embeddings.append(temp)
merged_dim += dim_E
else:
large_inputs.append(column)
print('Large feature %s is treated as continuous' % name)
mm = MinMaxScaler()
raw_data = raw_data.reshape((len(raw_data), 1))
test_raw_data = test_raw_data.reshape((len(test_raw_data), 1))
mm.fit(np.concatenate((raw_data, test_raw_data), axis=0))
train_dict[name] = mm.transform(raw_data)
test_dict[name] = mm.transform(test_raw_data)
merged_dim += 1
continuous_features = discovery_continuous_map(feature_file, dataset_names)
continuous_inputs = Input(shape=(len(continuous_features), ),
name='continuous')
merged_inputs.append(continuous_inputs)
raw_data = X[continuous_features.keys()].as_matrix()
test_raw_data = test_X[continuous_features.keys()].as_matrix()
mm = MinMaxScaler()
mm.fit(np.concatenate((raw_data, test_raw_data), axis=0))
train_dict['continuous'] = mm.transform(raw_data)
test_dict['continuous'] = mm.transform(test_raw_data)
merged_dim += len(continuous_features)
print('merge input_dim for this dataset = %s' % merged_dim)
merge = concatenate(embeddings + large_inputs + [continuous_inputs],
name='merge_features')
h1 = Dense(400, activation='relu', name='hidden1')(merge)