def class_before():
print('Gathering before classification')
train['Class'] = train['Class'].replace(
['back.', 'land.', 'neptune.', 'pod.', 'smurf.', 'teardrop.'], 'dos.')
train['Class'] = train['Class'].replace(
['buffer_overflow.', 'rootkit.', 'loadmodule.', 'perl.'], 'u2r.')
train['Class'] = train['Class'].replace([
'ftp_write.', 'guess_passwd.', 'imap.', 'multihop.', 'phf.', 'spy.',
'warezclient.', 'warezmaster.'
], 'r2l.')
train['Class'] = train['Class'].replace(
['ipsweep.', 'nmap.', 'portsweep.', 'satan.'], 'probe.')
test['Class'] = test['Class'].replace([
'udpstorm.', 'processtable.', 'mailbomb.', 'apache2.', 'back.',
'land.', 'neptune.', 'pod.', 'smurf.', 'teardrop.'
], 'dos.')
test['Class'] = test['Class'].replace([
'httptunnel.', 'xterm.', 'sqlattack.', 'ps.', 'buffer_overflow.',
'rootkit.', 'loadmodule.', 'perl.'
], 'u2r.')
test['Class'] = test['Class'].replace([
'xsnoop.', 'xlock.', 'sendmail.', 'named.', 'snmpgetattack.',
'snmpguess.', 'worm.', 'ftp_write.', 'guess_passwd.', 'imap.',
'multihop.', 'phf.', 'spy.', 'warezclient.', 'warezmaster.'
], 'r2l.')
test['Class'] = test['Class'].replace(
['saint.', 'mscan.', 'ipsweep.', 'nmap.', 'portsweep.', 'satan.'],
'probe.')
train_x, train_y, test_x, test_y = train_test(train, test)
# Fit model
classifier = GaussianNB()
c = classifier.fit(train_x, train_y)
# Predict training and test set results
train_pred = c.predict(train_x)
test_pred = c.predict(test_x)
train_accuracy, test_accuracy = result(train_y, train_pred, test_y,
test_pred)
train_accuracy = round(train_accuracy * 100, 2)
test_accuracy = round(test_accuracy * 100, 2)
return 'train: ', train_accuracy, ' test:', test_accuracy
save_dir = os.path.join(os.getcwd(), 'models')
model_name = 'double_conv_8filters_PSNR'
model_fname = save_dir + f'/{model_name}.h5'
# dataset preprocessing TODO : Save the two dataset for faster loading time?
# dataset = pr.cifar_download_and_scale()
# blurred = pr.blur_input(dataset, k_size=3)
#
# np.save('data/cifar10' , dataset)
# np.save('data/cifar10_blurred_ksize3', blurred)
dataset = np.load('data/cifar10.npy')[:size]
blurred = np.load('data/cifar10_blurred_ksize3.npy')[:size]
x_train, x_test, y_train, y_test = pr.train_test(blurred, dataset)
print(x_train.shape, x_test.shape, y_train.shape, y_test.shape)
inp = Input(shape=(32, 32, 3))
x = Conv2D(kernel_size=(3, 3), strides=(1, 1), filters=8, padding='same')(inp)
x = Conv2D(kernel_size=(3, 3), strides=(1, 1), filters=3, padding='same')(x)
model = Model(inp, x)
model.summary()
opt = tf.keras.optimizers.Adam(
learning_rate=0.001, # keras standard params
beta_1=0.9,
beta_2=0.999,
epsilon=1e-7)
from metrics import SSIM, PSNR, SSIM_loss, SSIM_multiscale_loss, MIX, SSIM_PSNR
import preprocessing as pr
from callbacks import CustomCB
# Constants
batch_size = 64
epochs = 50
size = 10000 # number of images used
cwd = os.getcwd()
dataset = np.load('data/cifar10.npy')[:size]
blurred = np.load('data/cifar10_blur_sigma0-3.npy')[:size]
x_train, x_test, y_train, y_test = pr.train_test(blurred[:size], dataset[:])
metrics = ['mean_squared_error', 'mean_absolute_error', PSNR, SSIM, MIX]
def create_model_unetlike():
inp = Input(shape=(32, 32, 3))
conv1 = Conv2D(kernel_size=(3, 3),
strides=(1, 1),
filters=8,
padding='same',
activation='relu')(inp)
mp1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv1)
# 16x16x8
conv2 = Conv2D(kernel_size=(3, 3),
import pandas as pd
from mlflow import log_artifacts, log_metric, log_param
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from baseline_model import base_model
from config import config # insert . to run from outside
from preprocessing import feature_vectorizer, load_dataset, preprocessing, train_test
# Load dataset from its path
dataset = load_dataset(config.RAW_DATAPATH, config.DATASET_NAME)
# Clean the data
dataset = preprocessing(dataset)
# Train test split the loaded dataset
X_train, X_test, y_train, y_test = train_test(dataset)
# Obtain feature vectors of the dataset
X_train, X_test = feature_vectorizer(X_train, X_test, TfidfVectorizer)
alpha = float(sys.argv[1])
with mlflow.start_run():
f1_score, accuracy = base_model(
X_train, X_test, y_train, y_test, MultinomialNB, alpha
)
print("Multinomial Naive Bayes model")
print(" f1_score: %s" % f1_score)
print(" accuracy: %s" % accuracy)
mlflow.log_param("alpha ", alpha)