def class_before(): print('Gathering before classification') train['Class'] = train['Class'].replace( ['back.', 'land.', 'neptune.', 'pod.', 'smurf.', 'teardrop.'], 'dos.') train['Class'] = train['Class'].replace( ['buffer_overflow.', 'rootkit.', 'loadmodule.', 'perl.'], 'u2r.') train['Class'] = train['Class'].replace([ 'ftp_write.', 'guess_passwd.', 'imap.', 'multihop.', 'phf.', 'spy.', 'warezclient.', 'warezmaster.' ], 'r2l.') train['Class'] = train['Class'].replace( ['ipsweep.', 'nmap.', 'portsweep.', 'satan.'], 'probe.') test['Class'] = test['Class'].replace([ 'udpstorm.', 'processtable.', 'mailbomb.', 'apache2.', 'back.', 'land.', 'neptune.', 'pod.', 'smurf.', 'teardrop.' ], 'dos.') test['Class'] = test['Class'].replace([ 'httptunnel.', 'xterm.', 'sqlattack.', 'ps.', 'buffer_overflow.', 'rootkit.', 'loadmodule.', 'perl.' ], 'u2r.') test['Class'] = test['Class'].replace([ 'xsnoop.', 'xlock.', 'sendmail.', 'named.', 'snmpgetattack.', 'snmpguess.', 'worm.', 'ftp_write.', 'guess_passwd.', 'imap.', 'multihop.', 'phf.', 'spy.', 'warezclient.', 'warezmaster.' ], 'r2l.') test['Class'] = test['Class'].replace( ['saint.', 'mscan.', 'ipsweep.', 'nmap.', 'portsweep.', 'satan.'], 'probe.') train_x, train_y, test_x, test_y = train_test(train, test) # Fit model classifier = GaussianNB() c = classifier.fit(train_x, train_y) # Predict training and test set results train_pred = c.predict(train_x) test_pred = c.predict(test_x) train_accuracy, test_accuracy = result(train_y, train_pred, test_y, test_pred) train_accuracy = round(train_accuracy * 100, 2) test_accuracy = round(test_accuracy * 100, 2) return 'train: ', train_accuracy, ' test:', test_accuracy
save_dir = os.path.join(os.getcwd(), 'models') model_name = 'double_conv_8filters_PSNR' model_fname = save_dir + f'/{model_name}.h5' # dataset preprocessing TODO : Save the two dataset for faster loading time? # dataset = pr.cifar_download_and_scale() # blurred = pr.blur_input(dataset, k_size=3) # # np.save('data/cifar10' , dataset) # np.save('data/cifar10_blurred_ksize3', blurred) dataset = np.load('data/cifar10.npy')[:size] blurred = np.load('data/cifar10_blurred_ksize3.npy')[:size] x_train, x_test, y_train, y_test = pr.train_test(blurred, dataset) print(x_train.shape, x_test.shape, y_train.shape, y_test.shape) inp = Input(shape=(32, 32, 3)) x = Conv2D(kernel_size=(3, 3), strides=(1, 1), filters=8, padding='same')(inp) x = Conv2D(kernel_size=(3, 3), strides=(1, 1), filters=3, padding='same')(x) model = Model(inp, x) model.summary() opt = tf.keras.optimizers.Adam( learning_rate=0.001, # keras standard params beta_1=0.9, beta_2=0.999, epsilon=1e-7)
from metrics import SSIM, PSNR, SSIM_loss, SSIM_multiscale_loss, MIX, SSIM_PSNR import preprocessing as pr from callbacks import CustomCB # Constants batch_size = 64 epochs = 50 size = 10000 # number of images used cwd = os.getcwd() dataset = np.load('data/cifar10.npy')[:size] blurred = np.load('data/cifar10_blur_sigma0-3.npy')[:size] x_train, x_test, y_train, y_test = pr.train_test(blurred[:size], dataset[:]) metrics = ['mean_squared_error', 'mean_absolute_error', PSNR, SSIM, MIX] def create_model_unetlike(): inp = Input(shape=(32, 32, 3)) conv1 = Conv2D(kernel_size=(3, 3), strides=(1, 1), filters=8, padding='same', activation='relu')(inp) mp1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv1) # 16x16x8 conv2 = Conv2D(kernel_size=(3, 3),
import pandas as pd from mlflow import log_artifacts, log_metric, log_param from sklearn.ensemble import RandomForestClassifier from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.naive_bayes import MultinomialNB from baseline_model import base_model from config import config # insert . to run from outside from preprocessing import feature_vectorizer, load_dataset, preprocessing, train_test # Load dataset from its path dataset = load_dataset(config.RAW_DATAPATH, config.DATASET_NAME) # Clean the data dataset = preprocessing(dataset) # Train test split the loaded dataset X_train, X_test, y_train, y_test = train_test(dataset) # Obtain feature vectors of the dataset X_train, X_test = feature_vectorizer(X_train, X_test, TfidfVectorizer) alpha = float(sys.argv[1]) with mlflow.start_run(): f1_score, accuracy = base_model( X_train, X_test, y_train, y_test, MultinomialNB, alpha ) print("Multinomial Naive Bayes model") print(" f1_score: %s" % f1_score) print(" accuracy: %s" % accuracy) mlflow.log_param("alpha ", alpha)