def evaluator(args): is_flow_cache_experiment = True K = 10 samplerdir, classifier_name, benign_threshold = args print('treshold at ', benign_threshold) clf_dir = get_classifier_dir(samplerdir, classifier_name, class_weight=None) gt_classes_str = pd.read_csv(join(samplerdir, '{}fold_0.csv'.format(K)), usecols=['Label'])['Label'].unique() gt_classes = sorted(encode_label(gt_classes_str)) C = len(gt_classes) cm_any_sum = np.zeros((C, C), dtype=float) cm_majority_sum = np.zeros((C, C), dtype=float) cm_all_sum = np.zeros((C, C), dtype=float) col_names = ['Timestamp'] + get_cols4eval() for test_index in range(K): runs_dir = join(clf_dir, 'K_{}/log/{}'.format(K, test_index)) if is_flow_cache_experiment: runs_dir = replace_w_unlimited_FC(runs_dir) clf = load_classifier(classifier_name, runs_dir) test_csv_file = join(samplerdir, '{}fold_{}.csv'.format(K, test_index)) df = pd.read_csv(test_csv_file, usecols=col_names, dtype=get_dtype4normalized()) #,skiprows=skip_idx) df['Day'] = df['Timestamp'].map(lambda x: x[:2]).astype( str) # type string df = df.sort_values(by=['Flow ID', 'Day', 'Label' ]) #used when deriving flow level metric pred_per_record = predict_proba_per_record(df, clf, benign_threshold) flowids, flowlabels_str, grouped = group_data(df) y = encode_label(flowlabels_str) pred_any, pred_maj, pred_all = evaluate_per_flow( grouped, y, pred_per_record) any_cm = confusion_matrix(y, pred_any) majority_cm = confusion_matrix(y, pred_maj) all_cm = confusion_matrix(y, pred_all) cm_any_sum += any_cm cm_majority_sum += majority_cm cm_all_sum += all_cm #gt_classes = np.unique(y) result_logger_ids18( join(clf_dir, 'K_{}_benign_threshold_{}'.format(K, benign_threshold)), gt_classes, (any_cm, majority_cm, all_cm), 'fold_{}_'.format(test_index)) result_logger_ids18( join(clf_dir, 'K_{}_benign_threshold_{}'.format(K, benign_threshold)), gt_classes, (cm_any_sum, cm_majority_sum, cm_all_sum), 'fold_avg_'.format(K))
def load_folds(dataroot, fold_prefix, K): df_list = [ pd.read_csv(join(dataroot, fold_prefix.format(i)), usecols=get_cols4ml(), dtype=get_dtype4normalized()) for i in range(K) ] fold_data = [ (df.drop(columns=['Label']).values, encode_label(df.Label.values)) \ for df in df_list] return fold_data
def __init__(self, csv_file): self.csv_file = csv_file self.num_records = self.get_num_records(csv_file) df = pd.read_csv(csv_file, engine='c', usecols=get_cols4ml(), dtype=get_dtype4normalized()) self.x = torch.FloatTensor(df.drop(columns=['Label']).values) self.y = torch.LongTensor(self.encode_label(df.Label.values))
def shuffle(fn): print(fn) df = pd.read_csv(fn, dtype=get_dtype4normalized(), engine='c') # 1min df = df.sample(frac=1, random_state = getSeed(), replace=False)# 20 sec fn_o = fn.replace('fold','shuffled_fold') tick = time.time() df.to_csv(fn_o,index=False, chunksize=10**4) # 7min print("Wrote in {:.2f}".format(time.time()-tick))
def __init__(self, csv_file, chunksize=10**4): self.csv_file = csv_file self.chunksize = chunksize self.seen_so_far = 0 # number of flow records seen so far self.seen_chunks = 0 self.iterableReader = pd.read_csv(csv_file, engine='c', usecols=get_cols4ml(), dtype=get_dtype4normalized(), chunksize=chunksize) label_to_id, id_to_label, _ = get_ids18_mappers() self.label_to_id = label_to_id
def evaluator(dataroot, classifier_name): print('evaluating ', ntpath.basename(dataroot)) test_csv_file = join(dataroot, 'fold_0.csv') result_test = subprocess.run(['wc', '-l', test_csv_file], stdout=subprocess.PIPE) test_records = int(result_test.stdout.split()[0]) # load Classifier class_weight = get_class_weights( dataroot) # because it is not important for evaluation num_class = 14 # because we remove Label,FlowID,Timestamp columns from X classifier_args, config = get_args(classifier_name, num_class, class_weight) pre_fingerprint = join(dataroot, 'c_{}'.format(classifier_name)) fingerprint = pre_fingerprint + config print('clf fingerprint', ntpath.basename(fingerprint)) classifier_args['runs_dir'] = join(fingerprint, 'log') clf = ClassifierLoader().load(classifier_args) # classifier loaded # load data col_names = get_cols4eval() col_names.append('Timestamp') df = pd.read_csv(test_csv_file, usecols=col_names, dtype=get_dtype4normalized()) print("Record distribution:") print(df.Label.value_counts()) df['Day'] = df['Timestamp'].map(lambda x: x[:2]).astype(str) # type string #group data df = df.sort_values(by=['Flow ID', 'Label' ]) # replaces ordering task in per_flow_eval flowids, flowlabels, grouped = group_data(df) y = encode_label(flowlabels) print("data is grouped and labels are encoded") pred_any, pred_maj, pred_all, _ = evaluate_per_flow(clf, y, grouped, df) any_cm = confusion_matrix(y, pred_any) maj_cm = confusion_matrix(y, pred_maj) all_cm = confusion_matrix(y, pred_all) any_acc = metrics.balanced_accuracy_score(y, pred_any) maj_acc = metrics.balanced_accuracy_score(y, pred_maj) all_acc = metrics.balanced_accuracy_score(y, pred_all) print(any_acc, maj_acc, all_acc) result_logger_ids18(fingerprint, np.unique(y), (any_cm, maj_cm, all_cm), 'test')
def evaluator(dataroot, classifier_name): K = 10 print("\nevaling ", dataroot) gt_num_class = pd.read_csv(join(dataroot, '{}fold_0.csv'.format(K)), usecols=['Label'])['Label'].nunique() # load Classifier classifier_args, config = get_args(classifier_name, class_weight=None) print("Balancing technique: ", classifier_args['balance']) pre_fingerprint = join(dataroot, 'c_{}'.format(classifier_name)) fingerprint = join(pre_fingerprint + config, 'K_{}'.format(K)) logdir = join(fingerprint, 'log') gt_classes = None for test_index in range(K): print("************* Testing holdout ", test_index, '************') runs_dir = join(logdir, '{}'.format(test_index)) print(runs_dir) print("with model: ", runs_dir) classifier_args['runs_dir'] = runs_dir loader = ClassifierLoader() clf = loader.load(classifier_args) # classifier loaded # load data col_names = get_cols4eval() col_names.append('Timestamp') test_csv_file = join(dataroot, '{}fold_{}.csv'.format(K, test_index)) df = pd.read_csv(test_csv_file, usecols=col_names, dtype=get_dtype4normalized(), nrows=4096) #,skiprows=skip_idx) df['Day'] = df['Timestamp'].map(lambda x: x[:2]).astype( str) # type string df = df.sort_values(by=['Flow ID', 'Day', 'Label']) # Done inference_times = predict(df, clf) for key, val in inference_times.items(): print(key, val) break
def worker(fn, cnt): reader = pd.read_csv(fn, usecols=get_cols4eval(), engine='c', dtype=get_dtype4normalized(), chunksize=10**6) # 1.5 min df = pd.concat([df for df in reader], sort=False) print(ntpath.basename(fn), df.Label.value_counts()) g = df.groupby(['Label'], sort=False) # 0.00 sec new_df = pd.DataFrame( g.apply(lambda x: x.sample(cnt, random_state=getSeed(), replace=True). reset_index(drop=True))) # 33 sec outfile = fn.replace(foldname_prefix, '{}bal_fold_'.format(K)) new_df = new_df.sample(frac=1, random_state=getSeed(), replace=False) # shuffling, 1min tick = time.time() new_df.to_csv(outfile, chunksize=10**5, index=False) print("Written in {:.2f} ".format(time.time() - tick)) # 3.5 mins for SFS
def evaluator(dataroot,classifier_name): K=10 print("\nevaling ",dataroot) gt_num_class = pd.read_csv(join(dataroot,'{}fold_0.csv'.format(K)),usecols=['Label'])['Label'].nunique() # load Classifier classifier_args, config = get_args(classifier_name, class_weight=None ) print("Balancing technique: ", classifier_args['balance']) pre_fingerprint = join(dataroot, 'c_{}'.format(classifier_name)) fingerprint = join(pre_fingerprint + config,'K_{}'.format(K)) logdir = join(fingerprint,'log') gt_classes = None for test_index in range(K): print("************* Testing holdout ", test_index,'************') runs_dir = join(logdir,'{}'.format(test_index)) print(runs_dir) print("with model: ",runs_dir) classifier_args['runs_dir'] = runs_dir loader = ClassifierLoader() clf = loader.load(classifier_args) # classifier loaded # load data col_names = get_cols4eval() col_names.append('Timestamp') test_csv_file = join(dataroot, '{}fold_{}.csv'.format(K,test_index)) df = pd.read_csv(test_csv_file,usecols=col_names,dtype=get_dtype4normalized(), nrows=10000)#,skiprows=skip_idx) x = df.drop(columns=['Flow ID','Label','Timestamp']).values # Done for i in range(10): inference_time = inference(clf,x) print(inference_time) break
def fold_worker(fn, b,m,d): #Assumptions: # 1. Benign records are in the beginning of each fold # 2. num of benign records is `b` # 3. num of duplicates made in over/under sampling is `d` # 4. num of malicious duplicated records `m` # then, using `m+d` benign records ensures we have equal #records against balancing case # which means we should remove last `b-(m+d)` benign records from fold K=5 # num iof folds num_to_exclude = (b - (m+d))//K tick = time.time() df = pd.read_csv(fn, engine='c', dtype=get_dtype4normalized()) # 4~5 min print("Read fold in {:.2f} min".format((time.time()-tick)/60)) N = df.shape[0] r_df = df[:N-num_to_exclude] sh_df = r_df.sample(frac=1,random_state=getSeed(), replace=False) outfile = fn.replace(in_fold, out_fold) assert fn!=outfile, "outfile is same as input file {}".format(ntpath.basename(fn)) sh_df.to_csv(outfile,chunksize=10**5, index=False)
def evaluator(dataroot, classifier_name): print(ntpath.basename(dataroot)) test_csv_file = join(dataroot, 'fold_0.csv') # load Classifier classifier_args, config = get_args(classifier_name, num_class='dummy', class_weight=None) print("Balancing technique: ", classifier_args['balance']) pre_fingerprint = join(dataroot, 'c_{}'.format(classifier_name)) fingerprint = pre_fingerprint + config logdir = join(fingerprint, 'log') if 'mem_const_exp' == 'mem_const_exp': # for mem constraint exp start = logdir.find('CSVs_r') end = logdir.find('_m_') CSV_dirname = logdir[start:end] logdir = logdir.replace(CSV_dirname, 'CSVs_r_1.0') # end print(logdir) classifier_args['runs_dir'] = logdir loader = ClassifierLoader() clf = loader.load(classifier_args) if 'noprint_clf_attr' == 'print_clf_attr' and 'tree' in classifier_name: print("maximum depth of the tree ", clf.tree_.max_depth) import matplotlib.pyplot as plt from sklearn.tree import plot_tree plt.figure() plot_tree(clf, filled=True) plt.savefig(join(logdir, 'tree_plot.png'), dpi=1000) return if 'norf_attr' == 'rf_attr' and 'forest' in classifier_name: depth = [est.tree_.max_depth for est in clf.estimators_] print(depth) depth = np.array(depth) print("forest depth", depth.mean(), depth.max(), depth.min()) print("maximum depth of the tree ", clf.base_estimator_.max_depth) return import matplotlib.pyplot as plt from sklearn.tree import plot_tree plt.figure() plot_tree(clf, filled=True) plt.savefig(join(logdir, 'tree_plot.png'), dpi=1000) return print("Classifier Loaded!") # classifier loaded # load data col_names = get_cols4eval() col_names.append('Timestamp') df = pd.read_csv(test_csv_file, usecols=col_names, dtype=get_dtype4normalized()) #,skiprows=skip_idx) df['Day'] = df['Timestamp'].map(lambda x: x[:2]).astype(str) # type string df = df.sort_values(by=['Flow ID', 'Day', 'Label']) print(df.Label.value_counts()) # Done pred_per_record = predict_per_record(df, clf) per_record_evaluation(df, pred_per_record) tick = time.time() flowids, flowlabels_str, grouped = group_data(df) print("Grouped in {:.2f} min".format((time.time() - tick) / 60)) y = encode_label(flowlabels_str) print("data is grouped and labels are encoded") pred_any, pred_maj, pred_all, y = evaluate_per_flow( clf, y, grouped, df, pred_per_record) gt_classes = np.unique(y) pred_classes = np.unique(pred_any) nunique_gt = len(gt_classes) nunique_pred = len(pred_classes) assert nunique_gt >= nunique_pred, "should not predict non existing class(es), but \n{} < \n{}".format( gt_classes, pred_classes) any_cm = confusion_matrix(y, pred_any) majority_cm = confusion_matrix(y, pred_maj) all_cm = confusion_matrix(y, pred_all) any_acc = metrics.balanced_accuracy_score(y, pred_any) maj_acc = metrics.balanced_accuracy_score(y, pred_maj) all_acc = metrics.balanced_accuracy_score(y, pred_all) print(any_acc, maj_acc, all_acc) result_logger_ids18(fingerprint, gt_classes, (any_cm, majority_cm, all_cm), 'test')
def classify(dataroot,classifier_name): K=5 balance = get_balancing_technique() train_data = [] #single fold 29M records # 4 folds 120M records # if 20M records require 5% RAM # then 120M records require 30% memory print("Reading the data...") tick=time.time() label_to_id, id_to_label, _ = get_ids18_mappers() num_train_records = 0 print("Reading 4 folds ") if balance=='with_loss' or balance=='no' or balance=='with_loss_sub': regex = 'r_fold_{}.csv' elif balance=='explicit': regex = 'bal_fold_{}.csv' for fold_index in tqdm(range(K)): if fold_index==0: continue reader = pd.read_csv(join(dataroot,regex.format(fold_index)),chunksize=10**6, usecols=get_cols4ml(), dtype=get_dtype4normalized())# 10**6 rows read in 9min, total is 29*10**6 # remove the extra header row for df in tqdm(reader): y_str = df.Label.values x = df.drop(columns=['Label']).values train_data.append((x,encode_label(y_str))) num_train_records +=df.shape[0] print(df.memory_usage(deep=True).sum()*(799902)/(1024*1024*1024 )) tock = time.time() print("read data in {:.2f}".format(tock-tick)) # 24min classifier_args, config = get_args(classifier_name, num_class='dummy', class_weight=None) pre_fingerprint = join(dataroot, 'c_{}'.format(classifier_name)) fingerprint = pre_fingerprint + config logdir = join(fingerprint,'log') ensure_dir(logdir) X_train = np.concatenate([fold[0] for fold in train_data ],axis=0) y_train = np.concatenate([fold[1] for fold in train_data ],axis=0) classifier_args['runs_dir']=logdir print("Start training") tick = time.time() clf= get_classifier(classifier_args) print("classes") print(np.unique(y_train)) clf.fit(X_train, y_train) fn = classifier_args['runs_dir']+'.pkl' pickle.dump(clf,open(fn,'wb')) print("Done training {} flow records in {:.2f} sec".format(y_train.shape[0],time.time()-tick))