def setUp(self): # Define data file and read X and y # Generate some data if the source data is missing this_directory = path.abspath(path.dirname(__file__)) mat_file = 'cardio.mat' try: mat = loadmat(path.join(*[this_directory, 'data', mat_file])) except TypeError: print('{data_file} does not exist. Use generated data'.format( data_file=mat_file)) X, y = generate_data(train_only=True) # load data except IOError: print('{data_file} does not exist. Use generated data'.format( data_file=mat_file)) X, y = generate_data(train_only=True) # load data else: X = mat['X'] y = mat['y'].ravel() X, y = check_X_y(X, y) self.X_train, self.X_test, self.y_train, self.y_test = \ train_test_split(X, y, test_size=0.4, random_state=42) self.base_estimators = [LOF(), LOF(), IForest(), COPOD()] self.clf = SUOD(base_estimators=self.base_estimators) self.clf.fit(self.X_train) self.roc_floor = 0.7
def model_init(self, model): """Model initialisation of a single model. """ if self.model == 'pca': self.models[model] = PCA(contamination=self.contamination) elif self.model == 'loda': self.models[model] = LODA(contamination=self.contamination) elif self.model == 'iforest': self.models[model] = IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination) elif self.model == 'cblof': self.models[model] = CBLOF(n_clusters=3, contamination=self.contamination) elif self.model == 'feature_bagging': self.models[model] = FeatureBagging( base_estimator=PCA(contamination=self.contamination), contamination=self.contamination) elif self.model == 'copod': self.models[model] = COPOD(contamination=self.contamination) elif self.model == 'hbos': self.models[model] = HBOS(contamination=self.contamination) else: self.models[model] = HBOS(contamination=self.contamination) self.custom_model_scalers[model] = MinMaxScaler()
def main(): scalers = ['no', 'std', 'minmax'] root = 'Unsupervised_Anamaly_Detection_csv' start = 0 counts = 90 CPUS = 3 CPUS_Models = 4 sklearn_models = [ 'AvgKNN', 'LargestKNN', 'MedKNN', 'PCA', 'COF', 'LODA', 'LOF', 'HBOS', 'MCD', 'AvgBagging', 'MaxBagging', 'IForest', 'CBLOF', 'COPOD', 'SOD', 'LSCPwithLODA', 'AveLMDD', 'VarLMDD', 'IqrLMDD', 'SoGaal', 'MoGaal', 'VAE', 'AutoEncoder' ] models = { 'BRM': BRM(bootstrap_sample_percent=70), 'GM': GaussianMixture(), 'IF': IsolationForest(), 'OCSVM': OneClassSVM(), 'EE': EllipticEnvelope(), 'AvgKNN': KNN(method='mean'), 'LargestKNN': KNN(method='largest'), 'MedKNN': KNN(method='median'), 'PCA': PCA(), 'COF': COF(), 'LODA': LODA(), 'LOF': LOF(), 'HBOS': HBOS(), 'MCD': MCD(), 'AvgBagging': FeatureBagging(combination='average'), 'MaxBagging': FeatureBagging(combination='max'), 'CBLOF': CBLOF(n_clusters=10, n_jobs=4), 'FactorAnalysis': FactorAnalysis(), 'KernelDensity': KernelDensity(), 'COPOD': COPOD(), 'SOD': SOD(), 'LSCPwithLODA': LSCP([LODA(), LODA()]), 'AveLMDD': LMDD(dis_measure='aad'), 'VarLMDD': LMDD(dis_measure='var'), 'IqrLMDD': LMDD(dis_measure='iqr'), 'SoGaal': SO_GAAL(), 'MoGaal': MO_GAAL(), 'VAE': VAE(encoder_neurons=[8, 4, 2]), 'AutoEncoder': AutoEncoder(hidden_neurons=[6, 3, 3, 6]), 'OCKRA': m_OCKRA(), } name = "30_Models" Parallel(n_jobs=CPUS) \ (delayed(runByScaler) (root, scaler, models, start, counts, other_models=sklearn_models, CPUS=CPUS_Models, save_name=name) for scaler in scalers)
def setUp(self): self.n_train = 200 self.n_test = 100 self.contamination = 0.1 self.roc_floor = 0.8 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, n_features=10, contamination=self.contamination, random_state=42) self.clf = COPOD(contamination=self.contamination, n_jobs=2) self.clf.fit(self.X_train) # get a copy from the single thread copy self.clf_ = COPOD(contamination=self.contamination) self.clf_.fit(self.X_train)
def setUp(self): self.n_train = 200 self.n_test = 100 self.contamination = 0.1 self.roc_floor = 0.8 self.X_train, self.y_train, self.X_test, self.y_test = generate_data( n_train=self.n_train, n_test=self.n_test, n_features=10, contamination=self.contamination, random_state=42) self.clf = COPOD(contamination=self.contamination) self.clf.fit(self.X_train)
def models_init(self): """Models initialisation. """ self.model = self.configuration.get('model', 'pca') if self.model == 'pca': self.models = { model: PCA(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'loda': self.models = { model: LODA(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'iforest': self.models = { model: IForest(n_estimators=50, bootstrap=True, behaviour='new', contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'cblof': self.models = { model: CBLOF(n_clusters=3, contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'feature_bagging': self.models = { model: FeatureBagging( base_estimator=PCA(contamination=self.contamination), contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'copod': self.models = { model: COPOD(contamination=self.contamination) for model in self.models_in_scope } elif self.model == 'hbos': self.models = { model: HBOS(contamination=self.contamination) for model in self.models_in_scope } else: self.models = { model: HBOS(contamination=self.contamination) for model in self.models_in_scope } self.custom_model_scalers = { model: MinMaxScaler() for model in self.models_in_scope }
def pred_COPOD(self, comp_with="openaq"): self.comp_with = comp_with if comp_with == "openaq": if self.X_o == []: pred = [] else: self.clf = COPOD() self.clf.fit(self.X_o) pred = self.clf.labels_ elif comp_with == "cams": pred = [] for each_X in self.X_c: self.clf = COPOD() self.clf.fit(each_X) pred.append(self.clf.labels_[-1]) A_location, B_location, C_location = self.pred_location(pred) return A_location, B_location, C_location
def remove_outlier(data, x, row, contamination): ''' data: which kind of data you are passing x : 0 for live and 1 for spoof ''' # 0 indicates all the live images x1 = np.where(data[:, 512]==x) #512 x2 = data[x1][0:row, :] train_features_x = x2[:,0:511] clf = COPOD(contamination = contamination) clf.fit(train_features_x) z = clf.labels_ z = np.asarray(z).reshape(row,1) z_final = np.hstack((x2, z)) x1_2 = np.where(z_final[:, 513]==0) x2_2 = z_final[x1_2][:, :] return x2_2
if __name__ == "__main__": contamination = 0.1 # percentage of outliers n_train = 200 # number of training points n_test = 100 # number of testing points # Generate sample data X_train, y_train, X_test, y_test = \ generate_data(n_train=n_train, n_test=n_test, n_features=2, contamination=contamination, random_state=42) # train COPOD detector clf_name = 'COPOD' clf = COPOD() clf.fit(X_train) # get the prediction labels and outlier scores of the training data y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers) y_train_scores = clf.decision_scores_ # raw outlier scores # get the prediction on the test data y_test_pred = clf.predict(X_test) # outlier labels (0 or 1) y_test_scores = clf.decision_function(X_test) # outlier scores # evaluate and print the results print("\nOn Training Data:") evaluate_print(clf_name, y_train, y_train_scores) print("\nOn Test Data:") evaluate_print(clf_name, y_test, y_test_scores)
'AvgKNN': KNN(method='mean'), 'LargestKNN': KNN(method='largest'), 'MedKNN': KNN(method='median'), 'PCA': PCA(), 'COF': COF(), 'LODA': LODA(), 'LOF': LOF(), 'HBOS': HBOS(), 'MCD': MCD(), 'AvgBagging': FeatureBagging(combination='average'), 'MaxBagging': FeatureBagging(combination='max'), 'IForest': IForest(), 'CBLOF': CBLOF(n_clusters=10, n_jobs=4), 'FactorAnalysis': FactorAnalysis(), 'KernelDensity': KernelDensity(), 'COPOD': COPOD(), 'SOD': SOD(), 'LSCPwithLODA': LSCP([LODA(), LODA()]), 'AveLMDD': LMDD(dis_measure='aad'), 'VarLMDD': LMDD(dis_measure='var'), 'IqrLMDD': LMDD(dis_measure='iqr'), 'SoGaal': SO_GAAL(), #'MoGaal':MO_GAAL(), 'VAE': VAE(encoder_neurons=[8, 4, 2]), 'AutoEncoder': AutoEncoder(hidden_neurons=[6, 3, 3, 6]) } models = { 'XGBOD': XGBOD(), 'BRM': BRM(), 'GM': GaussianMixture(),
def detect_anomaly(df_floats: pd.DataFrame, train_size: float, outliers_rate: float, classifier: str, plot: bool = False): """ Return binary classified outlier and raw outlier score. Performs training of anomaly detection model on subset of dataset and returns binary label and decision score for whole dataset. Parameters ---------- df_floats: pd.DataFrame with elements as floats. train_size: proportion of dataset to be used for training anomaly detection model. outliers_rate: proportion of training set to be considered outlier. classifier: string representing name of anomaly detection algorithm. plot: plots 2d contourf of anomaly detection scores. Returns ------- y_labels: numpy array of the same length as df_floats that assigns 0/1 (inlier/outlier) to each observation according to fitted model. y_scores: numpy array of the same length as df_floats that assigns outlier scores to each observation according to fitted model. """ if df_floats.shape[0] < 8: raise Warning( 'Not enough measurements. Please use DataFrame with at last 10 measurements.' ) if train_size > 1: train_size = train_size / 100 # TODO: Find out empirical way to set contamination level - Tukey's method if outliers_rate >= 1: outliers_rate = outliers_rate / 100 random_state = np.random.RandomState(42) # TODO: Perform scaling of data ONLY for AKNN, CBLOF, HBOS, KNN, OCSVM. Other classifiers are not influenced. classifiers = { 'Average KNN (AKNN)': KNN(method='mean', contamination=outliers_rate), 'Cluster-based Local Outlier Factor (CBLOF)': CBLOF(contamination=outliers_rate, check_estimator=False, random_state=random_state), 'Copula based Outlier Detection (COPOD)': COPOD(contamination=outliers_rate), 'Histogram-base Outlier Detection (HBOS)': HBOS(contamination=outliers_rate), 'Isolation Forest (IForest)': IForest(contamination=outliers_rate, random_state=random_state), 'K Nearest Neighbors (KNN)': KNN(contamination=outliers_rate), 'One-Class SVM (OCSVM)': OCSVM(contamination=outliers_rate), 'Principal component analysis (PCA)': PCA(contamination=outliers_rate) } scaler = MinMaxScaler(feature_range=(0, 1)) scaled = scaler.fit_transform(df_floats) df_scaled = pd.DataFrame(scaled, index=df_floats.index, columns=df_floats.columns) x_train, x_test = train_test_split(df_scaled, train_size=train_size) if classifier == 'all': raise Warning('This option is currently unsupported.' '\nPlease use one of those classifiers:' '\n{}.'.format(list(classifiers.keys()))) # for i, (clf_name, clf) in enumerate(classifiers.items()): # # fit model # clf.fit(x_train) # # prediction of a datapoint category outlier or inlier # y_labels = clf.predict(df_scaled) # plot_outlier_detection(df_scaled, y_labels, clf, clf_name, scaler) else: clf_name = '' for name in classifiers.keys(): if classifier in name: clf_name = name break if clf_name: # print("\nUsed classifier: {}".format(clf_name)) clf = classifiers.get(clf_name) clf.fit(x_train) y_labels = clf.predict( df_scaled) # binary labels (0: inliers, 1: outliers) y_scores = clf.decision_function(df_scaled) # raw outlier scores else: raise NameError('Unknown classifier. ' 'Please use one of those: {}.'.format( list(classifiers.keys()))) if plot: plot_outlier_detection(df_scaled, y_labels, clf, clf_name, scaler) return y_labels, y_scores
def compare(inputdata, labels, n_clusters, dset_name): """ Compute the AUC, Fgap, Frank score on all conventional outlier detectors for the given dataset Args: inputdata: input data labels: ground truth outlier labels n_clusters: number of clusters, for some cluster-based detectors dset_name: dataset Returns: AUC, Fgap, Frank """ print( "Competing with conventional unsupervised outlier detection algorithms..." ) random_state = np.random.RandomState(1) if inputdata.shape[1] < 64: AEneurons = [16, 8, 8, 16] VAEneurons = [16, 8, 4], [4, 8, 16] else: AEneurons = [64, 32, 32, 64] VAEneurons = [128, 64, 32], [32, 64, 128] classifiers = { 'PCA': PCA(random_state=random_state), 'AutoEncoder': AutoEncoder(batch_size=100, hidden_neurons=AEneurons, random_state=random_state), 'VAE': VAE(batch_size=100, encoder_neurons=VAEneurons[0], decoder_neurons=VAEneurons[1], random_state=random_state), 'COPOD': COPOD(), 'Iforest': IForest(random_state=random_state), 'AutoEncoder': AutoEncoder(batch_size=100, random_state=random_state), 'VAE': VAE(batch_size=100, random_state=random_state), 'LODA': LODA(), 'OCSVM': OCSVM(), 'ABOD': ABOD(n_neighbors=20), 'Fb': FeatureBagging(random_state=random_state), 'CBLOF': CBLOF(n_clusters=n_clusters, check_estimator=False, random_state=random_state), 'LOF': LOF(), 'COF': COF() } for clf_name, clf in classifiers.items(): print(f"Using {clf_name} method") starttime = time.time() clf.fit(inputdata) time_taken = time.time() - starttime test_scores = clf.decision_scores_ # -----fix some broken scores----- # for i in range(len(test_scores)): cur = test_scores[i] if np.isnan(cur) or not np.isfinite(cur): test_scores[i] = 0 np.save(f'{dset_name}/{clf_name}_raw.npy', test_scores) auc = roc_auc_score(labels, test_scores) print('AUC:', auc) fetch(normalize(test_scores), f'../datasets/{dset_name.upper()}_Y.npy', f'{dset_name}/attribute.npy') print('time_taken:', time_taken)
def execute(self): evaluation_results = [] print("Loading training data...") data = pd.DataFrame() for i, chunk in enumerate( pd.read_csv(self.input_file, header=None, chunksize=self.chunk_size)): print("Reading chunk: %d" % (i + 1)) #print(chunk) data = data.append(chunk) input_dimensionality = len(data.columns) - 1 print("Input Dimensionality: %d" % (input_dimensionality)) positive_data = data[data[len(data.columns) - 1] == 1].iloc[:, :len(data.columns) - 1] negative_data = data[data[len(data.columns) - 1] == -1].iloc[:, :len(data.columns) - 1] training_data = positive_data.sample(frac=0.70) positive_validation_data = positive_data.drop(training_data.index) if self.neg_cont and self.neg_cont > 0: print("Negative Contamination: %0.4f" % (self.neg_cont)) num_negative = math.floor( self.neg_cont * (len(negative_data) + len(positive_validation_data))) negative_data = data.sample(frac=1, random_state=200)[ data[len(data.columns) - 1] == -1].iloc[:num_negative, :len(data.columns) - 1] negative_validation_data = negative_data.copy() temp_positive = positive_validation_data.copy() temp_positive[input_dimensionality] = 1 temp_negative = negative_data.copy() temp_negative[input_dimensionality] = -1 validation_data_with_labels = pd.concat([temp_positive, temp_negative], ignore_index=True) validation_data = validation_data_with_labels.iloc[:, :len(data.columns ) - 1] validation_labels = validation_data_with_labels.iloc[:, -1:].values # Convert to tensor positive_data = torch.tensor(positive_data.values).float().to( self.device) negative_data = torch.tensor(negative_data.values).float().to( self.device) training_data = torch.tensor(training_data.values).float() validation_data = torch.tensor(validation_data.values).float() print("Validation Data:") print(validation_data) ## AE-D TRAINING ## print("Initializing autoencoder...") net = Autoencoder(layers=self.layers, device=self.device, add_syn=self.add_syn) net.to(self.device) print(net) print("Training Stochastic Autoencoder...") net.fit(training_data, epochs=self.epochs, lr=self.lr, batch_size=self.batch_size) predictions = net.predict(validation_data) tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc = performance_metrics( validation_labels, predictions) r = ["AE-D", tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc] evaluation_results.append(r) print("AE-D Results:") print( tabulate([r], [ "ALGO", "TP", "TN", "FP", "FN", "TPR", "TNR", "PPV", "NPV", "TS", "PT", "ACC", "F1", "MCC" ], tablefmt="grid")) # Convert back to CPU before other methods validation_data = validation_data.cpu() # Train only linear classifiers if self.eval_cat == "linear": print("Initiating training for linear detectors...") ## MCD ## print("Training MCD...") result = train_and_evaluate_classifier("MCD", MCD(), validation_data, validation_labels) evaluation_results.append(result) ## ROBUST COVARIANCE ## print("Training Robust Covariance...") result = train_and_evaluate_classifier("ROB-COV", EllipticEnvelope(), validation_data, validation_labels) evaluation_results.append(result) ## ONE CLASS SVM TRAINING ## print("Training OneClassSVM...") result = train_and_evaluate_classifier( "OC-SVM", svm.OneClassSVM(gamma="auto"), validation_data, validation_labels) evaluation_results.append(result) elif self.eval_cat == "prob": ## ABOD ## #print("Training ABOD...") #result = train_and_evaluate_classifier("ABOD", ABOD(), validation_data, validation_labels) #evaluation_results.append(result) ## SOS ## #print("Training SOS...") #result = train_and_evaluate_classifier("SOS", SOS(), validation_data, validation_labels) #evaluation_results.append(result) ## COPOD ## print("Training COPOD...") result = train_and_evaluate_classifier("COPOD", COPOD(), validation_data, validation_labels) evaluation_results.append(result) elif self.eval_cat == "ensemble": ## ISOLATION FOREST TRAINING ## print("Training Isolation Forest...") result = train_and_evaluate_classifier( "ISO-F", IsolationForest(random_state=0), validation_data, validation_labels) evaluation_results.append(result) ## LODA ## print("Training LODA...") result = train_and_evaluate_classifier("LODA", LODA(), validation_data, validation_labels) evaluation_results.append(result) ## LSCP ## # print("Training LSCP...") # result = train_and_evaluate_classifier("LSCP", LSCP([LOF(), LOF()]), validation_data, validation_labels) # evaluation_results.append(result) elif self.eval_cat == "proximity": ## LOCAL OUTLIER FACTOR ## print("Training Local Outlier Factor...") result = train_and_evaluate_classifier( "LOC-OF", LocalOutlierFactor(novelty=True), validation_data, validation_labels) evaluation_results.append(result) ## CBLOF ## print("Training CBLOF...") result = train_and_evaluate_classifier("CBLOF", CBLOF(), validation_data, validation_labels) evaluation_results.append(result) ## HBOS ## print("Training HBOS...") result = train_and_evaluate_classifier("HBOS", HBOS(), validation_data, validation_labels) evaluation_results.append(result) elif self.eval_cat == "nn": ## VAE ## print("Training VAE...") result = train_and_evaluate_classifier( "VAE", VAE(encoder_neurons=self.layers, decoder_neurons=self.layers.reverse()), validation_data, validation_labels) evaluation_results.append(result) ## SO_GAAL ## print("Training SO_GAAL...") result = train_and_evaluate_classifier( "SO_GAAL", SO_GAAL(lr_d=self.lr, stop_epochs=self.epochs), validation_data, validation_labels) evaluation_results.append(result) ## MO_GAAL ## print("Training MO_GAAL...") result = train_and_evaluate_classifier( "MO_GAAL", MO_GAAL(lr_d=self.lr, stop_epochs=self.epochs), validation_data, validation_labels) evaluation_results.append(result) ## EVALUATE RESULTS ## if self.eval_cat != "none": print("Aggregated Results:") print( tabulate(evaluation_results, [ "ALGO", "TP", "TN", "FP", "FN", "TPR", "TNR", "PPV", "NPV", "TS", "PT", "ACC", "F1", "MCC" ], tablefmt="grid")) ## DATASET METRICS ## len_training_data_points = len(training_data) len_positive_validations = len(positive_validation_data) len_negative_validations = len(negative_validation_data) len_validations = len_positive_validations + len_negative_validations metrics_results = [ ["Training Data Points", len_training_data_points], ["# Normal Points", len_positive_validations], ["# Anomalies", len_negative_validations], [ "Contamination Percentage", math.floor((len_negative_validations / len_validations) * 100) ] ] ## EVALUATE RESULTS ## print(tabulate(metrics_results, ["Metric", "Value"], tablefmt="grid")) if self.printout: print("Saving results to %s" % (self.printout)) df = pd.DataFrame(evaluation_results) df.to_csv(self.printout, header=None, index=False)
''' Author: Christian O'Leary Email: [email protected] ''' import numpy as np from pyod.models.copod import COPOD from emmv import emmv_scores rng = np.random.RandomState(42) NUM_COLS = 2 # Generate train data X = 0.3 * rng.randn(100, NUM_COLS) X_train = np.r_[X + 2, X - 2] # Generate some regular novel observations X = 0.3 * rng.randn(20, NUM_COLS) X_regular = np.r_[X + 2, X - 2] # Generate some abnormal novel observations X_outliers = rng.uniform(low=-4, high=4, size=(20, NUM_COLS)) # fit the model model = COPOD() model.fit(X_train) # Get EM & MV scores X_test = np.concatenate((X_regular, X_outliers), axis=0) test_scores = emmv_scores(model, X_test) print('Excess Mass score;', test_scores['em']) print('Mass Volume score:', test_scores['mv'])
def main(): # PART 1: # Getting the predictions for each classifier # SK means: The classifier is from sklearn or works like sklearn # PY means: The classifier is from pyod or works like pyod models = { 'SK_EE': EllipticEnvelope(), 'SK_GM': GaussianMixture(), 'SK_IF': IsolationForest(), 'SK_OCSVM': OneClassSVM(), 'SK_FA': FactorAnalysis(), 'SK_KD': KernelDensity(), 'PY_PCA': PCA(), 'PY_COF': COF(), 'PY_LODA': LODA(), 'PY_LOF': LOF(), 'PY_HBOS': HBOS(), 'PY_MCD': MCD(), 'PY_AvgKNN': KNN(method='mean'), 'PY_LargestKNN': KNN(method='largest'), 'PY_MedKNN': KNN(method='median'), 'PY_AvgBagging': FeatureBagging(combination='average'), 'PY_MaxBagging': FeatureBagging(combination='max'), 'PY_CBLOF': CBLOF(n_clusters=10, n_jobs=4), 'PY_COPOD': COPOD(), 'PY_SOD': SOD(), 'PY_LSCPwithLODA': LSCP([LODA(), LODA()]), 'PY_AveLMDD': LMDD(dis_measure='aad'), 'PY_VarLMDD': LMDD(dis_measure='var'), 'PY_IqrLMDD': LMDD(dis_measure='iqr'), 'PY_VAE': VAE(encoder_neurons=[8, 4, 2]), 'PY_AutoEncoder': AutoEncoder(hidden_neurons=[6, 3, 3, 6]), 'SK_BRM': BRM(bootstrap_sample_percent=70), 'SK_OCKRA': m_OCKRA(), 'PY_SoGaal': SO_GAAL(), 'PY_MoGaal': MO_GAAL() } ranker = ADRanker(data="datasets", models=models) ranker.get_predictions() # PART 2: # After predictions, we can evaluate our classifiers using different scores # You can add manually a new metric by modifying 'metrics.py' ranker.get_scores(scores={'auc': Metrics.get_roc, 'ave': Metrics.get_ave}) # PART 3: # Finally, it is time to summarize the results by plotting different graphs # You can add your own graphs by modifying ' plots.py' plot = Plots() plot.make_plot_basic(paths=[ 'results/scores/auc/no/results.csv', 'results/scores/auc/minmax/results.csv', 'results/scores/auc/std/results.csv', 'results/scores/ave/no/results.csv', 'results/scores/ave/minmax/results.csv', 'results/scores/ave/std/results.csv' ], scalers=[ 'Without scaler', 'Min max scaler', 'Standard scaler', 'Without scaler', 'Min max scaler', 'Standard scaler' ]) plot.make_cd_plot( paths=[ 'results/scores/auc/minmax/results.csv', 'results/scores/ave/no/results.csv', 'results/scores/auc/no/results.csv', 'results/scores/ave/no/results.csv', 'results/scores/auc/std/results.csv', 'results/scores/ave/std/results.csv' ], names=[ 'CD auc minmax scale', 'CD ave minmax scale', 'CD auc no scale', 'CD ave no scale', 'CD auc std scale', 'CD ave std scale' ], titles=[ 'CD diagram - AUC with min max scaling', 'CD diagram - Average precision with min max scaling', 'CD diagram - AUC without scaling', 'CD diagram - Average precision without scaling', 'CD diagram - AUC with standard scaling', 'CD diagram - Average precision with standard scaling' ])
def evaluation_od_train(x, y, data_name, model_name="iforest", chosen_subspace=None): """ using anomaly detector to yield anomaly score for each subspace, generate two files: the subspaces with the highest anomaly score & lof score for each subspace :param x: data matrix :param y: class information :param data_name: the data set name, using for naming the ground truth file :param model_name: anomaly detector name, default: lof :param chosen_subspace: use this to only evaluate a subset of the power set of full feature space :return: df: a ground-truth map using anomaly idx as key and ground truth feature subspace as value. """ global chosen_model dim = x.shape[1] ano_idx = np.where(y == 1)[0] n_ano = len(ano_idx) # get all the possible feature subset or just use given subset list f_subsets = utils.get_subset_candidate(dim, chosen_subspace) # score anomalies in each subspace, generate the score matrix n_subsets = len(f_subsets) score_matrix = np.zeros([n_ano, n_subsets]) for i in tqdm(range(n_subsets)): subset = f_subsets[i] x_subset = x[:, subset] if model_name == "iforest": clf = IForest() clf.fit(x_subset) od_score = clf.decision_scores_ elif model_name == "copod": clf = COPOD() clf.fit(x_subset) od_score = clf.decision_scores_ elif model_name == "hbos": clf = HBOS() clf.fit(x_subset) od_score = clf.decision_scores_ else: raise ValueError("unsupported od model") od_score = utils.min_max_norm(od_score) score_matrix[:, i] = od_score[ano_idx] if not os.path.exists(eva_root + "data_od_evaluation/"): os.makedirs(eva_root + "data_od_evaluation/") # score matrix to df anomaly_score_df = pd.DataFrame(data=score_matrix, columns=[str(s) for s in f_subsets]) col_name = anomaly_score_df.columns.tolist() col_name.insert(0, 'ano_idx') anomaly_score_df["ano_idx"] = ano_idx anomaly_score_df = anomaly_score_df.reindex(columns=col_name) path1 = eva_root + "data_od_evaluation/" + data_name + "_score_" + model_name + ".csv" anomaly_score_df.to_csv(path1, index=False) # get the ground truth (one subspace for each anomaly that the anomaly can obtain the highest anomaly score) g_truth_df = pd.DataFrame(columns=["ano_idx", "exp_subspace"]) exp_subspaces = [] for ii, ano_score in enumerate(score_matrix): max_score_idx = int(np.argmax(ano_score)) exp_subset = str(f_subsets[max_score_idx]) exp_subspaces.append(exp_subset) g_truth_df["ano_idx"] = ano_idx g_truth_df["exp_subspace"] = exp_subspaces g_truth_df.astype({"exp_subspace": "object"}) path2 = eva_root + "data_od_evaluation/" + data_name + "_gt_" + model_name + ".csv" g_truth_df.to_csv(path2, index=False) return anomaly_score_df, g_truth_df
generate_data(n_train=n_train, n_test=n_test, n_features=2, contamination=contamination, random_state=42) # train SUOD clf_name = 'SUOD' # initialized a group of outlier detectors for acceleration detector_list = [ LOF(n_neighbors=15), LOF(n_neighbors=20), LOF(n_neighbors=25), LOF(n_neighbors=35), COPOD(), IForest(n_estimators=100), IForest(n_estimators=200) ] # decide the number of parallel process, and the combination method clf = SUOD(base_estimators=detector_list, n_jobs=2, combination='average', verbose=False) # or to use the default detectors # clf = SUOD(n_jobs=2, combination='average', # verbose=False) clf.fit(X_train)
contamination = 0.1 # percentage of outliers n_train = 2000 # number of training points n_test = 100 # number of testing points # Generate sample data X_train, X_test, y_train, y_test = \ generate_data(n_train=n_train, n_test=n_test, n_features=2, contamination=contamination, random_state=42) # train COPOD detector # you could try parallel version as well. clf_name = 'COPOD_parallel' clf = COPOD(n_jobs=2) clf.fit(X_train) # get the prediction labels and outlier scores of the training data y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers) y_train_scores = clf.decision_scores_ # raw outlier scores # get the prediction on the test data y_test_pred = clf.predict(X_test) # outlier labels (0 or 1) y_test_scores = clf.decision_function(X_test) # outlier scores # evaluate and print the results print("\nOn Training Data:") evaluate_print(clf_name, y_train, y_train_scores) print("\nOn Test Data:") evaluate_print(clf_name, y_test, y_test_scores)
def fit(self, X, shrink_cols = True, data_scaler = preprocessing.MaxAbsScaler(), quick_methods = True, slow_methods = False, nn_methods = False, contamination = 0.05, use_score_rank = False, random_state = None, verbose = 0): if len(X.shape) > 2: X = X.reshape(X.shape[0], X.shape[1]*X.shape[2]) elif len(X.shape) > 3: raise ValueError("Expected number of dimensions: 2 or 3") if shrink_cols: X = X[:,~np.all(X == 0, axis=0)] log.info('zero columns shrinked') if data_scaler: X = data_scaler.fit_transform(X) log.info(f'used {data_scaler} data scaler') #log.info(X[0:1,:]) n_rows = X.shape[0] n_features = X.shape[1] log.info (f'n_rows = {n_rows}, n_features = {n_features}') quick_scores = np.zeros([n_rows, 0]) slow_scores = np.zeros([n_rows, 0]) nn_scores = np.zeros([n_rows, 0]) if quick_methods: # Define anomaly detection tools to be compared quick_classifiers = { 'PCA_randomized': PCA(contamination=contamination, random_state=random_state, standardization = False, svd_solver = 'randomized'), 'PCA_full': PCA(contamination=contamination, random_state=random_state, standardization = False, svd_solver = 'full'), 'COPOD': COPOD(contamination=contamination), f'HBOS': HBOS(contamination=contamination), f'HBOS_{200}': HBOS(contamination=contamination, n_bins = 200), f'HBOS_{300}': HBOS(contamination=contamination, n_bins = 300), 'LODA': LODA(contamination=contamination), 'LODA_200': LODA(contamination=contamination, n_random_cuts = 200), 'LODA_300': LODA(contamination=contamination, n_random_cuts = 300), 'IForest_100': IForest(contamination=contamination, random_state=random_state, n_estimators = 100, bootstrap = False, n_jobs = -1), 'IForest_200': IForest(contamination=contamination, random_state=random_state, n_estimators = 200, bootstrap = False, n_jobs = -1), 'IForest_bootstrap': IForest(contamination = contamination, random_state=random_state, n_estimators = 150, bootstrap = True, n_jobs = -1), #'MCD': # MCD(contamination=contamination, random_state=random_state, assume_centered = False), #'MCD_centered': # MCD(contamination=contamination, random_state=random_state, assume_centered = True), f'CBLOF_16': CBLOF(contamination=contamination, random_state=random_state, n_clusters = 16), f'CBLOF_24': CBLOF(contamination=contamination, random_state=random_state, n_clusters = 24), f'CBLOF_32': CBLOF(contamination=contamination, random_state=random_state, n_clusters = 32) } quick_scores = np.zeros([n_rows, len(quick_classifiers)]) for i, (clf_name, clf) in enumerate(quick_classifiers.items()): log.info(f'{i+1} - fitting {clf_name}') try: clf.fit(X) quick_scores[:, i] = clf.decision_scores_ except: log.info(traceback.print_exc()) else: log.info(f'Base detector {i+1}/{len(quick_classifiers)} is fitted for prediction') quick_scores = np.nan_to_num(quick_scores) if slow_methods: # initialize a set of detectors for LSCP detector_list = [LOF(n_neighbors=10), LOF(n_neighbors=15), LOF(n_neighbors=20)] slow_classifiers = { #'Angle-based Outlier Detector (ABOD)': #too slow and nan results # ABOD(contamination=contamination), #'One-class SVM (OCSVM)': # OCSVM(contamination=contamination, cache_size = 2000, shrinking = False, tol = 1e-2), #'LSCP': #slow and no parallel # LSCP(detector_list, contamination=contamination, random_state=random_state, local_region_size = 30), #'Feature Bagging': #ensemble #no real par # FeatureBagging(LOF(n_neighbors=20), contamination=contamination, # random_state=random_state, n_jobs = -1), #'SOS' : # too memory inefficient # SOS(contamination=contamination), #'COF': # memory inefficient # COF(contamination=contamination), #'SOD': # SOD(contamination = contamination), #'KNN': # KNN(contamination=contamination, n_jobs = -1), #'KNN_50': # KNN(contamination=contamination, leaf_size = 50, n_jobs = -1), #'KNN_70': # KNN(contamination=contamination, leaf_size = 70, n_jobs = -1), 'LOF_4': LOF(n_neighbors=4, contamination=contamination, n_jobs = -1), 'LOF_5': LOF(n_neighbors=5, contamination=contamination, n_jobs = -1), 'LOF_6': LOF(n_neighbors=6, contamination=contamination, n_jobs = -1), 'LOF_7': LOF(n_neighbors=7, contamination=contamination, n_jobs = -1), 'LOF_8': LOF(n_neighbors=8, contamination=contamination, n_jobs = -1), 'LOF_9': LOF(n_neighbors=9, contamination=contamination, n_jobs = -1), 'LOF_10': LOF(n_neighbors=10, contamination=contamination, n_jobs = -1), 'LOF_12': LOF(n_neighbors=12, contamination=contamination, n_jobs = -1), 'LOF_14': LOF(n_neighbors=14, contamination=contamination, n_jobs = -1), 'LOF_16': LOF(n_neighbors=16, contamination=contamination, n_jobs = -1), 'LOF_18': LOF(n_neighbors=18, contamination=contamination, n_jobs = -1), 'LOF_20': LOF(n_neighbors=20, contamination=contamination, n_jobs = -1), 'LOF_22': LOF(n_neighbors=22, contamination=contamination, n_jobs = -1) } slow_scores = np.zeros([n_rows, len(slow_classifiers)]) for i, (clf_name, clf) in enumerate(slow_classifiers.items()): log.info(f'{i+1} - fitting {clf_name}') try: clf.fit(X) slow_scores[:, i] = clf.decision_scores_ except: log.info(traceback.print_exc()) else: log.info(f'Base detector {i+1}/{len(slow_classifiers)} is fitted for prediction') slow_scores = np.nan_to_num(slow_scores) if nn_methods: nn_classifiers = {} n_list = [1024, 512, 256, 128, 64, 32, 16, 8, 4, 2] n_idx = next(x[0] for x in enumerate(n_list) if x[1] < n_features) for i in range(3,6): n_enc = n_list[n_idx:n_idx+i-1] n_dec = n_enc[::-1] n_enc_dec = n_enc + n_dec nn_classifiers[f'FULL_AE_{len(n_enc + n_dec)}'] = {'clf': self.full_autoencoder, 'hidden_layers' : n_enc_dec } nn_classifiers[f'VAE_{len(n_enc_dec)}'] = {'clf': VAE(contamination = contamination, random_state = random_state, encoder_neurons = n_enc, decoder_neurons = n_dec, preprocessing = False, epochs = 32, verbosity = verbose), 'hidden_layers' : n_enc + n_dec } nn_scores = np.zeros([n_rows, len(nn_classifiers)]) for i, (clf_name, clf) in enumerate(nn_classifiers.items()): log.info(f'''{i+1} - fitting {clf_name} with layers {clf['hidden_layers']}''') try: if clf['clf'] == self.full_autoencoder: nn_scores[:, i] = clf['clf'](X, neurons_list = clf['hidden_layers'], verbose = verbose) else: clf['clf'].fit(X) nn_scores[:, i] = clf['clf'].decision_scores_ except: log.info(traceback.print_exc()) else: log.info(f'Base detector {i+1}/{len(nn_classifiers)} is fitted for prediction') nn_scores = np.nan_to_num(nn_scores) all_scores = np.concatenate((quick_scores, slow_scores, nn_scores), axis=1) all_scores = all_scores[:,~np.all(all_scores == 0, axis=0)] log.info(f'total scores = {all_scores.shape[1]}') all_scores_norm = np.copy(all_scores) if use_score_rank: all_scores_norm = np.apply_along_axis(rank_fun, 0, all_scores_norm) log.info(f'score rank applied') all_scores_norm = preprocessing.MinMaxScaler().fit_transform(all_scores_norm) if all_scores_norm.shape[1] >= 12: score_by_aom = aom(all_scores_norm, method = 'dynamic', n_buckets = round(all_scores_norm.shape[1]/4)) score_by_moa = moa(all_scores_norm, method = 'dynamic', n_buckets = round(all_scores_norm.shape[1]/4)) score_by_avg = np.mean(all_scores_norm, axis = 1) score_by_max = np.max(all_scores_norm, axis = 1) else: score_by_avg = np.mean(all_scores_norm, axis = 1) score_by_max = np.max(all_scores_norm, axis = 1) score_by_aom = score_by_avg score_by_moa = score_by_max return score_by_aom, score_by_moa, score_by_max, score_by_avg, all_scores, all_scores_norm
print('\nFinished ' + self.model_name) return None if __name__ == '__main__': # Specify the root directory rootDir = 'G:/My Drive/Github/ml-group-col/One-Class-models/Anomaly_Datasets_csv/' # specify the random state rs = 10 # Save how to run the models models = [ (IsolationForest(random_state=rs), 'ISOF'), (EllipticEnvelope(random_state=rs), 'EE'), (LMDD(dis_measure='aad', random_state=rs), 'AAD_LMDD'), (COPOD(), 'COPOD'), (FeatureBagging(combination='average', random_state=rs), 'AVE_Bagging'), # n_jobs (LMDD(dis_measure='iqr', random_state=rs), 'IQR_LMDD'), (KNN(method='largest'), 'Largest_KNN'), # n_jobs (LODA(), 'LODA'), (FeatureBagging(combination='max', n_jobs=-1, random_state=rs), 'MAX_Bagging'), (MCD(random_state=rs), 'MCD'), (XGBOD(random_state=rs), 'XGBOD'), # n_jobs (GaussianMixture(random_state=rs), 'GMM'), (LocalOutlierFactor(novelty=True), 'LOF'), (KNN(method='median'), 'Median_KNN'), # n_jobs (KNN(method='mean'), 'Avg_KNN'), # n_jobs (CBLOF(n_clusters=10, random_state=rs), 'CBLOF'), (HBOS(), 'HBOS'),