def pca(self, X_train, n_components=None, contamination=None):
"""
Train PCA model from PYOD
Parameters
__________
X_train: scaled training data
contamination: percentage of anomalies in the data
n_components: number of components to transform
Returns
________
Anomaly scores
"""
model = PCAOD(n_components=n_components, contamination=contamination)
model.fit(X_train)
# Predict raw anomaly score
labels = model.predict(X_train) # outlier labels (0 or 1)
pca_anomaly_scores = model.decision_function(X_train) # outlier scores
pca_anomaly_scores = self.min_max_scaler(pca_anomaly_scores)
return pca_anomaly_scores, labels
def detect(file, amountanom, realtime):
"""
Functon to apply a very simple anomaly detector
amountanom: The top number of anomalies we want to print
realtime: If we want to read the conn.log file in real time (not working)
"""
# Create a zeek reader on a given log file. Thanks brothon
reader = bro_log_reader.BroLogReader(file, tail=realtime)
# Create a Pandas dataframe from reader
bro_df = pd.DataFrame(reader.readrows())
# In case you need a label, due to some models being able to work in a semisupervized mode, then put it here. For now everything is 'normal', but we are not using this for detection
bro_df['label'] = 'normal'
# Change the datetime delta value to seconds. Scikit does not now how to work with timedeltas
bro_df['durationsec'] = bro_df.duration.apply(lambda x: x.total_seconds())
# Replace the rows without data (with '-') with -1. Even though this may add a bias in the algorithms, is better than not using the lines.
bro_df['orig_bytes'] = bro_df['orig_bytes'].replace(to_replace='-',
value=-1)
bro_df['resp_bytes'] = bro_df['resp_bytes'].replace(to_replace='-',
value=-1)
bro_df['resp_pkts'] = bro_df['resp_pkts'].replace(to_replace='-', value=-1)
bro_df['orig_ip_bytes'] = bro_df['orig_ip_bytes'].replace(to_replace='-',
value=-1)
bro_df['resp_ip_bytes'] = bro_df['resp_ip_bytes'].replace(to_replace='-',
value=-1)
# Add the columns from the log file that we know are numbers. This is only for conn.log files.
X_train = bro_df[[
'durationsec', 'orig_bytes', 'id.resp_p', 'resp_bytes',
'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes'
]]
# Our y is the label. But we are not using it now.
y = bro_df.label
# The X_test is where we are going to search for anomalies. In our case, its the same set of data than X_train.
X_test = X_train
#################
# Select a model from below
# ABOD class for Angle-base Outlier Detection. For an observation, the variance of its weighted cosine scores to all neighbors could be viewed as the outlying score.
#clf = ABOD()
# LOF
#clf = LOF()
# CBLOF
#clf = CBLOF()
# LOCI
#clf = LOCI()
# LSCP
#clf = LSCP()
# MCD
#clf = MCD()
# OCSVM
#clf = OCSVM()
# PCA. Good and fast!
clf = PCA()
# SOD
#clf = SOD()
# SO_GAAL
#clf = SO_GALL()
# SOS
#clf = SOS()
# XGBOD
#clf = XGBOD()
# KNN
# Good results but slow
#clf = KNN()
#clf = KNN(n_neighbors=10)
#################
# Fit the model to the train data
clf.fit(X_train)
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# Convert the ndarrays of scores and predictions to pandas series
scores_series = pd.Series(y_test_scores)
pred_series = pd.Series(y_test_pred)
# Now use the series to add a new column to the X test
X_test['score'] = scores_series.values
X_test['pred'] = pred_series.values
# Add the score to the bro_df also. So we can show it at the end
bro_df['score'] = X_test['score']
# Keep the positive predictions only. That is, keep only what we predict is an anomaly.
X_test_predicted = X_test[X_test.pred == 1]
# Keep the top X amount of anomalies
top10 = X_test_predicted.sort_values(by='score',
ascending=False).iloc[:amountanom]
## Print the results
# Find the predicted anomalies in the original bro dataframe, where the rest of the data is
df_to_print = bro_df.iloc[top10.index]
print('\nFlows of the top anomalies')
# Only print some columns, not all, so its easier to read.
df_to_print = df_to_print.drop([
'conn_state', 'history', 'local_orig', 'local_resp', 'missed_bytes',
'ts', 'tunnel_parents', 'uid', 'label'
],
axis=1)
print(df_to_print)
# Generate sample data
X_train, y_train, X_test, y_test = \
generate_data(n_train=n_train,
n_test=n_test,
n_features=2,
contamination=contamination,
random_state=42)
# train PCA detector
clf_name = 'PCA'
clf = PCA()
clf.fit(X_train)
# get the prediction labels and outlier scores of the training data
y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_ # raw outlier scores
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# evaluate and print the results
print("\nOn Training Data:")
evaluate_print(clf_name, y_train, y_train_scores)
print("\nOn Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)
# visualize the results
visualize(clf_name, X_train, y_train, X_test, y_test, y_train_pred,
y_test_pred, show_figure=True, save_figure=False)
print('{data_file} does not exist. Use generated data'.format(
data_file=mat_file))
X, y = generate_data(train_only=True) # load data
except IOError:
print('{data_file} does not exist. Use generated data'.format(
data_file=mat_file))
X, y = generate_data(train_only=True) # load data
else:
X = mat['X']
y = mat['y'].ravel()
for t in range(ite):
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.4)
# standardizing data for processing
X_train_norm, X_test_norm = standardizer(X_train, X_test)
# initialize 20 base detectors for combination
clf = PCA()
clf.fit(X_train_norm)
train_scores = clf.decision_scores_
test_scores = clf.decision_function(X_test_norm)
print()
evaluate_print('PCA Train', y_train, train_scores)
evaluate_print('PCA Test', y_test, test_scores)
class TestPCA(unittest.TestCase):
def setUp(self):
self.n_train = 200
self.n_test = 100
self.contamination = 0.1
self.roc_floor = 0.8
self.X_train, self.y_train, self.X_test, self.y_test = generate_data(
n_train=self.n_train,
n_test=self.n_test,
n_features=10,
contamination=self.contamination,
random_state=42)
self.clf = PCA(contamination=self.contamination, random_state=42)
self.clf.fit(self.X_train)
def test_parameters(self):
assert (hasattr(self.clf, 'decision_scores_')
and self.clf.decision_scores_ is not None)
assert (hasattr(self.clf, 'labels_') and self.clf.labels_ is not None)
assert (hasattr(self.clf, 'threshold_')
and self.clf.threshold_ is not None)
assert (hasattr(self.clf, '_mu') and self.clf._mu is not None)
assert (hasattr(self.clf, '_sigma') and self.clf._sigma is not None)
assert (hasattr(self.clf, 'selected_components_')
and self.clf.selected_components_ is not None)
assert (hasattr(self.clf, 'selected_w_components_')
and self.clf.selected_w_components_ is not None)
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
# check performance
assert (roc_auc_score(self.y_test, pred_scores) >= self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_fit_predict_score(self):
self.clf.fit_predict_score(self.X_test, self.y_test)
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='roc_auc_score')
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='prc_n_score')
with assert_raises(NotImplementedError):
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='something')
def test_predict_rank(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, self.X_train.shape[0] + 1)
assert_array_less(-0.1, pred_ranks)
def test_predict_rank_normalized(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test, normalized=True)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, 1.01)
assert_array_less(-0.1, pred_ranks)
def test_model_clone(self):
clone_clf = clone(self.clf)
def tearDown(self):
pass
"""### Angle-based Outlier Detector (Probabilistic Based Model)""" from pyod.models import abod clf_abod = abod.ABOD(contamination=0.1, n_neighbors=5, method='fast') clf_abod.fit(X) y_pred = clf_abod.predict(X) # outlier labels (0 or 1) y_scores = clf_abod.decision_function(X) # outlier scores colors = np.array(['#377eb8', '#ff7f00']) plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[(y_pred - 1) // 2]) clf_abod.fit_predict_score(X[:, 0].reshape(-1,1), y_pred, scoring='roc_auc_score') """### Linear Model PCA""" from pyod.models.pca import PCA clf_pca = PCA() clf_pca.fit(X) y_pred = clf_pca.predict(X) # outlier labels (0 or 1) y_scores = clf_pca.decision_function(X) # outlier scores y_pred colors = np.array(['#377eb8', '#ff7f00']) plt.scatter(X[:, 0], X[:, 1], s=10, color=colors[(y_pred - 1) // 2]) clf_pca.fit_predict_score(X[:, 0].reshape(-1,1), y_pred, scoring='roc_auc_score')
def detect(file, amountanom, realtime, dumptocsv):
"""
Function to apply a very simple anomaly detector
amountanom: The top number of anomalies we want to print
realtime: If we want to read the conn.log file in real time (not working)
"""
# Create a Pandas dataframe from the conn.log
bro_df = pd.read_csv(
file,
sep="\t",
comment='#',
names=[
'ts', 'uid', 'id.orig_h', 'id.orig_p', 'id.resp_h', 'id.resp_p',
'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes',
'conn_state', 'local_orig', 'local_resp', 'missed_bytes',
'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts',
'resp_ip_bytes', 'tunnel_parents'
])
# In case you need a label, due to some models being able to work in a
# semisupervized mode, then put it here. For now everything is
# 'normal', but we are not using this for detection
bro_df['label'] = 'normal'
# Replace the rows without data (with '-') with 0.
# Even though this may add a bias in the algorithms,
# is better than not using the lines.
# Also fill the no values with 0
# Finally put a type to each column
bro_df['orig_bytes'].replace('-', '0', inplace=True)
bro_df['orig_bytes'] = bro_df['orig_bytes'].fillna(0).astype('int32')
bro_df['resp_bytes'].replace('-', '0', inplace=True)
bro_df['resp_bytes'] = bro_df['resp_bytes'].fillna(0).astype('int32')
bro_df['resp_pkts'].replace('-', '0', inplace=True)
bro_df['resp_pkts'] = bro_df['resp_pkts'].fillna(0).astype('int32')
bro_df['orig_ip_bytes'].replace('-', '0', inplace=True)
bro_df['orig_ip_bytes'] = bro_df['orig_ip_bytes'].fillna(0).astype('int32')
bro_df['resp_ip_bytes'].replace('-', '0', inplace=True)
bro_df['resp_ip_bytes'] = bro_df['resp_ip_bytes'].fillna(0).astype('int32')
bro_df['duration'].replace('-', '0', inplace=True)
bro_df['duration'] = bro_df['duration'].fillna(0).astype('float64')
# Save dataframe to disk as CSV
if dumptocsv != "None":
bro_df.to_csv(dumptocsv)
# Add the columns from the log file that we know are numbers. This is only for conn.log files.
X_train = bro_df[[
'duration', 'orig_bytes', 'id.resp_p', 'resp_bytes', 'orig_ip_bytes',
'resp_pkts', 'resp_ip_bytes'
]]
# Our y is the label. But we are not using it now.
y = bro_df.label
# The X_test is where we are going to search for anomalies. In our case, its the same set of data than X_train.
X_test = X_train
#################
# Select a model from below
# ABOD class for Angle-base Outlier Detection. For an observation, the
# variance of its weighted cosine scores to all neighbors could be
# viewed as the outlying score.
# clf = ABOD()
# LOF
# clf = LOF()
# CBLOF
# clf = CBLOF()
# LOCI
# clf = LOCI()
# LSCP
# clf = LSCP()
# MCD
# clf = MCD()
# OCSVM
# clf = OCSVM()
# PCA. Good and fast!
clf = PCA()
# SOD
# clf = SOD()
# SO_GAAL
# clf = SO_GALL()
# SOS
# clf = SOS()
# XGBOD
# clf = XGBOD()
# KNN
# Good results but slow
# clf = KNN()
# clf = KNN(n_neighbors=10)
#################
# Fit the model to the train data
clf.fit(X_train)
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# Convert the ndarrays of scores and predictions to pandas series
scores_series = pd.Series(y_test_scores)
pred_series = pd.Series(y_test_pred)
# Now use the series to add a new column to the X test
X_test['score'] = scores_series.values
X_test['pred'] = pred_series.values
# Add the score to the bro_df also. So we can show it at the end
bro_df['score'] = X_test['score']
# Keep the positive predictions only. That is, keep only what we predict is an anomaly.
X_test_predicted = X_test[X_test.pred == 1]
# Keep the top X amount of anomalies
top10 = X_test_predicted.sort_values(by='score',
ascending=False).iloc[:amountanom]
# Print the results
# Find the predicted anomalies in the original bro dataframe, where the rest of the data is
df_to_print = bro_df.iloc[top10.index]
print('\nFlows of the top anomalies')
# Only print some columns, not all, so its easier to read.
df_to_print = df_to_print.drop([
'conn_state', 'history', 'local_orig', 'local_resp', 'missed_bytes',
'ts', 'tunnel_parents', 'uid', 'label'
],
axis=1)
print(df_to_print)
#划分测试集和训练集
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.33)
#使用pyod中的PCA算法拟合数据
clf_name = 'PCA'
clf = PCA()
clf.fit(X_train)
#预测得到由0和1组成的数组，1表示离群点，0表示飞离群点
y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_ # raw outlier scores，The outlier scores of the training data.
#预测样本是不是离群点，返回0和1 的数组
y_test_pred = clf.predict(X_test)
y_test_scores = clf.decision_function(
X_test) # outlier scores，The anomaly score of the input samples.
#使用sklearn中的roc_auc_score方法得到auc值，即roc曲线下面的面积
try:
sumAuc_train += sklearn.metrics.roc_auc_score(y_train,
y_train_scores,
average='macro')
sumAuc_test += sklearn.metrics.roc_auc_score(y_test,
y_test_scores,
average='macro')
#s=precision_score(y_train, y_train_scores, average='macro')
i += 1
print(sumAuc_train, sumAuc_test)
except ValueError:
print('1')
pass
class PCA(CollectiveBaseDetector):
"""PCA-based outlier detection with both univariate and multivariate
time series data. TS data will be first transformed to tabular format.
For univariate data, it will be in shape of [valid_length, window_size].
for multivariate data with d sequences, it will be in the shape of
[valid_length, window_size].
Parameters
----------
window_size : int
The moving window size.
step_size : int, optional (default=1)
The displacement for moving window.
contamination : float in (0., 0.5), optional (default=0.1)
The amount of contamination of the data set,
i.e. the proportion of outliers in the data set. Used when fitting to
define the threshold on the decision function.
n_components : int, float, None or string
Number of components to keep. It should be smaller than the window_size.
if n_components is not set all components are kept::
n_components == min(n_samples, n_features)
if n_components == 'mle' and svd_solver == 'full', Minka\'s MLE is used
to guess the dimension
if ``0 < n_components < 1`` and svd_solver == 'full', select the number
of components such that the amount of variance that needs to be
explained is greater than the percentage specified by n_components
n_components cannot be equal to n_features for svd_solver == 'arpack'.
n_selected_components : int, optional (default=None)
Number of selected principal components
for calculating the outlier scores. It is not necessarily equal to
the total number of the principal components. If not set, use
all principal components.
copy : bool (default True)
If False, data passed to fit are overwritten and running
fit(X).transform(X) will not yield the expected results,
use fit_transform(X) instead.
whiten : bool, optional (default False)
When True (False by default) the `components_` vectors are multiplied
by the square root of n_samples and then divided by the singular values
to ensure uncorrelated outputs with unit component-wise variances.
Whitening will remove some information from the transformed signal
(the relative variance scales of the components) but can sometime
improve the predictive accuracy of the downstream estimators by
making their data respect some hard-wired assumptions.
svd_solver : string {'auto', 'full', 'arpack', 'randomized'}
auto :
the solver is selected by a default policy based on `X.shape` and
`n_components`: if the input data is larger than 500x500 and the
number of components to extract is lower than 80% of the smallest
dimension of the data, then the more efficient 'randomized'
method is enabled. Otherwise the exact full SVD is computed and
optionally truncated afterwards.
full :
run exact full SVD calling the standard LAPACK solver via
`scipy.linalg.svd` and select the components by postprocessing
arpack :
run SVD truncated to n_components calling ARPACK solver via
`scipy.sparse.linalg.svds`. It requires strictly
0 < n_components < X.shape[1]
randomized :
run randomized SVD by the method of Halko et al.
tol : float >= 0, optional (default .0)
Tolerance for singular values computed by svd_solver == 'arpack'.
iterated_power : int >= 0, or 'auto', (default 'auto')
Number of iterations for the power method computed by
svd_solver == 'randomized'.
random_state : int, RandomState instance or None, optional (default None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`. Used when ``svd_solver`` == 'arpack' or 'randomized'.
weighted : bool, optional (default=True)
If True, the eigenvalues are used in score computation.
The eigenvectors with small eigenvalues comes with more importance
in outlier score calculation.
standardization : bool, optional (default=True)
If True, perform standardization first to convert
data to zero mean and unit variance.
See http://scikit-learn.org/stable/auto_examples/preprocessing/plot_scaling_importance.html
Attributes
----------
decision_scores_ : numpy array of shape (n_samples,)
The outlier scores of the training data.
The higher, the more abnormal. Outliers tend to have higher
scores. This value is available once the detector is
fitted.
threshold_ : float
The threshold is based on ``contamination``. It is the
``n_samples * contamination`` most abnormal samples in
``decision_scores_``. The threshold is calculated for generating
binary outlier labels.
labels_ : int, either 0 or 1
The binary labels of the training data. 0 stands for inliers
and 1 for outliers/anomalies. It is generated by applying
``threshold_`` on ``decision_scores_``.
"""
def __init__(self,
window_size,
step_size=1,
contamination=0.1,
n_components=None,
n_selected_components=None,
copy=True,
whiten=False,
svd_solver='auto',
tol=0.0,
iterated_power='auto',
random_state=None,
weighted=True,
standardization=True):
super(PCA, self).__init__(contamination=contamination)
self.window_size = window_size
self.step_size = step_size
# parameters for PCA
self.n_components = n_components
self.n_selected_components = n_selected_components
self.copy = copy
self.whiten = whiten
self.svd_solver = svd_solver
self.tol = tol
self.iterated_power = iterated_power
self.random_state = random_state
self.weighted = weighted
self.standardization = standardization
# initialize a kNN model
self.model_ = PCA_PYOD(
n_components=self.n_components,
n_selected_components=self.n_selected_components,
contamination=self.contamination,
copy=self.copy,
whiten=self.whiten,
svd_solver=self.svd_solver,
tol=self.tol,
iterated_power=self.iterated_power,
random_state=self.random_state,
weighted=self.weighted,
standardization=self.standardization)
def fit(self, X: np.array) -> object:
"""Fit detector. y is ignored in unsupervised methods.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
y : Ignored
Not used, present for API consistency by convention.
Returns
-------
self : object
Fitted estimator.
"""
X = check_array(X).astype(np.float)
# first convert it into submatrices, and flatten it
sub_matrices, self.left_inds_, self.right_inds_ = get_sub_matrices(
X,
self.window_size,
self.step_size,
return_numpy=True,
flatten=True,
flatten_order='F')
# if self.n_components > sub_matrices.shape[1]:
# raise ValueError('n_components exceeds window_size times the number of sequences.')
# fit the PCA model
self.model_.fit(sub_matrices)
self.decision_scores_ = self.model_.decision_scores_
self._process_decision_scores()
return self
def decision_function(self, X: np.array):
"""Predict raw anomaly scores of X using the fitted detector.
The anomaly score of an input sample is computed based on the fitted
detector. For consistency, outliers are assigned with
higher anomaly scores.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples. Sparse matrices are accepted only
if they are supported by the base estimator.
Returns
-------
anomaly_scores : numpy array of shape (n_samples,)
The anomaly score of the input samples.
"""
check_is_fitted(self, ['model_'])
X = check_array(X).astype(np.float)
# first convert it into submatrices, and flatten it
sub_matrices, X_left_inds, X_right_inds = get_sub_matrices(
X,
self.window_size,
self.step_size,
return_numpy=True,
flatten=True,
flatten_order='F')
# return the prediction result by PCA
return self.model_.decision_function(
sub_matrices), X_left_inds.ravel(), X_right_inds.ravel()
data_dict = load_dataset(
dataset,
subdataset,
"all",
)
x_train = data_dict["train"]
x_test = data_dict["test"]
x_test_labels = data_dict["test_labels"]
start = time.time()
# data preprocessing for MSCRED
od = PCA()
od.fit(x_train)
# get outlier scores
anomaly_score = od.decision_function(x_test)
anomaly_label = x_test_labels
end = time.time()
time = end - start
# Make evaluation
evaluate_all(anomaly_score, anomaly_label)
salience = compute_salience(anomaly_score, anomaly_label)
print('time')
print(' ', time)
print('salience')
print(' ', salience)
class TestPCA(unittest.TestCase):
def setUp(self):
self.n_train = 100
self.n_test = 50
self.contamination = 0.1
self.roc_floor = 0.5
self.X_train, self.y_train, self.X_test, self.y_test = generate_data(
n_train=self.n_train, n_test=self.n_test,
contamination=self.contamination, random_state=42)
self.clf = PCA(contamination=self.contamination)
self.clf.fit(self.X_train)
def test_sklearn_estimator(self):
check_estimator(self.clf)
def test_parameters(self):
if not hasattr(self.clf, 'decision_scores_') or \
self.clf.decision_scores_ is None:
self.assertRaises(AttributeError, 'decision_scores_ is not set')
if not hasattr(self.clf, 'labels_') or self.clf.labels_ is None:
self.assertRaises(AttributeError, 'labels_ is not set')
if not hasattr(self.clf, 'threshold_') or self.clf.threshold_ is None:
self.assertRaises(AttributeError, 'threshold_ is not set')
if not hasattr(self.clf, 'selected_components_') or \
self.clf.selected_components_ is None:
self.assertRaises(AttributeError,
'selected_components_ is not set')
if not hasattr(self.clf, 'selected_w_components_') or \
self.clf.selected_w_components_ is None:
self.assertRaises(AttributeError,
'selected_w_components_ is not set')
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
assert_greater(roc_auc_score(self.y_test, pred_scores), self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_fit_predict_score(self):
self.clf.fit_predict_score(self.X_test, self.y_test)
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='roc_auc_score')
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='prc_n_score')
with assert_raises(NotImplementedError):
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='something')
def test_predict_rank(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=2)
assert_array_less(pred_ranks, self.X_train.shape[0] + 1)
assert_array_less(-0.1, pred_ranks)
def test_predict_rank_normalized(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test, normalized=True)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=2)
assert_array_less(pred_ranks, 1.01)
assert_array_less(-0.1, pred_ranks)
def tearDown(self):
pass
clf_mcd = MCD() clf_lof = LOF() clf_cblof = CBLOF() # clf_lscp = LSCP([clf_knn, clf_pca, clf_mcd ]) # clf_ae = AutoEncoder(epochs=50) clf_mcd.fit(encodings_train) clf_pca.fit(encodings_train) clf_knn.fit(encodings_train) clf_lof.fit(encodings_train) clf_cblof.fit(encodings_train) # clf_lscp.fit(encodings_train) # clf_ae.fit(encodings_train) anomaly_scores_mcd = clf_mcd.decision_function(encodings_train) anomaly_scores_pca = clf_pca.decision_function(encodings_train) anomaly_scores_knn = clf_knn.decision_function(encodings_train) anomaly_scores_lof = clf_lof.decision_function(encodings_train) anomaly_scores_cblof = clf_cblof.decision_function(encodings_train) # anomaly_scores_lscp = clf_lscp.decision_function(encodings_train) # anomaly_scores_ae = clf_ae.predict_proba(encodings_train) # y_test_scores = [] # for x,_ in test_loader: # encodings_test = encoder(torch.Tensor(x).to(device)) # probs = clf.predict_proba(encodings_test.detach().cpu().numpy()) # y_test_scores.extend(probs[:,0]) # y_test_scores = np.array(y_test_scores) y_ind_1 = np.argwhere(y_window.reshape(-1, ) == 1) y_ind_3 = np.argwhere(y_window.reshape(-1, ) == 3)
class TestPCA(unittest.TestCase):
def setUp(self):
self.n_train = 100
self.n_test = 50
self.contamination = 0.1
self.roc_floor = 0.5
self.X_train, self.y_train, self.X_test, self.y_test = generate_data(
n_train=self.n_train,
n_test=self.n_test,
contamination=self.contamination)
self.clf = PCA(contamination=self.contamination)
self.clf.fit(self.X_train)
def test_sklearn_estimator(self):
check_estimator(self.clf)
def test_parameters(self):
if not hasattr(self.clf, 'decision_scores_') or \
self.clf.decision_scores_ is None:
self.assertRaises(AttributeError, 'decision_scores_ is not set')
if not hasattr(self.clf, 'labels_') or self.clf.labels_ is None:
self.assertRaises(AttributeError, 'labels_ is not set')
if not hasattr(self.clf, 'threshold_') or self.clf.threshold_ is None:
self.assertRaises(AttributeError, 'threshold_ is not set')
if not hasattr(self.clf, 'selected_components_') or \
self.clf.selected_components_ is None:
self.assertRaises(AttributeError,
'selected_components_ is not set')
if not hasattr(self.clf, 'selected_w_components_') or \
self.clf.selected_w_components_ is None:
self.assertRaises(AttributeError,
'selected_w_components_ is not set')
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
# TODO: turn off performance check before a better data generation
# method is available.
# check performance
# assert_greater(roc_auc_score(self.y_test, pred_scores), self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_evaluate(self):
self.clf.fit_predict_evaluate(self.X_test, self.y_test)
def tearDown(self):
pass
# Generate sample data
X_train, y_train, X_test, y_test = \
generate_data(n_train=n_train,
n_test=n_test,
n_features=2,
contamination=contamination,
random_state=42)
# train PCA detector
clf_name = 'PCA'
clf = PCA()
clf.fit(X_train)
# get the prediction labels and outlier scores of the training data
y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_ # raw outlier scores
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# evaluate and print the results
print("\nOn Training Data:")
evaluate_print(clf_name, y_train, y_train_scores)
print("\nOn Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)
# visualize the results
visualize(clf_name, X_train, y_train, X_test, y_test, y_train_pred,
y_test_pred, show_figure=True, save_figure=False)
class TestPCA(unittest.TestCase):
def setUp(self):
self.n_train = 100
self.n_test = 50
self.contamination = 0.1
self.roc_floor = 0.5
self.X_train, self.y_train, self.X_test, self.y_test = generate_data(
n_train=self.n_train, n_test=self.n_test,
contamination=self.contamination, random_state=42)
self.clf = PCA(contamination=self.contamination, random_state=42)
self.clf.fit(self.X_train)
def test_sklearn_estimator(self):
check_estimator(self.clf)
def test_parameters(self):
assert_true(hasattr(self.clf, 'decision_scores_') and
self.clf.decision_scores_ is not None)
assert_true(hasattr(self.clf, 'labels_') and
self.clf.labels_ is not None)
assert_true(hasattr(self.clf, 'threshold_') and
self.clf.threshold_ is not None)
assert_true(hasattr(self.clf, '_mu') and
self.clf._mu is not None)
assert_true(hasattr(self.clf, '_sigma') and
self.clf._sigma is not None)
assert_true(hasattr(self.clf, 'selected_components_') and
self.clf.selected_components_ is not None)
assert_true(hasattr(self.clf, 'selected_w_components_') and
self.clf.selected_w_components_ is not None)
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
assert_greater(roc_auc_score(self.y_test, pred_scores), self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_fit_predict_score(self):
self.clf.fit_predict_score(self.X_test, self.y_test)
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='roc_auc_score')
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='prc_n_score')
with assert_raises(NotImplementedError):
self.clf.fit_predict_score(self.X_test, self.y_test,
scoring='something')
def test_predict_rank(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, self.X_train.shape[0] + 1)
assert_array_less(-0.1, pred_ranks)
def test_predict_rank_normalized(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test, normalized=True)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, 1.01)
assert_array_less(-0.1, pred_ranks)
def tearDown(self):
pass