def train_model(station: Station) -> LSCP:
t1 = time.time()
log.info(f'Training model for {station}...')
log.info('Loading training observations')
observations_select = Observation.select(
Observation.time,
Observation.sample_frequency,
Observation.sample_count,
Observation.rms,
Observation.crest,
Observation.peak_to_peak,
Observation.kurtosis,
).where(Observation.station == station, Observation.is_training)
obs_data = []
for observation in observations_select:
obs_data.append([
observation.rms, observation.peak_to_peak, observation.kurtosis,
observation.crest
])
log.info('Fitting LSCP model')
lscp = LSCP([KNN()] * 5 + [LOF()] * 5 + [PCA()] * 5, contamination=0.03)
lscp.fit(X=obs_data)
log.info(f'Trained model in {time.time() - t1}')
return lscp
def compute_ensemble_components(self, data_array):
detector_list = []
feature_index = np.array([i for i in range(data_array.shape[1])])
for i in range(self.ensemble_size):
# Randomly sample feature size
feature_size = np.random.randint(self.dim_start, self.dim_end)
# Randomly select features
selected_features = np.random.choice(feature_index, feature_size)
detector_list.append(kNN_LSCP(neighbor_size=self.neighbor, selected_features=selected_features))
clf = LSCP(detector_list)
clf.fit(data_array)
score = clf.decision_scores_
return [score, ]
generate_data(n_train=n_train,
n_test=n_test,
n_features=2,
contamination=contamination,
random_state=42)
# train lscp
clf_name = 'LSCP'
detector_list = [
LOF(n_neighbors=15),
LOF(n_neighbors=20),
LOF(n_neighbors=25),
LOF(n_neighbors=35)
]
clf = LSCP(detector_list, random_state=42)
clf.fit(X_train)
# get the prediction labels and outlier scores of the training data
y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_ # raw outlier scores
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# evaluate and print the results
print("\nOn Training Data:")
evaluate_print(clf_name, y_train, y_train_scores)
print("\nOn Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)
def identify_outliers(df,
features,
contamination=0.1,
algorithms=['Isolation Forest']):
"""Cleans the outliers.
Outlier detection using LSCP: Locally selective combination in parallel outlier ensembles.
https://arxiv.org/abs/1812.01528
Parameters
----------
features : list
List of feature names.
df : DataFrame
The data to be examined.
contamination : float in (0., 0.5)
the proportion of outliers in the data set.
algorithms: list
list with at the names of least 2 algorithms to be used during LSCP. A list of supported algorithms:
['Isolation Forest', 'Cluster-based Local Outlier Factor', 'Minimum Covariance Determinant (MCD)',
'Principal Component Analysis (PCA)', 'Angle-based Outlier Detector (ABOD)',
'Histogram-base Outlier Detection (HBOS)', 'K Nearest Neighbors (KNN)', 'Local Outlier Factor (LOF)',
'Feature Bagging', 'One-class SVM (OCSVM)']
Returns
-------
df_sorted : DataFrame
Original data with 3 new columns: anomaly_score, probability and prediction. Sorted on descending anomaly score.
df_styled: DataFrame
Styled version of df_sorted for use in Jupyter Notebook (i.e. display(df_styled)).
"""
df_numeric = df.select_dtypes(
include=[np.number]) # keep only numeric type features
X = np.asarray(df_numeric)
classifiers = {
'Isolation Forest': IForest,
'Cluster-based Local Outlier Factor': CBLOF,
'Minimum Covariance Determinant (MCD)': MCD,
'Principal Component Analysis (PCA)': PCA,
'Angle-based Outlier Detector (ABOD)': ABOD,
'Histogram-base Outlier Detection (HBOS)': HBOS,
'K Nearest Neighbors (KNN)': knn,
'Local Outlier Factor (LOF)': LOF,
'Feature Bagging': FeatureBagging,
'One-class SVM (OCSVM)': OCSVM,
}
if len(algorithms) > 1:
selected_classifiers = [classifiers[x]() for x in algorithms]
clf = LSCP(selected_classifiers, contamination=contamination)
else:
clf = classifiers[algorithms[0]](contamination=contamination)
clf.fit(X)
y_pred = clf.predict(X)
y_predict_proba = clf.predict_proba(X, method='unify')
y_predict_proba = [item[1] for item in y_predict_proba]
outlier_index, = np.where(y_pred == 1)
anomaly_score = clf.decision_function(X)
anomaly_score = pd.DataFrame(anomaly_score, columns=['anomaly_score'])
y_predict_proba = pd.DataFrame(y_predict_proba, columns=['probability'])
prediction = pd.DataFrame(y_pred, columns=['prediction'])
df.columns = features
df_with_anomaly_score = pd.concat(
[df, anomaly_score, y_predict_proba, prediction], axis=1)
df_sorted = df_with_anomaly_score.sort_values(by='anomaly_score',
ascending=False)
cm = sns.diverging_palette(220, 10, sep=80, n=7, as_cmap=True)
df_styled = df_sorted.style.background_gradient(cmap=cm, subset=['anomaly_score']) \
.apply(lambda x: ['background: MistyRose' if x.name in outlier_index.tolist() else '' for i in x], axis=1,
subset=df_sorted.columns[:-3])
return df_sorted, df_styled
class LSCP(BaseAggregator):
""" Locally Selection Combination in Parallel Outlier Ensembles
LSCP is an unsupervised parallel outlier detection ensemble which selects
competent detectors in the local region of a test instance. This
implementation uses an Average of Maximum strategy. First, a heterogeneous
list of base detectors is fit to the training data and then generates a
pseudo ground truth for each train instance is generated by
taking the maximum outlier score.
For each test instance:
1) The local region is defined to be the set of nearest training points in
randomly sampled feature subspaces which occur more frequently than
a defined threshold over multiple iterations.
2) Using the local region, a local pseudo ground truth is defined and the
pearson correlation is calculated between each base detector's training
outlier scores and the pseudo ground truth.
3) A histogram is built out of pearson correlation scores; detectors in
the largest bin are selected as competent base detectors for the given
test instance.
4) The average outlier score of the selected competent detectors is taken
to be the final score.
See :cite:`zhao2019lscp` for details.
Parameters
----------
base_estimators : list, length must be greater than 1
Base unsupervised outlier detectors from PyOD. (Note: requires fit and
decision_function methods)
local_region_size : int, optional (default=30)
Number of training points to consider in each iteration of the local
region generation process (30 by default).
local_max_features : float in (0.5, 1.), optional (default=1.0)
Maximum proportion of number of features to consider when defining the
local region (1.0 by default).
n_bins : int, optional (default=10)
Number of bins to use when selecting the local region
random_state : RandomState, optional (default=None)
A random number generator instance to define the state of the random
permutations generator.
contamination : float in (0., 0.5), optional (default=0.1)
The amount of contamination of the data set, i.e.
the proportion of outliers in the data set. Used when fitting to
define the threshold on the decision function (0.1 by default).
pre_fitted: bool, optional (default=False)
Whether the base estimators are trained. If True, `fit`
process may be skipped.
Attributes
----------
decision_scores_ : numpy array of shape (n_samples,)
The outlier scores of the training data.
The higher, the more abnormal. Outliers tend to have higher
scores. This value is available once the detector is fitted.
threshold_ : float
The threshold is based on ``contamination``. It is the
``n_samples * contamination`` most abnormal samples in
``decision_scores_``. The threshold is calculated for generating
binary outlier labels.
labels_ : int, either 0 or 1
The binary labels of the training data. 0 stands for inliers
and 1 for outliers/anomalies. It is generated by applying
``threshold_`` on ``decision_scores_``.
"""
def __init__(self, base_estimators, local_region_size=30,
local_max_features=1.0, n_bins=10,
random_state=None, contamination=0.1, pre_fitted=False):
super(LSCP, self).__init__(base_estimators=base_estimators,
pre_fitted=pre_fitted)
if not (0. < contamination <= 0.5):
raise ValueError("contamination must be in (0, 0.5], "
"got: %f" % contamination)
self.contamination = contamination
self.base_estimators = base_estimators
self.local_region_size = local_region_size
self.local_max_features = local_max_features
self.n_bins = n_bins
self.random_state = random_state
def fit(self, X, y=None):
"""Fit detector. y is optional for unsupervised methods.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
y : numpy array of shape (n_samples,), optional (default=None)
The ground truth of the input samples (labels).
"""
# Validate inputs X and y
X = check_array(X)
self._set_n_classes(y)
self.detector_ = PyOD_LSCP(detector_list=self.base_estimators,
local_region_size=self.local_region_size,
local_max_features=self.local_max_features,
n_bins=self.n_bins,
random_state=self.random_state,
contamination=self.contamination)
self.detector_.fit(X)
self.decision_scores_ = self.detector_.decision_scores_
self._process_decision_scores()
return self
def decision_function(self, X):
"""Predict raw anomaly scores of X using the fitted detector.
The anomaly score of an input sample is computed based on the fitted
detector. For consistency, outliers are assigned with
higher anomaly scores.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples. Sparse matrices are accepted only
if they are supported by the base estimator.
Returns
-------
anomaly_scores : numpy array of shape (n_samples,)
The anomaly score of the input samples.
"""
check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
X = check_array(X)
return self.detector_.decision_function(X)
def predict(self, X):
"""Predict if a particular sample is an outlier or not.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
Returns
-------
outlier_labels : numpy array of shape (n_samples,)
For each observation, tells whether or not
it should be considered as an outlier according to the
fitted model. 0 stands for inliers and 1 for outliers.
"""
check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
X = check_array(X)
return self._detector_predict(X)
def predict_proba(self, X, proba_method='linear'):
"""Predict the probability of a sample being outlier. Two approaches
are possible:
1. simply use Min-max conversion to linearly transform the outlier
scores into the range of [0,1]. The model must be
fitted first.
2. use unifying scores, see :cite:`kriegel2011interpreting`.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
proba_method : str, optional (default='linear')
Probability conversion method. It must be one of
'linear' or 'unify'.
Returns
-------
outlier_labels : numpy array of shape (n_samples,)
For each observation, tells whether or not
it should be considered as an outlier according to the
fitted model. Return the outlier probability, ranging
in [0,1].
"""
check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
X = check_array(X)
return self._detector_predict_proba(X, proba_method)
class TestLSCP(unittest.TestCase):
def setUp(self):
# Define data file and read X and y
# Generate some data if the source data is missing
this_directory = path.abspath(path.dirname(__file__))
mat_file = 'cardio.mat'
try:
mat = loadmat(path.join(*[this_directory, 'data', mat_file]))
except TypeError:
print('{data_file} does not exist. Use generated data'.format(
data_file=mat_file))
X, y = generate_data(train_only=True) # load data
except IOError:
print('{data_file} does not exist. Use generated data'.format(
data_file=mat_file))
X, y = generate_data(train_only=True) # load data
else:
X = mat['X']
y = mat['y'].ravel()
X, y = check_X_y(X, y)
self.X_train, self.X_test, self.y_train, self.y_test = \
train_test_split(X, y, test_size=0.4, random_state=42)
self.detector_list = [LOF(), LOF()]
self.clf = LSCP(self.detector_list)
self.clf.fit(self.X_train)
self.roc_floor = 0.6
def test_parameters(self):
assert (hasattr(self.clf, 'decision_scores_')
and self.clf.decision_scores_ is not None)
assert (hasattr(self.clf, 'labels_') and self.clf.labels_ is not None)
assert (hasattr(self.clf, 'threshold_')
and self.clf.threshold_ is not None)
assert (hasattr(self.clf, '_mu') and self.clf._mu is not None)
assert (hasattr(self.clf, '_sigma') and self.clf._sigma is not None)
assert (hasattr(self.clf, 'detector_list')
and self.clf.detector_list is not None)
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
# check performance
assert (roc_auc_score(self.y_test, pred_scores) >= self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert (pred_proba.min() >= 0)
assert (pred_proba.max() <= 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_fit_predict_score(self):
self.clf.fit_predict_score(self.X_test, self.y_test)
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='roc_auc_score')
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='prc_n_score')
with assert_raises(NotImplementedError):
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='something')
def test_predict_rank(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, self.X_train.shape[0] + 1)
assert_array_less(-0.1, pred_ranks)
def test_predict_rank_normalized(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test, normalized=True)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, 1.01)
assert_array_less(-0.1, pred_ranks)
def tearDown(self):
pass
# In[6]:
data195061 = df[(df['CarId'] == '195061')]
x = data195061['Time']
y = data195061['Speed diff']
plt.figure(figsize=(10, 4))
plt.plot(x, y, label='Car 195061')
plt.xlabel('Time')
plt.ylabel('Speed diff')
plt.show()
# In[7]:
lscp = LSCP(detector_list=[MCD(), MCD()])
lscp.fit(df['Speed diff'].values.reshape(-1, 1))
xx = np.linspace(df['Speed diff'].min(), df['Speed diff'].max(),
len(df)).reshape(-1, 1)
anomaly_score = lscp.decision_function(xx)
outlier = lscp.predict(xx)
plt.figure(figsize=(10, 4))
plt.plot(xx, anomaly_score, label='anomaly score')
plt.ylabel('anomaly score')
plt.xlabel('Speed diff')
plt.show()
# In[8]:
df.loc[df['Speed diff'] > 10]
# ### Analyse graphique
lineData = line.strip().split(' ')
lineData = list(map(lambda x: float(x), lineData))
dataMat.append(lineData)
return (np.array(dataMat))
data = data_loadDataSet()
X_train, y_train, X_test, y_test = generate_data(n_train=50,
n_test=50,
contamination=0.1,
random_state=42)
X_train, X_test = standardizer(X_train, X_test)
detector_list = [LOF(n_neighbors=10), LOF(n_neighbors=15)]
clf = LSCP(detector_list)
clf.fit(X_train)
clf.fit(data)
y_train_scores = clf.decision_scores_
sort_factor = argsort(y_train_scores, kind='quicksort')
print(sort_factor)
sort_factors = sort_factor[::-1]
print(sort_factors)
np.savetxt(r'C:\Users\zz\Desktop\res\lscp\D1_2.txt',
sort_factors,
fmt='%f',
delimiter=' ')
# count = 0
# num = 97
# ot = 0
class TestLSCP(unittest.TestCase):
def setUp(self):
self.n_train = 100
self.n_test = 50
self.contamination = 0.1
self.roc_floor = 0.6
self.X_train, self.y_train, self.X_test, self.y_test = generate_data(
n_train=self.n_train,
n_test=self.n_test,
contamination=self.contamination,
random_state=42)
self.X_train, self.X_test = standardizer(self.X_train, self.X_test)
self.detector_list = [LOF(), LOF()]
self.clf = LSCP(self.detector_list, contamination=self.contamination)
self.clf.fit(self.X_train)
def test_parameters(self):
assert_true(
hasattr(self.clf, 'decision_scores_')
and self.clf.decision_scores_ is not None)
assert_true(
hasattr(self.clf, 'labels_') and self.clf.labels_ is not None)
assert_true(
hasattr(self.clf, 'threshold_')
and self.clf.threshold_ is not None)
assert_true(hasattr(self.clf, '_mu') and self.clf._mu is not None)
assert_true(
hasattr(self.clf, '_sigma') and self.clf._sigma is not None)
assert_true(
hasattr(self.clf, 'detector_list')
and self.clf.detector_list is not None)
def test_train_scores(self):
assert_equal(len(self.clf.decision_scores_), self.X_train.shape[0])
def test_prediction_scores(self):
pred_scores = self.clf.decision_function(self.X_test)
# check score shapes
assert_equal(pred_scores.shape[0], self.X_test.shape[0])
# check performance
assert_greater(roc_auc_score(self.y_test, pred_scores), self.roc_floor)
def test_prediction_labels(self):
pred_labels = self.clf.predict(self.X_test)
assert_equal(pred_labels.shape, self.y_test.shape)
def test_prediction_proba(self):
pred_proba = self.clf.predict_proba(self.X_test)
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_linear(self):
pred_proba = self.clf.predict_proba(self.X_test, method='linear')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_unify(self):
pred_proba = self.clf.predict_proba(self.X_test, method='unify')
assert_greater_equal(pred_proba.min(), 0)
assert_less_equal(pred_proba.max(), 1)
def test_prediction_proba_parameter(self):
with assert_raises(ValueError):
self.clf.predict_proba(self.X_test, method='something')
def test_fit_predict(self):
pred_labels = self.clf.fit_predict(self.X_train)
assert_equal(pred_labels.shape, self.y_train.shape)
def test_fit_predict_score(self):
self.clf.fit_predict_score(self.X_test, self.y_test)
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='roc_auc_score')
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='prc_n_score')
with assert_raises(NotImplementedError):
self.clf.fit_predict_score(self.X_test,
self.y_test,
scoring='something')
def test_predict_rank(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, self.X_train.shape[0] + 1)
assert_array_less(-0.1, pred_ranks)
def test_predict_rank_normalized(self):
pred_socres = self.clf.decision_function(self.X_test)
pred_ranks = self.clf._predict_rank(self.X_test, normalized=True)
# assert the order is reserved
assert_allclose(rankdata(pred_ranks), rankdata(pred_socres), atol=3)
assert_array_less(pred_ranks, 1.01)
assert_array_less(-0.1, pred_ranks)
def tearDown(self):
pass
n_train = 200 # number of training points
n_test = 100 # number of testing points
# Generate sample data
X_train, y_train, X_test, y_test = \
generate_data(n_train=n_train,
n_test=n_test,
contamination=contamination,
random_state=42)
X_train, X_test = standardizer(X_train, X_test)
# train lscp
clf_name = 'LSCP'
detector_list = [LOF(), LOF()]
clf = LSCP(detector_list, random_state=42)
clf.fit(X_train)
# get the prediction labels and outlier scores of the training data
y_train_pred = clf.labels_ # binary labels (0: inliers, 1: outliers)
y_train_scores = clf.decision_scores_ # raw outlier scores
# get the prediction on the test data
y_test_pred = clf.predict(X_test) # outlier labels (0 or 1)
y_test_scores = clf.decision_function(X_test) # outlier scores
# evaluate and print the results
print("\nOn Training Data:")
evaluate_print(clf_name, y_train, y_train_scores)
print("\nOn Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)