def detect_anomalies(kills):
num_neighbors = min(KILL_NUM_NEIGHBORS, len(kills) - 1)
contam = min(float(KILL_MAX_ANOM) / len(kills), 0.2)
lof = LocalOutlierFactor(num_neighbors, metric="manhattan", contamination=contam)
kill_vals = np.array([[k.value / 1e6] for k in kills])
res = lof.fit_predict(kill_vals)
return [kills[i] for i in np.nditer(np.where(res == -1))]
def fit(self, X, y=None):
"""Fit detector. y is optional for unsupervised methods.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
y : numpy array of shape (n_samples,), optional (default=None)
The ground truth of the input samples (labels).
"""
# validate inputs X and y (optional)
X = check_array(X)
self._set_n_classes(y)
self.detector_ = LocalOutlierFactor(n_neighbors=self.n_neighbors,
algorithm=self.algorithm,
leaf_size=self.leaf_size,
metric=self.metric,
p=self.p,
metric_params=self.metric_params,
contamination=self.contamination,
n_jobs=self.n_jobs)
self.detector_.fit(X=X, y=y)
# Invert decision_scores_. Outliers comes with higher outlier scores
self.decision_scores_ = invert_order(
self.detector_.negative_outlier_factor_)
self._process_decision_scores()
return self
def main(argv):
config = read_parser(argv, Inputs, InputsOpt_Defaults)
if config['mode'] == 'get_abbrennen':
print('Select Signals AE')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
print('Select Signals Trigger')
root = Tk()
root.withdraw()
root.update()
FilepathsT = filedialog.askopenfilenames()
root.destroy()
T_inis = []
T_ends = []
Durations = []
for filepath, filepathT in zip(Filepaths, FilepathsT):
filename = os.path.basename(filepath)[:-5]
signal = load_signal(filepath, config['channel_ae'])
trigger = load_signal(filepathT, config['channel_trigger'])
fig, ax = plt.subplots(nrows=2, ncols=1)
length_signal = len(signal)
length_trigger = len(trigger)
t_signal = np.arange(length_signal) / config['fs_ae']
t_trigger = np.arange(length_trigger) / config['fs_trigger']
# ax[0].plot(t_signal, signal)
# ax[1].plot(t_trigger, trigger)
# print(config['fs_trigger'])
flag = 'ON'
idx_ini = -1
while flag == 'ON':
idx_ini += 1
if trigger[idx_ini] <= config['threshold']:
t_ini = t_trigger[idx_ini]
flag = 'OFF'
flag = 'ON'
idx_end = idx_ini + int(
config['seg_lockout'] * config['fs_trigger'])
while flag == 'ON':
idx_end += 1
if trigger[idx_end] > config['threshold']:
t_end = t_trigger[idx_end]
flag = 'OFF'
# print(t_ini)
# print(t_end)
# plt.show()
T_inis.append(t_ini)
T_ends.append(t_end)
Durations.append(t_end - t_ini)
abbrennen = signal[int(t_ini *
config['fs_ae']):int(t_end *
config['fs_ae'])]
mydict = {config['channel_ae']: abbrennen}
scipy.io.savemat(filename + '_Abbrennen' + '.mat', mydict)
print('Initial_Times: ', T_inis)
print('Ending Times: ', T_ends)
print('Durations: ', Durations)
elif config['mode'] == 'get_nachbehandlung':
print('Select Signals AE')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
print('Select Signals Trigger')
root = Tk()
root.withdraw()
root.update()
FilepathsT = filedialog.askopenfilenames()
root.destroy()
T_inis = []
T_ends = []
Durations = []
for filepath, filepathT in zip(Filepaths, FilepathsT):
filename = os.path.basename(filepath)[:-5]
signal = load_signal(filepath, config['channel_ae'])
trigger = load_signal(filepathT, config['channel_trigger'])
fig, ax = plt.subplots(nrows=2, ncols=1)
length_signal = len(signal)
length_trigger = len(trigger)
t_signal = np.arange(length_signal) / config['fs_ae']
t_trigger = np.arange(length_trigger) / config['fs_trigger']
# ax[0].plot(t_signal, signal)
# ax[1].plot(t_trigger, trigger)
# print(config['fs_trigger'])
flag = 'ON'
idx_ini = -1
while flag == 'ON':
idx_ini += 1
if trigger[idx_ini] <= config['threshold']:
t_ini = t_trigger[idx_ini]
flag = 'OFF'
flag = 'ON'
idx_end = idx_ini + int(
config['seg_lockout'] * config['fs_trigger'])
while flag == 'ON':
idx_end += 1
if trigger[idx_end] > config['threshold']:
t_end = t_trigger[idx_end]
flag = 'OFF'
T_inis.append(t_ini)
T_ends.append(t_end)
Durations.append(t_end - t_ini)
# abbrennen = signal[int(t_ini*config['fs_ae']) : int(t_end*config['fs_ae'])]
nachbehandlung = signal[int(t_end *
config['fs_ae']):int((t_end + 10) *
config['fs_ae'])]
mydict = {config['channel_ae']: nachbehandlung}
scipy.io.savemat(filename + '_Nachbehandlung10' + '.mat', mydict)
print('Initial_Times: ', T_inis)
print('Ending Times: ', T_ends)
print('Durations: ', Durations)
elif config['mode'] == 'plot_features':
print('Waehlen XLS von Klasse 1 (gut)')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
print('Waehlen XLS von Klasse 1 -1 (schlecht)')
root = Tk()
root.withdraw()
root.update()
Filepaths2 = filedialog.askopenfilenames()
root.destroy()
names_features = [
'amax', 'count', 'crest', 'dc', 'dura', 'freq', 'kurt', 'ra',
'rise', 'rms'
]
fig, ax = plt.subplots(nrows=2, ncols=2)
# names_features = ['count', 'dura']
# plt.xlabel('Überschwingungen', fontsize=14), plt.ylabel('Dauer [us]', fontsize=14)
# names_features = ['amax', 'rms']
# plt.xlabel('Max. Amplitude [mV]', fontsize=14), plt.ylabel('RMS [mV]', fontsize=14)
# names_features = ['rise', 'kurt']
# plt.xlabel('Anstiegszeit [us]', fontsize=14), plt.ylabel('Kurtosis', fontsize=14)
# names_features = ['crest', 'freq']
# plt.xlabel('Crest Faktor', fontsize=14), plt.ylabel('Hauptfrequenz [kHz]', fontsize=14)
Dict_Features = {}
for feature in names_features:
Dict_Features[feature] = []
Labels = []
for filepath in Filepaths:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
Labels += [1 for i in range(len(mydict[element]))]
for filepath in Filepaths2:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
Labels += [-1 for i in range(len(mydict[element]))]
n_samples = len(Dict_Features[names_features[0]])
n_features = len(names_features)
Idx_Gut = [i for i in range(n_samples) if Labels[i] == 1]
Idx_Schlecht = [i for i in range(n_samples) if Labels[i] == -1]
alpha = 0.5
ax[0][0].scatter(np.array(Dict_Features['count'])[Idx_Gut],
np.array(Dict_Features['dura'])[Idx_Gut],
color='blue',
marker='s',
alpha=alpha,
label='Gut')
ax[0][0].scatter(np.array(Dict_Features['count'])[Idx_Schlecht],
np.array(Dict_Features['dura'])[Idx_Schlecht],
color='red',
alpha=alpha,
label='Schlecht')
ax[0][0].set_xlabel('Überschwingungen [-]',
fontsize=14), ax[0][0].set_ylabel('Dauer [us]',
fontsize=14)
ax[1][0].scatter(np.array(Dict_Features['amax'])[Idx_Gut],
np.array(Dict_Features['rise'])[Idx_Gut],
color='blue',
marker='s',
alpha=alpha,
label='Gut')
ax[1][0].scatter(np.array(Dict_Features['amax'])[Idx_Schlecht],
np.array(Dict_Features['rise'])[Idx_Schlecht],
color='red',
alpha=alpha,
label='Schlecht')
ax[1][0].set_xlabel('Max. Amplitude [mV]',
fontsize=14), ax[1][0].set_ylabel(
'Anstiegszeit [us]', fontsize=14)
ax[0][1].scatter(np.array(Dict_Features['kurt'])[Idx_Gut],
np.array(Dict_Features['rms'])[Idx_Gut],
color='blue',
marker='s',
alpha=alpha,
label='Gut')
ax[0][1].scatter(np.array(Dict_Features['kurt'])[Idx_Schlecht],
np.array(Dict_Features['rms'])[Idx_Schlecht],
color='red',
alpha=alpha,
label='Schlecht')
ax[0][1].set_xlabel('Kurtosis [-]',
fontsize=14), ax[0][1].set_ylabel('RMS Wert [mV]',
fontsize=14)
ax[1][1].scatter(np.array(Dict_Features['crest'])[Idx_Gut],
np.array(Dict_Features['freq'])[Idx_Gut],
color='blue',
marker='s',
alpha=alpha,
label='Gut')
ax[1][1].scatter(np.array(Dict_Features['crest'])[Idx_Schlecht],
np.array(Dict_Features['freq'])[Idx_Schlecht],
color='red',
alpha=alpha,
label='Schlecht')
ax[1][1].set_xlabel('Crest Faktor [-]',
fontsize=14), ax[1][1].set_ylabel(
'Hauptfrequenz [kHz]', fontsize=14)
ax[0][0].tick_params(axis='both', labelsize=12)
ax[1][0].tick_params(axis='both', labelsize=12)
ax[0][1].tick_params(axis='both', labelsize=12)
ax[1][1].tick_params(axis='both', labelsize=12)
ax[0][0].legend(fontsize=12)
ax[0][1].legend(fontsize=12)
ax[1][0].legend(fontsize=12)
ax[1][1].legend(fontsize=12)
plt.tight_layout()
plt.show()
elif config['mode'] == 'svm_one_class_valid':
print('Waehlen XLS von Klasse 1 (gut)')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
# print('Waehlen XLS von Klasse 1 -1 (schlecht)')
# root = Tk()
# root.withdraw()
# root.update()
# Filepaths2 = filedialog.askopenfilenames()
# root.destroy()
names_features = [
'amax', 'count', 'crest', 'dc', 'dura', 'freq', 'kurt', 'ra',
'rise', 'rms'
]
Dict_Features = {}
for feature in names_features:
Dict_Features[feature] = []
# Labels = []
for filepath in Filepaths:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
# Labels += [1 for i in range(len(mydict[element]))]
n_samples = len(Dict_Features[names_features[0]])
n_features = len(names_features)
Features = np.zeros((n_samples, n_features))
count = 0
for feature in names_features:
Features[:, count] = Dict_Features[feature]
count += 1
scaler = StandardScaler()
scaler.fit(Features)
Features = scaler.transform(Features)
from sklearn.svm import OneClassSVM
from sklearn.model_selection import train_test_split
Normals = []
Anormals = []
for i in range(10):
print('Validation n°', i)
X_train, X_valid = train_test_split(Features,
test_size=0.25,
random_state=None)
clf = OneClassSVM(nu=0.5, kernel='sigmoid')
clf.fit(X_train)
y_pred = clf.predict(X_valid)
normal = 0
anormal = 0
for element in y_pred:
if element == 1:
normal += 1
elif element == -1:
anormal += 1
else:
print('error 9475')
sys.exit()
normal = normal / len(y_pred)
anormal = anormal / len(y_pred)
# print('normal ', normal)
# print('anormal ', anormal)
Normals.append(normal)
Anormals.append(anormal)
print('Normal rate: ', np.mean(np.array(Normals)))
print('Anormal rate: ', np.mean(np.array(Anormals)))
# for filepath in Filepaths2:
# mydict = pd.read_excel(filepath)
# mydict = mydict.to_dict(orient='list')
# for element in names_features:
# Dict_Features[element] += mydict[element]
elif config['mode'] == 'svm_one_class_test':
print('Waehlen XLS von Klasse 1 (gut)')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
names_features = [
'amax', 'count', 'crest', 'dc', 'dura', 'freq', 'kurt', 'ra',
'rise', 'rms'
]
Dict_Features = {}
for feature in names_features:
Dict_Features[feature] = []
# Labels = []
for filepath in Filepaths:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
# Labels += [1 for i in range(len(mydict[element]))]
n_samples = len(Dict_Features[names_features[0]])
n_features = len(names_features)
Features = np.zeros((n_samples, n_features))
count = 0
for feature in names_features:
Features[:, count] = Dict_Features[feature]
count += 1
scaler = StandardScaler()
scaler.fit(Features)
Features = scaler.transform(Features)
from sklearn.svm import OneClassSVM
from sklearn.model_selection import train_test_split
clf = OneClassSVM(nu=0.01, kernel='sigmoid')
clf.fit(Features)
print('Waehlen XLS von Klasse -1 (schlecht)')
root = Tk()
root.withdraw()
root.update()
Filepaths2 = filedialog.askopenfilenames()
root.destroy()
Dict_Features2 = {}
for feature in names_features:
Dict_Features2[feature] = []
for filepath in Filepaths2:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features2[element] += mydict[element]
n_samples2 = len(Dict_Features2[names_features[0]])
n_features2 = len(names_features)
Features2 = np.zeros((n_samples2, n_features2))
count = 0
for feature in names_features:
fact = np.random.randn()
print(fact)
Features2[:,
count] = np.array(Dict_Features2[feature]) * fact * 1000
count += 1
# scaler = StandardScaler()
# scaler.fit(Features2)
Features2 = scaler.transform(Features2)
y_pred = clf.predict(Features2)
normal = 0
anormal = 0
for element in y_pred:
if element == 1:
normal += 1
elif element == -1:
anormal += 1
else:
print('error 9475')
sys.exit()
normal = normal / len(y_pred)
anormal = anormal / len(y_pred)
print('Normal rate: ', normal)
print('Anormal rate: ', anormal)
elif config['mode'] == 'novelty_valid':
print('Waehlen XLS von Klasse 1 (gut)')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
# print('Waehlen XLS von Klasse 1 -1 (schlecht)')
# root = Tk()
# root.withdraw()
# root.update()
# Filepaths2 = filedialog.askopenfilenames()
# root.destroy()
names_features = [
'amax', 'count', 'crest', 'dc', 'dura', 'freq', 'kurt', 'ra',
'rise', 'rms'
]
Dict_Features = {}
for feature in names_features:
Dict_Features[feature] = []
# Labels = []
for filepath in Filepaths:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
# Labels += [1 for i in range(len(mydict[element]))]
n_samples = len(Dict_Features[names_features[0]])
n_features = len(names_features)
Features = np.zeros((n_samples, n_features))
count = 0
for feature in names_features:
Features[:, count] = Dict_Features[feature]
count += 1
# scaler = StandardScaler()
# scaler.fit(Features)
# Features = scaler.transform(Features)
# from sklearn.decomposition import PCA
# pca = PCA(n_components=4)
# Features = pca.fit_transform(Features)
# print(pca.explained_variance_ratio_ )
from sklearn.neighbors import LocalOutlierFactor
from sklearn.model_selection import train_test_split
# clf = LocalOutlierFactor(n_neighbors=20, novelty=True, contamination=0.1)
# clf.fit(Features)
Normals = []
Anormals = []
for i in range(20):
print('Validation n°', i)
X_train, X_valid = train_test_split(Features,
test_size=0.25,
random_state=None)
clf = LocalOutlierFactor(n_neighbors=20,
novelty=True,
contamination='auto')
clf.fit(X_train)
y_pred = clf.predict(X_valid)
normal = 0
anormal = 0
for element in y_pred:
if element == 1:
normal += 1
elif element == -1:
anormal += 1
else:
print('error 9475')
sys.exit()
normal = normal / len(y_pred)
anormal = anormal / len(y_pred)
# print('normal ', normal)
# print('anormal ', anormal)
Normals.append(normal)
Anormals.append(anormal)
print('Normal rate: ', np.mean(np.array(Normals)))
print('Anormal rate: ', np.mean(np.array(Anormals)))
print('Normal STD: ', np.std(np.array(Normals)))
print('Anormal STD: ', np.std(np.array(Anormals)))
# for filepath in Filepaths2:
# mydict = pd.read_excel(filepath)
# mydict = mydict.to_dict(orient='list')
# for element in names_features:
# Dict_Features[element] += mydict[element]
elif config['mode'] == 'novelty_test':
# import sklearn
# print('The scikit-learn version is {}.'.format(sklearn.__version__))
# sys.exit()
print('Waehlen XLS von Klasse 1 (gut)')
root = Tk()
root.withdraw()
root.update()
Filepaths = filedialog.askopenfilenames()
root.destroy()
names_features = [
'amax', 'count', 'crest', 'dc', 'dura', 'freq', 'kurt', 'ra',
'rise', 'rms'
]
Dict_Features = {}
for feature in names_features:
Dict_Features[feature] = []
# Labels = []
for filepath in Filepaths:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features[element] += mydict[element]
# Labels += [1 for i in range(len(mydict[element]))]
n_samples = len(Dict_Features[names_features[0]])
n_features = len(names_features)
Features = np.zeros((n_samples, n_features))
count = 0
for feature in names_features:
Features[:, count] = Dict_Features[feature]
count += 1
# scaler = StandardScaler()
# scaler.fit(Features)
# Features = scaler.transform(Features)
# from sklearn.decomposition import PCA
# pca = PCA(n_components=4)
# Features = pca.fit_transform(Features)
# print(pca.explained_variance_ratio_ )
from sklearn.neighbors import LocalOutlierFactor
from sklearn.model_selection import train_test_split
clf = LocalOutlierFactor(n_neighbors=20,
novelty=True,
contamination='auto')
clf.fit(Features)
print('Waehlen XLS von Klasse -1 (schlecht)')
root = Tk()
root.withdraw()
root.update()
Filepaths2 = filedialog.askopenfilenames()
root.destroy()
Dict_Features2 = {}
for feature in names_features:
Dict_Features2[feature] = []
for filepath in Filepaths2:
mydict = pd.read_excel(filepath)
mydict = mydict.to_dict(orient='list')
for element in names_features:
Dict_Features2[element] += mydict[element]
n_samples2 = len(Dict_Features2[names_features[0]])
n_features2 = len(names_features)
Features2 = np.zeros((n_samples2, n_features2))
count = 0
for feature in names_features:
# fact = np.random.randn()
# print(fact)
Features2[:, count] = np.array(Dict_Features2[feature])
count += 1
# scaler = StandardScaler()
# scaler.fit(Features2)
# Features2 = scaler.transform(Features2)
# Features2 = pca.transform(Features2)
y_pred = clf.predict(Features2)
normal = 0
anormal = 0
for element in y_pred:
if element == 1:
normal += 1
elif element == -1:
anormal += 1
else:
print('error 9475')
sys.exit()
normal = normal / len(y_pred)
anormal = anormal / len(y_pred)
print('Normal rate: ', normal)
print('Anormal rate: ', anormal)
else:
print('wrong_mode')
return
happy.Economy)
happy1.Family = np.where(happy1.Family.isna(), np.mean(happy.Family),
happy.Family)
happy1.Health = np.where(happy1.Health.isna(), np.mean(happy.Health),
happy.Health)
happy1.Freedom = np.where(happy1.Freedom.isna(), np.mean(happy.Freedom),
happy.Freedom)
happy1.Trust = np.where(happy1.Trust.isna(), np.mean(happy.Trust), happy.Trust)
happy1.dtypes
from sklearn.neighbors import LocalOutlierFactor
happy2 = happy1.drop(columns=["Rating", "Grade"])
from sklearn.neighbors import LocalOutlierFactor
lof1 = LocalOutlierFactor()
lof1.fit(happy2)
x = happy2.loc[lof1.negative_outlier_factor_ > -2, :]
y = happy1[["Rating", "Grade"]][lof1.negative_outlier_factor_ > -2]
from sklearn.model_selection import train_test_split
tr_x, te_x, tr_y, te_y = train_test_split(x.drop(columns="Score"),
x.Score,
test_size=0.3,
random_state=1234)
from sklearn.preprocessing import MinMaxScaler
minmax1 = MinMaxScaler()
tr_xs = tr_x.copy()
te_xs = te_x.copy()
import numpy as np
import matplotlib.pyplot as plt
from sklearn.neighbors import LocalOutlierFactor
np.random.seed(42)
# Generate train data
X = 0.3 * np.random.randn(100, 2)
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))
X = np.r_[X + 2, X - 2, X_outliers]
# fit the model
clf = LocalOutlierFactor(n_neighbors=20)
y_pred = clf.fit_predict(X)
y_pred_outliers = y_pred[200:]
false_positives = (y_pred[:200] == -1)
false_negatives = (y_pred[200:] == 1)
errors = np.concatenate((false_positives, false_negatives), axis=0)
# plot the level sets of the decision function
xx, yy = np.meshgrid(np.linspace(-5, 5, 50), np.linspace(-5, 5, 50))
Z = clf._decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.title("Local Outlier Factor (LOF)")
plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r)
a = plt.scatter(X[:200, 0],
cnf_matix = confusion_matrix(y, y_pred) precision = cnf_matix[1][1] / (cnf_matix[0][1] + cnf_matix[1][1]) recall = cnf_matix[1][1] / (cnf_matix[1][1] + cnf_matix[1][0]) print(precision, recall) # Split into anomaly and normal examples y_norm_idxs = (y == 0).squeeze() x_norm = x[y_norm_idxs] # Labeled examples x_norm = x_norm.reshape(-1, 1) x_an = x[~y_norm_idxs] # anormal x_an = x_an.reshape(-1, 1) #SVM on class without anormal clf = svm.OneClassSVM(nu=.1, kernel='rbf', gamma=.1) clf.fit(x_norm) y_pred = clf.predict(x_new) y_pred = np.array([1 if x == -1 else 0 for x in y_pred]) cnf_matix = confusion_matrix(y, y_pred) precision = cnf_matix[1][1] / (cnf_matix[0][1] + cnf_matix[1][1]) recall = cnf_matix[1][1] / (cnf_matix[1][1] + cnf_matix[1][0]) print(precision, recall) #Local outlier facto clf = LocalOutlierFactor(n_neighbors=20, contamination=0.1) y_pred = clf.fit_predict(x_new) y_pred = np.array([1 if x == -1 else 0 for x in y_pred]) cnf_matix = confusion_matrix(y, y_pred) precision = cnf_matix[1][1] / (cnf_matix[0][1] + cnf_matix[1][1]) recall = cnf_matix[1][1] / (cnf_matix[1][1] + cnf_matix[1][0]) print(precision, recall) print(confusion_matrix(y, y_pred))
target = "Class"
#define a random state
state = np.random.RandomState(42)
X = data1[columns]
Y = data1[target]
X_outliers = state.uniform(low=0, high=1, size=(X.shape[0], X.shape(1))
print(X.shape)
print(Y.shape)
#define the outlier detection methods
classifiers = {
"Isolation Forest":IsolationForest(n_estimators=100, max_samples=len(X), contamination=outlier_fraction,random_state=state, verbose=0),
"Local Outlier Fractor":LocalOutlierFactor(novelty=True, n_neighbors=20, algorithm='auto', leaf_size=30, metric='minkowski', p=2, metric_params=None, contamination=outlier_fraction),
"Support Vector Machine":OneClassSVM(kernel='rbf', degree=3, gamma=0.1,nu=0.05, max_iter=-1, random_state=state)
}
type(classifiers)
n_outliers = len(La_Nina)
for i, (clf_name,clf) in enumerate(classifiers.items()):
#fir the data and tag outliers
if clf_name == "Local Outlier Factor":
y_pred = clf.fit_predict(X)
scores_prediction = clf.negative_outlier_factor_
elif clf_name == "Support Vector Machine":
clf.fit(X)
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.neighbors import LocalOutlierFactor
data = pd.read_csv("creditcard.csv")
# sampling random 50000 points
data_50000 = data.sample(n=50000)
data_50000.to_csv("NewCreditCard.csv")
newData = pd.read_csv("NewCreditCard.csv")
FinalData = newData.drop("Unnamed: 0", axis=1)
lof = LocalOutlierFactor(n_neighbors=2,
algorithm='auto',
metric='minkowski',
p=2,
metric_params=None,
contamination=0.5,
n_jobs=1)
outlierArray = lof.fit_predict(FinalData)
print(outlierArray)
countOutliers = 0
countInliers = 0
for i in range(50000):
if outlierArray[i] == -1:
countOutliers += 1
else:
countInliers += 1
print("Total number of outliers = " + str(countOutliers))
print("Total number of inliers = " + str(countInliers))
from sklearn.metrics import classification_report, accuracy_score
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
#define random state
state = 1
#define the outlier detection methods
classifier = {
"IsolationForest":
IsolationForest(max_samples=len(X),
contamination=fruad_fraction,
random_state=state),
"LocalOutlierFactor":
LocalOutlierFactor(n_neighbors=20, contamination=fruad_fraction)
}
n_outliers = len(Fraud)
for i, (clf_name, clf) in enumerate(classifier.items()):
if clf_name == "LocalOutlierFactor":
y_pred = clf.fit_predict(X)
scores_pred = clf.negetive_outlier_factor_
else:
y_pred = clf.fit(X)
scores_pred = clf.decision_function(X)
y_pred = clf.predict(X)
# Reshape the prediction values to 0 for valid, 1 for fraud
y_pred[y_pred == 1] = 0
X1 = np.column_stack((c, timeindex10, timeindex100))
mean10 = cal_mean(c)
std10 = cal_std(c)
mean100, std100 = cal_mean_std_100days(c)
X2 = np.column_stack((c, mean10, std10, mean100, std100))
label = cal_label(c, mean10, mean100, std100)
MaxYield = cal_MaximumYield(h, l)
CloseYield = cal_CloseYield(c)
ATR = cal_ATR(h, l, c)
OBV = cal_OBV(c, v)
X3 = np.column_stack((c, MaxYield, CloseYield, ATR, OBV, label))
clf = LocalOutlierFactor(n_neighbors=20)
y_score1 = clf.fit_predict(X1)
y_score2 = clf.fit_predict(X2)
y_score3 = clf.fit_predict(X3)
X1 = np.column_stack((X1, y_score1))
X2 = np.column_stack((X2, y_score2))
X3 = np.column_stack((X3, y_score3))
dataframe1 = pd.DataFrame(
X1, columns=['ClosePrice', 'TimeIndex10', 'TimeIndex100', 'y_score'])
dataframe2 = pd.DataFrame(
X2,
columns=['ClosePrice', 'Mean10', 'Std10', 'Mean100', 'Std100', 'y_score'])
dataframe3 = pd.DataFrame(X3,
columns=[
'ClosePrice', 'MaxYiekd', 'CloseYield', 'ATR',
def __init__(self, **kwargs):
self._model = LocalOutlierFactor(**kwargs)
X_train, y_train = X_train[mask, :], y_train[mask]
# summarize the shape of the updated training dataset
print(X_train.shape, y_train.shape)
# fit the model
model = LinearRegression()
model.fit(X_train, y_train)
# evaluate the model
yhat = model.predict(X_test)
# evaluate predictions
mae = mean_absolute_error(y_test, yhat)
print('Minimum Covariance Determinant MAE: %.3f' % mae)
####====> Local Outlier Factor
print('Local Outlier Factor anomaly detection')
# identify outliers in the training dataset
lof = LocalOutlierFactor()
yhat = lof.fit_predict(X_train)
# select all rows that are not outliers
mask = yhat != -1
X_train, y_train = X_train[mask, :], y_train[mask]
# summarize the shape of the updated training dataset
print(X_train.shape, y_train.shape)
# fit the model
model = LinearRegression()
model.fit(X_train, y_train)
# evaluate the model
yhat = model.predict(X_test)
# evaluate predictions
mae = mean_absolute_error(y_test, yhat)
print('Local Outlier Factor : %.3f' % mae)
# import pdb; pdb.set_trace()
# self.train_data = torch.from_numpy(features[0:200000])
# self.test_data = torch.from_numpy(features[200001:284807])
# self.train_labels = torch.from_numpy(labels[0:200000])
# self.test_labels = torch.from_numpy(labels[200001:284807])
train_data = torch.from_numpy(features[0:100000])
test_data = torch.from_numpy(features[100000:138046])
train_labels = torch.from_numpy(labels[0:100000])
test_labels = torch.from_numpy(labels[100000:138046])
# knn = KNeighborsClassifier(n_neighbors=10, algorithm='ball_tree').fit(train_data, train_labels)
# kmeans = KMeans(n_clusters=2, random_state=0).fit(train_data)
OCSVM = OneClassSVM(gamma='auto', nu=0.02).fit(train_data)
isolation_forest = IsolationForest(random_state=0).fit(train_data)
local_outliar = LocalOutlierFactor(n_neighbors=20,
novelty=True).fit(train_data)
models = [OCSVM, isolation_forest, local_outliar]
names = ["ocsvm", "isolation_forest" "local_outliar"]
for model, name in zip(models, names):
print(
"==================================================================================="
)
print(name)
predicted = model.predict(test_data)
scores = 1 - model.score_samples(test_data)
predicted = [mod(x) for x in predicted]
# print(predicted[0:10])
# print(scores[0:10])
def get_lof(db: pd.DataFrame) -> list:
lof = LocalOutlierFactor()
yhat_lof = lof.fit_predict(db)
return yhat_lof == -1
df2 = df2[df2["Job Title"].isin(emp_counts[emp_counts > 3000].index)]
df2['Salary Paid'] = df2['Salary Paid'].apply(lambda x:x.split('.')[0].strip()).replace({'\$':'', ',':''}, regex=True)
FirAtt_lst = df2['Job Title'].unique()
SecAtt_lst = df2['Employer'].unique()
ThrAtt_lst = df2['Calendar Year'].unique()
################################### Forming a context #######################################
Orgn_Ctx = df2.loc[df2['Job Title'].isin([FirAtt_lst[0],FirAtt_lst[1],FirAtt_lst[2],FirAtt_lst[3], FirAtt_lst[4]]) & \
df2['Employer'].isin([SecAtt_lst[0],SecAtt_lst[1], SecAtt_lst[2],SecAtt_lst[3], SecAtt_lst[4], SecAtt_lst[5]]) & \
df2['Calendar Year'].isin([ThrAtt_lst[0],ThrAtt_lst[1],ThrAtt_lst[2],ThrAtt_lst[3],ThrAtt_lst[4]])]
####################### Finding an outlier in the selected context #######################
clf = LocalOutlierFactor(n_neighbors=20)
Sal_outliers = clf.fit_predict(Orgn_Ctx['Salary Paid'].values.reshape(-1,1))
Queried_ID =Orgn_Ctx.iloc[Sal_outliers.argmin()][1]
print '\n\n Outlier\'s ID in the selected context is: ', Queried_ID
################# Exploring Contexts larger than the original to find the maximal #################
FirAtt_Sprset = sum(map(lambda r: list(combinations(FirAtt_lst[5:], r)), range(1, len(FirAtt_lst[5:])+1)), [])
SecAtt_Sprset = sum(map(lambda r: list(combinations(SecAtt_lst[6:], r)), range(1, len(SecAtt_lst[6:])+1)), [])
ThrAtt_Sprset = sum(map(lambda r: list(combinations(ThrAtt_lst[5:], r)), range(1, len(ThrAtt_lst[5:])+1)), [])
Sub_pop = []
Sub_pop_count = 0
Epsilon = 0.1 ### Privacy Parameter
output = []
context = []
import numpy as np
import matplotlib.pyplot as plt
from sklearn.neighbors import LocalOutlierFactor
print(__doc__)
np.random.seed(42)
# Generate train data
X = 0.3 * np.random.randn(100, 2)
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))
X = np.r_[X + 2, X - 2, X_outliers]
# fit the model
clf = LocalOutlierFactor(n_neighbors=20)
y_pred = clf.fit_predict(X)
y_pred_outliers = y_pred[200:]
# plot the level sets of the decision function
xx, yy = np.meshgrid(np.linspace(-5, 5, 50), np.linspace(-5, 5, 50))
Z = clf._decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.title("Local Outlier Factor (LOF)")
plt.contourf(xx, yy, Z, cmap=plt.cm.Blues_r)
a = plt.scatter(X[:200, 0], X[:200, 1], c='white')
b = plt.scatter(X[200:, 0], X[200:, 1], c='red')
plt.axis('tight')
plt.xlim((-5, 5))
gamma = 0.1
num_neighbors = 35
# Construct the data set
offset = 2
data_inliers_1 = 0.3 * np.random.randn(num_inliers // 2, 2) - offset
data_inliers_2 = 0.3 * np.random.randn(num_inliers // 2, 2) + offset
data_inliers = np.r_[data_inliers_1, data_inliers_2]
data_outliers = np.random.uniform(low=-5, high=5, size=(num_outliers, 2))
data = np.r_[data_inliers, data_outliers]
# Construct the classifiers.
ensemble = dict(oneclasssvm=OneClassSVM(kernel="rbf", gamma=gamma, nu=contamination),
elliptic_envelope=EllipticEnvelope(contamination=contamination),
isolation_forest=IsolationForest(contamination=contamination, max_samples=num_samples),
local_outlier_factor=LocalOutlierFactor(n_neighbors=num_neighbors, contamination=contamination))
ensemble_predicted_data = dict()
# Fit the data for different classifiers
for name, clf in ensemble.items():
if name.startswith("local_outlier_factor"):
predicted_data = clf.fit_predict(data)
else:
clf.fit(data)
predicted_data = clf.predict(data)
ensemble_predicted_data[name] = predicted_data
# Perform outlier detection
inlier_predicted_data = data[predicted_data == 1]
outlier_predicted_data = data[predicted_data == -1]
num_inliers_predicted = inlier_predicted_data.shape[0]
if dataset_name == 'SA':
lb = LabelBinarizer()
x1 = lb.fit_transform(X[:, 1].astype(str))
x2 = lb.fit_transform(X[:, 2].astype(str))
x3 = lb.fit_transform(X[:, 3].astype(str))
X = np.c_[X[:, :1], x1, x2, x3, X[:, 4:]]
y = (y != b'normal.').astype(int)
if dataset_name == 'http' or dataset_name == 'smtp':
y = (y != b'normal.').astype(int)
X = X.astype(float)
print('LocalOutlierFactor processing...')
model = LocalOutlierFactor(n_neighbors=20)
tstart = time()
model.fit(X)
fit_time = time() - tstart
scoring = -model.negative_outlier_factor_ # the lower, the more normal
fpr, tpr, thresholds = roc_curve(y, scoring)
AUC = auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1,
label=('ROC for %s (area = %0.3f, train-time: %0.2fs)'
% (dataset_name, AUC, fit_time)))
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
n_outliers = int(outliers_fraction * n_samples)
n_inliers = n_samples - n_outliers
# define outlier/anomaly detection methods to be compared
anomaly_algorithms = [("Robust covariance",
EllipticEnvelope(contamination=outliers_fraction)),
("One-Class SVM",
svm.OneClassSVM(nu=outliers_fraction,
kernel="rbf",
gamma=0.1)),
("Isolation Forest",
IsolationForest(behaviour='new',
contamination=outliers_fraction,
random_state=42)),
("Local Outlier Factor",
LocalOutlierFactor(n_neighbors=35,
contamination=outliers_fraction))]
# Define datasets
blobs_params = dict(random_state=0, n_samples=n_inliers, n_features=2)
datasets = [
make_blobs(centers=[[0, 0], [0, 0]], cluster_std=0.5, **blobs_params)[0],
make_blobs(centers=[[2, 2], [-2, -2]],
cluster_std=[0.5, 0.5],
**blobs_params)[0],
make_blobs(centers=[[2, 2], [-2, -2]],
cluster_std=[1.5, .3],
**blobs_params)[0],
4. * (make_moons(n_samples=n_samples, noise=.05, random_state=0)[0] -
np.array([0.5, 0.25])),
14. * (np.random.RandomState(42).rand(n_samples, 2) - 0.5)
]
#############################
# Çok Değişkenli Aykırı Değer Analizi: Local Outlier Factor
#############################
# Gözlemleri bulundukları konumda yoğunluk tabanlı skorlayarak
# buna göre aykırı değer olabilecek değerleri tanımlayabilmemize imkan sağlıyor.
df = sns.load_dataset('diamonds')
df = df.select_dtypes(include=['float64', 'int64'])
df = df.dropna()
df.head()
for col in df.columns:
print(col, check_outlier(df, col))
clf = LocalOutlierFactor(n_neighbors=20)
clf.fit_predict(df)
df_scores = clf.negative_outlier_factor_
df_scores[0:5]
np.sort(df_scores)[0:5]
scores = pd.DataFrame(np.sort(df_scores))
scores.plot(stacked=True, xlim=[0, 20], style='.-')
plt.show()
esik_deger = np.sort(df_scores)[3]
df[df_scores < esik_deger]
df[df_scores < esik_deger].shape
for row in cur:
vec = row
vecs.append(vec)
print(vec)
scaler = StandardScaler()
#scaler=MinMaxScaler()
scaler.fit(vecs)
vecs = scaler.transform(vecs)
end = time.time()
conn.close()
print(end - start)
print(len(vecs))
for vec in vecs:
print(vec)
start = time.time()
clf = LocalOutlierFactor(n_neighbors=1, novelty=True, metric='cosine')
clf.fit(vecs)
print("Fitting done")
pkl_filename = "lof_distinct_model.pkl"
pickle.dump(clf, open(pkl_filename, 'wb'))
pkl_scaler = "lof_scaler.pkl"
pickle.dump(scaler, open(pkl_scaler, 'wb'))
end = time.time()
# print(end-start)
X_scores = clf.negative_outlier_factor_
print(X_scores)
'''
plt.title("Local Outlier Factor (LOF)")
plt.scatter(X[:, 0], X[:, 1], color='k', s=3., label='Data points')
# plot circles with radius proportional to the outlier scores
def data_representation(X, y, x, data_rep):
print("Dataset Representation:", x)
df = pd.DataFrame(X)
n_samples = df.shape[0]
n_features = df.shape[1]
n_classes = set(y)
n_classes = len(n_classes)
c = Counter(y)
values = np.array(list(c.values())) / n_samples
class_weights_min = values.min()
class_weights_mean = values.mean()
class_weights_max = values.max()
mean_min = df.mean().min()
mean_mean = df.mean().mean()
mean_max = df.mean().max()
t_mean_min = stats.trim_mean(X, 0.1).min()
t_mean_mean = stats.trim_mean(X, 0.1).mean()
t_mean_max = stats.trim_mean(X, 0.1).max()
median_min = df.median().min()
median_mean = df.median().mean()
median_max = df.median().max()
sem_min = df.sem().min()
sem_mean = df.sem().mean()
sem_max = df.sem().max()
std_min = df.std().min()
std_mean = df.std().mean()
std_max = df.std().max()
mad_min = df.mad().min()
mad_mean = df.mad().mean()
mad_max = df.mad().max()
var_min = df.var().min()
var_mean = df.var().mean()
var_max = df.var().max()
skew_min = df.skew().min()
skew_mean = df.skew().mean()
skew_max = df.skew().max()
kurt_min = df.kurtosis().min()
kurt_mean = df.kurtosis().mean()
kurt_max = df.kurtosis().max()
p_corr = df.corr(method='pearson')
np.fill_diagonal(p_corr.values, 0)
p_corr_min = p_corr.min().min()
p_corr_mean = p_corr.mean().mean()
p_corr_max = p_corr.max().max()
k_corr = df.corr(method='kendall')
np.fill_diagonal(k_corr.values, 0)
k_corr_min = k_corr.min().min()
k_corr_mean = k_corr.mean().mean()
k_corr_max = k_corr.max().max()
s_corr = df.corr(method='spearman')
np.fill_diagonal(s_corr.values, 0)
s_corr_min = s_corr.min().min()
s_corr_mean = s_corr.mean().mean()
s_corr_max = s_corr.max().max()
cov_min = df.cov().min().min()
cov_mean = df.cov().mean().mean()
cov_max = df.cov().max().max()
variation = stats.variation(X)
variation_min = variation.min()
variation_mean = variation.mean()
variation_max = variation.max()
z_score = stats.zscore(X)
z_score_min = z_score.min()
z_score_mean = z_score.mean()
z_score_max = z_score.max()
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
iqr_min = IQR.min()
iqr_mean = IQR.mean()
iqr_max = IQR.max()
iqr_mul_outliers_sum = ((df < (Q1 - 1.5 * IQR)) |
(df > (Q3 + 1.5 * IQR))).any(axis=1).sum()
iqr_mul_outliers_per = iqr_mul_outliers_sum / n_samples
iqr_uni_outliers_sum = ((df < (Q1 - 1.5 * IQR)) |
(df > (Q3 + 1.5 * IQR))).sum().sum()
iqr_uni_outliers_per = iqr_uni_outliers_sum / (n_samples * n_features)
z = np.abs(stats.zscore(X))
z = pd.DataFrame(z)
z_mul_outliers_sum = (z > 3).any(axis=1).sum()
z_mul_outliers_per = z_mul_outliers_sum / n_samples
z_uni_outliers_sum = (z > 3).sum().sum()
z_uni_outliers_per = z_uni_outliers_sum / (n_samples * n_features)
X_entr_min = entropy(df).min()
X_entr_mean = entropy(df).mean()
X_entr_max = entropy(df).max()
y_entr = entropy(y)
mutual_info_min = mutual_info_classif(X, y, random_state=1).min()
mutual_info_mean = mutual_info_classif(X, y, random_state=1).mean()
mutual_info_max = mutual_info_classif(X, y, random_state=1).max()
if (mutual_info_mean != 0):
en = y_entr / mutual_info_mean
ns = (X_entr_mean - mutual_info_mean) / mutual_info_mean
else:
en = 0
ns = 0
clf = IsolationForest(behaviour="new",
contamination='auto',
random_state=1)
if_anomalies = clf.fit_predict(X)
if_an = np.where(if_anomalies == -1)
if_an_sum = len(if_an[0])
if_an_per = if_an_sum / n_samples
clf = LocalOutlierFactor(contamination='auto')
lof_anomalies = clf.fit_predict(X)
lof_an = np.where(lof_anomalies == -1)
lof_an_sum = len(lof_an[0])
lof_an_per = lof_an_sum / n_samples
clf = svm.OneClassSVM(gamma='scale')
svm_anomalies = clf.fit_predict(X)
svm_an = np.where(svm_anomalies == -1)
svm_an_sum = len(svm_an[0])
svm_an_per = svm_an_sum / n_samples
pca = PCA(n_components=2, random_state=1)
X_pca = pca.fit_transform(X)
pca_ev_sum = pca.explained_variance_ratio_.sum()
pca_ev_min = pca.explained_variance_ratio_.min()
pca_ev_mean = pca.explained_variance_ratio_.mean()
pca_ev_max = pca.explained_variance_ratio_.max()
pca_sv_sum = pca.singular_values_.sum()
pca_sv_min = pca.singular_values_.min()
pca_sv_mean = pca.singular_values_.mean()
pca_sv_max = pca.singular_values_.max()
pca_nv = pca.noise_variance_
tsvd = TruncatedSVD(n_components=2, random_state=1)
tsvd.fit_transform(X)
tsvd_ev_sum = tsvd.explained_variance_ratio_.sum()
tsvd_ev_min = tsvd.explained_variance_ratio_.min()
tsvd_ev_mean = tsvd.explained_variance_ratio_.mean()
tsvd_ev_max = tsvd.explained_variance_ratio_.max()
tsvd_sv_sum = tsvd.singular_values_.sum()
tsvd_sv_min = tsvd.singular_values_.min()
tsvd_sv_mean = tsvd.singular_values_.mean()
tsvd_sv_max = tsvd.singular_values_.max()
anova = f_classif(X, y)
anova_f_min = anova[0].min()
anova_f_mean = anova[0].mean()
anova_f_max = anova[0].max()
anova_sum = len(anova[1][anova[1] < 0.05])
anova_per = len(anova[1][anova[1] < 0.05]) / n_features
U, singv, VT = svd(X)
singv_min = singv.min()
singv_mean = singv.mean()
singv_max = singv.max()
bestfeatures = SelectKBest(score_func=chi2, k=2)
fit = bestfeatures.fit(X, y)
dfscores = fit.scores_
chi2_based_scores_min = dfscores.min()
chi2_based_scores_mean = dfscores.mean()
chi2_based_scores_max = dfscores.max()
estimator = LogisticRegression(random_state=1)
selector = RFECV(estimator, step=1, cv=5)
og_X = pd.DataFrame(X)
new_X = selector.fit_transform(og_X, y)
rfecv_per_optimal_feat = selector.n_features_ / n_features
dt_best = DecisionTreeClassifier(max_depth=1,
splitter='best',
random_state=1)
scores = cross_val_score(dt_best, X, y, cv=5)
dt_best_acc = scores.mean()
dt_rnd = DecisionTreeClassifier(max_depth=1,
splitter='random',
random_state=1)
scores = cross_val_score(dt_rnd, X, y, cv=5)
dt_rnd_acc = scores.mean()
kn1 = KNeighborsClassifier(n_neighbors=1)
scores = cross_val_score(kn1, X, y, cv=5)
kn1_acc = scores.mean()
lda = LinearDiscriminantAnalysis()
scores = cross_val_score(lda, X, y, cv=5)
lda_acc = scores.mean()
nb = GaussianNB()
scores = cross_val_score(nb, X, y, cv=5)
nb_acc = scores.mean()
tmp = [
n_samples, n_features, n_classes, class_weights_min,
class_weights_mean, class_weights_max, mean_min, mean_mean, mean_max,
t_mean_min, t_mean_mean, t_mean_max, median_min, median_mean,
median_max, sem_min, sem_mean, sem_max, std_min, std_mean, std_max,
mad_min, mad_mean, mad_max, var_min, var_mean, var_max, skew_min,
skew_mean, skew_max, kurt_min, kurt_mean, kurt_max, p_corr_min,
p_corr_mean, p_corr_max, k_corr_min, k_corr_mean, k_corr_max,
s_corr_min, s_corr_mean, s_corr_max, cov_min, cov_mean, cov_max,
variation_min, variation_mean, variation_max, z_score_min,
z_score_mean, z_score_max, iqr_min, iqr_mean, iqr_max,
iqr_mul_outliers_sum, iqr_mul_outliers_per, iqr_uni_outliers_sum,
iqr_uni_outliers_per, z_mul_outliers_sum, z_mul_outliers_per,
z_uni_outliers_sum, z_uni_outliers_per, X_entr_min, X_entr_mean,
X_entr_max, y_entr, mutual_info_min, mutual_info_mean, mutual_info_max,
en, ns, if_an_sum, if_an_per, lof_an_sum, lof_an_per, svm_an_sum,
svm_an_per, pca_ev_sum, pca_ev_min, pca_ev_mean, pca_ev_max,
pca_sv_sum, pca_sv_min, pca_sv_mean, pca_sv_max, pca_nv, tsvd_ev_sum,
tsvd_ev_min, tsvd_ev_mean, tsvd_ev_max, tsvd_sv_sum, tsvd_sv_min,
tsvd_sv_mean, tsvd_sv_max, anova_f_min, anova_f_mean, anova_f_max,
anova_sum, anova_per, singv_min, singv_mean, singv_max,
chi2_based_scores_min, chi2_based_scores_mean, chi2_based_scores_max,
rfecv_per_optimal_feat, dt_best_acc, dt_rnd_acc, kn1_acc, lda_acc,
nb_acc
]
tmp = np.asarray(tmp)
tmp = tmp.reshape(1, len(tmp))
data_rep.append(tmp)
def outlier_detector(data,
features,
feature1,
feature2,
threshold,
plotting=True):
x = data[features]
clf = LocalOutlierFactor()
y_pred = clf.fit_predict(x)
X_score = clf.negative_outlier_factor_
outlier_score = pd.DataFrame()
outlier_score["score"] = X_score
outlier_score.head()
filter1 = outlier_score["score"] < threshold
outlier_index = outlier_score[filter1].index.tolist()
x_len = len(x.drop(outlier_index))
if plotting == True:
fig, ax = plt.subplots(1, 1, figsize=(13, 8))
plt.scatter(x[feature1],
x[feature2],
color="k",
s=6,
label="Data Points")
f1_index = x.columns.get_loc(feature1)
f2_index = x.columns.get_loc(feature2)
plt.scatter(x.iloc[outlier_index, f1_index],
x.iloc[outlier_index, f2_index],
color="red",
s=30,
label="Outlier")
radius = (X_score.max() - X_score) / (X_score.max() - X_score.min())
outlier_score["radius"] = radius
plt.scatter(x[feature1],
x[feature2],
s=1000 * radius,
edgecolor="b",
facecolors="none",
label="Outlier Score")
plt.legend()
plt.xlabel("{}".format(feature1))
plt.ylabel("{}".format(feature2))
plt.grid(True, alpha=0.4)
plt.text(0.66,
0.1,
"Number of Outliers:" + str(len(data) - x_len),
horizontalalignment='left',
verticalalignment='top',
transform=ax.transAxes,
fontsize=18,
color="black")
plt.title("Outlier Detection Plot")
plt.show()
x = x.drop(outlier_index)
print("Number of Outliers(Number of Dropped Rows): {}".format(
len(data) - x_len))
print("Min Outlier Score: {}".format(np.min(outlier_score["score"])))
return x, outlier_score["score"]
def build_model(accs_normal1, accs_bearing1, accs_gear1):
N = 1024 * 2
normal1_datas = utils.spliteAcc2fft(accs_normal1, N, freq)
bearing1_datas = utils.spliteAcc2fft(accs_bearing1, N, freq)
gear1_datas = utils.spliteAcc2fft(accs_gear1, N, freq)
n_sample_out = 200
normal_datas_in, normal_datas_out = normal1_datas[
n_sample_out:], normal1_datas[:n_sample_out]
bearing_datas_in, bearing_datas_out = bearing1_datas[
n_sample_out:], bearing1_datas[:n_sample_out]
gear_datas_in, gear_datas_out = gear1_datas[
n_sample_out:], gear1_datas[:n_sample_out]
datas = np.r_[normal_datas_in, bearing_datas_in, gear_datas_in]
labels = np.r_[
np.zeros(normal_datas_in.shape[0]), # 0 for inlier, 1 for outlier
np.ones(bearing_datas_in.shape[0]),
np.ones(gear_datas_in.shape[0])]
train_datas, test_datas, train_labels, test_labels = utils.split_train_test(
datas=datas, labels=labels, frac=0.8)
for n_neighbor in [20, 40, 60, 100]:
for n_contamination in [0.05, 0.1]:
lof_model = LocalOutlierFactor(n_neighbors=n_neighbor,
contamination=n_contamination)
lof_model.fit(
train_datas
) # create_lof_model(train_datas.shape[0] // 3).fit(train_datas)
y_score = -lof_model._decision_function(test_datas)
# Compute ROC curve and ROC area for each class
fpr, tpr, thresholds = roc_curve(test_labels, y_score)
threshold = get_best_threshold_roc(fpr=fpr,
tpr=tpr,
thresholds=thresholds)
roc_auc = auc(fpr, tpr)
# y_score_test = -lof_model._decision_function(test_datas)
y_pred = np.zeros(test_labels.shape[0])
y_pred[y_score >= threshold] = 1
f1 = f1_score(test_labels, y_pred)
# select best model with best roc_auc
if f1 > best_test_score:
best_test_score = f1
best_model = lof_model
best_threshold = threshold
print(
'n_neighbor: %d, n_contamination: %f, roc_auc score: %.3f, f1 score: %.3f'
% (n_neighbor, n_contamination, roc_auc, f1))
# # save best model to disk
# filename = 'finalized_model_1.sav'
# joblib.dump(best_model, filename)
print('[Test phase] START ')
out_test_datas = np.vstack(
[normal_datas_out, bearing_datas_out, gear_datas_out])
out_test_labels = np.hstack([
np.zeros(normal_datas_out.shape[0]), # 0 for inlier, 1 for outlier
np.ones(bearing_datas_out.shape[0]),
np.ones(gear_datas_out.shape[0])
])
# y_score = -best_model.negative_outlier_factor_
y_score_test = -best_model._decision_function(out_test_datas)
fpr, tpr, thresholds = roc_curve(out_test_labels, y_score_test)
roc_auc = auc(fpr, tpr)
y_pred = np.zeros(out_test_labels.shape[0])
y_pred[y_score_test >= best_threshold] = 1
f1 = f1_score(out_test_labels, y_pred)
print('[Test phase] roc_auc score: %.3f, f1 score: %.3f ' % (roc_auc, f1))
def DFS_Alg(Org_Vec, Queue, Data_to_write, Epsilon, max_ctx):
Visited = []
contexts = [Org_Vec]
termination_threshold = 500
Terminator = 0
while len(Visited) < 100:
Terminator += 1
if (Terminator > termination_threshold):
break
BFS_Vec = np.zeros(len(Org_Vec))
for i in range(len(Org_Vec)):
BFS_Vec[i] = Stack[len(Stack) - 1][3][i]
Visited.append(np.zeros(len(Org_Vec)))
for i in range(len(Org_Vec)):
Visited[len(Visited) - 1][i] = Stack[len(Stack) - 1][3][i]
Queue.append(Stack[len(Stack) - 1])
BFS_Flp = np.zeros(len(Org_Vec))
sub_q = []
for Flp_bit in range(0, (len(Org_Vec))):
Sub_Sal_list = []
Sub_ID_list = []
for i in range(len(BFS_Vec)):
BFS_Flp[i] = BFS_Vec[i]
BFS_Flp[Flp_bit] = 1 - BFS_Flp[Flp_bit]
BFS_Ctx = df2.loc[df2['Weapon'].isin(FirAtt_lst[np.where(BFS_Flp[0:len(FirAtt_lst)] == 1)].tolist()) &\
df2['State'].isin(SecAtt_lst[np.where(BFS_Flp[len(FirAtt_lst):len(FirAtt_lst)+len(SecAtt_lst)] == 1)].tolist()) &\
df2['AgencyType'].isin(ThrAtt_lst[np.where(BFS_Flp[len(FirAtt_lst)+len(SecAtt_lst):len(FirAtt_lst)+len(SecAtt_lst)+len(ThrAtt_lst)] == 1)].tolist())]
if ((not any(np.array_equal(BFS_Flp[:], x[:])
for x in Visited)) and
(not any(np.array_equal(BFS_Flp[:], x[:]) for x in contexts))
and (BFS_Ctx.shape[0] > 20)):
for row in range(BFS_Ctx.shape[0]):
Sub_Sal_list.append(BFS_Ctx.iloc[row, 4])
Sub_ID_list.append(BFS_Ctx.iloc[row, 0])
Sub_Sal_arr = np.array(Sub_Sal_list)
clf = LocalOutlierFactor(n_neighbors=20)
Sub_Sal_outliers = clf.fit_predict(Sub_Sal_arr.reshape(-1, 1))
for outlier_finder in range(0, len(Sub_ID_list)):
if ((Sub_Sal_outliers[outlier_finder] == -1)
and (Sub_ID_list[outlier_finder] == Queried_ID)):
Sub_Score = mp.exp(Epsilon * (BFS_Ctx.shape[0]))
sub_q.append([
Flp_bit, Sub_Score, BFS_Ctx.shape[0],
np.zeros(len(Org_Vec))
])
for i in range(len(sub_q[len(sub_q) - 1][3])):
sub_q[len(sub_q) - 1][3][i] = BFS_Flp[i]
# Sampling from sub_queue(sampling in each layer)
if not sub_q:
Stack.remove(Stack[len(Stack) - 1])
else:
Sub_elements = [elem[0] for elem in sub_q]
Sub_probabilities = []
for prob in sub_q:
Sub_probabilities.append(prob[1] /
(sum([prob[1] for prob in sub_q])))
SubRes = np.random.choice(Sub_elements, 1, p=Sub_probabilities)
for child in range(0, len(sub_q)):
if sub_q[child][0] == SubRes[0]:
Q_indx = child
Stack.append([
len(Stack), sub_q[Q_indx][1], sub_q[Q_indx][2],
np.zeros(len(BFS_Vec))
])
for i in range(len(BFS_Vec)):
Stack[len(Stack) - 1][3][i] = sub_q[Q_indx][3][i]
contexts.append(np.zeros(len(Org_Vec)))
for i in range(len(Org_Vec)):
contexts[len(contexts) - 1][i] = sub_q[Q_indx][3][i]
# Exp mechanism on the visited nodes
for i in range(len(Queue)):
Queue[i][0] = i
elements = [elem for elem in range(len(Queue))]
probabilities = []
for prob in Queue:
probabilities.append(prob[1] / (sum([prob[1] for prob in Queue])))
Res = np.random.choice(elements, 1, p=probabilities)
Data_to_write.append(Queue[Res[0]][2] / max_ctx)
return
def _get_pipeline(self):
return [('scaler', StandardScaler()),
('model',
LocalOutlierFactor(n_jobs=self.conf.n_jobs, novelty=True))]
def removeOutliers(train,
labels=None,
opt='isolation',
cont='auto',
rerun=100,
outlier_importance=20,
max_features=0.2,
max_samples=0.2,
random_state=0,
**kwargs):
# Set seed and data size
n1, m = train.shape
np.random.seed(random_state)
# Merge into one dataset with labels
if labels is None: data = train
else: data = pd.concat([train, labels], axis=1)
# Define functions for interation of estimators
def IterateResults(estimator, data, rerun):
score = np.zeros(n1)
print("Outlier detection: Iterating", opt, "estimator", rerun,
"times.")
print("Cummulative outliers found")
def resample_score(seed):
np.random.seed(seed)
return estimator.fit(data).decision_function(data)
mapping = map(resample_score, range(random_state,
random_state + rerun))
for i in mapping:
# Give more weights to outliers found
i[i < 0] = i[i < 0] * outlier_importance
score += i
print((score < 0).sum(), end="->")
print("Done!")
return score / rerun
def MahalanobisDist(data):
def is_pos_def(A):
if np.allclose(A, A.T):
try:
np.linalg.cholesky(A)
return True
except np.linalg.LinAlgError:
return False
else:
return False
covar = np.cov(data, rowvar=False)
if is_pos_def(covar):
covar_inv = np.linalg.inv(covar)
if is_pos_def(covar_inv):
mean = np.mean(data, axis=0)
diff = data - mean
md = np.sqrt(diff.dot(covar_inv).dot(diff.T).diagonal())
return md
else:
print(
"Error: Inverse of Covariance Matrix is not positive definite!"
)
else:
print("Error: Covariance Matrix is not positive definite!")
# Choose method
if opt == 'isolation':
from sklearn.ensemble import IsolationForest
estim = IsolationForest(contamination=cont,
behaviour='new',
max_samples=max_samples,
max_features=max_features,
n_estimators=50,
n_jobs=-1,
**kwargs)
decision = estim.fit(data).predict(data)
if (rerun > 0):
decision = IterateResults(estim, data, rerun)
if opt == 'lof':
from sklearn.neighbors import LocalOutlierFactor
estim = LocalOutlierFactor(contamination=cont,
n_neighbors=55,
n_jobs=-1)
decision = estim.fit_predict(data)
if opt == 'svm':
from sklearn.svm import OneClassSVM
if cont == 'auto':
cont = 0.01
estim = OneClassSVM(nu=cont, gamma='scale', tol=1e-3)
decision = estim.fit(data).predict(data)
if opt == 'covariance':
if cont == 'auto': cont = 4
MD = MahalanobisDist(data.values)
std = np.std(MD)
mean = np.mean(MD)
k = 3. * std if True else 2. * std
high, low = mean + k, mean - k
decision = (MD >= high) * (-2) + (MD <= low) * (-2) + 1
# Print summary information
index = decision < 0
print("Outlier values: ", round(index.sum() * 100 / n1, 3), "% (",
index.sum(), "/", n1, ")")
print("Outlier values", opt, "method indecies:")
for i in data[index].index:
print(i, end=' ')
print()
if index.sum() / n1 > 0.1:
print("Warning! More than 10% of training observations deleted!")
# Discard outliers
out = data[np.invert(index)]
if labels is None:
return out
else:
train = out.iloc[:, 0:m]
labels = pd.DataFrame(out.iloc[:, -1])
return (train, labels)
# UserID_le = preprocessing.LabelEncoder()
# preprocessed_features["UserID"] = UserID_le.fit_transform(preprocessed_features["UserID"])
# TerminalSN_le = preprocessing.LabelEncoder()
# preprocessed_features["TerminalSN"] = TerminalSN_le.fit_transform(preprocessed_features["TerminalSN"])
# EventID_le = preprocessing.LabelEncoder()
# preprocessed_features["EventID"] = EventID_le.fit_transform(preprocessed_features["EventID"])
# Split data set, not needed as it is unsupervised
# train_features, test_features = train_test_split(preprocessed_features, test_size=0.2)
# Begin Training
neigh = LocalOutlierFactor(n_neighbors=300,
leaf_size=100,
novelty=False,
algorithm="auto",
contamination=0.01)
train_outliers = neigh.fit_predict(preprocessed_features) # On training data
# Compile into Data Frame for print
outlier_result_df = pd.DataFrame()
# outlier_result_df["UserID"] = UserID_le.inverse_transform(preprocessed_features["UserID"])
# outlier_result_df["TerminalSN"] = TerminalSN_le.inverse_transform(preprocessed_features["TerminalSN"])
# outlier_result_df["Timestamps"] = raw_df["TIMESTAMPS"]
outlier_result_df["Time_Of_Day"] = preprocessed_features["Time_Of_Day"]
outlier_result_df["Outlier"] = train_outliers
print(outlier_result_df)
# Get Percentage of Outliers
outlier_percentage = len(outlier_result_df.loc[outlier_result_df["Outlier"] ==
def clean_points(point_cloud):
clf = LocalOutlierFactor(n_neighbors=50, contamination='auto')
y_pred = clf.fit_predict(point_cloud)
mask = ((y_pred + 1) / 2).astype(bool)
return clf, mask
#Import the algorithms
from sklearn.metrics import classification_report, accuracy_score
from sklearn.ensemble import IsolationForest
from sklearn.neighbors import LocalOutlierFactor
#Initializing random state
state = 1
#Putting the classifiers inside a dictionary
classifiers = {
"IsloationForest":
IsolationForest(max_samples=len(X),
contamination=outlier_frac,
random_state=state),
"Local Outlier Factor":
LocalOutlierFactor(n_neighbors=20, contamination=outlier_frac)
}
# In[30]:
#Fit the model
n_outliers = len(Fraud)
#Running the loop
for i, (clf_name, clf) in enumerate(classifiers.items()):
if clf_name == "Local Outlier Factor":
y_pred = clf.fit_predict(X)
scores_pred = clf.negative_outlier_factor_
else:
clf.fit(X)
scores_pred = clf.decision_function(X)
plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True) from adtk.detector import LevelShiftAD levelshift_ad = LevelShiftAD() anomalies = levelshift_ad.fit_detect(s_train) plot(s_train, anomaly_pred=anomalies, ap_color='red', ap_marker_on_curve=True) from adtk.detector import MinClusterDetector from sklearn.cluster import KMeans min_cluster_detector = MinClusterDetector(KMeans(n_clusters=5)) anomalies = min_cluster_detector.fit_detect(df) plot(df, anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_color='red', ap_alpha=0.3, curve_group='all'); from adtk.detector import OutlierDetector from sklearn.neighbors import LocalOutlierFactor outlier_detector = OutlierDetector(LocalOutlierFactor(contamination=0.05)) anomalies = outlier_detector.fit_detect(df) plot(df, anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_color='red', ap_alpha=0.3, curve_group='all'); from adtk.detector import RegressionAD from sklearn.linear_model import LinearRegression regression_ad = RegressionAD(regressor=LinearRegression(), target="data2", c=3.0) anomalies = regression_ad.fit_detect(df) plot(df, anomaly_pred=anomalies, ts_linewidth=2, ts_markersize=3, ap_color='red', ap_alpha=0.3, curve_group='all'); from adtk.transformer import RollingAggregate s_transformed = RollingAggregate(agg='count', window=5).transform(df.iloc[:,1]) plot(s_transformed, ts_linewidth=2, ts_markersize=3);
np.random.seed(42)
# Generate train data
X_inliers = 0.3 * np.random.randn(100, 2)
X_inliers = np.r_[X_inliers + 2, X_inliers - 2]
# Generate some outliers
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))
X = np.r_[X_inliers, X_outliers]
n_outliers = len(X_outliers)
ground_truth = np.ones(len(X), dtype=int)
ground_truth[-n_outliers:] = -1
# fit the model for outlier detection (default)
clf = LocalOutlierFactor(n_neighbors=20, contamination=0.1)
# use fit_predict to compute the predicted labels of the training samples
# (when LOF is used for outlier detection, the estimator has no predict,
# decision_function and score_samples methods).
y_pred = clf.fit_predict(X)
n_errors = (y_pred != ground_truth).sum()
X_scores = clf.negative_outlier_factor_
plt.title("Local Outlier Factor (LOF)")
plt.scatter(X[:, 0], X[:, 1], color='k', s=3., label='Data points')
# plot circles with radius proportional to the outlier scores
radius = (X_scores.max() - X_scores) / (X_scores.max() - X_scores.min())
plt.scatter(X[:, 0], X[:, 1], s=1000 * radius, edgecolors='r',
facecolors='none', label='Outlier scores')
plt.axis('tight')
plt.xlim((-5, 5))
def __init__(self):
self.clf = LocalOutlierFactor(novelty=True, contamination=0.1)
self.scaler = StandardScaler()
try:
for ne in range(nb_exp):
print 'exp num:', ne
X, y = sh(X, y)
X_train = X[:n_samples_train, :]
X_test = X[n_samples_train:, :]
y_train = y[:n_samples_train]
y_test = y[n_samples_train:]
# # training only on normal data:
# X_train = X_train[y_train == 0]
# y_train = y_train[y_train == 0]
print('LocalOutlierFactor processing...')
model = LocalOutlierFactor(n_neighbors=20)
tstart = time()
model.fit(X_train)
fit_time += time() - tstart
tstart = time()
scoring = -model.decision_function(X_test) # the lower,the more normal
predict_time += time() - tstart
fpr_, tpr_, thresholds_ = roc_curve(y_test, scoring)
if fit_time + predict_time > max_time:
raise TimeoutError
f = interp1d(fpr_, tpr_)
tpr += f(x_axis)
tpr[0] = 0.
def getLocalFactor(_df):
clf = LocalOutlierFactor(
contamination=OUTLIER_FRACTION,
n_jobs=definitions.getNumberOfCore()
)
return clf.fit_predict(_df)
class LOF(BaseDetector):
"""Wrapper of scikit-learn LOF Class with more functionalities.
Unsupervised Outlier Detection using Local Outlier Factor (LOF).
The anomaly score of each sample is called Local Outlier Factor.
It measures the local deviation of density of a given sample with
respect to its neighbors.
It is local in that the anomaly score depends on how isolated the object
is with respect to the surrounding neighborhood.
More precisely, locality is given by k-nearest neighbors, whose distance
is used to estimate the local density.
By comparing the local density of a sample to the local densities of
its neighbors, one can identify samples that have a substantially lower
density than their neighbors. These are considered outliers.
See :cite:`breunig2000lof` for details.
Parameters
----------
n_neighbors : int, optional (default=20)
Number of neighbors to use by default for `kneighbors` queries.
If n_neighbors is larger than the number of samples provided,
all samples will be used.
algorithm : {'auto', 'ball_tree', 'kd_tree', 'brute'}, optional
Algorithm used to compute the nearest neighbors:
- 'ball_tree' will use BallTree
- 'kd_tree' will use KDTree
- 'brute' will use a brute-force search.
- 'auto' will attempt to decide the most appropriate algorithm
based on the values passed to :meth:`fit` method.
Note: fitting on sparse input will override the setting of
this parameter, using brute force.
leaf_size : int, optional (default=30)
Leaf size passed to `BallTree` or `KDTree`. This can
affect the speed of the construction and query, as well as the memory
required to store the tree. The optimal value depends on the
nature of the problem.
metric : string or callable, default 'minkowski'
metric used for the distance computation. Any metric from scikit-learn
or scipy.spatial.distance can be used.
If 'precomputed', the training input X is expected to be a distance
matrix.
If metric is a callable function, it is called on each
pair of instances (rows) and the resulting value recorded. The callable
should take two arrays as input and return one value indicating the
distance between them. This works for Scipy's metrics, but is less
efficient than passing the metric name as a string.
Valid values for metric are:
- from scikit-learn: ['cityblock', 'cosine', 'euclidean', 'l1', 'l2',
'manhattan']
- from scipy.spatial.distance: ['braycurtis', 'canberra', 'chebyshev',
'correlation', 'dice', 'hamming', 'jaccard', 'kulsinski',
'mahalanobis', 'matching', 'minkowski', 'rogerstanimoto',
'russellrao', 'seuclidean', 'sokalmichener', 'sokalsneath',
'sqeuclidean', 'yule']
See the documentation for scipy.spatial.distance for details on these
metrics:
http://docs.scipy.org/doc/scipy/reference/spatial.distance.html
p : integer, optional (default = 2)
Parameter for the Minkowski metric from
sklearn.metrics.pairwise.pairwise_distances. When p = 1, this is
equivalent to using manhattan_distance (l1), and euclidean_distance
(l2) for p = 2. For arbitrary p, minkowski_distance (l_p) is used.
See http://scikit-learn.org/stable/modules/generated/sklearn.metrics.pairwise.pairwise_distances
metric_params : dict, optional (default = None)
Additional keyword arguments for the metric function.
contamination : float in (0., 0.5), optional (default=0.1)
The amount of contamination of the data set, i.e. the proportion
of outliers in the data set. When fitting this is used to define the
threshold on the decision function.
n_jobs : int, optional (default = 1)
The number of parallel jobs to run for neighbors search.
If ``-1``, then the number of jobs is set to the number of CPU cores.
Affects only kneighbors and kneighbors_graph methods.
Attributes
----------
n_neighbors_ : int
The actual number of neighbors used for `kneighbors` queries.
decision_scores_ : numpy array of shape (n_samples,)
The outlier scores of the training data.
The higher, the more abnormal. Outliers tend to have higher
scores. This value is available once the detector is
fitted.
threshold_ : float
The threshold is based on ``contamination``. It is the
``n_samples * contamination`` most abnormal samples in
``decision_scores_``. The threshold is calculated for generating
binary outlier labels.
labels_ : int, either 0 or 1
The binary labels of the training data. 0 stands for inliers
and 1 for outliers/anomalies. It is generated by applying
``threshold_`` on ``decision_scores_``.
"""
def __init__(self, n_neighbors=20, algorithm='auto', leaf_size=30,
metric='minkowski', p=2, metric_params=None,
contamination=0.1, n_jobs=1):
super(LOF, self).__init__(contamination=contamination)
self.n_neighbors = n_neighbors
self.algorithm = algorithm
self.leaf_size = leaf_size
self.metric = metric
self.p = p
self.metric_params = metric_params
self.n_jobs = n_jobs
# noinspection PyIncorrectDocstring
def fit(self, X, y=None):
"""Fit detector. y is optional for unsupervised methods.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The input samples.
y : numpy array of shape (n_samples,), optional (default=None)
The ground truth of the input samples (labels).
"""
# validate inputs X and y (optional)
X = check_array(X)
self._set_n_classes(y)
self.detector_ = LocalOutlierFactor(n_neighbors=self.n_neighbors,
algorithm=self.algorithm,
leaf_size=self.leaf_size,
metric=self.metric,
p=self.p,
metric_params=self.metric_params,
contamination=self.contamination,
n_jobs=self.n_jobs)
self.detector_.fit(X=X, y=y)
# Invert decision_scores_. Outliers comes with higher outlier scores
self.decision_scores_ = invert_order(
self.detector_.negative_outlier_factor_)
self._process_decision_scores()
return self
def decision_function(self, X):
"""Predict raw anomaly score of X using the fitted detector.
The anomaly score of an input sample is computed based on different
detector algorithms. For consistency, outliers are assigned with
larger anomaly scores.
Parameters
----------
X : numpy array of shape (n_samples, n_features)
The training input samples. Sparse matrices are accepted only
if they are supported by the base estimator.
Returns
-------
anomaly_scores : numpy array of shape (n_samples,)
The anomaly score of the input samples.
"""
check_is_fitted(self, ['decision_scores_', 'threshold_', 'labels_'])
# Invert outlier scores. Outliers comes with higher outlier scores
# noinspection PyProtectedMember
if _sklearn_version_20():
return invert_order(self.detector_._score_samples(X))
else:
return invert_order(self.detector_._decision_function(X))
@property
def n_neighbors_(self):
"""The actual number of neighbors used for kneighbors queries.
Decorator for scikit-learn LOF attributes.
"""
return self.detector_.n_neighbors_
columns = 4
train_len = mt.floor(len(df) / 3 * 2)
test_len = mt.floor(len(df) / 3 * 1)
train_data = [
tuple(values) for values in df.iloc[:train_len, 0:columns].values
]
df2 = pd.read_csv(
rand_file,
#index_col=0,
parse_dates=True)
test_data = [tuple(values) for values in df2.iloc[:, 0:columns].values]
clf = LocalOutlierFactor(n_neighbors=12,
novelty=True,
contamination=0.05)
y_pred_X = clf.fit(train_data)
get_thr_outlier = threshold_loop(test_data, 1, 1.5, .01,
only_file_name)
#print(get_thr_outlier)
#print(only_file_name)
#writer.writerows([only_file_name])
writer.writerow(get_thr_outlier)
#print(get_thr_outlier)
#loop=loop+1
#if loop==3:
#break
n_samples_test = n_samples - n_samples_train
X_train = X[:n_samples_train, :]
X_test = X[n_samples_train:, :]
y_train = y[:n_samples_train]
y_test = y[n_samples_train:]
# training and testing only on normal data:
X_train = X_train[y_train == 0]
y_train = y_train[y_train == 0]
X_test = X_test[y_test == 0]
y_test = y_test[y_test == 0]
# define models:
iforest = IsolationForest()
lof = LocalOutlierFactor(n_neighbors=20)
ocsvm = OneClassSVM()
lim_inf = X.min(axis=0)
lim_sup = X.max(axis=0)
volume_support = (lim_sup - lim_inf).prod()
t = np.arange(0, 100 / volume_support, 0.01 / volume_support)
axis_alpha = np.arange(alpha_min, alpha_max, 0.0001)
unif = np.random.uniform(lim_inf, lim_sup,
size=(n_generated, n_features))
# fit:
print('IsolationForest processing...')
iforest = IsolationForest()
iforest.fit(X_train)
s_X_iforest = iforest.decision_function(X_test)
print(__doc__)
np.random.seed(42)
xx, yy = np.meshgrid(np.linspace(-5, 5, 500), np.linspace(-5, 5, 500))
# Generate normal (not abnormal) training observations
X = 0.3 * np.random.randn(100, 2)
X_train = np.r_[X + 2, X - 2]
# Generate new normal (not abnormal) observations
X = 0.3 * np.random.randn(20, 2)
X_test = np.r_[X + 2, X - 2]
# Generate some abnormal novel observations
X_outliers = np.random.uniform(low=-4, high=4, size=(20, 2))
# fit the model for novelty detection (novelty=True)
clf = LocalOutlierFactor(n_neighbors=20, novelty=True, contamination=0.1)
clf.fit(X_train)
# DO NOT use predict, decision_function and score_samples on X_train as this
# would give wrong results but only on new unseen data (not used in X_train),
# e.g. X_test, X_outliers or the meshgrid
y_pred_test = clf.predict(X_test)
y_pred_outliers = clf.predict(X_outliers)
n_error_test = y_pred_test[y_pred_test == -1].size
n_error_outliers = y_pred_outliers[y_pred_outliers == 1].size
# plot the learned frontier, the points, and the nearest vectors to the plane
Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.title("Novelty Detection with LOF")
plt.contourf(xx, yy, Z, levels=np.linspace(Z.min(), 0, 7), cmap=plt.cm.PuBu)
