def print_metrics(y, y_pred):
print("Precision @10:{}".format(
np.round(precision_n_scores(y, y_pred, n=10), 4)))
print("Precision @20:{}".format(
np.round(precision_n_scores(y, y_pred, n=20), 4)))
print("Precision @50:{}".format(
np.round(precision_n_scores(y, y_pred, n=50), 4)))
print("Precision @100:{}".format(
np.round(precision_n_scores(y, y_pred, n=100), 4)))
print("Precision @1000:{}".format(
np.round(precision_n_scores(y, y_pred, n=1000), 4)))
print("ROC AUC Score:{}".format(np.round(roc_auc_score(y, y_pred), 4)))
print("Recall Score:{}".format(np.round(recall_score(y, y_pred), 4)))
def calculate(method, total_roc, total_prn, x_train, x_test, y_train, y_test):
if method == 'KNN':
clf = KNN()
elif method == 'CBLOF':
clf = CBLOF()
elif method == 'PCA':
clf = PCA()
else:
clf = IForest()
clf.fit(x_train) # 使用x_train训练检测器clf
# 返回训练数据x_train上的异常标签和异常分值
y_train_pred = clf.labels_ # 返回训练数据上的分类标签 (0: 正常值, 1: 异常值)
y_train_scores = clf.decision_scores_ # 返回训练数据上的异常值 (分值越大越异常)
print("On train Data:")
evaluate_print(method, y_train, y_train_scores)
# 用训练好的clf来预测未知数据中的异常值
y_test_pred = clf.predict(x_test) # 返回未知数据上的分类标签 (0: 正常值, 1: 异常值)
y_test_scores = clf.decision_function(x_test) # 返回未知数据上的异常值 (分值越大越异常)
print("On Test Data:")
evaluate_print(method, y_test, y_test_scores)
y_true = column_or_1d(y_test)
y_pred = column_or_1d(y_test_scores)
check_consistent_length(y_true, y_pred)
roc = np.round(roc_auc_score(y_true, y_pred), decimals=4),
prn = np.round(precision_n_scores(y_true, y_pred), decimals=4)
total_roc.append(roc)
total_prn.append(prn)
def print_score(picture_names, test_scores, y_test):
count = 0
correct_human = 0
correct_not_human = 0
wrong_human = 0
wrong_not_human = 0
for file in picture_names:
is_human = data_set_labels.loc[file][0] == 1
is_not_human = data_set_labels.loc[file][0] == 0
output = test_scores[count]
if is_human:
if output > 0.50:
correct_human += 1
else:
wrong_human += 1
if is_not_human:
if output > 0.50:
wrong_not_human += 1
else:
correct_not_human += 1
count += 1
correct = correct_human + correct_not_human
roc = round(roc_auc_score(y_test, test_scores), ndigits=4)
prn = round(precision_n_scores(y_test, test_scores), ndigits=4)
return roc, prn, correct / count
def evaluate_print(clf_name, y, y_pred):
"""Utility function for evaluating and printing the results for examples.
Default metrics include ROC and Precision @ n
Parameters
----------
clf_name : str
The name of the detector.
y : list or numpy array of shape (n_samples,)
The ground truth. Binary (0: inliers, 1: outliers).
y_pred : list or numpy array of shape (n_samples,)
The raw outlier scores as returned by a fitted model.
"""
y = column_or_1d(y)
y_pred = column_or_1d(y_pred)
check_consistent_length(y, y_pred)
print('{clf_name} ROC:{roc}, precision @ rank n:{prn}, ap:{prn}'.format(
clf_name=clf_name,
roc=np.round(roc_auc_score(y, y_pred), decimals=4),
prn=np.round(precision_n_scores(y, y_pred), decimals=4),
ap=np.round(average_precision_score(y, y_pred), decimals=4)))
def evaluation(y, y_scores, method):
'''
评估函数，y为groundtruth，y_scores为预测值，返回PR曲线，ROC曲线和AUC
'''
if isinstance(y_scores, dict):
colors = ['r', 'g', 'b', '#FF1493', '#483D8B']
plt.figure(figsize=(7, 7))
i = 0
for algo in y_scores:
pre_scr = y_scores[algo]
print(algo, pre_scr.shape)
fpr_level = np.arange(0, 1, 0.01)
fpr, tpr, threshold = metrics.roc_curve(y, pre_scr)
interp = interpolate.interp1d(fpr, tpr)
tpr_at_fpr = [interp(x) for x in fpr_level]
roc_auc = metrics.auc(fpr, tpr)
plt.plot(fpr,
tpr,
color=colors[i],
label='%s ROC(area = %0.2f)' % (algo, roc_auc))
i += 1
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.title('Models Compare' + '-ROC')
plt.plot([0, 1], [0, 1], color='navy', linestyle='--')
plt.legend(loc="lower right")
else:
fpr_level = np.arange(0, 1, 0.01)
fpr, tpr, threshold = metrics.roc_curve(y, y_scores)
interp = interpolate.interp1d(fpr, tpr)
tpr_at_fpr = [interp(x) for x in fpr_level]
roc_auc = metrics.auc(fpr, tpr)
precision, recall, _ = metrics.precision_recall_curve(y, y_scores)
pr_auc = round(precision_n_scores(y, y_scores), ndigits=4)
# pr_auc_t = metrics.auc(recall, precision)
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
plt.xlabel('FPR')
plt.ylabel('TPR')
plt.title(method + '-ROC')
plt.plot([0, 1], [0, 1], color='navy', linestyle='--')
plt.plot(fpr,
tpr,
color='r',
label='ROC curve (area = %0.2f)' % roc_auc)
plt.legend(loc="lower right")
plt.subplot(1, 2, 2)
plt.plot(recall,
precision,
marker='.',
label='precision @ rank n: %0.2f)' % pr_auc)
plt.legend(loc="upper right")
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title(method + '-PR')
plt.show()
def evaluate_print(clf_name, y, y_pred):
"""
Utility function for evaluating and printing the results for examples
Internal use only
:param clf_name: The name of the detector
:type clf_name: str
:param y: The ground truth
:type y: list or array of shape (n_samples,)
:param y_pred: The predicted outlier scores
:type y: list or array of shape (n_samples,)
"""
y = column_or_1d(y)
y_pred = column_or_1d(y_pred)
print('{clf_name} ROC:{roc}, precision @ rank n:{prn}'.format(
clf_name=clf_name,
roc=np.round(roc_auc_score(y, y_pred), decimals=4),
prn=np.round(precision_n_scores(y, y_pred), decimals=4)))
def _calculateMetrics(predictionScores, predictionLabels, groundTruth):
labels = predictionLabels.tolist()
fp = 0
fn = 0
tp = 0
tn = 0
for i in range(len(groundTruth)):
prediction = labels[i]
actual = groundTruth[i]
if prediction > 0 and actual == 0: fp += 1
elif prediction == 0 and actual > 0: fn += 1
elif prediction == 0 and actual == 0: tn += 1
elif prediction > 0 and actual > 0: tp += 1
return {
"p_at_n": precision_n_scores(groundTruth, predictionScores.tolist()),
"fp": fp,
"fn": fn,
"auc": roc_auc_score(groundTruth, predictionScores),
"avg_precision": average_precision_score(groundTruth, predictionScores)
}
def train(doc_list, dataset_name, clf_name):
model_roc = []
model_prc = []
if clf_name == "PCA":
clf = PCA()
elif clf_name == "MCD":
clf = MCD()
elif clf_name == "LOF":
clf = LOF()
elif clf_name == "KNN":
clf = KNN()
elif clf_name == "LODA":
clf = LODA()
for i in range(10):
data = pd.read_csv(doc_list[i], header=0, index_col=0)
train_x = data.drop(drop + ground_truth, axis=1).values
train_y = np.array([
transfor[x]
for x in list(_flatten(data[ground_truth].values.tolist()))
])
clf.fit(train_x)
predict = clf.decision_scores_
roc = roc_auc_score(train_y, predict)
prc = precision_n_scores(train_y, predict)
if ((i + 1) % 200 == 0):
print("第" + str(i + 1) + "个文件结果:")
evaluate_print(clf_name, train_y, predict)
model_roc.append(roc)
model_prc.append(prc)
model_roc_avg = np.mean(model_roc)
model_prc_avg = np.mean(model_prc)
print("模型" + clf_name + "在数据集" + dataset_name + "的平均roc_auc为" +
str(round(model_roc_avg, 4)) + ",平均prc为" +
str(round(model_prc_avg, 4)) + "。")
return model_roc_avg, model_prc_avg
def evaluate(self, X_test, y_test):
pred_score = self.decision_function(X_test)
prec_n = (precision_n_scores(y_test, pred_score))
print("precision@n", prec_n)
def test_precision_n_scores(self):
assert_equal(precision_score(self.y, self.manual_labels),
precision_n_scores(self.y, self.labels_))
pca_roc = []
pca_prn = []
pca_time = []
rp_roc = []
rp_prn = []
rp_time = []
for j in range(n_iter):
start = time.time()
clf = LOF() # change this to other detection algorithms
clf.fit(X)
y_train_scores = clf.decision_scores_
original_time.append(time.time() - start)
original_roc.append(roc_auc_score(y, y_train_scores))
original_prn.append(precision_n_scores(y, y_train_scores))
X_transformer, _ = jl_fit_transform(X, dim_new, "basic")
start = time.time()
clf.fit(X_transformer)
y_train_scores = clf.decision_scores_
basic_time.append(time.time() - start)
basic_roc.append(roc_auc_score(y, y_train_scores))
basic_prn.append(precision_n_scores(y, y_train_scores))
X_transformer, _ = jl_fit_transform(X, dim_new, "discrete")
start = time.time()
clf.fit(X_transformer)
y_train_scores = clf.decision_scores_
discrete_time.append(time.time() - start)
discrete_roc.append(roc_auc_score(y, y_train_scores))
'K Nearest Neighbors (KNN)': 2,
'Average K Nearest Neighbors (AvgKNN)': 3,
'Median K Nearest Neighbors (MedKNN)': 4,
'Subspace Outlier Detection (SOD)': 5
}
for clf_name, clf in classifiers.items():
t0 = time()
clf.fit(X_norm)
scores = clf.decision_function(X_norm)
t1 = time()
duration = round(t1 - t0, ndigits=4)
roc = round(roc_auc_score(y, scores), ndigits=4)
ap = round(average_precision_score(y, scores), ndigits=4)
prn = round(precision_n_scores(y, scores), ndigits=4)
print('{clf_name} ROC:{roc}, AP:{ap}, precision @ rank n:{prn}, '
'execution time: {duration}s'.format(
clf_name=clf_name, roc=roc, ap=ap, prn=prn, duration=duration))
time_mat[i, classifiers_indices[clf_name]] = duration
roc_mat[i, classifiers_indices[clf_name]] = roc
ap_mat[i, classifiers_indices[clf_name]] = ap
prn_mat[i, classifiers_indices[clf_name]] = prn
time_list = time_list + np.mean(time_mat, axis=0).tolist()
temp_df = pd.DataFrame(time_list).transpose()
temp_df.columns = df_columns
time_df = pd.concat([time_df, temp_df], axis=0)
'KNN': KNN(contamination=contam),
'LOF': LOF(contamination=contam),
'PCA': PCA(contamination=contam),
'LODA': LODA(contamination=contam)
}
for cls in classifiers:
clf = classifiers[cls]
t0 = time.time()
x_item = standardizer(x_item)
clf.fit(x_item)
y_scores = clf.decision_function(x_item)
t1 = time.time()
duration = round(t1 - t0, ndigits=4)
roc = round(roc_auc_score(y_item, y_scores), ndigits=4)
prn = round(precision_n_scores(y_item, y_scores),
ndigits=4)
results[cls].append(roc)
print(
'benchmark id:{bench_id}, model:{clf_name}, ROC:{roc}, precision @ rank n:{prn}, '
'execution time: {duration}s'.format(
bench_id=bench_id,
clf_name=cls,
roc=roc,
prn=prn,
duration=duration))
csv_writer.writerow([bench_id, cls, roc, prn, duration])
f.close()
y_train_scores = clf.decision_scores_ # 返回训练数据上的异常值 (分值越大越异常)
print("On train Data:")
evaluate_print(clf_name, y_train, y_train_scores)
# 用训练好的clf来预测未知数据中的异常值
y_test_pred = clf.predict(X_test) # 返回未知数据上的分类标签 (0: 正常值, 1: 异常值)
y_test_scores = clf.decision_function(X_test) # 返回未知数据上的异常值 (分值越大越异常)
print("On Test Data:")
evaluate_print(clf_name, y_test, y_test_scores)
y_true = column_or_1d(y_test)
y_pred = column_or_1d(y_test_scores)
check_consistent_length(y_true, y_pred)
roc = np.round(roc_auc_score(y_true, y_pred), decimals=4),
prn = np.round(precision_n_scores(y_true, y_pred), decimals=4)
knn_roc.append(roc)
knn_prn.append(prn)
clf_name = 'LOF'
clf = LOF() # 初始化检测器clf
clf.fit(X_train) # 使用X_train训练检测器clf
# 返回训练数据X_train上的异常标签和异常分值
y_train_pred = clf.labels_ # 返回训练数据上的分类标签 (0: 正常值, 1: 异常值)
y_train_scores = clf.decision_scores_ # 返回训练数据上的异常值 (分值越大越异常)
print("On train Data:")
evaluate_print(clf_name, y_train, y_train_scores)
# 用训练好的clf来预测未知数据中的异常值
y_train_pred,
y_test_pred,
show_figure=True,
save_figure=False)
# ### 用roc和prn评价准确率
# In[21]:
from sklearn.metrics import roc_auc_score
from pyod.utils.utility import precision_n_scores
# In[22]:
train_roc = np.round(roc_auc_score(y_train, y_train_pred), decimals=4)
train_prn = np.round(precision_n_scores(y_train, y_train_pred), decimals=4)
test_roc = np.round(roc_auc_score(y_test, y_test_pred), decimals=4)
test_prn = np.round(precision_n_scores(y_test, y_test_pred), decimals=4)
# In[23]:
print(train_roc)
print(train_prn)
print(test_roc)
print(test_prn)
# ## 2.2 展示用多个算法对批量数据进行检测与评价（有些benchmark的训练集或测试集只有全为异常值或全为非异常值，无法用roc_auc_score进行评价，故设为0）
# ### 新建一个dict，用于保存结果
# In[24]:
'COD_B': 2,
'COD_S': 3,
'COD_M': 4,
'COD': 5
}
for clf_name, clf in classifiers.items():
t0 = time()
clf.fit(X_train_norm)
test_scores = clf.decision_function(X_test_norm)
t1 = time()
duration = round(t1 - t0, ndigits=4)
test_scores = np.nan_to_num(test_scores)
roc = round(roc_auc_score(y_test, test_scores), ndigits=4)
prn = round(precision_n_scores(y_test, test_scores), ndigits=4)
ap = round(average_precision_score(y_test, test_scores), ndigits=4)
print('{clf_name} ROC:{roc}, precision @ rank n:{prn}, AP:{ap},'
'execution time: {duration}s'.format(
clf_name=clf_name, roc=roc, prn=prn, ap=ap, duration=duration))
time_mat[i, classifiers_indices[clf_name]] = duration
roc_mat[i, classifiers_indices[clf_name]] = roc
prn_mat[i, classifiers_indices[clf_name]] = prn
ap_mat[i, classifiers_indices[clf_name]] = ap
time_list = time_list + np.mean(time_mat, axis=0).tolist()
temp_df = pd.DataFrame(time_list).transpose()
temp_df.columns = df_columns
time_df = pd.concat([time_df, temp_df], axis=0)
k = k_list[i]
clf = KNN(n_neighbors=k, method='largest')
clf.fit(X_train_norm)
train_scores[:, i] = clf.decision_scores_
test_scores[:, i] = clf.decision_function(X_test_norm)
# decision scores have to be normalized before combination
train_scores_norm, test_scores_norm = standardizer(
train_scores, test_scores)
# combination by average
comb_by_average = average(test_scores_norm)
roc_average.append(roc_auc_score(y_test, comb_by_average))
prn_average.append(precision_n_scores(y_test, comb_by_average))
print('ite', t + 1, 'comb by average,', 'ROC:',
roc_auc_score(y_test, comb_by_average), 'precision@n_train:',
precision_n_scores(y_test, comb_by_average))
# combination by max
comb_by_maximization = maximization(test_scores_norm)
roc_maximization.append(roc_auc_score(y_test, comb_by_maximization))
prn_maximization.append(
precision_n_scores(y_test, comb_by_maximization))
print('ite', t + 1, 'comb by max,', 'ROC:',
roc_auc_score(y_test, comb_by_maximization),
'precision@n_train:',
precision_n_scores(y_test, comb_by_maximization))
# combination by aom
#kNN
clf_name = 'kNN'
clf = KNN(method='median')
# In[4]:
#用训练集训练
clf.fit(X_train)
y_train_pred = clf.labels_
y_train_scores = clf.decision_scores_
y_test_pred = clf.predict(X_test)
y_test_scores = clf.decision_function(X_test)
#评价性能
roc_train = round(roc_auc_score(y_train, y_train_scores), 4)
prn_train = round(precision_n_scores(y_train, y_train_scores), ndigits=4)
roc_test = round(roc_auc_score(y_test, y_test_scores), 4)
prn_test = round(precision_n_scores(y_test, y_test_scores), ndigits=4)
# In[5]:
#输出计算得到的roc_auc和precision @ rank n
print("\nOn Train Data:")
print(clf_name, 'roc:', roc_train, 'precision @ rank n:', prn_train)
print("\nOn Test Data:")
print(clf_name, 'roc:', roc_test, 'precision @ rank n:', prn_test)
# In[6]:
#初始化绘制散点图所使用的数据点坐标
X_train_groundtruth_d2_0 = X_train_d2[y_train[:scatter_num][:] == 0]
def compute_precision_at_n(self, rst, ground_truth, n=None):
y_scores = np.array(rst)
return precision_n_scores(ground_truth, y_scores, n)
'MCD', 'OCSVM', 'PCA']
for j in range(n_iter):
stat_mat = np.zeros([len(classifiers), 10])
for i, (clf_name, clf) in enumerate(classifiers.items()):
################## original version
clf.fit(X_train)
pseudo_labels = clf.decision_scores_
# replace nan by mean
np_mean = np.nanmean(pseudo_labels)
pseudo_labels[np.isnan(pseudo_labels)] = np_mean
print('Iter', j + 1, i + 1, clf_name, '|', 'train stat',
np.round(roc_auc_score(y_train, pseudo_labels), decimals=4), '|',
np.round(precision_n_scores(y_train, pseudo_labels), decimals=4))
stat_mat[i, 0] = np.round(roc_auc_score(y_train, pseudo_labels),
decimals=4)
stat_mat[i, 1] = np.round(precision_n_scores(y_train, pseudo_labels),
decimals=4)
################## xgb train scores
regressor = RandomForestRegressor()
regressor.fit(X_train, pseudo_labels)
pseudo_scores = regressor.predict(X_train)
print('Iter', j + 1, i + 1, 'kd', clf_name, '|', 'train stat',
np.round(roc_auc_score(y_train, pseudo_scores), decimals=4), '|',
np.round(precision_n_scores(y_train, pseudo_scores), decimals=4))
def evaluate(self, X_test, y_test):
pred_score = self.sample_scores(X_test)
prec_n = (precision_n_scores(y_test, pred_score))
print("precision@n", prec_n)
# train a HBOS detector (default version)
clf = HBOS()
clf.fit(X_train)
# get the prediction on the training data
y_train_pred = clf.y_pred
y_train_score = clf.decision_scores
# get the prediction on the test data
y_test_pred = clf.predict(X_test)
y_test_score = clf.decision_function(X_test)
print('Train ROC:{roc}, precision@n:{prn}'.format(
roc=roc_auc_score(y_train, y_train_score),
prn=precision_n_scores(y_train, y_train_score)))
print('Test ROC:{roc}, precision@n:{prn}'.format(
roc=roc_auc_score(y_test, y_test_score),
prn=precision_n_scores(y_test, y_test_score)))
#######################################################################
# Visualizations
# initialize the log directory if it does not exist
pathlib.Path('example_figs').mkdir(parents=True, exist_ok=True)
# plot the results
fig = plt.figure(figsize=(12, 10))
ax = fig.add_subplot(221)
plt.scatter(X_train[:, 0], X_train[:, 1], c=c_train)
plt.title('Train ground truth')
def _eval_classifier(y_test, y_fit):
auc = round(roc_auc_score(y_test, y_fit), ndigits=4)
topn = round(precision_n_scores(y_test, y_fit), ndigits=4)
return auc, topn
k = k_list[i]
clf = Knn(n_neighbors=k, method='largest')
clf.fit(X_train_norm)
train_scores[:, i] = clf.decision_scores.ravel()
test_scores[:, i] = clf.decision_function(X_test_norm).ravel()
# scores have to be normalized before combination
train_scores_norm, test_scores_norm = standardizer(train_scores,
test_scores)
# combination by mean
comb_by_mean = np.mean(test_scores_norm, axis=1)
roc_mean.append(roc_auc_score(y_test, comb_by_mean))
prn_mean.append(precision_n_scores(y_test, comb_by_mean))
print('ite', t + 1, 'comb by mean,',
'ROC:', roc_auc_score(y_test, comb_by_mean),
'precision@n:', precision_n_scores(y_test, comb_by_mean))
# combination by max
comb_by_max = np.max(test_scores_norm, axis=1)
roc_max.append(roc_auc_score(y_test, comb_by_max))
prn_max.append(precision_n_scores(y_test, comb_by_max))
print('ite', t + 1, 'comb by max,', 'ROC:',
roc_auc_score(y_test, comb_by_max),
'precision@n:', precision_n_scores(y_test, comb_by_max))
# combination by aom
comb_by_aom = aom(test_scores_norm, 5, 20)
roc_aom.append(roc_auc_score(y_test, comb_by_aom))
'K Nearest Neighbors (KNN)': 5,
'Local Outlier Factor (LOF)': 6,
'Minimum Covariance Determinant (MCD)': 7,
'One-class SVM (OCSVM)': 8,
'Principal Component Analysis (PCA)': 9,
'Locally Selective Combination (LSCP)': 10}
for clf_name, clf in classifiers.items():
t0 = time()
clf.fit(X_train_norm)
test_scores = clf.decision_function(X_test_norm)
t1 = time()
duration = round(t1 - t0, ndigits=4)
roc = round(roc_auc_score(y_test, test_scores), ndigits=4)
prn = round(precision_n_scores(y_test, test_scores), ndigits=4)
print('{clf_name} ROC:{roc}, precision @ rank n:{prn}, '
'execution time: {duration}s'.format(
clf_name=clf_name, roc=roc, prn=prn, duration=duration))
time_mat[i, classifiers_indices[clf_name]] = duration
roc_mat[i, classifiers_indices[clf_name]] = roc
prn_mat[i, classifiers_indices[clf_name]] = prn
time_list = time_list + np.mean(time_mat, axis=0).tolist()
temp_df = pd.DataFrame(time_list).transpose()
temp_df.columns = df_columns
time_df = pd.concat([time_df, temp_df], axis=0)
roc_list = roc_list + np.mean(roc_mat, axis=0).tolist()
