def plot_discrimination_threshold(clf, data='spam', outpath=None):
if data == 'spam':
X, y = load_spam()
elif data == 'churn':
X, y = load_churn()
else:
raise ValueError("no dataset loader '{}'".format(data))
_, ax = plt.subplots()
visualizer = DiscriminationThreshold(clf, ax=ax)
visualizer.fit(X, y)
visualizer.poof(outpath=outpath)
def plot_optimal_threshold(model, x_train, y_train):
"""
"""
# Visualization Threshold
visualizer = DiscriminationThreshold(model)
visualizer.fit(x_train, y_train) # Fit the data to the visualizer
visualizer.show()
def discrimination_threshold(self) -> None:
visualizer = DiscriminationThreshold(self.trained_model)
visualizer.fit(self.X_test,
self.y_test) # Fit the data to the visualizer
save_dir = f"{self.plots_dir}/discrimination_plot_{self.model_id}.png"
visualizer.show(outpath=save_dir)
if not LOCAL:
upload_to_s3(save_dir,
f'plots/discrimination_plot_{self.model_id}.png',
bucket=S3_BUCKET_NAME)
plt.clf()
def discrimination_thersold(model, classes, X_train, Y_train, X_test, Y_test):
from yellowbrick.classifier import DiscriminationThreshold
# Instantiate the classification model and visualizer
viz = DiscriminationThreshold(model)
# visualizer.fit(X, y) # Fit the training data to the visualizer
# visualizer.poof() # Draw/show/poof the data
viz.fit(X_train, Y_train)
# viz.score(X_test, Y_test)
viz.poof()
def plot_discrimination_threshold(clf, data='spam', outpath=None):
if data == 'spam':
X, y = load_spam()
elif data == 'churn':
X, y = load_churn()
else:
raise ValueError("no dataset loader '{}'".format(data))
_, ax = plt.subplots()
visualizer = DiscriminationThreshold(clf, ax=ax)
visualizer.fit(X, y)
visualizer.poof(outpath=outpath)
def selectDiscr():
data_path = "labeled_data.csv"
data = pd.read_csv(data_path)
# We create the preprocessing pipelines for both numeric and categorical data.
numeric_features = ['count_reviews', 'rating']
numeric_transformer = Pipeline(
steps=[('imputer',
SimpleImputer(strategy='median')), ('scaler',
StandardScaler())])
categorical_features = ['product_category']
categorical_transformer = Pipeline(
steps=[('imputer',
SimpleImputer(strategy='constant', fill_value='missing')
), ('onehot', OneHotEncoder(handle_unknown='ignore'))])
preprocessor = ColumnTransformer(
transformers=[('num', numeric_transformer, numeric_features
), ('cat', categorical_transformer,
categorical_features)])
viz = DiscriminationThreshold(LogisticRegression())
clf = VisualPipeline(steps=[
('preprocessor', preprocessor),
#('classifier', LogisticRegression(solver='lbfgs')),
('viz', viz)
])
X = data.drop('label', axis=1)
y = data['label']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = clf.fit(X_train, y_train)
model.poof()
def classification(fname="classification.png"):
# Create side-by-side axes grid
_, axes = plt.subplots(ncols=2, figsize=(18, 6))
# Add ClassificationReport to the reft
data = load_spam(split=True)
oz = ClassificationReport(MultinomialNB(),
classes=["ham", "spam"],
ax=axes[0])
oz.fit(data.X.train, data.y.train)
oz.score(data.X.test, data.y.test)
oz.finalize()
# Add DiscriminationThreshold to the right
data = load_spam(split=False)
oz = DiscriminationThreshold(LogisticRegression(), ax=axes[1])
oz.fit(data.X, data.y)
oz.finalize()
# Save figure
path = os.path.join(FIGURES, fname)
plt.tight_layout()
plt.savefig(path)
def evaluate_visualizer(self, classes=None, params={}):
LOGGER.info('Initializing plot model')
if os.path.isdir(os.path.join(os.getcwd(), 'visualizer/')) == False:
os.makedirs(os.path.join(os.getcwd(), 'visualizer/'))
if classes is None:
classes = pd.value_counts(self.y.values.flatten()).index.tolist()
visualizers = []
for idx, (name_model, estimator) in enumerate(self.estimator.items()):
X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split(
self.X,
self.y,
test_size=0.2,
stratify=self.y,
random_state=24)
try:
LOGGER.info('Visualizer ClassificationReport')
visualizer = ClassificationReport(model=estimator,
classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = ClassificationReport(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR ClassificationReport')
try:
LOGGER.info('Visualizer ConfusionMatrix')
visualizer = ConfusionMatrix(model=estimator, classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = ConfusionMatrix(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR ConfusionMatrix')
try:
LOGGER.info('Visualizer ROCAUC')
visualizer = ROCAUC(model=estimator, classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = ROCAUC(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR ROCAUC')
try:
LOGGER.info('Visualizer PrecisionRecallCurve')
visualizer = PrecisionRecallCurve(model=estimator,
per_class=True,
classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = PrecisionRecallCurve(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR PrecisionRecallCurve')
try:
LOGGER.info('Visualizer ClassPredictionError')
visualizer = ClassPredictionError(model=estimator,
classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = ClassPredictionError(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR ClassPredictionError')
try:
LOGGER.info('Visualizer Discrimination Threshold')
visualizer = DiscriminationThreshold(model=estimator,
classes=classes)
if visualizer.__class__.__name__ in params.keys():
visualizer = DiscriminationThreshold(
**params[visualizer.__class__.__name__])
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.show(outpath=os.path.join(
os.getcwd(),
f'visualizer/{visualizer.__class__.__name__}_{estimator.__class__.__name__}.png'
))
plt.cla()
except:
LOGGER.warn('ERROR Discrimination Threshold')
# %%
classification_report(clf, X, y)
# %%
visualizer = ROCAUC(clf, classes=class_names)
visualizer.score(X, y)
visualizer.poof()
# %%
visualizer = ClassPredictionError(clf, classes=class_names)
visualizer.score(X, y)
visualizer.poof()
# %%
visualizer = DiscriminationThreshold(clf)
visualizer.fit(X, y)
visualizer.poof()
# %%
keep = [263, 268, 287, 288, 300, 302, 307, 308, 313, 315]
# %%
seed = 15
test_size = 0.33
Xt, Xv, yt, yv = \
sklearn.model_selection.train_test_split(
X[keep], y, test_size=test_size, stratify=y, random_state=seed)
# %%
explainer = shap.TreeExplainer(clf)
ax3 = fig3.add_subplot(111)
ax3.set_title('Scoreverteilung')
score = np.linspace(0, 1, 1000)
ax3.plot(score, num_tp1, "-", label = "Signal")
ax3.plot(score, num_tn1, "-", label = "Background")
ax3.set_ylabel("Anzahl")
ax3.set_xlabel("Scorecut")
ax3.legend()
fig3.savefig("plots/forest/Scoredistribution.pdf")
# precision recall threshold curve
# https://www.kaggle.com/kevinarvai/fine-tuning-a-classifier-in-scikit-learn, http://www.scikit-yb.org/en/latest/api/classifier/threshold.html
from yellowbrick.classifier import DiscriminationThreshold
fig5 = plt.figure(5)
ax5 = fig5.add_subplot(111)
visualizer = DiscriminationThreshold(forest, exclude = ("queue_rate", "fscore"), ax = ax5)
visualizer.fit(data_train_X, data_train_y) # Fit the training data to the visualizer
visualizer.poof(outpath="plots/forest/precrecathresh.pdf") # Draw/show/poof the data
print(time.clock())
print(confusion_matrix(expected, (predicted_probs[:,1] > 0.3).astype(bool)))
from sklearn.metrics import classification_report
print(classification_report(expected, (predicted_probs[:,1] > 0.3).astype(bool)))
# http://www.scikit-yb.org/en/latest/api/features/importances.html
from yellowbrick.features.importances import FeatureImportances
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]
def supervised_prediction(algorithm, training_x, testing_x, training_y,
testing_y, cols, cf, threshold_plot):
#model
algorithm.fit(training_x, training_y)
predictions = algorithm.predict(testing_x)
probabilities = algorithm.predict_proba(testing_x)
#coeffs
if cf == "coefficients":
coefficients = pd.DataFrame(algorithm.coef_.ravel())
elif cf == "features":
coefficients = pd.DataFrame(algorithm.feature_importances_)
column_df = pd.DataFrame(cols)
coef_sumry = (pd.merge(coefficients,
column_df,
left_index=True,
right_index=True,
how="left"))
coef_sumry.columns = ["coefficients", "features"]
coef_sumry = coef_sumry.sort_values(by="coefficients", ascending=False)
print(algorithm)
print("\n Classification report : \n",
classification_report(testing_y, predictions))
print("Accuracy Score : ", accuracy_score(testing_y, predictions))
#confusion matrix
conf_matrix = confusion_matrix(testing_y, predictions)
#roc_auc_score
model_roc_auc = roc_auc_score(testing_y, predictions)
print("Area under curve : ", model_roc_auc, "\n")
fpr, tpr, thresholds = roc_curve(testing_y, probabilities[:, 1])
#plot confusion matrix
trace1 = go.Heatmap(z=conf_matrix,
x=["Not churn", "Churn"],
y=["Not churn", "Churn"],
showscale=False,
colorscale="Picnic",
name="matrix")
#plot roc curve
trace2 = go.Scatter(x=fpr,
y=tpr,
name="Roc : " + str(model_roc_auc),
line=dict(color=('rgb(22, 96, 167)'), width=2))
trace3 = go.Scatter(x=[0, 1],
y=[0, 1],
line=dict(color=('rgb(205, 12, 24)'),
width=2,
dash='dot'))
#plot coeffs
trace4 = go.Bar(x=coef_sumry["features"],
y=coef_sumry["coefficients"],
name="coefficients",
marker=dict(color=coef_sumry["coefficients"],
colorscale="Picnic",
line=dict(width=.6, color="black")))
#subplots
fig = tls.make_subplots(
rows=2,
cols=2,
specs=[[{}, {}], [{
'colspan': 2
}, None]],
subplot_titles=('Confusion Matrix',
'Receiver operating characteristic',
'Feature Importances'))
fig.append_trace(trace1, 1, 1)
fig.append_trace(trace2, 1, 2)
fig.append_trace(trace3, 1, 2)
fig.append_trace(trace4, 2, 1)
fig['layout'].update(showlegend=False,
title="Model performance",
autosize=False,
height=900,
width=800,
plot_bgcolor='rgba(240,240,240, 0.95)',
paper_bgcolor='rgba(240,240,240, 0.95)',
margin=dict(b=195))
fig["layout"]["xaxis2"].update(dict(title="false positive rate"))
fig["layout"]["yaxis2"].update(dict(title="true positive rate"))
fig["layout"]["xaxis3"].update(
dict(showgrid=True, tickfont=dict(size=10), tickangle=90))
py.iplot(fig)
if threshold_plot == True:
visualizer = DiscriminationThreshold(algorithm)
visualizer.fit(training_x, training_y)
visualizer.poof()
def discrimination_thresholding(xx,yy,estimatorss,**kwargs):
vz = DiscriminationThreshold(estimatorss, classes=['Reach, 1 Reach, or L/R Reach', 'Null, Multiple Reaches, Or Multiple Arms'],
cmap="YlGn", size=(600, 360), **kwargs)
vz.fit(xx,yy)
vz.score(xx,yy)
vz.show()
visualizer = ClassPredictionError(model)
visualizer.score(X_test, y_test)
visualizer.show()
# 分类报告
visualizer = ClassificationReport(model)
visualizer.score(X_test, y_test)
visualizer.show()
# 混淆矩阵
visualizer = ConfusionMatrix(model)
visualizer.score(X_test, y_test)
visualizer.show()
# 阈值选择
visualizer = DiscriminationThreshold(model)
visualizer.fit(X_train, y_train)
visualizer.show()
# 学习率
visualizer = LearningCurve(model, scoring='f1_weighted')
visualizer.fit(X_train, y_train)
visualizer.show()
# 交叉验证
visualizer = CVScores(model, cv=5, scoring='f1_weighted')
visualizer.fit(X_train, y_train)
visualizer.show()
# 特征重要性
visualizer = FeatureImportances(model)
### ROC-AUC from yellowbrick.classifier import ROCAUC visualizer = ROCAUC(LogisticRegression(), classes=classes) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.poof() ### Class Prediction Error from yellowbrick.classifier import ClassPredictionError visualizer = ClassPredictionError(LogisticRegression(), classes=classes) visualizer.fit(X_train, y_train) visualizer.score(X_test, y_test) visualizer.poof() ### Discrimination Threshold # Only works for binary classification from yellowbrick.classifier import DiscriminationThreshold visualizer = DiscriminationThreshold(LogisticRegression()) visualizer.fit(X, y) visualizer.poof()
def train_and_evaluate_classifier(
algorithm,
training_x,
testing_x,
training_y,
testing_y,
cols,
cf='coefficients',
threshold_plot=True,
):
"""
Обучение классификатора на тренировочных данных, оценка прогноза на тестовых,
и визуализация некоторых метрик качества прогноза.
algorithm - использованный алгоритм с методами fit, predict и predict_proba
training_x - данные для предсказывающих переменных (обучение)
testing_x - данные для предсказывающих переменных (тест)
training_y - целевая переменная (обучение)
training_y - целевая переменная (тест)
cf - ["coefficients","features"](коэффициенты для логрегрессии, параметры для деревьев)
threshold_plot - если True, возвращает для модели threshold plot
"""
# модель
algorithm.fit(training_x, training_y)
predictions = algorithm.predict(testing_x)
probabilities = algorithm.predict_proba(testing_x)
# коэффициенты
if cf == "coefficients":
coefficients = pd.DataFrame(algorithm.coef_.ravel())
elif cf == "features":
coefficients = pd.DataFrame(algorithm.feature_importances_)
else:
raise ValueError(
'`coefficients` value must be one of {`coefficients`, `features`}')
column_df = pd.DataFrame(cols)
coef_sumry = (pd.merge(coefficients,
column_df,
left_index=True,
right_index=True,
how="left"))
coef_sumry.columns = ["coefficients", "features"]
coef_sumry = coef_sumry.sort_values(by="coefficients", ascending=False)
print(algorithm)
print("\n Отчет по классфицикации : \n",
classification_report(testing_y, predictions))
print("Точность : ", accuracy_score(testing_y, predictions))
# confusion_matrix
conf_matrix = confusion_matrix(testing_y, predictions)
# roc_auc_score
model_roc_auc = roc_auc_score(testing_y, predictions)
print("Площадь под кривой : ", model_roc_auc, "\n")
fpr, tpr, thresholds = roc_curve(testing_y, probabilities[:, 1])
# готовим confusion_matrix для рисования
trace1 = go.Heatmap(
z=conf_matrix,
x=["Пользователи", "Отток"],
y=["Пользователи", "Отток"],
showscale=False,
colorscale="Picnic",
name="matrix",
)
# готовим roc_curve для рисования
trace2 = go.Scatter(x=fpr,
y=tpr,
name="Roc : " + str(model_roc_auc),
line=dict(color=('rgb(22, 96, 167)'), width=2))
trace3 = go.Scatter(x=[0, 1],
y=[0, 1],
line=dict(color=('rgb(205, 12, 24)'),
width=2,
dash='dot'))
# готовим коэффициенты для рисования
trace4 = go.Bar(x=coef_sumry["features"],
y=coef_sumry["coefficients"],
name="coefficients",
marker=dict(color=coef_sumry["coefficients"],
colorscale="Picnic",
line=dict(width=.6, color="black")))
# рисуем
fig = tls.make_subplots(rows=2,
cols=2,
specs=[[{}, {}], [{
'colspan': 2
}, None]])
fig.append_trace(trace1, 1, 1)
fig.append_trace(trace2, 1, 2)
fig.append_trace(trace3, 1, 2)
fig.append_trace(trace4, 2, 1)
fig['layout'].update(
showlegend=False,
autosize=False,
height=900,
width=800,
plot_bgcolor='rgba(240,240,240, 0.95)',
paper_bgcolor='rgba(240,240,240, 0.95)',
margin=dict(b=195),
)
fig["layout"]["xaxis2"].update(dict(title="false positive rate"))
fig["layout"]["yaxis2"].update(dict(title="true positive rate"))
fig["layout"]["xaxis3"].update(
dict(showgrid=True, tickfont=dict(size=10), tickangle=90))
py.iplot(fig)
if threshold_plot:
visualizer = DiscriminationThreshold(algorithm)
visualizer.fit(training_x, training_y)
visualizer.poof()
def discrimination():
X, y = load_spam()
oz = DiscriminationThreshold(LogisticRegression(solver="lbfgs"),
ax=newfig())
oz.fit(X, y)
savefig(oz, "discrimination_threshold")
def telecom_churn_prediction_alg(algorithm,
training_x,
testing_x,
training_y,
testing_y,
threshold_plot=True):
# model
algorithm.fit(training_x, training_y)
predictions = algorithm.predict(testing_x)
probabilities = algorithm.predict_proba(testing_x)
print(algorithm)
print("\n Classification report : \n",
classification_report(testing_y, predictions))
print("Accuracy Score : ", accuracy_score(testing_y, predictions))
# confusion matrix
conf_matrix = confusion_matrix(testing_y, predictions)
# roc_auc_score
model_roc_auc = roc_auc_score(testing_y, predictions)
print("Area under curve : ", model_roc_auc)
fpr, tpr, thresholds = roc_curve(testing_y, probabilities[:, 1])
# plot roc curve
trace1 = go.Scatter(
x=fpr,
y=tpr,
name="Roc : " + str(model_roc_auc),
line=dict(color=('rgb(22, 96, 167)'), width=2),
)
trace2 = go.Scatter(x=[0, 1],
y=[0, 1],
line=dict(color=('rgb(205, 12, 24)'),
width=2,
dash='dot'))
# plot confusion matrix
trace3 = go.Heatmap(z=conf_matrix,
x=["Not churn", "Churn"],
y=["Not churn", "Churn"],
showscale=False,
colorscale="Blues",
name="matrix",
xaxis="x2",
yaxis="y2")
layout = go.Layout(
dict(title="Model performance",
autosize=False,
height=500,
width=800,
showlegend=False,
plot_bgcolor="rgb(243,243,243)",
paper_bgcolor="rgb(243,243,243)",
xaxis=dict(title="false positive rate",
gridcolor='rgb(255, 255, 255)',
domain=[0, 0.6],
ticklen=5,
gridwidth=2),
yaxis=dict(title="true positive rate",
gridcolor='rgb(255, 255, 255)',
zerolinewidth=1,
ticklen=5,
gridwidth=2),
margin=dict(b=200),
xaxis2=dict(domain=[0.7, 1],
tickangle=90,
gridcolor='rgb(255, 255, 255)'),
yaxis2=dict(anchor='x2', gridcolor='rgb(255, 255, 255)')))
data = [trace1, trace2, trace3]
fig = go.Figure(data=data, layout=layout)
py.iplot(fig)
if threshold_plot == True:
visualizer = DiscriminationThreshold(algorithm)
visualizer.fit(training_x, training_y)
visualizer.poof()
# Compute the accuracy: accuracy
accuracy = float(np.sum(preds==y_test))/y_test.shape[0]
print("accuracy: %f" % (accuracy))
#%%
#Measuign performance
train_score2= xgbc.score(X_train, y_train)
test_score2 = xgbc.score(X_test, y_test)
print(train_score1)
print(test_score1)
#%%
#Measuign performance
print(confusion_matrix(y_test, preds))
print(classification_report(y_test, preds))
print(accuracy_score(y_test, preds))
#%%
#ROC
# Instantiate the visualizer with the classification model
visualizer = ROCAUC(xgbc, classes=["will not default", "will default"])
visualizer.fit(X_train, y_train) # Fit the training data to the visualizer
visualizer.score(X_test, y_test) # Evaluate the model on the test data
visualizer.show()
#%%
visualizer = DiscriminationThreshold(xgbc)
visualizer.fit(X, y)
# Fit the data to the visualizer
visualizer.show()
LR = LogisticRegression()
LR.fit(X_train,y_train)
evaluate_model(LR,X_test,y_test,True) # evaluate model
# let's look discimination
pred = LR.predict(X_test)
proba = LR.predict_proba(X_test) # discrimination threshold is 0.5, let's find best disc. threshhold.
proba = pd.DataFrame(proba,columns=["0","1"])
proba["Selected Class"] = pred
# try to best threshold to maximize f1 score
vis = DiscriminationThreshold(LR)
vis.fit(X_train,y_train)
vis.poof() # algorithm trys to maximize f1 score
# threshold = 0.29
# KNN
from sklearn.neighbors import KNeighborsClassifier
k_scores = {}
for k in range(1,30,2):
KNN = KNeighborsClassifier(n_neighbors=k)
KNN.fit(X_train,y_train)
k_scores[k] = [KNN.score(X_test,y_test),roc_auc_score(y_test,KNN.predict(X_test))]
def telecom_churn_prediction(algorithm,
name,
X_train,
X_test,
y_train,
y_test,
cols,
cf=None,
plot=False,
threshold=False):
#model
start = time() # Get start time
algorithm.fit(X_train, y_train)
end = time() # Get end time
# Calculate the training time
train_time = round(end - start, 4)
#predict
start = time() # Get start time
predictions_test = algorithm.predict(X_test)
end = time() # Get end time
# Calculate the training time
pred_time = round(end - start, 4)
predictions_train = algorithm.predict(X_train)
probabilities = algorithm.predict_proba(X_test)
#coeffs
if cf != None:
if cf == "coefficients":
coefficients = pd.DataFrame(algorithm.coef_.ravel())
elif cf == "features":
coefficients = pd.DataFrame(algorithm.feature_importances_)
column_df = pd.DataFrame(cols)
coef_sumry = (pd.merge(coefficients,
column_df,
left_index=True,
right_index=True,
how="left"))
coef_sumry.columns = ["coefficients", "features"]
coef_sumry = coef_sumry.sort_values(by="coefficients", ascending=False)
print(algorithm)
print("\n Classification report : \n",
classification_report(y_test, predictions_test))
#confusion matrix
conf_matrix = confusion_matrix(y_test, predictions_test)
#roc_auc_score
model_roc_auc = roc_auc_score(y_test, predictions_test)
print('train')
print("Accuracy Score : ", accuracy_score(y_train, predictions_train))
print("Area under curve : ", roc_auc_score(y_train, predictions_train),
"\n")
print('test')
print("Accuracy Score :", accuracy_score(y_test, predictions_test))
print("Area under curve : ", model_roc_auc, "\n")
fpr, tpr, thresholds = roc_curve(y_test, probabilities[:, 1])
accuracy = accuracy_score(y_test, predictions_test)
recallscore = recall_score(y_test, predictions_test)
precision = precision_score(y_test, predictions_test)
roc_auc_train = roc_auc_score(y_train, predictions_train)
roc_auc_test = roc_auc_score(y_test, predictions_test)
f1score = f1_score(y_test, predictions_test)
result = pd.DataFrame({
"Model": [name],
"Accuracy_score": [accuracy],
"Recall_score": [recallscore],
"Precision": [precision],
"f1_score": [f1score],
"Area_under_curve(train)": [roc_auc_train],
"Area_under_curve(test)": [roc_auc_test],
"train_time": [train_time],
'pred_time': [pred_time]
})
if cf != None:
plt.figure(figsize=(12, 8))
#plot confusion matrix
plt.subplot(221)
plt.grid(b=None) #無網格
plot_confusion_matrix(conf_matrix, ["Not churn", "Churn"])
plt.subplot(222)
#plot roc curve
plt.plot(fpr, tpr, label="ROC Curve")
plt.title('Receiver operating characteristic')
plt.xlabel("false positive rate")
plt.ylabel("true positive rate (recall)")
#plot coeffs
sns.set(font_scale=1)
plt.subplot(212)
plt.title('Feature Importances')
plt.xticks(rotation='90')
sns.barplot(coef_sumry['features'], coef_sumry['coefficients'])
plt.subplots_adjust(top=1.2,
bottom=0.2,
left=0.10,
right=0.95,
hspace=0.25,
wspace=0.35)
if threshold == True:
#plot threshold
plt.figure(figsize=(14, 4))
visualizer = DiscriminationThreshold(algorithm)
visualizer.fit(X_train, y_train)
visualizer.poof()
elif cf == None:
plt.figure(figsize=(12, 4))
#plot confusion matrix
plt.subplot(121)
plt.grid(b=None) #無網格
plot_confusion_matrix(conf_matrix, ["Not churn", "Churn"])
plt.subplot(122)
#plot roc curve
plt.plot(fpr, tpr, label="ROC Curve")
plt.title('Receiver operating characteristic')
plt.xlabel("false positive rate")
plt.ylabel("true positive rate (recall)")
plt.subplots_adjust(top=1.2,
bottom=0.2,
left=0.10,
right=0.95,
hspace=0.25,
wspace=0.35)
return result
rf_probas = rf.predict_proba(X_test)[:, 1]
plot_precision_recall(y_test, rf_probas)
from yellowbrick.classifier import PrecisionRecallCurve
viz = PrecisionRecallCurve(rf)
viz.fit(X_train, y_train)
viz.score(X_test, y_test)
viz.poof()
# Discimination Threshold - probability or score at which the positive class is chosen over the negative class
from yellowbrick.classifier import DiscriminationThreshold
viz = DiscriminationThreshold(rf)
viz.fit(X_train, y_train)
viz.poof()
# Average Precision
from sklearn.metrics import average_precision_score
average_precision_score(
y_test,
rf.predict_proba(X_test)[:, 1]) # slice to give probs of class 1
# AUC and ROC curve
from sklearn.metrics import roc_auc_score
fig["layout"]["yaxis2"].update(dict(title = "true positive rate"))
fig["layout"]["xaxis3"].update(dict(showgrid = True,tickfont = dict(size = 10),tickangle = 90))
py.iplot(fig)
# Usually, we assign an object to a class if the probabily of belonging to this class is above 0.5
# However, this threshold can be adjusted and this function allows to find the optimal value given some metrics (recall, precison, f1, queue rate)
# Find optimal threshold
optimal_idx = np.argmax(tpr - fpr)
optimal_threshold = thresholds[optimal_idx]
# The optimal threshold is 0.30.
# Better method to find optimal threshold
visualizer = DiscriminationThreshold(classifier)
visualizer.fit(X_train,y_train)
visualizer.poof()
########################################## IMPROVEMENTS
# Implement SMOTE
"""
# SMOTE is not appropriate as it does not deal with dummy variables.
sm = SMOTE()
X_smote, y_smote = sm.fit_resample(X_train, y_train)
refit=True,
random_state=random_state,
verbose=True)
grid_search.fit(X, y, **fit_params)
opt_parameters = grid_search.best_params_
lgbm_clf = lgbm.LGBMClassifier(**opt_parameters)
# In[89]:
model_performance(lgbm_clf, 'LightGBM')
scores_table(lgbm_clf, 'LightGBM')
# In[90]:
visualizer = DiscriminationThreshold(lgbm_clf)
visualizer.fit(X, y)
visualizer.poof()
# In[91]:
knn_clf = KNeighborsClassifier()
voting_clf = VotingClassifier(estimators=[('lgbm_clf', lgbm_clf),
('knn', KNeighborsClassifier())],
voting='soft',
weights=[1, 1])
params = {'knn__n_neighbors': np.arange(1, 30)}
def discrimination_threshold(ax=None):
data = load_spam(return_dataset=True)
X, y = data.to_pandas()
viz = DiscriminationThreshold(RandomForestClassifier(n_estimators=10), ax=ax)
return tts_plot(viz, X, y, score=False)