def test_calibration_display_default_labels(pyplot, name, expected_label):
prob_true = np.array([0, 1, 1, 0])
prob_pred = np.array([0.2, 0.8, 0.8, 0.4])
y_prob = np.array([])
viz = CalibrationDisplay(prob_true, prob_pred, y_prob, estimator_name=name)
viz.plot()
assert viz.line_.get_label() == expected_label
def test_calibration_display_ref_line(pyplot, iris_data_binary):
# Check that `ref_line` only appears once
X, y = iris_data_binary
lr = LogisticRegression().fit(X, y)
dt = DecisionTreeClassifier().fit(X, y)
viz = CalibrationDisplay.from_estimator(lr, X, y)
viz2 = CalibrationDisplay.from_estimator(dt, X, y, ax=viz.ax_)
labels = viz2.ax_.get_legend_handles_labels()[1]
assert labels.count("Perfectly calibrated") == 1
def test_calibration_display_label_class_plot(pyplot):
# Checks that when instantiating `CalibrationDisplay` class then calling
# `plot`, `self.estimator_name` is the one given in `plot`
prob_true = np.array([0, 1, 1, 0])
prob_pred = np.array([0.2, 0.8, 0.8, 0.4])
y_prob = np.array([])
name = "name one"
viz = CalibrationDisplay(prob_true, prob_pred, y_prob, estimator_name=name)
assert viz.estimator_name == name
name = "name two"
viz.plot(name=name)
assert viz.line_.get_label() == name
def test_calibration_display_default_labels(pyplot, name, expected_label):
prob_true = np.array([0, 1, 1, 0])
prob_pred = np.array([0.2, 0.8, 0.8, 0.4])
y_prob = np.array([])
viz = CalibrationDisplay(prob_true, prob_pred, y_prob, estimator_name=name)
viz.plot()
expected_legend_labels = [] if name is None else [name]
expected_legend_labels.append("Perfectly calibrated")
legend_labels = viz.ax_.get_legend().get_texts()
assert len(legend_labels) == len(expected_legend_labels)
for labels in legend_labels:
assert labels.get_text() in expected_legend_labels
def test_calibration_display_non_binary(pyplot, iris_data, constructor_name):
X, y = iris_data
clf = DecisionTreeClassifier()
clf.fit(X, y)
y_prob = clf.predict_proba(X)
if constructor_name == "from_estimator":
msg = "to be a binary classifier, but got"
with pytest.raises(ValueError, match=msg):
CalibrationDisplay.from_estimator(clf, X, y)
else:
msg = "y should be a 1d array, got an array of shape"
with pytest.raises(ValueError, match=msg):
CalibrationDisplay.from_predictions(y, y_prob)
def test_calibration_display_pos_label(
pyplot, iris_data_binary, pos_label, expected_pos_label
):
"""Check the behaviour of `pos_label` in the `CalibrationDisplay`."""
X, y = iris_data_binary
lr = LogisticRegression().fit(X, y)
viz = CalibrationDisplay.from_estimator(lr, X, y, pos_label=pos_label)
y_prob = lr.predict_proba(X)[:, expected_pos_label]
prob_true, prob_pred = calibration_curve(y, y_prob, pos_label=pos_label)
assert_allclose(viz.prob_true, prob_true)
assert_allclose(viz.prob_pred, prob_pred)
assert_allclose(viz.y_prob, y_prob)
assert (
viz.ax_.get_xlabel()
== f"Mean predicted probability (Positive class: {expected_pos_label})"
)
assert (
viz.ax_.get_ylabel()
== f"Fraction of positives (Positive class: {expected_pos_label})"
)
expected_legend_labels = [lr.__class__.__name__, "Perfectly calibrated"]
legend_labels = viz.ax_.get_legend().get_texts()
assert len(legend_labels) == len(expected_legend_labels)
for labels in legend_labels:
assert labels.get_text() in expected_legend_labels
def test_calibration_display_compute(pyplot, iris_data_binary, n_bins, strategy):
# Ensure `CalibrationDisplay.from_predictions` and `calibration_curve`
# compute the same results. Also checks attributes of the
# CalibrationDisplay object.
X, y = iris_data_binary
lr = LogisticRegression().fit(X, y)
viz = CalibrationDisplay.from_estimator(
lr, X, y, n_bins=n_bins, strategy=strategy, alpha=0.8
)
y_prob = lr.predict_proba(X)[:, 1]
prob_true, prob_pred = calibration_curve(
y, y_prob, n_bins=n_bins, strategy=strategy
)
assert_allclose(viz.prob_true, prob_true)
assert_allclose(viz.prob_pred, prob_pred)
assert_allclose(viz.y_prob, y_prob)
assert viz.estimator_name == "LogisticRegression"
# cannot fail thanks to pyplot fixture
import matplotlib as mpl # noqa
assert isinstance(viz.line_, mpl.lines.Line2D)
assert viz.line_.get_alpha() == 0.8
assert isinstance(viz.ax_, mpl.axes.Axes)
assert isinstance(viz.figure_, mpl.figure.Figure)
assert viz.ax_.get_xlabel() == "Mean predicted probability"
assert viz.ax_.get_ylabel() == "Fraction of positives"
assert viz.line_.get_label() == "LogisticRegression"
def my_calibration_plot(model, calibrated_model, X_data, Y_data):
fig = plt.figure(figsize=(10, 10))
gs = GridSpec(1, 1)
ax_calibration_curve = fig.add_subplot(gs[0])
calibration_displays = {}
display = CalibrationDisplay.from_estimator(model,
X_data,
Y_data,
n_bins=10,
name='No calibrated',
ax=ax_calibration_curve,
color='red')
calibration_displays['No calibrated'] = display
ax = fig.add_subplot(gs[0])
probs_calib = calibrated_model.predict_proba(X_data)[:, 1]
fop_calib, mpv_calib = calibration_curve(Y_data,
probs_calib,
n_bins=10,
normalize=True)
ax.plot(mpv_calib, fop_calib, marker='.', color='blue', label='Calibrated')
plt.legend()
plt.show()
def test_calibration_display_pos_label(
pyplot, iris_data_binary, pos_label, expected_pos_label
):
"""Check the behaviour of `pos_label` in the `CalibrationDisplay`."""
X, y = iris_data_binary
lr = LogisticRegression().fit(X, y)
viz = CalibrationDisplay.from_estimator(lr, X, y, pos_label=pos_label)
y_prob = lr.predict_proba(X)[:, expected_pos_label]
prob_true, prob_pred = calibration_curve(y, y_prob, pos_label=pos_label)
assert_allclose(viz.prob_true, prob_true)
assert_allclose(viz.prob_pred, prob_pred)
assert_allclose(viz.y_prob, y_prob)
assert (
viz.ax_.get_xlabel()
== f"Mean predicted probability (Positive class: {expected_pos_label})"
)
assert (
viz.ax_.get_ylabel()
== f"Fraction of positives (Positive class: {expected_pos_label})"
)
assert viz.line_.get_label() == "LogisticRegression"
def test_plot_calibration_curve_pipeline(pyplot, iris_data_binary):
# Ensure pipelines are supported by CalibrationDisplay.from_estimator
X, y = iris_data_binary
clf = make_pipeline(StandardScaler(), LogisticRegression())
clf.fit(X, y)
viz = CalibrationDisplay.from_estimator(clf, X, y)
assert clf.__class__.__name__ in viz.line_.get_label()
assert viz.estimator_name == clf.__class__.__name__
def test_calibration_display_label_class_plot(pyplot):
# Checks that when instantiating `CalibrationDisplay` class then calling
# `plot`, `self.estimator_name` is the one given in `plot`
prob_true = np.array([0, 1, 1, 0])
prob_pred = np.array([0.2, 0.8, 0.8, 0.4])
y_prob = np.array([])
name = "name one"
viz = CalibrationDisplay(prob_true, prob_pred, y_prob, estimator_name=name)
assert viz.estimator_name == name
name = "name two"
viz.plot(name=name)
expected_legend_labels = [name, "Perfectly calibrated"]
legend_labels = viz.ax_.get_legend().get_texts()
assert len(legend_labels) == len(expected_legend_labels)
for labels in legend_labels:
assert labels.get_text() in expected_legend_labels
def test_plot_calibration_curve_pipeline(pyplot, iris_data_binary):
# Ensure pipelines are supported by CalibrationDisplay.from_estimator
X, y = iris_data_binary
clf = make_pipeline(StandardScaler(), LogisticRegression())
clf.fit(X, y)
viz = CalibrationDisplay.from_estimator(clf, X, y)
expected_legend_labels = [viz.estimator_name, "Perfectly calibrated"]
legend_labels = viz.ax_.get_legend().get_texts()
assert len(legend_labels) == len(expected_legend_labels)
for labels in legend_labels:
assert labels.get_text() in expected_legend_labels
def test_calibration_display_validation(pyplot, iris_data, iris_data_binary):
X, y = iris_data
X_binary, y_binary = iris_data_binary
reg = LinearRegression().fit(X, y)
msg = "'estimator' should be a fitted classifier"
with pytest.raises(ValueError, match=msg):
CalibrationDisplay.from_estimator(reg, X, y)
clf = LinearSVC().fit(X, y)
msg = "response method predict_proba is not defined in"
with pytest.raises(ValueError, match=msg):
CalibrationDisplay.from_estimator(clf, X, y)
clf = LogisticRegression()
with pytest.raises(NotFittedError):
CalibrationDisplay.from_estimator(clf, X, y)
def multi_class_calibration(y_true,
y_prob,
n_bins=5,
strategy='uniform',
names=None,
ref_line=True,
ax=None,
**kwargs):
"""
Displays a multi-class Calibration Curve (one line per class in one-v-rest setup)
:param y_true: True Labels: note that labels must be sequential starting from 0
:param y_prob: Predicted Probabilities
:param n_bins: Number of bins to use (see CalibrationDisplay.from_predictions)
:param strategy: Strategy for bins (see CalibrationDisplay.from_predictions)
:param names: Class names to use
:param ref_line: Whether to plot reference line (see CalibrationDisplay.from_predictions)
:param ax: Axes to draw on (see CalibrationDisplay.from_predictions)
:param kwargs: Keyword arguments passed on to plot
:return: Dict of Calibration Displays (by name)
"""
# Iterate over classes
names = utils.default(names, np.arange(y_prob.shape[1]))
displays = {}
for cls, name in zip(range(y_prob.shape[1]), names):
_y_true = (y_true == cls).astype(
int) # Get positive class for this label
_y_prob = y_prob[:, cls] # Get probability assigned to this class
displays[name] = CalibrationDisplay.from_predictions(_y_true,
_y_prob,
n_bins=n_bins,
strategy=strategy,
name=name,
ref_line=ref_line,
ax=ax,
**kwargs)
return displays
]
# %%
fig = plt.figure(figsize=(10, 10))
gs = GridSpec(4, 2)
colors = plt.cm.get_cmap("Dark2")
ax_calibration_curve = fig.add_subplot(gs[:2, :2])
calibration_displays = {}
for i, (clf, name) in enumerate(clf_list):
clf.fit(X_train, y_train)
display = CalibrationDisplay.from_estimator(
clf,
X_test,
y_test,
n_bins=10,
name=name,
ax=ax_calibration_curve,
color=colors(i),
)
calibration_displays[name] = display
ax_calibration_curve.grid()
ax_calibration_curve.set_title("Calibration plots (Naive Bayes)")
# Add histogram
grid_positions = [(2, 0), (2, 1), (3, 0), (3, 1)]
for i, (_, name) in enumerate(clf_list):
row, col = grid_positions[i]
ax = fig.add_subplot(gs[row, col])
def eiras(filename1, filename2):
"""
Getting result of applying one file model on another.
Arguments:two different datasets,filename1 for training set, filename2 for test set.
Output: parameters that measure the performance of the exsiting model using filename1.
"""
bysy = pd.read_csv(filename1)
szyy = pd.read_csv(filename2)
train = bysy.values
valid = szyy.values
g = train.shape[1]
X_train = train[:, :g - 1]
y_train = train[:, g - 1]
X_test = valid[0:, :g - 1]
y_test = valid[0:, g - 1]
feature_name = valid[-1:0:1, :g - 1]
models = [{
'label':
'LR',
'model':
linear_model.LogisticRegression(penalty='l1', solver='liblinear'),
'rank':
'A',
}, {
'label': 'SVM',
'model': svm.SVC(kernel='linear', gamma=1, C=1, probability=True),
'rank': 'B',
}, {
'label': 'GBDT',
'model': GradientBoostingClassifier(),
'rank': 'C',
}, {
'label':
'XGboost',
'model':
XGBClassifier(subsample=1,
eta=0.05,
eval_metric=['logloss', 'auc', 'error'],
use_label_encoder=False),
'rank':
'D',
}]
def Find_Optimal_Cutoff(TPR, FPR, threshold):
y = TPR - FPR
Youden_index = np.argmax(y) # Only the first occurrence is returned.
optimal_threshold = threshold[Youden_index]
point = [FPR[Youden_index], TPR[Youden_index]]
return optimal_threshold, point
def importance(model, X_train, feature_name):
try:
shap_values = shap.TreeExplainer(model).shap_values(X_train)
#shap.summary_plot(shap_values, X_train,feature_names=feature_name)
#shap.plots.bar(shap_values)
except Exception:
return None
n_bootstraps = 1000
rng_seed = 42 # control reproducibility
bootstrapped_scores = []
rng = np.random.RandomState(rng_seed)
AUC = []
CI_lower = []
CI_upper = []
Specificity = []
Sensitivity = []
Accuracy = []
Optimal = []
Point = []
Youden = []
for m in models:
model = m['model'] # select the model
label = m['label']
model.fit(X_train, y_train) # train the model
y_pred = model.predict(X_test) # predict the test data
y_pro = model.predict_proba(X_test)
y_prob = model.predict_proba(X_test)[:, 1]
# Compute False postive rate=fpr, and True positive rate=tpr
fpr, tpr, thresholds = metrics.roc_curve(
y_test,
model.predict_proba(X_test)[:, 1])
# Calculate Area under the curve to display on the plot
auc = roc_auc_score(y_test, model.predict_proba(X_test)[:, 1])
plt.plot(fpr, tpr, label='%s (AUC = %0.2f)' % (m['label'], auc))
#using optimal_threshold function to find best sensitivity and specificity trade-off
optimal_threshold, point = Find_Optimal_Cutoff(tpr, fpr, thresholds)
best_y_pred = (model.predict_proba(X_test)[:, 1] >=
optimal_threshold).astype(bool)
# compute sensitivity,specifictiy and accuracy
#confusion = confusion_matrix(y_test,best_y_pred)
#disp = ConfusionMatrixDisplay(confusion_matrix=confusion,display_labels=model.classes_)
#disp.plot()
#plt.show()
tn, fp, fn, tp = confusion_matrix(y_test, best_y_pred).ravel()
sensitivity = float(tp) / float(tp + fn)
specificity = float(tn) / float(tn + fp)
youden = (sensitivity + specificity) - 1
accuracy = metrics.accuracy_score(y_test,
best_y_pred,
normalize=True,
sample_weight=None)
#importance
#importance(model,X_train,feature_name)
#
#compute CI using bootstraps
yy_pred = y_pro[:, 1]
for i in range(n_bootstraps):
indices = rng.randint(0, len(yy_pred), len(yy_pred))
if len(np.unique(y_test[indices])) < 2:
continue
score = roc_auc_score(y_test[indices], yy_pred[indices])
bootstrapped_scores.append(score)
sorted_scores = np.array(bootstrapped_scores)
sorted_scores.sort()
confidence_lower = sorted_scores[int(0.05 * len(sorted_scores))]
confidence_upper = sorted_scores[int(0.95 * len(sorted_scores))]
print(
label,
"Confidence interval for the score: [{:0.3f} - {:0.3}]".format(
confidence_lower, confidence_upper))
Sensitivity.append(sensitivity)
Specificity.append(specificity)
Accuracy.append(accuracy)
AUC.append(auc)
CI_lower.append(confidence_lower)
CI_upper.append(confidence_upper)
Youden.append(youden)
Optimal.append(optimal_threshold)
plt.plot([0, 1], [0, 1], 'k--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('1-特异率', fontproperties=myfont)
plt.ylabel('敏感率', fontproperties=myfont)
plt.legend(loc="lower right")
#plt.show()
plt.savefig('C:/Users/mjdee/Desktop/JI-2020/ML/the_roc_mimic.jpg',
bbox_inches='tight',
dpi=600)
plt.close()
#plot calibration_curve
fig = plt.figure(figsize=(10, 10))
gs = GridSpec(4, 2)
ax_calibration_curve = fig.add_subplot(gs[:2, :2])
calibration_displays = {}
colors = plt.cm.get_cmap("Dark2")
i = 0
for m in (models):
model = m['model']
label = m['label']
model.fit(X_train, y_train)
display = CalibrationDisplay.from_estimator(
model,
X_test,
y_test,
n_bins=10,
name=label,
ax=ax_calibration_curve,
color=colors(i),
)
i = i + 1
calibration_displays[label] = display
ax_calibration_curve.grid()
ax_calibration_curve.set_title("A Calibration plots")
#plot calibration_histograms
grid_positions = [(2, 0), (2, 1), (3, 0), (3, 1)]
i = 0
for m in (models):
model = m['model']
label = m['label']
rank = m['rank']
row, col = grid_positions[i]
ax = fig.add_subplot(gs[row, col])
ax.hist(calibration_displays[label].y_prob,
range=(0, 1),
bins=10,
label=name,
color=colors(i))
i = i + 1
ax.set(title=str(rank) + str(label),
xlabel="Mean predicted probability",
ylabel="Count")
plt.tight_layout()
plt.savefig('C:/Users/mjdee/Desktop/JI-2020/ML/the_calibration_mimic.jpg',
bbox_inches='tight',
dpi=600)
plt.show()
return Sensitivity, Specificity, Accuracy, AUC, CI_upper, CI_lower, Youden, Optimal
