def get_preds_probas(est: ClassifierMixin, X_test: DataFrame, y_test: Series, mapper_dict: Dict) -> DataFrame: """ Get prediction probabilities (if available) or return true and predicted labels """ df_preds = DataFrame(est.predict(X_test), index=X_test.index) if hasattr(est.named_steps["clf"], "predict_proba"): # Get prediction probabilities (if available) df_probas = DataFrame(est.predict_proba(X_test), index=X_test.index) # Append prediction and prediction probabilities df_summ = concat([df_preds, df_probas], axis=1) df_summ.columns = ["predicted_label"] + [ f"probability_of_{i}" for i in range(0, len(np.unique(y_test))) ] # Get label (class) with maximum prediction probability for each row df_summ["max_class_number_manually"] = df_probas.idxmax(axis=1) df_summ["probability_of_max_class"] = df_probas.max(axis=1) # Compare .predict_proba() and manually extracted prediction # probability lhs = df_summ["max_class_number_manually"] rhs = df_summ["predicted_label"].replace(mapper_dict) assert (lhs == rhs).eq(True).all() else: df_summ = df_preds.copy() # Get true label df_summ.insert(0, "true_label", y_test) return df_summ
def test_probabilities(model: ClassifierMixin, X: np.array, y: pd.Series, bins: int = 10, threshold: float = 0.5): """Print confusion matrix based on class probability.""" probs = [p[1] for p in model.predict_proba(X)] print('\tProbabilities') df = pd.DataFrame({'prob': probs, 'label': y}) step = 1 / bins cut_labels = [round(step * f, 1) for f in range(10)] by_prob = (df.groupby(pd.cut(df['prob'], bins, labels=cut_labels)).agg(['sum', 'count'])['label']) print('\t\tprobs\t1\t0\tacc') for index, row in by_prob.iloc[::-1].iterrows(): ones = row['sum'] if math.isnan(ones): ones = 0 else: ones = int(ones) count = row['count'] zeros = int(count) - ones if count > 0: acc = zeros / count if index < threshold else ones / count else: acc = 0.0 print(f'\t\t{index}\t{ones}\t{zeros}\t{acc:.3f}')
def _predict_oof_model( self, estimator: ClassifierMixin, X: ArrayLike, ) -> NDArray: """ Predict probabilities of a test set from a fitted estimator. Parameters ---------- estimator : ClassifierMixin Fitted estimator. X : ArrayLike Test set. Returns ------- ArrayLike Predicted probabilities. """ y_pred_proba = estimator.predict_proba(X) # we enforce y_pred_proba to contain all labels included y if len(estimator.classes_) != self.n_classes_: y_pred_proba = self._fix_number_of_classes( estimator.classes_, y_pred_proba ) return y_pred_proba
def plot_decision_boundary( X: pd.DataFrame, y: pd.Series, clf: ClassifierMixin = sklearn.linear_model.LogisticRegression(), title: str = "Decision Boundary Logistic Regression", legend_title: str = "Legend", h: float = 0.05, figsize: tuple = (11.7, 8.27), ): """Generate a simple plot of the decision boundary of a classifier. Parameters ---------- X : array-like, shape (n_samples, n_features) Classifier vector, where n_samples is the number of samples and n_features is the number of features. y : array-like, shape (n_samples) Target relative to X for classification. Datatype should be integers. clf : scikit-learn algorithm An object that has the `predict` and `predict_proba` methods h : int (default: 0.05) Step size in the mesh title : string Title for the plot. legend_title : string Legend title for the plot. figsize: tuple (default: (11.7, 8.27)) Width and height of the figure in inches Returns ------- boundaries: Figure Properties of the figure can be changed later, e.g. use `boundaries.axes[0].set_ylim(0,100)` to change ylim ax: Axes The axes associated with the boundaries Figure. Examples -------- >>> import seaborn as sns >>> from sklearn.svm import SVC >>> data = sns.load_dataset("iris") >>> # convert the target from string to category to numeric as sklearn cannot handle strings as target >>> y = data["species"] >>> X = data[["sepal_length", "sepal_width"]] >>> clf = SVC(kernel="rbf", gamma=2, C=1, probability=True) >>> _ = plot_decision_boundary(X=X, y=y, clf=clf, title = 'Decision Boundary', legend_title = "Species") """ if X.shape[1] != 2: raise ValueError("X must contains only two features.") if not (pd.api.types.is_integer_dtype(y) or pd.api.types.is_object_dtype(y) or pd.api.types.is_categorical_dtype(y)): raise TypeError( "The target variable y can only have the following dtype: [int, object, category]." ) label_0 = X.columns.tolist()[0] label_1 = X.columns.tolist()[1] X = X.copy() y = y.copy() X = X.values y = y.astype("category").cat.codes.values # full_col_list = list(sns.color_palette("husl", len(np.unique(y)))) full_col_list = list(sns.color_palette()) if len(np.unique(y)) > len(full_col_list): raise ValueError( "More labels in the data then colors in the color list. Either reduce the number of labels or expend the color list" ) sub_col_list = full_col_list[0:len(np.unique(y))] cmap_bold = ListedColormap(sub_col_list) # Try to include a mapping in a later release (+ show categorical labels in the legend) _ = clf.fit(X, y) # Plot the decision boundary. For that, we will assign a color to each # point in the mesh [x_min, x_max]x[y_min, y_max]. x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1 y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1 xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h)) Z = clf.predict(np.c_[xx.ravel(), yy.ravel()]) Z = Z.reshape(xx.shape) Z_proba = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()]) Z_max = Z_proba.max(axis=1) # Take the class with highest probability Z_max = Z_max.reshape(xx.shape) # Put the result into a color plot boundaries, ax = plt.subplots(figsize=figsize) _ = ax.contour(xx, yy, Z, cmap=cmap_bold) _ = ax.scatter(xx, yy, s=(Z_max**2 / h), c=Z, cmap=cmap_bold, alpha=1, edgecolors="none") # Plot also the training points training = ax.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, edgecolors="black") _ = plt.xlim(xx.min(), xx.max()) _ = plt.ylim(yy.min(), yy.max()) _ = plt.title(title) _ = plt.subplots_adjust(right=0.8) _ = plt.xlabel(label_0) _ = plt.ylabel(label_1) # Add legend colors leg1 = plt.legend( *training.legend_elements(), frameon=False, fontsize=12, borderaxespad=0, bbox_to_anchor=(1, 0.5), handlelength=2, handletextpad=1, title=legend_title, ) # Add legend sizes l1 = plt.scatter([], [], c="black", s=0.4**2 / h, edgecolors="none") l2 = plt.scatter([], [], c="black", s=0.6**2 / h, edgecolors="none") l3 = plt.scatter([], [], c="black", s=0.8**2 / h, edgecolors="none") l4 = plt.scatter([], [], c="black", s=1**2 / h, edgecolors="none") labels = ["0.4", "0.6", "0.8", "1"] _ = plt.legend( [l1, l2, l3, l4], labels, frameon=False, fontsize=12, borderaxespad=0, bbox_to_anchor=(1, 1), handlelength=2, handletextpad=1, title="Probabilities", scatterpoints=1, ) _ = plt.gca().add_artist(leg1) return boundaries, ax