def test_cmap(self): np.random.seed(0) clf = LogisticRegression() clf.fit(self.X, self.y) probas = clf.predict_proba(self.X) plot_precision_recall(self.y, probas, cmap='nipy_spectral') plot_precision_recall(self.y, probas, cmap=plt.cm.nipy_spectral)
def test_classes_to_plot(self): np.random.seed(0) clf = LogisticRegression() clf.fit(self.X, self.y) probas = clf.predict_proba(self.X) plot_precision_recall(self.y, probas, classes_to_plot=[0, 1]) plot_precision_recall(self.y, probas, classes_to_plot=np.array([0, 1]))
def test_plot_micro(self): np.random.seed(0) clf = LogisticRegression() clf.fit(self.X, self.y) probas = clf.predict_proba(self.X) plot_precision_recall(self.y, probas, plot_micro=True) plot_precision_recall(self.y, probas, plot_micro=False)
def prc(self, predicted_probs, plot_name, file_name): """ A method that plots a precision recall curve for the given model. """ plot_precision_recall(self.results_convert(self.y), predicted_probs, title=plot_name) plt.savefig(file_name)
def create_precision_recall_chart(classifier, X_test, y_test, y_pred_proba=None): """Create precision recall chart. Tip: Check Sklearn-Neptune integration `documentation <https://docs-beta.neptune.ai/essentials/integrations/machine-learning-frameworks/sklearn>`_ for the full example. Args: classifier (:obj:`classifier`): | Fitted sklearn classifier object X_test (:obj:`ndarray`): | Testing data matrix y_test (:obj:`ndarray`): | The classification target for testing y_pred_proba (:obj:`ndarray`, optional, default is ``None``): | Classifier predictions probabilities on test data. Returns: ``neptune.types.File`` object that you can assign to run's ``base_namespace``. Examples: .. code:: python3 import neptune.new.integrations.sklearn as npt_utils rfc = RandomForestClassifier() rfc.fit(X_train, y_train) run = neptune.init(project='my_workspace/my_project') run['visuals/precision_recall'] = npt_utils.create_precision_recall_chart(rfc, X_test, y_test) """ assert is_classifier( classifier), 'classifier should be sklearn classifier.' chart = None if y_pred_proba is None: try: y_pred_proba = classifier.predict_proba(X_test) except Exception as e: print( 'Did not log Precision-Recall chart: this classifier does not provide predictions probabilities.' 'Error {}'.format(e)) return chart try: fig, ax = plt.subplots() plot_precision_recall(y_test, y_pred_proba, ax=ax) chart = neptune.types.File.as_image(fig) plt.close(fig) except Exception as e: print('Did not log Precision-Recall chart. Error {}'.format(e)) return chart
def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''): """Creates and logs Precision Recall curve and Average precision score to Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs Precision Recall curve and Average precision score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_precision_recall_auc neptune.init() with neptune.create_experiment(): log_precision_recall_auc(y_test, y_test_pred) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune expect_not_a_run(_exp) avg_precision = sk_metrics.average_precision_score(y_true, y_pred[:, 1]) _exp.log_metric(prefix + 'avg_precision', avg_precision) fig, ax = plt.subplots() plt_metrics.plot_precision_recall(y_true, y_pred, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def test_ax(self): np.random.seed(0) clf = LogisticRegression() clf.fit(self.X, self.y) probas = clf.predict_proba(self.X) fig, ax = plt.subplots(1, 1) out_ax = plot_precision_recall(self.y, probas) assert ax is not out_ax out_ax = plot_precision_recall(self.y, probas, ax=ax) assert ax is out_ax
def log_precision_recall_chart(classifier, X_test, y_test, y_pred_proba=None, experiment=None): """Log precision recall chart. Make sure you created an experiment by using ``neptune.create_experiment()`` before you use this method. Tip: Check `Neptune documentation <https://docs.neptune.ai/integrations/scikit_learn.html>`_ for the full example. Args: classifier (:obj:`classifier`): | Fitted sklearn classifier object X_test (:obj:`ndarray`): | Testing data matrix y_test (:obj:`ndarray`): | The classification target for testing y_pred_proba (:obj:`ndarray`, optional, default is ``None``): | Classifier predictions probabilities on test data. experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``): | Neptune ``Experiment`` object to control to which experiment you log the data. | If ``None``, log to currently active, and most recent experiment. Returns: ``None`` Examples: .. code:: python3 rfc = RandomForestClassifier() rfc.fit(X_train, y_train) neptune.init('my_workspace/my_project') neptune.create_experiment() log_precision_recall_chart(rfc, X_test, y_test) """ assert is_classifier(classifier), 'classifier should be sklearn classifier.' exp = _validate_experiment(experiment) if y_pred_proba is None: try: y_pred_proba = classifier.predict_proba(X_test) except Exception as e: print('Did not log Precision-Recall chart: this classifier does not provide predictions probabilities.' 'Error {}'.format(e)) return try: fig, ax = plt.subplots() plot_precision_recall(y_test, y_pred_proba, ax=ax) exp.log_image('charts_sklearn', fig, image_name='Precision Recall Curve') plt.close(fig) except Exception as e: print('Did not log Precision-Recall chart. Error {}'.format(e))
def plot_learning_curve(self, save_dir): # Plot learning curve for the classifier est = self.model est.set_params(**self.params) _, axes = plt.subplots(3, 3, figsize=(20, 12), dpi=200, constrained_layout=True) # plt.tight_layout() _train_ax = [axes[0][0], axes[0][1], axes[0][2]] plot_learning_curve(est, "{} - Learning curves (Train)".format(self.symbol), self.X_train, self.y_train, axes=_train_ax, cv=self.cv) n_classes = np.unique(self.y_test).shape[0] if hasattr(self.cv_estimator, 'predict_proba'): y_train_proba = self.cv_estimator.predict_proba(self.X_train) axes[1][0].set_title("{} - ROC (Train)".format(self.symbol)) plot_roc(self.y_train, y_train_proba, n_classes, ax=axes[1][0]) axes[1][1].set_title("{} - Precision/Recall (Train)".format( self.symbol)) plot_precision_recall(self.y_train, y_train_proba, ax=axes[1][1]) axes[1][2].set_title("{} - Confusion matrix (Train)".format( self.symbol)) plot_confusion_matrix(self.cv_estimator, self.X_train, self.y_train, cmap='Blues', ax=axes[1][2]) if hasattr(self.cv_estimator, 'predict_proba'): y_test_proba = self.cv_estimator.predict_proba(self.X_test) axes[2][0].set_title("{} - ROC (Test)".format(self.symbol)) plot_roc(self.y_test, y_test_proba, ax=axes[2][0]) axes[2][1].set_title("{} - Precision/Recall (Test)".format( self.symbol)) plot_precision_recall(self.y_test, y_test_proba, ax=axes[2][1]) axes[2][2].set_title("{} - Confusion matrix (Test)".format( self.symbol)) plot_confusion_matrix(self.cv_estimator, self.X_test, self.y_test, cmap='Oranges', ax=axes[2][2]) curve_path = '{}{}_learning_curve.png'.format(save_dir, self.symbol) plt.savefig(curve_path) plt.close()
def prc(self, predicted_probs, plot_name, file_name): plot_precision_recall(self.y_test, predicted_probs, title=plot_name) plt.savefig(file_name)
def test_array_like(self): plot_precision_recall([0, 1], [[0.8, 0.2], [0.2, 0.8]]) plot_precision_recall([0, 'a'], [[0.8, 0.2], [0.2, 0.8]]) plot_precision_recall(['b', 'a'], [[0.8, 0.2], [0.2, 0.8]])
sorted(cancer.target_names) print(classification_report(y_test, y_pred)) # Specify a threshold y_pred_thresh = rf.predict_proba(X_test)[:, 1] > 0.85 # 0.85 as threshold print(classification_report(y_test, y_pred_thresh)) ### Precision-Recall curve from scikitplot.metrics import plot_precision_recall rf_probas = rf.predict_proba(X_test)[:, 1] plot_precision_recall(y_test, rf_probas) from yellowbrick.classifier import PrecisionRecallCurve viz = PrecisionRecallCurve(rf) viz.fit(X_train, y_train) viz.score(X_test, y_test) viz.poof() # Discimination Threshold - probability or score at which the positive class is chosen over the negative class from yellowbrick.classifier import DiscriminationThreshold viz = DiscriminationThreshold(rf) viz.fit(X_train, y_train) viz.poof()
def plot_classification_metrics(y_true, y_pred_proba, classes_to_plot=None, threshold=None, plot_micro=True): """ Plot ROC Curve, Precision-Recall Curve and Confusion Matrix Parameters ---------- y_true : array-like, shape (n_samples) Ground truth labels. y_pred_proba : array-like, shape (n_samples, n_classes) Prediction probabilities or decision scores for each class returned by a classifier. classes_to_plot : list-like, optional Classes for which the ROC curve should be plotted. e.g. [0, 'cold']. If the specified class does not exist, it will be ignored. If ``None``, all classes will be plotted. threshold : None or float if a float is set, it will be used as the decision threshold for binary classification plot_micro : bool whether to plot the averaged roc_curve and the precision-recall curve using average method 'micro' Returns ------- fig : :class:`matplotlib.figure.Figure` object axs : Axes object or array of Axes objects. """ fig = plt.figure(dpi=100, figsize=(10.5, 8)) ax1 = plt.subplot2grid((4, 4), (0, 0), rowspan=2, colspan=2) ax2 = plt.subplot2grid((4, 4), (0, 2), rowspan=2, colspan=2) ax3 = plt.subplot2grid((4, 4), (2, 0), rowspan=2, colspan=2) # region Plot ROC Curve plot_roc(y_true, y_pred_proba, plot_macro=False, plot_micro=plot_micro, classes_to_plot=classes_to_plot, ax=ax1) # endregion # region plot Precision-Recall Curve plot_precision_recall(y_true, y_pred_proba, plot_micro=plot_micro, classes_to_plot=classes_to_plot, ax=ax2) ax2.legend(loc='lower right') # endregion # region Plot Confusion Matrix y_pred_idx = np.argmax(y_pred_proba, axis=-1) labels = np.sort(np.unique(y_true)) y_pred = labels[y_pred_idx] plot_confusion_matrix(y_true, y_pred, normalize=True, ax=ax3) # endregion axs = [ax1, ax2, ax3] if threshold: # region Plot Confusion Matrix labels = np.sort(np.unique(y_true)) assert len(labels) == 2, """Problem is not binary classification but decision threshold is set""" ax4 = plt.subplot2grid((4, 4), (2, 2), rowspan=2, colspan=2) is_positive = y_pred_proba[:, 1] > threshold y_pred = labels[is_positive.astype('int')] plot_confusion_matrix(y_true, y_pred, normalize=True, ax=ax4) ax4.set_title('Confusion Matrix with adjusted ' 'decision threshold: {:.2}'.format(threshold)) # update color limit im3 = ax3.get_images()[0] clim = im3.get_clim() im4 = ax4.get_images()[0] im4.set_clim(clim) axs.append(ax4) # endregion fig.tight_layout() return fig, axs
'y_pred': y_pred, 'y_pred_probability': y_pred_proba.max(axis=1) }) log_table('predictions', df) # Log model performance visualizations import matplotlib.pyplot as plt from scikitplot.metrics import plot_roc, plot_precision_recall fig, ax = plt.subplots() plot_roc(y_test, y_pred_proba, ax=ax) neptune.log_image('model-performance-visualizations', fig, image_name='ROC') fig, ax = plt.subplots() plot_precision_recall(y_test, y_pred_proba, ax=ax) neptune.log_image('model-performance-visualizations', fig, image_name='precision recall') plt.close('all') # Log train data sample (images per class) for j, class_name in enumerate(class_names): plt.figure(figsize=(10, 10)) label_ = np.where(y_train == j) for i in range(9): plt.subplot(3, 3, i + 1) plt.xticks([]) plt.yticks([]) plt.grid(False)
def test_string_classes(self): np.random.seed(0) clf = LogisticRegression() clf.fit(self.X, convert_labels_into_string(self.y)) probas = clf.predict_proba(self.X) plot_precision_recall(convert_labels_into_string(self.y), probas)
def predict(model, data_loader, pos_loss_weight, verbose=0, tag='', xlfile_name=None, prefix=None): #DOC OK """ This function is used to evaluate a binary classifier. Params: ------ model: torch.nn.Module the model to be used for prediction data_loader: torch.utils.data.DataLoader the test data real_loaders pos_loss_weight: int/float a number to be passed as a the weight for the positive class for BCEWithLogitsLoss verbose: int if 0 : only metrics will be printed if 1 : verbose 0 + plotting precision-recall curve if 2 : verbose 1 + visualize 40 images of wrong and correct predictions. if 3: verbose 2 + visualize the confusion matrix tag: string a string to include in the save figure name. xlfile_name: str the path where the results Excel file is located. prefix: list the values to append results to, usually are the parameters used to get these results. Returns: ------- tuple(predicted labels, the loss value, F1 score) Notes: ------ Sigmoid is applied on the output of the model, so no need to include it in the model. """ # set the device device = Classifier.device matplotlib.use('Agg') #TODO disable when you want to show figures mode = model.training # set the mode to evaluation model.eval() np_pred_labels = np.array([]) np_true_labels = np.array([]) np_probs = np.array([]) # for all batches for idx ,(imgs, labels) in enumerate(data_loader): imgs, labels = imgs.to(device), labels.to(device).float() # start testing with no auto grad with torch.set_grad_enabled(False): output = model(imgs).view(-1) #get the output batch probs = torch.sigmoid(output) preds = torch.gt(probs, 0.5).float() loss = auto_weight_BCEWithLogitsLoss(output, labels, pos_loss_weight) # stack all the true labels, predicted labels, and probabilities for all batches together np_true_labels = np.hstack((np_true_labels, labels.data.cpu().numpy())).astype(np.int) np_pred_labels = np.hstack((np_pred_labels, preds.data.cpu().numpy())).astype(np.int) np_probs = np.hstack((np_probs, probs.data.cpu().numpy())) if verbose >= 0: #print metrics Pr, R, F1, S = precision_recall_fscore_support(y_true=np_true_labels, y_pred=np_pred_labels, average=None) average_precision = average_precision_score(y_true=np_true_labels, y_score=np_probs) print('-' * 10 + '(class 0): ' + ' Loss= {:.5f} Precision = {:.4f} Recall = {:.4f} F1 = {:.4f} Support = {}'.format(loss.item(), Pr[0], R[0], F1[0], S[0])) print('-'*10+ '(class 1): ' + ' Loss= {:.5f} Precision = {:.4f} Recall = {:.4f} F1 = {:.4f} Support = {}'.format(loss.item(), Pr[1], R[1], F1[1], S[1])) print('\naverage precision {:.4f}'.format(average_precision)) # area under precision-recall curve can be useful here if verbose >= 1: #plot precision recall curve two_class_probs = np.stack((1-np_probs, np_probs), axis=1) #stack positive class and negative class probabilities plot_precision_recall(y_true=np_true_labels, y_probas= two_class_probs) # plot precision recall curve fpr, tpr, thresholds = roc_curve(y_true= np_true_labels, y_score= np_probs, pos_label=1) roc_auc = roc_auc_score(y_true=np_true_labels, y_score=np_probs) plt.figure() plt.plot(fpr, tpr) plt.xlabel('FPR'), plt.ylabel('TPR'), plt.title('ROC Curve for model\n' + os.path.split(model.path)[1]), plt.legend(['auc = {:.4f}'.format(roc_auc)]) plt.savefig(os.path.join(model.path, model.id + tag + '_ROC' )) if verbose >= 2: #show some images (20 correctly predicted and 20 mistakenly predicted) visualize_model(num_images=[20,20], data_loader=data_loader, true_labels=np_true_labels, pred_labels=np_pred_labels ,classes=['Tis', 'Les']) if verbose==3: plot_confusion_matrix(y_true=np_true_labels, y_pred=np_pred_labels, classes=np.array(['Tis', 'Les'])) plt.savefig(os.path.join(model.path, model.id + tag+ '_confusion')) plt.show() #return the model to the mode it came with model.train(mode) # if there is an excel file name, write the results there if xlfile_name: print('writing metrics to the excel file...') write_xl(xlfile_name, prefix+[Pr[1], R[1], F1[1], roc_auc]) return np_pred_labels, loss.item(), F1
def main(): neptune.init(api_token=os.getenv('NEPTUNE_API_TOKEN'), project_qualified_name=os.getenv('NEPTUNE_PROJECT')) train_idx = pd.read_csv(TRAIN_IDX_PATH, nrows=NROWS) valid_idx = pd.read_csv(VALID_IDX_PATH, nrows=NROWS) features = pd.read_csv(FEATURES_PATH, nrows=NROWS) train = pd.merge(train_idx, features, on='SK_ID_CURR') valid = pd.merge(valid_idx, features, on='SK_ID_CURR') all_params = { 'num_boost_round': NUM_BOOST_ROUND, 'early_stopping_rounds': EARLY_STOPPING_ROUNDS, **LGBM_PARAMS } with neptune.create_experiment(name='model training', params=all_params, tags=['lgbm'], upload_source_files=get_filepaths(), properties={ 'features_path': FEATURES_PATH, 'features_version': md5_hash(FEATURES_PATH), 'train_split_version': md5_hash(TRAIN_IDX_PATH), 'valid_split_version': md5_hash(VALID_IDX_PATH), }): results = train_evaluate(train, valid, LGBM_PARAMS, callbacks=[neptune_monitor()]) train_score, valid_score = results['train_score'], results[ 'valid_score'] train_preds, valid_preds = results['train_preds'], results[ 'valid_preds'] neptune.send_metric('train_auc', train_score) neptune.send_metric('valid_auc', valid_score) train_pred_path = os.path.join(PREDICTION_DIRPATH, 'train_preds.csv') train_preds.to_csv(train_pred_path, index=None) neptune.send_artifact(train_pred_path) valid_pred_path = os.path.join(PREDICTION_DIRPATH, 'valid_preds.csv') valid_preds.to_csv(valid_pred_path, index=None) neptune.send_artifact(valid_pred_path) model_path = os.path.join(MODEL_DIRPATH, 'model.pkl') joblib.dump(results['model'], model_path) neptune.set_property('model_path', model_path) neptune.set_property('model_version', md5_hash(model_path)) neptune.send_artifact(model_path) if PACKAGE_TO_PROD: saved_path = CreditDefaultClassifier.pack( model=results['model']).save(PRODUCTION_DIRPATH) neptune.set_property('production_model_path', saved_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_confusion_matrix(valid_preds['TARGET'], valid_preds['preds_pos'] > 0.5, ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'conf_matrix.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_roc(valid_preds['TARGET'], valid_preds[['preds_neg', 'preds_pos']], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'roc_auc.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) sk_metrics.plot_precision_recall( valid_preds['TARGET'], valid_preds[['preds_neg', 'preds_pos']], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'prec_recall.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path) fig, ax = plt.subplots(figsize=(16, 12)) plot_prediction_distribution(valid_preds['TARGET'], valid_preds['preds_pos'], ax=ax) plot_path = os.path.join(REPORTS_DIRPATH, 'preds_dist.png') fig.savefig(plot_path) neptune.send_image('diagnostics', plot_path)