def log_roc_auc_chart(classifier,
X_train,
X_test,
y_train,
y_test,
experiment=None):
"""Log ROC-AUC chart.
Make sure you created an experiment by using ``neptune.create_experiment()`` before you use this method.
Tip:
Check `Neptune documentation <https://docs.neptune.ai/integrations/scikit_learn.html>`_ for the full example.
Args:
classifier (:obj:`classifier`):
| Fitted sklearn classifier object
X_train (:obj:`ndarray`):
| Training data matrix
X_test (:obj:`ndarray`):
| Testing data matrix
y_train (:obj:`ndarray`):
| The classification target for training
y_test (:obj:`ndarray`):
| The classification target for testing
experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``):
| Neptune ``Experiment`` object to control to which experiment you log the data.
| If ``None``, log to currently active, and most recent experiment.
Returns:
``None``
Examples:
.. code:: python3
rfc = RandomForestClassifier()
rfc.fit(X_train, y_train)
neptune.init('my_workspace/my_project')
exp = neptune.create_experiment()
log_roc_auc_chart(rfc, X_train, X_test, y_train, y_test, experiment=exp)
"""
assert is_classifier(
classifier), 'classifier should be sklearn classifier.'
exp = _validate_experiment(experiment)
try:
fig, ax = plt.subplots()
visualizer = ROCAUC(classifier, is_fitted=True, ax=ax)
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.finalize()
exp.log_image('charts_sklearn', fig, image_name='ROC-AUC')
plt.close(fig)
except Exception as e:
print('Did not log ROC-AUC chart. Error {}'.format(e))
def create_roc_auc_chart(classifier, X_train, X_test, y_train, y_test):
"""Create ROC-AUC chart.
Tip:
Check Sklearn-Neptune integration
`documentation <https://docs-beta.neptune.ai/essentials/integrations/machine-learning-frameworks/sklearn>`_
for the full example.
Args:
classifier (:obj:`classifier`):
| Fitted sklearn classifier object
X_train (:obj:`ndarray`):
| Training data matrix
X_test (:obj:`ndarray`):
| Testing data matrix
y_train (:obj:`ndarray`):
| The classification target for training
y_test (:obj:`ndarray`):
| The classification target for testing
Returns:
``neptune.types.File`` object that you can assign to run's ``base_namespace``.
Examples:
.. code:: python3
import neptune.new.integrations.sklearn as npt_utils
rfc = RandomForestClassifier()
rfc.fit(X_train, y_train)
run = neptune.init(project='my_workspace/my_project')
run['visuals/roc_auc'] = npt_utils.create_roc_auc_chart(rfc, X_train, X_test, y_train, y_test)
"""
assert is_classifier(
classifier), 'classifier should be sklearn classifier.'
chart = None
try:
fig, ax = plt.subplots()
visualizer = ROCAUC(classifier, is_fitted=True, ax=ax)
visualizer.fit(X_train, y_train)
visualizer.score(X_test, y_test)
visualizer.finalize()
chart = neptune.types.File.as_image(fig)
plt.close(fig)
except Exception as e:
print('Did not log ROC-AUC chart. Error {}'.format(e))
return chart
def evaluate_model(clf_, X_tr, X_te, y_tr, y_te, cls_rpt_tr=False, show=True, cls_labels=None, binary=False):
"""Takes any classifier, train/test data for X/y, labels for graph (optional).
Will output (if show) a Sklearn Classification Report and Confusion Matrix
along with a Yellowbrick ROC/AUC curve and Feature Importance graph (if a tree).
Otherwise will return training/test predictions."""
import sklearn.metrics as metrics
import matplotlib.pyplot as plt
from yellowbrick.classifier import ROCAUC
## Fit and predict
y_hat_trn, y_hat_tes = fit_n_pred(clf_, X_tr, X_te, y_tr)
if show:
## Classification Report / Scores
if cls_rpt_tr:
print('Classification Report Train')
print(metrics.classification_report(y_tr,y_hat_trn))
else:
print('Classification Report Test')
print(metrics.classification_report(y_te,y_hat_tes))
## Confusion Matrix
fig, ax = plt.subplots(figsize=(10,5), ncols=2)
metrics.plot_confusion_matrix(clf_,X_te,y_te,cmap="YlOrRd",
normalize='true',ax=ax[0])
ax[0].set(title='Confusion Matrix Test Data')
ax[0].grid(False)
roc = ROCAUC(clf_, classes=cls_labels, ax=ax[1])
roc.fit(X_tr, y_tr)
roc.score(X_te, y_te)
roc.finalize()
plt.tight_layout()
plt.show()
if binary:
try:
imps = plot_importance(clf_, X_tr)
except:
imps = None
else:
return y_hat_trn, y_hat_tes