def __init__(self,
experiment=None,
log_study=False,
log_charts=False,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None): # pylint: disable=W0621
self.exp = experiment if experiment else neptune
self.log_study = log_study
expect_not_a_run(self.exp)
if log_charts:
message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)
log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True
self.log_optimization_history = log_optimization_history
self.log_contour = log_contour
self.log_parallel_coordinate = log_parallel_coordinate
self.log_slice = log_slice
self.params = params
def __init__(self, experiment=None, log_checkpoint=True):
self._exp = experiment if experiment else neptune
expect_not_a_run(self._exp)
self.log_checkpoint = log_checkpoint
self._iteration = 0
def log_class_metrics_by_threshold(y_true,
y_pred_pos,
experiment=None,
channel_name='metrics_by_threshold',
prefix=''):
"""Creates metric/threshold charts for each metric and logs them to Neptune.
Metrics for which charsta re created and logged are: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score',
'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value',
'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate'
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metrics_by_threshold'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Logs metric/threshold charts to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_class_metrics_by_threshold
neptune.init()
with neptune.create_experiment():
log_class_metrics_by_threshold(y_test, y_test_pred[:,1])
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred_pos.shape
) == 1, 'y_pred_pos needs to be 1D prediction for positive class'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
figs = _plot_class_metrics_by_threshold(y_true, y_pred_pos)
for fig in figs:
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()
def __init__(self, learn=None, experiment=None, prefix=''):
self._exp = experiment if experiment else neptune
self._prefix = prefix
expect_not_a_run(self._exp)
if learn is not None:
super().__init__(learn)
def _log_plot_objective(results, experiment, name='diagnostics'):
try:
expect_not_a_run(experiment)
fig = plt.figure(figsize=(16, 12))
fig = axes2fig(sk_plots.plot_objective(results), fig=fig)
experiment.log_image(name, fig)
except Exception as e:
print('Could not create the objective chart due to error: {}'.format(e))
def log_precision_recall_auc(y_true,
y_pred,
experiment=None,
channel_name='metric_charts',
prefix=''):
"""Creates and logs Precision Recall curve and Average precision score to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Logs Precision Recall curve and Average precision score to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_precision_recall_auc
neptune.init()
with neptune.create_experiment():
log_precision_recall_auc(y_test, y_test_pred)
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred.shape
) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
avg_precision = sk_metrics.average_precision_score(y_true, y_pred[:, 1])
_exp.log_metric(prefix + 'avg_precision', avg_precision)
fig, ax = plt.subplots()
plt_metrics.plot_precision_recall(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()
def log_confusion_matrix(y_true,
y_pred_class,
experiment=None,
channel_name='metric_charts',
prefix=''):
"""Creates a confusion matrix figure and logs it in Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Log confusion matrix to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_confusion_matrix
neptune.init()
with neptune.create_experiment():
log_confusion_matrix(y_test, y_test_pred[:,1]>0.5)
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred_class.shape
) == 1, 'y_pred_class needs to be 1D class prediction with values 0, 1'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
fig, ax = plt.subplots()
_plot_confusion_matrix(y_true, y_pred_class, ax=ax)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()
def log_results(results, experiment=None, log_plots=True, log_pickle=True):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), convergence plot ('diagnostics' log),
evaluations plot ('diagnostics' log), and objective plot ('diagnostics' log).
Args:
results('scipy.optimize.OptimizeResult'): Results object that is typically an output
| of the function like `skopt.forest_minimize(...)`
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_plots: ('bool'): If True skopt plots will be logged to Neptune.
log_pickle: ('bool'): if True pickled skopt results object will be logged to Neptune.
Examples:
Run skopt training::
...
results = skopt.forest_minimize(objective, space,
base_estimator='ET', n_calls=100, n_random_starts=10)
Initialize Neptune::
import neptune
neptune.init(api_token='ANONYMOUS',
project_qualified_name='shared/showroom')
neptune.create_experiment(name='optuna sweep')
Send best parameters to Neptune::
import neptunecontrib.monitoring.skopt as sk_utils
sk_utils.log_results(results)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1065/logs
"""
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
_log_best_score(results, _exp)
_log_best_parameters(results, _exp)
if log_plots:
_log_plot_convergence(results, _exp)
_log_plot_evaluations(results, _exp)
_log_plot_regret(results, _exp)
_log_plot_objective(results, _exp)
if log_pickle:
_log_results_object(results, _exp)
def log_class_metrics(y_true, y_pred_class, experiment=None, prefix=''):
"""Calculates and logs all class-based metrics to Neptune.
Metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef',
'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value',
'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate'
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Log class metrics to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_class_metrics
neptune.init()
with neptune.create_experiment():
log_class_metrics(y_test, y_test_pred[:,1]>0.5)
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred_class.shape
) == 1, 'y_pred_class needs to be 1D class prediction with values 0, 1'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
scores = _class_metrics(y_true, y_pred_class)
for metric_name, score in scores.items():
_exp.log_metric(prefix + metric_name, score)
def _validate_experiment(experiment):
if experiment is not None:
expect_not_a_run(experiment)
if not isinstance(experiment, neptune.experiments.Experiment):
ValueError('Passed experiment is not Neptune experiment. Create one by using "create_experiment()"')
else:
try:
experiment = neptune.get_experiment()
except neptune.exceptions.NeptuneNoExperimentContextException:
raise neptune.exceptions.NeptuneNoExperimentContextException()
return experiment
def log_prediction_distribution(y_true,
y_pred_pos,
experiment=None,
channel_name='metric_charts',
prefix=''):
"""Generates prediction distribution plot from predictions and true labels.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Plot prediction distribution::
from neptunecontrib.monitoring.metrics import log_prediction_distribution
log_prediction_distribution(y_test, y_test_pred[:, 1])
"""
assert len(
y_pred_pos.shape
) == 1, 'y_pred_pos needs to be 1D prediction for positive class'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
fig, ax = plt.subplots()
_plot_prediction_distribution(y_true, y_pred_pos, ax=ax)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()
def log_brier_loss(y_true, y_pred_pos, experiment=None, prefix=''):
"""Calculates and logs brier loss to Neptune.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Logs Brier score to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_brier_loss
neptune.init()
with neptune.create_experiment():
log_brier_loss(y_test, y_test_pred[:,1])
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred_pos.shape
) == 1, 'y_pred_pos needs to be 1D prediction for positive class'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
brier = sk_metrics.brier_score_loss(y_true, y_pred_pos)
_exp.log_metric(prefix + 'brier_loss', brier)
def log_study_info(study,
experiment=None,
log_study=True,
log_charts=True,
log_optimization_history=False,
log_contour=False,
log_parallel_coordinate=False,
log_slice=False,
params=None):
"""Logs runs results and parameters to neptune.
Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric),
best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts
('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder.
Args:
study('optuna.study.Study'): Optuna study object after training is completed.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
log_study('bool'): Whether optuna study object should be logged as pickle. Default is True.
log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged.
By default all charts are sent.
To not log any charts set log_charts=False.
If you want to log a particular chart change the argument for that chart explicitly.
For example log_charts=False and log_slice=True will log only the slice plot to Neptune.
log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True.
log_contour('bool'): Whether optuna contour plot should be logged. Default is True.
log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True.
log_slice('bool'): Whether optuna slice chart should be logged. Default is True.
params(`list`): List of parameters to be visualized. Default is all parameters.
Examples:
Initialize neptune_monitor::
import neptune
import neptunecontrib.monitoring.optuna as opt_utils
neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME')
neptune.create_experiment(name='optuna sweep')
neptune_callback = opt_utils.NeptuneCallback()
Run Optuna training passing monitor as callback::
...
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100, callbacks=[neptune_callback])
opt_utils.log_study_info(study)
You can explore an example experiment in Neptune:
https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts
"""
import optuna.visualization as vis
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
_exp.log_metric('best_score', study.best_value)
_exp.set_property('best_parameters', study.best_params)
if log_charts:
message = """log_charts argument is depraceted and will be removed in future releases.
Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly.
"""
warnings.warn(message)
log_optimization_history = True
log_contour = True
log_parallel_coordinate = True
log_slice = True
if log_study:
pickle_and_log_artifact(study, 'study.pkl', experiment=_exp)
if log_optimization_history:
log_chart(name='optimization_history',
chart=vis.plot_optimization_history(study),
experiment=_exp)
if log_contour:
log_chart(name='contour',
chart=vis.plot_contour(study, params=params),
experiment=_exp)
if log_parallel_coordinate:
log_chart(name='parallel_coordinate',
chart=vis.plot_parallel_coordinate(study, params=params),
experiment=_exp)
if log_slice:
log_chart(name='slice',
chart=vis.plot_slice(study, params=params),
experiment=_exp)
def __init__(self, experiment=None):
self.exp = experiment if experiment else neptune
expect_not_a_run(self.exp)
def _log_results_object(results, experiment=None):
expect_not_a_run(experiment)
experiment.log_artifact(_export_results_object(results), 'results.pkl')
def __init__(self, experiment=None, prefix=''):
super().__init__()
self._exp = experiment if experiment else neptune
self._prefix = prefix
expect_not_a_run(self._exp)
def _log_plot_evaluations(results, experiment, name='diagnostics'):
expect_not_a_run(experiment)
fig = plt.figure(figsize=(16, 12))
fig = axes2fig(sk_plots.plot_evaluations(results, bins=10), fig=fig)
experiment.log_image(name, fig)
def _log_plot_regret(results, experiment, name='diagnostics'):
expect_not_a_run(experiment)
fig, ax = plt.subplots()
sk_plots.plot_regret(results, ax=ax)
experiment.log_image(name, fig)
def _log_best_parameters(results, experiment):
expect_not_a_run(experiment)
named_params = ([(dimension.name, param) for dimension, param in zip(results.space, results.x)])
experiment.set_property('best_parameters', str(named_params))
def neptune_callback(log_model=True,
log_importance=True,
max_num_features=None,
log_tree=None,
experiment=None,
**kwargs):
"""XGBoost callback for Neptune experiments.
This is XGBoost callback that automatically logs training and evaluation metrics, feature importance chart,
visualized trees and trained Booster to Neptune.
Check Neptune documentation for the `full example <https://docs.neptune.ai/integrations/xgboost.html>`_.
Make sure you created an experiment before you start XGBoost training using ``neptune.create_experiment()``
(`check our docs <https://docs.neptune.ai/api-reference/neptune/projects/index.html
#neptune.projects.Project.create_experiment>`_).
You need to install graphviz and graphviz Python interface for ``log_tree`` feature to work.
Check `Graphviz <https://graphviz.org/download/>`_ and
`Graphviz Python interface <https://graphviz.readthedocs.io/en/stable/manual.html#installation>`_
for installation info.
Integration works with ``xgboost>=1.2.0``.
Tip:
Use this `Google Colab <https://colab.research.google.com//github/neptune-ai/neptune-examples/blob/master/
integrations/xgboost/docs/Neptune-XGBoost.ipynb>`_
run it as a "`neptuner`" user - zero setup, it just works.
Note:
If you use early stopping, make sure to log model, feature importance and trees on your own.
Neptune logs these artifacts only after last iteration, which you may not reach because of early stop.
Args:
log_model (:obj:`bool`, optional, default is ``True``):
| Log booster to Neptune after last boosting iteration.
| If you run xgb.cv, log booster for all folds.
log_importance (:obj:`bool`, optional, default is ``True``):
| Log feature importance to Neptune as image after last boosting iteration.
| Specify number of features using ``max_num_features`` parameter below.
| If you run xgb.cv, log feature importance for each folds' booster.
max_num_features (:obj:`int`, optional, default is ``None``):
| Plot top ``max_num_features`` features on the importance plot.
| If ``None``, plot all features.
log_tree (:obj:`list` of :obj:`int`, optional, default is ``None``):
| Log specified trees to Neptune as images after last boosting iteration.
| If you run xgb.cv, log specified trees for each folds' booster.
| Default is ``None`` - do not log any tree.
experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``):
| For advanced users only. Pass Neptune ``Experiment``
object if you want to control to which experiment data is logged.
| If ``None``, log to currently active, and most recent experiment.
kwargs:
Parametrize XGBoost functions used in this callback:
`xgboost.plot_importance <https://xgboost.readthedocs.io/en/latest/python/python_api.html
?highlight=plot_tree#xgboost.plot_importance>`_
and `xgboost.to_graphviz <https://xgboost.readthedocs.io/en/latest/python/python_api.html
?highlight=plot_tree#xgboost.to_graphviz>`_.
Returns:
:obj:`callback`, function that you can pass directly to the XGBoost callbacks list, for example to the
``xgboost.cv()``
(`see docs <https://xgboost.readthedocs.io/en/latest/python/python_api.html?highlight=plot_tree#xgboost.cv>`_)
or ``XGBClassifier.fit()``
(`check docs <https://xgboost.readthedocs.io/en/latest/python/python_api.html?highlight=plot_tree
#xgboost.XGBClassifier.fit>`_).
Examples:
``xgb.train`` examples
.. code:: python3
# basic usage
xgb.train(param, dtrain, num_round, watchlist,
callbacks=[neptune_callback()])
# do not log model
xgb.train(param, dtrain, num_round, watchlist,
callbacks=[neptune_callback(log_model=False)])
# log top 5 features' importance chart
xgb.train(param, dtrain, num_round, watchlist,
callbacks=[neptune_callback(max_num_features=5)])
``xgb.cv`` examples
.. code:: python3
# log 5 trees per each folds' booster
xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7,
callbacks=neptune_callback(log_tree=[0,1,2,3,4]))
# log only metrics
xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7,
callbacks=[neptune_callback(log_model=False,
log_importance=False,
max_num_features=None,
log_tree=None)])
# log top 3 features per each folds' booster and first tree
xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7,
callbacks=[neptune_callback(log_model=False,
max_num_features=3,
log_tree=[0,])])
``sklearn`` API examples
.. code:: python3
# basic usage with early stopping
xgb.XGBRegressor().fit(X_train, y_train,
early_stopping_rounds=10,
eval_metric=['mae', 'rmse', 'rmsle'],
eval_set=[(X_test, y_test)],
callbacks=[neptune_callback()])
# do not log model
clf = xgb.XGBRegressor()
clf.fit(X_train, y_train,
eval_metric=['mae', 'rmse', 'rmsle'],
eval_set=[(X_test, y_test)],
callbacks=[neptune_callback(log_model=False)])
y_pred = clf.predict(X_test)
# log 8 trees
reg = xgb.XGBRegressor(**params)
reg.fit(X_train, y_train,
eval_metric=['mae', 'rmse', 'rmsle'],
eval_set=[(X_test, y_test)],
callbacks=[neptune_callback(log_tree=[0,1,2,3,4,5,6,7])])
"""
if experiment:
_exp = experiment
else:
try:
neptune.get_experiment()
_exp = neptune
except neptune.exceptions.NeptuneNoExperimentContextException:
raise neptune.exceptions.NeptuneNoExperimentContextException()
expect_not_a_run(_exp)
assert isinstance(log_model, bool),\
'log_model must be bool, got {} instead. Check log_model parameter.'.format(type(log_model))
assert isinstance(log_importance, bool),\
'log_importance must be bool, got {} instead. Check log_importance parameter.'.format(type(log_importance))
if max_num_features is not None:
assert isinstance(max_num_features, int),\
'max_num_features must be int, got {} instead. ' \
'Check max_num_features parameter.'.format(type(max_num_features))
if log_tree is not None:
if isinstance(log_tree, tuple):
log_tree = list(log_tree)
assert isinstance(log_tree, list),\
'log_tree must be list of int, got {} instead. Check log_tree parameter.'.format(type(log_tree))
def callback(env):
# Log metrics after iteration
for item in env.evaluation_result_list:
if len(item) == 2: # train case
_exp.log_metric(item[0], item[1])
if len(item) == 3: # cv case
_exp.log_metric('{}-mean'.format(item[0]), item[1])
_exp.log_metric('{}-std'.format(item[0]), item[2])
# Log booster, end of training
if env.iteration + 1 == env.end_iteration and log_model:
if env.cvfolds: # cv case
for i, cvpack in enumerate(env.cvfolds):
_log_model(cvpack.bst, 'cv-fold-{}-bst.model'.format(i),
_exp)
else: # train case
_log_model(env.model, 'bst.model', _exp)
# Log feature importance, end of training
if env.iteration + 1 == env.end_iteration and log_importance:
if env.cvfolds: # cv case
for i, cvpack in enumerate(env.cvfolds):
_log_importance(cvpack.bst,
max_num_features,
_exp,
title='cv-fold-{}'.format(i),
**kwargs)
else: # train case
_log_importance(env.model, max_num_features, _exp, **kwargs)
# Log trees, end of training
if env.iteration + 1 == env.end_iteration and log_tree:
if env.cvfolds:
for j, cvpack in enumerate(env.cvfolds):
_log_trees(cvpack.bst, log_tree,
'trees-cv-fold-{}'.format(j), _exp, **kwargs)
else:
_log_trees(env.model, log_tree, 'trees', _exp, **kwargs)
return callback
def log_ks_statistic(y_true,
y_pred,
experiment=None,
channel_name='metric_charts',
prefix=''):
"""Creates and logs KS statistics curve and KS statistics score to Neptune.
Kolmogorov-Smirnov statistics chart can be calculated for true positive rates (TPR) and true negative rates (TNR)
for each threshold and plotted on a chart.
The maximum distance from TPR to TNR can be treated as performance metric.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
channel_name(str): name of the neptune channel. Default is 'metric_charts'.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Create and log KS statistics curve and KS statistics score to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_ks_statistic
neptune.init()
with neptune.create_experiment():
log_ks_statistic(y_test, y_test_pred)
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred.shape
) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
res = binary_ks_curve(y_true, y_pred[:, 1])
ks_stat = res[3]
_exp.log_metric(prefix + 'ks_statistic', ks_stat)
fig, ax = plt.subplots()
plt_metrics.plot_ks_statistic(y_true, y_pred, ax=ax)
send_figure(fig, channel_name=prefix + channel_name, experiment=_exp)
plt.close()
def log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, sensitive_attributes,
favorable_label, unfavorable_label,
privileged_groups, unprivileged_groups,
experiment=None, prefix=''):
"""Creates fairness metric charts, calculates fairness classification metrics and logs them to Neptune.
Class-based metrics that are logged: 'true_positive_rate_difference','false_positive_rate_difference',
'false_omission_rate_difference', 'false_discovery_rate_difference', 'error_rate_difference',
'false_positive_rate_ratio', 'false_negative_rate_ratio', 'false_omission_rate_ratio',
'false_discovery_rate_ratio', 'error_rate_ratio', 'average_odds_difference', 'disparate_impact',
'statistical_parity_difference', 'equal_opportunity_difference', 'theil_index',
'between_group_theil_index', 'between_all_groups_theil_index', 'coefficient_of_variation',
'between_group_coefficient_of_variation', 'between_all_groups_coefficient_of_variation',
'generalized_entropy_index', 'between_group_generalized_entropy_index',
'between_all_groups_generalized_entropy_index'
Charts are logged to the 'metric_by_group' channel: 'confusion matrix', 'TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV',
'FDR', 'FOR', 'ACC', 'error_rate', 'selection_rate', 'power', 'precision', 'recall',
'sensitivity', 'specificity'.
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1.
y_pred_score (array-like, shape (n_samples)): Class predictions with values from 0 to 1. Default None.
sensitive_attributes (pandas.DataFrame, shape (n_samples, k)): datafame containing only sensitive columns.
favorable_label (str or int): label that is favorable, brings positive value to a person being classified.
unfavorable_label (str or int): label that is unfavorable, brings positive value to a person being classified.
privileged_groups (dict): dictionary with column names and list of values for those columns that
belong to the privileged groups.
unprivileged_groups (dict): dictionary with column names and list of values for those columns that
belong to the unprivileged groups.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test.
Log metrics and performance curves to Neptune::
import neptune
from neptunecontrib.monitoring.fairness import log_fairness_classification_metrics
neptune.init()
with neptune.create_experiment():
log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, test[['race']],
favorable_label='granted_parole',
unfavorable_label='not_granted_parole',
privileged_groups={'race':['Caucasian']},
privileged_groups={'race':['African-American','Hispanic]},
)
Check out this experiment https://ui.neptune.ai/jakub-czakon/model-fairness/e/MOD-92/logs.
"""
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
bias_info = {'favorable_label': favorable_label,
'unfavorable_label': unfavorable_label,
'protected_columns': sensitive_attributes.columns.tolist()}
privileged_info = _fmt_priveleged_info(privileged_groups, unprivileged_groups)
ground_truth_test = _make_dataset(sensitive_attributes, y_true, **bias_info, **privileged_info)
prediction_test = _make_dataset(sensitive_attributes, y_pred_class, y_pred_score, **bias_info, **privileged_info)
clf_metric = ClassificationMetric(ground_truth_test, prediction_test, **privileged_info)
_log_fairness_metrics(clf_metric, _exp, prefix)
fig = _plot_confusion_matrix_by_group(clf_metric, figsize=(12, 4))
plt.tight_layout()
plt.close()
send_figure(fig, channel_name=prefix + 'metrics_by_group')
group_metrics = ['TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV', 'FDR', 'FOR',
'ACC', 'error_rate', 'selection_rate', 'power',
'precision', 'recall', 'sensitivity', 'specificity']
for metric_name in group_metrics:
fig, ax = plt.subplots(figsize=(12, 8))
_plot_performance_by_group(clf_metric, metric_name, ax)
send_figure(fig, experiment=_exp, channel_name=prefix + 'metrics_by_group')
plt.close()
def log_binary_classification_metrics(y_true,
y_pred,
threshold=0.5,
experiment=None,
prefix=''):
"""Creates metric charts and calculates classification metrics and logs them to Neptune.
Class-based metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score',
'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value',
'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate'
For each class-based metric, a curve with metric/threshold is logged to 'metrics_by_threshold' channel.
Losses that are logged: 'brier_loss', 'log_loss'
Other metrics that are logged: 'roc_auc', 'ks_statistic', 'avg_precision'
Curves that are logged: 'roc_auc', 'precision_recall_curve', 'ks_statistic_curve', 'cumulative_gain_curve',
'lift_curve',
Args:
y_true (array-like, shape (n_samples)): Ground truth (correct) target values.
y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1.
experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None.
threshold (float): Threshold that calculates a class for class-based metrics. Default is 0.5.
prefix(str): Prefix that will be added before metric name when logged to Neptune.
Examples:
Train the model and make predictions on test::
from sklearn.datasets import make_classification
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
X, y = make_classification(n_samples=2000)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
model = RandomForestClassifier()
model.fit(X_train, y_train)
y_test_pred = model.predict_proba(X_test)
Log metrics and performance curves to Neptune::
import neptune
from neptunecontrib.monitoring.metrics import log_binary_classification_metrics
neptune.init()
with neptune.create_experiment():
log_binary_classification_metrics(y_test, y_test_pred, threshold=0.5)
Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs.
"""
assert len(
y_pred.shape
) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it'
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
log_confusion_matrix(y_true,
y_pred[:, 1] > threshold,
experiment=_exp,
prefix=prefix)
log_classification_report(y_true,
y_pred[:, 1] > threshold,
experiment=_exp,
prefix=prefix)
log_class_metrics(y_true,
y_pred[:, 1] > threshold,
experiment=_exp,
prefix=prefix)
log_class_metrics_by_threshold(y_true,
y_pred[:, 1],
experiment=_exp,
prefix=prefix)
log_roc_auc(y_true, y_pred, experiment=_exp, prefix=prefix)
log_precision_recall_auc(y_true, y_pred, experiment=_exp, prefix=prefix)
log_brier_loss(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix)
log_log_loss(y_true, y_pred, experiment=_exp, prefix=prefix)
log_ks_statistic(y_true, y_pred, experiment=_exp, prefix=prefix)
log_cumulative_gain(y_true, y_pred, experiment=_exp, prefix=prefix)
log_lift_curve(y_true, y_pred, experiment=_exp, prefix=prefix)
log_prediction_distribution(y_true,
y_pred[:, 1],
experiment=_exp,
prefix=prefix)
def neptune_monitor(experiment=None, prefix=''):
"""Logs lightGBM learning curves to Neptune.
Goes over the list of metrics and valid_sets passed to the `lgb.train`
object and logs them to a separate channels. For example with 'objective': 'multiclass'
and `valid_names=['train','valid']` there will be 2 channels created:
`train_multiclass_logloss` and `valid_multiclass_logloss`.
Args:
experiment(`neptune.experiments.Experiment`): Neptune experiment.
prefix(str): Prefix that should be added before the `metric_name`
and `valid_name` before logging to the appropriate channel.
Returns:
`func`: Callback function that should be passed to the `callbacks` parameter of
the `lgb.train` function.
Examples:
Prepare dataset::
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.datasets import load_wine
data = load_wine()
X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.1)
lgb_train = lgb.Dataset(X_train, y_train)
lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)
Define model parameters::
params = {'boosting_type': 'gbdt',
'objective': 'multiclass',
'num_class': 3,
'num_leaves': 31,
'learning_rate': 0.05,
'feature_fraction': 0.9
}
Define your Neptune monitor::
monitor = neptune_monitor()
Run `lgb.train` passing `neptune_monitor()` to the `callbacks` parameter::
gbm = lgb.train(params,
lgb_train,
num_boost_round=500,
valid_sets=[lgb_train, lgb_eval],
valid_names=['train','valid'],
callbacks=[monitor],
)
Note:
If you are running a k-fold validation it is a good idea to add the k-fold prefix
and pass it to the `neptune_monitor` function::
prefix='fold_{}'.format(fold_id)
monitor = neptune_monitor(prefix=prefix)
"""
_exp = experiment if experiment else neptune
expect_not_a_run(_exp)
def callback(env):
for name, loss_name, loss_value, _ in env.evaluation_result_list:
channel_name = '{}{}_{}'.format(prefix, name, loss_name)
_exp.send_metric(channel_name, x=env.iteration, y=loss_value)
return callback
