def __init__(self, experiment=None, log_study=False, log_charts=False, log_optimization_history=False, log_contour=False, log_parallel_coordinate=False, log_slice=False, params=None): # pylint: disable=W0621 self.exp = experiment if experiment else neptune self.log_study = log_study expect_not_a_run(self.exp) if log_charts: message = """log_charts argument is depraceted and will be removed in future releases. Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly. """ warnings.warn(message) log_optimization_history = True log_contour = True log_parallel_coordinate = True log_slice = True self.log_optimization_history = log_optimization_history self.log_contour = log_contour self.log_parallel_coordinate = log_parallel_coordinate self.log_slice = log_slice self.params = params
def __init__(self, experiment=None, log_checkpoint=True): self._exp = experiment if experiment else neptune expect_not_a_run(self._exp) self.log_checkpoint = log_checkpoint self._iteration = 0
def log_class_metrics_by_threshold(y_true, y_pred_pos, experiment=None, channel_name='metrics_by_threshold', prefix=''): """Creates metric/threshold charts for each metric and logs them to Neptune. Metrics for which charsta re created and logged are: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value', 'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate' Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metrics_by_threshold'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs metric/threshold charts to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_class_metrics_by_threshold neptune.init() with neptune.create_experiment(): log_class_metrics_by_threshold(y_test, y_test_pred[:,1]) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_pos.shape ) == 1, 'y_pred_pos needs to be 1D prediction for positive class' _exp = experiment if experiment else neptune expect_not_a_run(_exp) figs = _plot_class_metrics_by_threshold(y_true, y_pred_pos) for fig in figs: send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def __init__(self, learn=None, experiment=None, prefix=''): self._exp = experiment if experiment else neptune self._prefix = prefix expect_not_a_run(self._exp) if learn is not None: super().__init__(learn)
def _log_plot_objective(results, experiment, name='diagnostics'): try: expect_not_a_run(experiment) fig = plt.figure(figsize=(16, 12)) fig = axes2fig(sk_plots.plot_objective(results), fig=fig) experiment.log_image(name, fig) except Exception as e: print('Could not create the objective chart due to error: {}'.format(e))
def log_precision_recall_auc(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''): """Creates and logs Precision Recall curve and Average precision score to Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs Precision Recall curve and Average precision score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_precision_recall_auc neptune.init() with neptune.create_experiment(): log_precision_recall_auc(y_test, y_test_pred) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune expect_not_a_run(_exp) avg_precision = sk_metrics.average_precision_score(y_true, y_pred[:, 1]) _exp.log_metric(prefix + 'avg_precision', avg_precision) fig, ax = plt.subplots() plt_metrics.plot_precision_recall(y_true, y_pred, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_confusion_matrix(y_true, y_pred_class, experiment=None, channel_name='metric_charts', prefix=''): """Creates a confusion matrix figure and logs it in Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Log confusion matrix to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_confusion_matrix neptune.init() with neptune.create_experiment(): log_confusion_matrix(y_test, y_test_pred[:,1]>0.5) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_class.shape ) == 1, 'y_pred_class needs to be 1D class prediction with values 0, 1' _exp = experiment if experiment else neptune expect_not_a_run(_exp) fig, ax = plt.subplots() _plot_confusion_matrix(y_true, y_pred_class, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_results(results, experiment=None, log_plots=True, log_pickle=True): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), convergence plot ('diagnostics' log), evaluations plot ('diagnostics' log), and objective plot ('diagnostics' log). Args: results('scipy.optimize.OptimizeResult'): Results object that is typically an output | of the function like `skopt.forest_minimize(...)` experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_plots: ('bool'): If True skopt plots will be logged to Neptune. log_pickle: ('bool'): if True pickled skopt results object will be logged to Neptune. Examples: Run skopt training:: ... results = skopt.forest_minimize(objective, space, base_estimator='ET', n_calls=100, n_random_starts=10) Initialize Neptune:: import neptune neptune.init(api_token='ANONYMOUS', project_qualified_name='shared/showroom') neptune.create_experiment(name='optuna sweep') Send best parameters to Neptune:: import neptunecontrib.monitoring.skopt as sk_utils sk_utils.log_results(results) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1065/logs """ _exp = experiment if experiment else neptune expect_not_a_run(_exp) _log_best_score(results, _exp) _log_best_parameters(results, _exp) if log_plots: _log_plot_convergence(results, _exp) _log_plot_evaluations(results, _exp) _log_plot_regret(results, _exp) _log_plot_objective(results, _exp) if log_pickle: _log_results_object(results, _exp)
def log_class_metrics(y_true, y_pred_class, experiment=None, prefix=''): """Calculates and logs all class-based metrics to Neptune. Metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value', 'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate' Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Log class metrics to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_class_metrics neptune.init() with neptune.create_experiment(): log_class_metrics(y_test, y_test_pred[:,1]>0.5) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_class.shape ) == 1, 'y_pred_class needs to be 1D class prediction with values 0, 1' _exp = experiment if experiment else neptune expect_not_a_run(_exp) scores = _class_metrics(y_true, y_pred_class) for metric_name, score in scores.items(): _exp.log_metric(prefix + metric_name, score)
def _validate_experiment(experiment): if experiment is not None: expect_not_a_run(experiment) if not isinstance(experiment, neptune.experiments.Experiment): ValueError('Passed experiment is not Neptune experiment. Create one by using "create_experiment()"') else: try: experiment = neptune.get_experiment() except neptune.exceptions.NeptuneNoExperimentContextException: raise neptune.exceptions.NeptuneNoExperimentContextException() return experiment
def log_prediction_distribution(y_true, y_pred_pos, experiment=None, channel_name='metric_charts', prefix=''): """Generates prediction distribution plot from predictions and true labels. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Plot prediction distribution:: from neptunecontrib.monitoring.metrics import log_prediction_distribution log_prediction_distribution(y_test, y_test_pred[:, 1]) """ assert len( y_pred_pos.shape ) == 1, 'y_pred_pos needs to be 1D prediction for positive class' _exp = experiment if experiment else neptune expect_not_a_run(_exp) fig, ax = plt.subplots() _plot_prediction_distribution(y_true, y_pred_pos, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_brier_loss(y_true, y_pred_pos, experiment=None, prefix=''): """Calculates and logs brier loss to Neptune. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_pos (array-like, shape (n_samples)): Score predictions with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Logs Brier score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_brier_loss neptune.init() with neptune.create_experiment(): log_brier_loss(y_test, y_test_pred[:,1]) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred_pos.shape ) == 1, 'y_pred_pos needs to be 1D prediction for positive class' _exp = experiment if experiment else neptune expect_not_a_run(_exp) brier = sk_metrics.brier_score_loss(y_true, y_pred_pos) _exp.log_metric(prefix + 'brier_loss', brier)
def log_study_info(study, experiment=None, log_study=True, log_charts=True, log_optimization_history=False, log_contour=False, log_parallel_coordinate=False, log_slice=False, params=None): """Logs runs results and parameters to neptune. Logs all hyperparameter optimization results to Neptune. Those include best score ('best_score' metric), best parameters ('best_parameters' property), the study object itself as artifact, and interactive optuna charts ('contour', 'parallel_coordinate', 'slice', 'optimization_history') as artifacts in 'charts' sub folder. Args: study('optuna.study.Study'): Optuna study object after training is completed. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. log_study('bool'): Whether optuna study object should be logged as pickle. Default is True. log_charts('bool'): Deprecated argument. Whether all optuna visualizations charts should be logged. By default all charts are sent. To not log any charts set log_charts=False. If you want to log a particular chart change the argument for that chart explicitly. For example log_charts=False and log_slice=True will log only the slice plot to Neptune. log_optimization_history('bool'): Whether optuna optimization history chart should be logged. Default is True. log_contour('bool'): Whether optuna contour plot should be logged. Default is True. log_parallel_coordinate('bool'): Whether optuna parallel coordinate plot should be logged. Default is True. log_slice('bool'): Whether optuna slice chart should be logged. Default is True. params(`list`): List of parameters to be visualized. Default is all parameters. Examples: Initialize neptune_monitor:: import neptune import neptunecontrib.monitoring.optuna as opt_utils neptune.init(project_qualified_name='USER_NAME/PROJECT_NAME') neptune.create_experiment(name='optuna sweep') neptune_callback = opt_utils.NeptuneCallback() Run Optuna training passing monitor as callback:: ... study = optuna.create_study(direction='maximize') study.optimize(objective, n_trials=100, callbacks=[neptune_callback]) opt_utils.log_study_info(study) You can explore an example experiment in Neptune: https://ui.neptune.ai/o/shared/org/showroom/e/SHOW-1016/artifacts """ import optuna.visualization as vis _exp = experiment if experiment else neptune expect_not_a_run(_exp) _exp.log_metric('best_score', study.best_value) _exp.set_property('best_parameters', study.best_params) if log_charts: message = """log_charts argument is depraceted and will be removed in future releases. Please use log_optimization_history, log_contour, log_parallel_coordinate, log_slice, arguments explicitly. """ warnings.warn(message) log_optimization_history = True log_contour = True log_parallel_coordinate = True log_slice = True if log_study: pickle_and_log_artifact(study, 'study.pkl', experiment=_exp) if log_optimization_history: log_chart(name='optimization_history', chart=vis.plot_optimization_history(study), experiment=_exp) if log_contour: log_chart(name='contour', chart=vis.plot_contour(study, params=params), experiment=_exp) if log_parallel_coordinate: log_chart(name='parallel_coordinate', chart=vis.plot_parallel_coordinate(study, params=params), experiment=_exp) if log_slice: log_chart(name='slice', chart=vis.plot_slice(study, params=params), experiment=_exp)
def __init__(self, experiment=None): self.exp = experiment if experiment else neptune expect_not_a_run(self.exp)
def _log_results_object(results, experiment=None): expect_not_a_run(experiment) experiment.log_artifact(_export_results_object(results), 'results.pkl')
def __init__(self, experiment=None, prefix=''): super().__init__() self._exp = experiment if experiment else neptune self._prefix = prefix expect_not_a_run(self._exp)
def _log_plot_evaluations(results, experiment, name='diagnostics'): expect_not_a_run(experiment) fig = plt.figure(figsize=(16, 12)) fig = axes2fig(sk_plots.plot_evaluations(results, bins=10), fig=fig) experiment.log_image(name, fig)
def _log_plot_regret(results, experiment, name='diagnostics'): expect_not_a_run(experiment) fig, ax = plt.subplots() sk_plots.plot_regret(results, ax=ax) experiment.log_image(name, fig)
def _log_best_parameters(results, experiment): expect_not_a_run(experiment) named_params = ([(dimension.name, param) for dimension, param in zip(results.space, results.x)]) experiment.set_property('best_parameters', str(named_params))
def neptune_callback(log_model=True, log_importance=True, max_num_features=None, log_tree=None, experiment=None, **kwargs): """XGBoost callback for Neptune experiments. This is XGBoost callback that automatically logs training and evaluation metrics, feature importance chart, visualized trees and trained Booster to Neptune. Check Neptune documentation for the `full example <https://docs.neptune.ai/integrations/xgboost.html>`_. Make sure you created an experiment before you start XGBoost training using ``neptune.create_experiment()`` (`check our docs <https://docs.neptune.ai/api-reference/neptune/projects/index.html #neptune.projects.Project.create_experiment>`_). You need to install graphviz and graphviz Python interface for ``log_tree`` feature to work. Check `Graphviz <https://graphviz.org/download/>`_ and `Graphviz Python interface <https://graphviz.readthedocs.io/en/stable/manual.html#installation>`_ for installation info. Integration works with ``xgboost>=1.2.0``. Tip: Use this `Google Colab <https://colab.research.google.com//github/neptune-ai/neptune-examples/blob/master/ integrations/xgboost/docs/Neptune-XGBoost.ipynb>`_ run it as a "`neptuner`" user - zero setup, it just works. Note: If you use early stopping, make sure to log model, feature importance and trees on your own. Neptune logs these artifacts only after last iteration, which you may not reach because of early stop. Args: log_model (:obj:`bool`, optional, default is ``True``): | Log booster to Neptune after last boosting iteration. | If you run xgb.cv, log booster for all folds. log_importance (:obj:`bool`, optional, default is ``True``): | Log feature importance to Neptune as image after last boosting iteration. | Specify number of features using ``max_num_features`` parameter below. | If you run xgb.cv, log feature importance for each folds' booster. max_num_features (:obj:`int`, optional, default is ``None``): | Plot top ``max_num_features`` features on the importance plot. | If ``None``, plot all features. log_tree (:obj:`list` of :obj:`int`, optional, default is ``None``): | Log specified trees to Neptune as images after last boosting iteration. | If you run xgb.cv, log specified trees for each folds' booster. | Default is ``None`` - do not log any tree. experiment (:obj:`neptune.experiments.Experiment`, optional, default is ``None``): | For advanced users only. Pass Neptune ``Experiment`` object if you want to control to which experiment data is logged. | If ``None``, log to currently active, and most recent experiment. kwargs: Parametrize XGBoost functions used in this callback: `xgboost.plot_importance <https://xgboost.readthedocs.io/en/latest/python/python_api.html ?highlight=plot_tree#xgboost.plot_importance>`_ and `xgboost.to_graphviz <https://xgboost.readthedocs.io/en/latest/python/python_api.html ?highlight=plot_tree#xgboost.to_graphviz>`_. Returns: :obj:`callback`, function that you can pass directly to the XGBoost callbacks list, for example to the ``xgboost.cv()`` (`see docs <https://xgboost.readthedocs.io/en/latest/python/python_api.html?highlight=plot_tree#xgboost.cv>`_) or ``XGBClassifier.fit()`` (`check docs <https://xgboost.readthedocs.io/en/latest/python/python_api.html?highlight=plot_tree #xgboost.XGBClassifier.fit>`_). Examples: ``xgb.train`` examples .. code:: python3 # basic usage xgb.train(param, dtrain, num_round, watchlist, callbacks=[neptune_callback()]) # do not log model xgb.train(param, dtrain, num_round, watchlist, callbacks=[neptune_callback(log_model=False)]) # log top 5 features' importance chart xgb.train(param, dtrain, num_round, watchlist, callbacks=[neptune_callback(max_num_features=5)]) ``xgb.cv`` examples .. code:: python3 # log 5 trees per each folds' booster xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7, callbacks=neptune_callback(log_tree=[0,1,2,3,4])) # log only metrics xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7, callbacks=[neptune_callback(log_model=False, log_importance=False, max_num_features=None, log_tree=None)]) # log top 3 features per each folds' booster and first tree xgb.cv(param, dtrain, num_boost_round=num_round, nfold=7, callbacks=[neptune_callback(log_model=False, max_num_features=3, log_tree=[0,])]) ``sklearn`` API examples .. code:: python3 # basic usage with early stopping xgb.XGBRegressor().fit(X_train, y_train, early_stopping_rounds=10, eval_metric=['mae', 'rmse', 'rmsle'], eval_set=[(X_test, y_test)], callbacks=[neptune_callback()]) # do not log model clf = xgb.XGBRegressor() clf.fit(X_train, y_train, eval_metric=['mae', 'rmse', 'rmsle'], eval_set=[(X_test, y_test)], callbacks=[neptune_callback(log_model=False)]) y_pred = clf.predict(X_test) # log 8 trees reg = xgb.XGBRegressor(**params) reg.fit(X_train, y_train, eval_metric=['mae', 'rmse', 'rmsle'], eval_set=[(X_test, y_test)], callbacks=[neptune_callback(log_tree=[0,1,2,3,4,5,6,7])]) """ if experiment: _exp = experiment else: try: neptune.get_experiment() _exp = neptune except neptune.exceptions.NeptuneNoExperimentContextException: raise neptune.exceptions.NeptuneNoExperimentContextException() expect_not_a_run(_exp) assert isinstance(log_model, bool),\ 'log_model must be bool, got {} instead. Check log_model parameter.'.format(type(log_model)) assert isinstance(log_importance, bool),\ 'log_importance must be bool, got {} instead. Check log_importance parameter.'.format(type(log_importance)) if max_num_features is not None: assert isinstance(max_num_features, int),\ 'max_num_features must be int, got {} instead. ' \ 'Check max_num_features parameter.'.format(type(max_num_features)) if log_tree is not None: if isinstance(log_tree, tuple): log_tree = list(log_tree) assert isinstance(log_tree, list),\ 'log_tree must be list of int, got {} instead. Check log_tree parameter.'.format(type(log_tree)) def callback(env): # Log metrics after iteration for item in env.evaluation_result_list: if len(item) == 2: # train case _exp.log_metric(item[0], item[1]) if len(item) == 3: # cv case _exp.log_metric('{}-mean'.format(item[0]), item[1]) _exp.log_metric('{}-std'.format(item[0]), item[2]) # Log booster, end of training if env.iteration + 1 == env.end_iteration and log_model: if env.cvfolds: # cv case for i, cvpack in enumerate(env.cvfolds): _log_model(cvpack.bst, 'cv-fold-{}-bst.model'.format(i), _exp) else: # train case _log_model(env.model, 'bst.model', _exp) # Log feature importance, end of training if env.iteration + 1 == env.end_iteration and log_importance: if env.cvfolds: # cv case for i, cvpack in enumerate(env.cvfolds): _log_importance(cvpack.bst, max_num_features, _exp, title='cv-fold-{}'.format(i), **kwargs) else: # train case _log_importance(env.model, max_num_features, _exp, **kwargs) # Log trees, end of training if env.iteration + 1 == env.end_iteration and log_tree: if env.cvfolds: for j, cvpack in enumerate(env.cvfolds): _log_trees(cvpack.bst, log_tree, 'trees-cv-fold-{}'.format(j), _exp, **kwargs) else: _log_trees(env.model, log_tree, 'trees', _exp, **kwargs) return callback
def log_ks_statistic(y_true, y_pred, experiment=None, channel_name='metric_charts', prefix=''): """Creates and logs KS statistics curve and KS statistics score to Neptune. Kolmogorov-Smirnov statistics chart can be calculated for true positive rates (TPR) and true negative rates (TNR) for each threshold and plotted on a chart. The maximum distance from TPR to TNR can be treated as performance metric. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. channel_name(str): name of the neptune channel. Default is 'metric_charts'. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Create and log KS statistics curve and KS statistics score to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_ks_statistic neptune.init() with neptune.create_experiment(): log_ks_statistic(y_test, y_test_pred) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune expect_not_a_run(_exp) res = binary_ks_curve(y_true, y_pred[:, 1]) ks_stat = res[3] _exp.log_metric(prefix + 'ks_statistic', ks_stat) fig, ax = plt.subplots() plt_metrics.plot_ks_statistic(y_true, y_pred, ax=ax) send_figure(fig, channel_name=prefix + channel_name, experiment=_exp) plt.close()
def log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, sensitive_attributes, favorable_label, unfavorable_label, privileged_groups, unprivileged_groups, experiment=None, prefix=''): """Creates fairness metric charts, calculates fairness classification metrics and logs them to Neptune. Class-based metrics that are logged: 'true_positive_rate_difference','false_positive_rate_difference', 'false_omission_rate_difference', 'false_discovery_rate_difference', 'error_rate_difference', 'false_positive_rate_ratio', 'false_negative_rate_ratio', 'false_omission_rate_ratio', 'false_discovery_rate_ratio', 'error_rate_ratio', 'average_odds_difference', 'disparate_impact', 'statistical_parity_difference', 'equal_opportunity_difference', 'theil_index', 'between_group_theil_index', 'between_all_groups_theil_index', 'coefficient_of_variation', 'between_group_coefficient_of_variation', 'between_all_groups_coefficient_of_variation', 'generalized_entropy_index', 'between_group_generalized_entropy_index', 'between_all_groups_generalized_entropy_index' Charts are logged to the 'metric_by_group' channel: 'confusion matrix', 'TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV', 'FDR', 'FOR', 'ACC', 'error_rate', 'selection_rate', 'power', 'precision', 'recall', 'sensitivity', 'specificity'. Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred_class (array-like, shape (n_samples)): Class predictions with values 0 or 1. y_pred_score (array-like, shape (n_samples)): Class predictions with values from 0 to 1. Default None. sensitive_attributes (pandas.DataFrame, shape (n_samples, k)): datafame containing only sensitive columns. favorable_label (str or int): label that is favorable, brings positive value to a person being classified. unfavorable_label (str or int): label that is unfavorable, brings positive value to a person being classified. privileged_groups (dict): dictionary with column names and list of values for those columns that belong to the privileged groups. unprivileged_groups (dict): dictionary with column names and list of values for those columns that belong to the unprivileged groups. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test. Log metrics and performance curves to Neptune:: import neptune from neptunecontrib.monitoring.fairness import log_fairness_classification_metrics neptune.init() with neptune.create_experiment(): log_fairness_classification_metrics(y_true, y_pred_class, y_pred_score, test[['race']], favorable_label='granted_parole', unfavorable_label='not_granted_parole', privileged_groups={'race':['Caucasian']}, privileged_groups={'race':['African-American','Hispanic]}, ) Check out this experiment https://ui.neptune.ai/jakub-czakon/model-fairness/e/MOD-92/logs. """ _exp = experiment if experiment else neptune expect_not_a_run(_exp) bias_info = {'favorable_label': favorable_label, 'unfavorable_label': unfavorable_label, 'protected_columns': sensitive_attributes.columns.tolist()} privileged_info = _fmt_priveleged_info(privileged_groups, unprivileged_groups) ground_truth_test = _make_dataset(sensitive_attributes, y_true, **bias_info, **privileged_info) prediction_test = _make_dataset(sensitive_attributes, y_pred_class, y_pred_score, **bias_info, **privileged_info) clf_metric = ClassificationMetric(ground_truth_test, prediction_test, **privileged_info) _log_fairness_metrics(clf_metric, _exp, prefix) fig = _plot_confusion_matrix_by_group(clf_metric, figsize=(12, 4)) plt.tight_layout() plt.close() send_figure(fig, channel_name=prefix + 'metrics_by_group') group_metrics = ['TPR', 'TNR', 'FPR', 'FNR', 'PPV', 'NPV', 'FDR', 'FOR', 'ACC', 'error_rate', 'selection_rate', 'power', 'precision', 'recall', 'sensitivity', 'specificity'] for metric_name in group_metrics: fig, ax = plt.subplots(figsize=(12, 8)) _plot_performance_by_group(clf_metric, metric_name, ax) send_figure(fig, experiment=_exp, channel_name=prefix + 'metrics_by_group') plt.close()
def log_binary_classification_metrics(y_true, y_pred, threshold=0.5, experiment=None, prefix=''): """Creates metric charts and calculates classification metrics and logs them to Neptune. Class-based metrics that are logged: 'accuracy', 'precision', 'recall', 'f1_score', 'f2_score', 'matthews_corrcoef', 'cohen_kappa', 'true_positive_rate', 'true_negative_rate', 'positive_predictive_value', 'negative_predictive_value', 'false_positive_rate', 'false_negative_rate', 'false_discovery_rate' For each class-based metric, a curve with metric/threshold is logged to 'metrics_by_threshold' channel. Losses that are logged: 'brier_loss', 'log_loss' Other metrics that are logged: 'roc_auc', 'ks_statistic', 'avg_precision' Curves that are logged: 'roc_auc', 'precision_recall_curve', 'ks_statistic_curve', 'cumulative_gain_curve', 'lift_curve', Args: y_true (array-like, shape (n_samples)): Ground truth (correct) target values. y_pred (array-like, shape (n_samples, 2)): Predictions for classes 0 and 1 with values from 0 to 1. experiment(`neptune.experiments.Experiment`): Neptune experiment. Default is None. threshold (float): Threshold that calculates a class for class-based metrics. Default is 0.5. prefix(str): Prefix that will be added before metric name when logged to Neptune. Examples: Train the model and make predictions on test:: from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import train_test_split from sklearn.metrics import classification_report X, y = make_classification(n_samples=2000) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) model = RandomForestClassifier() model.fit(X_train, y_train) y_test_pred = model.predict_proba(X_test) Log metrics and performance curves to Neptune:: import neptune from neptunecontrib.monitoring.metrics import log_binary_classification_metrics neptune.init() with neptune.create_experiment(): log_binary_classification_metrics(y_test, y_test_pred, threshold=0.5) Check out this experiment https://ui.neptune.ai/o/neptune-ai/org/binary-classification-metrics/e/BIN-101/logs. """ assert len( y_pred.shape ) == 2, 'y_pred needs to be (n_samples, 2), use expand_prediction helper to format it' _exp = experiment if experiment else neptune expect_not_a_run(_exp) log_confusion_matrix(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix) log_classification_report(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix) log_class_metrics(y_true, y_pred[:, 1] > threshold, experiment=_exp, prefix=prefix) log_class_metrics_by_threshold(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix) log_roc_auc(y_true, y_pred, experiment=_exp, prefix=prefix) log_precision_recall_auc(y_true, y_pred, experiment=_exp, prefix=prefix) log_brier_loss(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix) log_log_loss(y_true, y_pred, experiment=_exp, prefix=prefix) log_ks_statistic(y_true, y_pred, experiment=_exp, prefix=prefix) log_cumulative_gain(y_true, y_pred, experiment=_exp, prefix=prefix) log_lift_curve(y_true, y_pred, experiment=_exp, prefix=prefix) log_prediction_distribution(y_true, y_pred[:, 1], experiment=_exp, prefix=prefix)
def neptune_monitor(experiment=None, prefix=''): """Logs lightGBM learning curves to Neptune. Goes over the list of metrics and valid_sets passed to the `lgb.train` object and logs them to a separate channels. For example with 'objective': 'multiclass' and `valid_names=['train','valid']` there will be 2 channels created: `train_multiclass_logloss` and `valid_multiclass_logloss`. Args: experiment(`neptune.experiments.Experiment`): Neptune experiment. prefix(str): Prefix that should be added before the `metric_name` and `valid_name` before logging to the appropriate channel. Returns: `func`: Callback function that should be passed to the `callbacks` parameter of the `lgb.train` function. Examples: Prepare dataset:: import lightgbm as lgb from sklearn.model_selection import train_test_split from sklearn.datasets import load_wine data = load_wine() X_train, X_test, y_train, y_test = train_test_split(data.data, data.target, test_size=0.1) lgb_train = lgb.Dataset(X_train, y_train) lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) Define model parameters:: params = {'boosting_type': 'gbdt', 'objective': 'multiclass', 'num_class': 3, 'num_leaves': 31, 'learning_rate': 0.05, 'feature_fraction': 0.9 } Define your Neptune monitor:: monitor = neptune_monitor() Run `lgb.train` passing `neptune_monitor()` to the `callbacks` parameter:: gbm = lgb.train(params, lgb_train, num_boost_round=500, valid_sets=[lgb_train, lgb_eval], valid_names=['train','valid'], callbacks=[monitor], ) Note: If you are running a k-fold validation it is a good idea to add the k-fold prefix and pass it to the `neptune_monitor` function:: prefix='fold_{}'.format(fold_id) monitor = neptune_monitor(prefix=prefix) """ _exp = experiment if experiment else neptune expect_not_a_run(_exp) def callback(env): for name, loss_name, loss_value, _ in env.evaluation_result_list: channel_name = '{}{}_{}'.format(prefix, name, loss_name) _exp.send_metric(channel_name, x=env.iteration, y=loss_value) return callback