def test_classifier_binary(): mlflow.sklearn.autolog() # use RandomForestClassifier that has method [predict_proba], so that we can test # logging of (1) log_loss and (2) roc_auc_score. model = sklearn.ensemble.RandomForestClassifier(max_depth=2, random_state=0, n_estimators=10) # use binary datasets to cover the test for roc curve & precision recall curve X, y_true = sklearn.datasets.load_breast_cancer(return_X_y=True) with mlflow.start_run() as run: model = fit_model(model, X, y_true, "fit") y_pred = model.predict(X) y_pred_prob = model.predict_proba(X) # For binary classification, y_score only accepts the probability of greater label y_pred_prob_roc = y_pred_prob[:, 1] run_id = run.info.run_id params, metrics, tags, artifacts = get_run_data(run_id) assert params == truncate_dict(stringify_dict_values(model.get_params(deep=True))) expected_metrics = { TRAINING_SCORE: model.score(X, y_true), "training_accuracy_score": sklearn.metrics.accuracy_score(y_true, y_pred), "training_precision_score": sklearn.metrics.precision_score( y_true, y_pred, average="weighted" ), "training_recall_score": sklearn.metrics.recall_score(y_true, y_pred, average="weighted"), "training_f1_score": sklearn.metrics.f1_score(y_true, y_pred, average="weighted"), "training_log_loss": sklearn.metrics.log_loss(y_true, y_pred_prob), } if _is_metric_supported("roc_auc_score"): expected_metrics["training_roc_auc_score"] = sklearn.metrics.roc_auc_score( y_true, y_score=y_pred_prob_roc, average="weighted", multi_class="ovo", ) assert metrics == expected_metrics assert tags == get_expected_class_tags(model) assert MODEL_DIR in artifacts client = mlflow.tracking.MlflowClient() artifacts = [x.path for x in client.list_artifacts(run_id)] plot_names = [] if _is_plotting_supported(): plot_names.extend( [ "{}.png".format("training_confusion_matrix"), "{}.png".format("training_roc_curve"), "{}.png".format("training_precision_recall_curve"), ] ) assert all(x in artifacts for x in plot_names) loaded_model = load_model_by_run_id(run_id) assert_predict_equal(loaded_model, model, X) # verify no figure is open assert len(plt.get_fignums()) == 0
def test_classifier_multi_class(): mlflow.sklearn.autolog() # use RandomForestClassifier that has method [predict_proba], so that we can test # logging of (1) log_loss and (2) roc_auc_score. model = sklearn.ensemble.RandomForestClassifier(max_depth=2, random_state=0, n_estimators=10) # use multi-class datasets to verify that roc curve & precision recall curve care not recorded X, y_true = get_iris() with mlflow.start_run() as run: model = fit_model(model, X, y_true, "fit") y_pred = model.predict(X) y_pred_prob = model.predict_proba(X) run_id = run.info.run_id params, metrics, tags, artifacts = get_run_data(run_id) assert params == truncate_dict(stringify_dict_values(model.get_params(deep=True))) expected_metrics = { TRAINING_SCORE: model.score(X, y_true), "training_accuracy_score": sklearn.metrics.accuracy_score(y_true, y_pred), "training_precision_score": sklearn.metrics.precision_score( y_true, y_pred, average="weighted" ), "training_recall_score": sklearn.metrics.recall_score(y_true, y_pred, average="weighted"), "training_f1_score": sklearn.metrics.f1_score(y_true, y_pred, average="weighted"), "training_log_loss": sklearn.metrics.log_loss(y_true, y_pred_prob), } if _is_metric_supported("roc_auc_score"): expected_metrics["training_roc_auc_score"] = sklearn.metrics.roc_auc_score( y_true, y_score=y_pred_prob, average="weighted", multi_class="ovo", ) assert metrics == expected_metrics assert tags == get_expected_class_tags(model) assert MODEL_DIR in artifacts client = mlflow.tracking.MlflowClient() artifacts = [x.path for x in client.list_artifacts(run_id)] plot_names = [] if _is_plotting_supported(): plot_names = ["{}.png".format("training_confusion_matrix")] assert all(x in artifacts for x in plot_names) loaded_model = load_model_by_run_id(run_id) assert_predict_equal(loaded_model, model, X)
def test_classifier(): mlflow.sklearn.autolog() # use RandomForestClassifier that has method [predict_proba], so that we can test # logging of (1) log_loss and (2) roc_auc_score. model = sklearn.ensemble.RandomForestClassifier(max_depth=2, random_state=0, n_estimators=10) X, y_true = get_iris() with mlflow.start_run() as run: model = fit_model(model, X, y_true, "fit") y_pred = model.predict(X) y_pred_prob = model.predict_proba(X) run_id = run.info.run_id params, metrics, tags, artifacts = get_run_data(run_id) assert params == truncate_dict( stringify_dict_values(model.get_params(deep=True))) expected_metrics = { TRAINING_SCORE: model.score(X, y_true), _METRICS_PREFIX + "accuracy_score": sklearn.metrics.accuracy_score(y_true, y_pred), _METRICS_PREFIX + "precision_score": sklearn.metrics.precision_score(y_true, y_pred, average="weighted"), _METRICS_PREFIX + "recall_score": sklearn.metrics.recall_score(y_true, y_pred, average="weighted"), _METRICS_PREFIX + "f1_score": sklearn.metrics.f1_score(y_true, y_pred, average="weighted"), _METRICS_PREFIX + "log_loss": sklearn.metrics.log_loss(y_true, y_pred_prob), } if _is_metric_supported("roc_auc_score"): expected_metrics[_METRICS_PREFIX + "roc_auc_score"] = sklearn.metrics.roc_auc_score( y_true, y_score=y_pred_prob, average="weighted", multi_class="ovo") assert metrics == expected_metrics assert tags == get_expected_class_tags(model) assert MODEL_DIR in artifacts loaded_model = load_model_by_run_id(run_id) assert_predict_equal(loaded_model, model, X)