class SkorchClassifierTests(unittest.TestCase): def setUp(self): model, X, y = get_skorch_classifier() self.explainer = ClassifierExplainer(model, X, y) def test_preds(self): self.assertIsInstance(self.explainer.preds, np.ndarray) def test_pred_probas(self): self.assertIsInstance(self.explainer.pred_probas(), np.ndarray) def test_permutation_importances(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) def test_shap_base_value(self): self.assertIsInstance(self.explainer.shap_base_value(), (np.floating, float)) def test_shap_values_shape(self): self.assertTrue(self.explainer.get_shap_values_df().shape == ( len(self.explainer), len(self.explainer.merged_cols))) def test_shap_values(self): self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_mean_abs_shap(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_calculate_properties(self): self.explainer.calculate_properties(include_interactions=False) def test_pdp_df(self): self.assertIsInstance(self.explainer.pdp_df("col1"), pd.DataFrame) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) def test_precision_df(self): self.assertIsInstance(self.explainer.get_precision_df(), pd.DataFrame) def test_lift_curve_df(self): self.assertIsInstance(self.explainer.get_liftcurve_df(), pd.DataFrame) def test_prediction_result_df(self): self.assertIsInstance(self.explainer.prediction_result_df(0), pd.DataFrame)
class ClassifierBaseExplainerTestsPipeline(unittest.TestCase): def setUp(self): #X, y = fetch_openml("titanic", version=1, as_frame=True, return_X_y=True) df = pd.read_csv(Path.cwd() / "tests" / "test_assets" / "pipeline_data.csv") X = df[['age', 'fare', 'embarked', 'sex', 'pclass']] y = df['survived'].astype(int) numeric_features = ['age', 'fare'] numeric_transformer = Pipeline( steps=[('imputer', SimpleImputer( strategy='median')), ('scaler', StandardScaler())]) categorical_features = ['embarked', 'sex', 'pclass'] categorical_transformer = Pipeline( steps=[('imputer', SimpleImputer( strategy='most_frequent')), ('ordinal', OrdinalEncoder())]) preprocessor = ColumnTransformer( transformers=[('num', numeric_transformer, numeric_features), ('cat', categorical_transformer, categorical_features)]) # Append classifier to preprocessing pipeline. # Now we have a full prediction pipeline. clf = Pipeline( steps=[('preprocessor', preprocessor), ('classifier', RandomForestClassifier())]) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2) clf.fit(X_train, y_train) self.explainer = ClassifierExplainer(clf, X_test, y_test) def test_columns_ranked_by_shap(self): self.assertIsInstance(self.explainer.columns_ranked_by_shap(), list) def test_permutation_importances(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics_descriptions(), dict) def test_mean_abs_shap_df(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_contrib_df(self): self.assertIsInstance(self.explainer.get_contrib_df(0), pd.DataFrame) self.assertIsInstance( self.explainer.get_contrib_df(X_row=self.explainer.X.iloc[[0]]), pd.DataFrame) def test_shap_base_value(self): self.assertIsInstance(self.explainer.shap_base_value(), (np.floating, float)) def test_shap_values_shape(self): self.assertTrue(self.explainer.get_shap_values_df().shape == ( len(self.explainer), len(self.explainer.merged_cols))) def test_shap_values(self): self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_pdp_df(self): self.assertIsInstance(self.explainer.pdp_df("age"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("sex"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("age", index=0), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("sex", index=0), pd.DataFrame)
class LogisticRegressionTests(unittest.TestCase): def setUp(self): X_train, y_train, X_test, y_test = titanic_survive() train_names, test_names = titanic_names() model = LogisticRegression() model.fit(X_train, y_train) self.explainer = ClassifierExplainer( model, X_test, y_test, shap='linear', cats=['Sex', 'Deck', 'Embarked'], labels=['Not survived', 'Survived'], idxs=test_names) def test_preds(self): self.assertIsInstance(self.explainer.preds, np.ndarray) def test_pred_percentiles(self): self.assertIsInstance(self.explainer.pred_percentiles(), np.ndarray) def test_columns_ranked_by_shap(self): self.assertIsInstance(self.explainer.columns_ranked_by_shap(), list) def test_permutation_importances(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics_descriptions(), dict) def test_mean_abs_shap_df(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_contrib_df(self): self.assertIsInstance(self.explainer.get_contrib_df(0), pd.DataFrame) self.assertIsInstance(self.explainer.get_contrib_df(0, topx=3), pd.DataFrame) def test_shap_base_value(self): self.assertIsInstance(self.explainer.shap_base_value(), (np.floating, float)) def test_shap_values_shape(self): self.assertTrue(self.explainer.get_shap_values_df().shape == ( len(self.explainer), len(self.explainer.merged_cols))) def test_shap_values(self): self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_mean_abs_shap(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_calculate_properties(self): self.explainer.calculate_properties(include_interactions=False) def test_pdp_df(self): self.assertIsInstance(self.explainer.pdp_df("Age"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Sex"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Deck"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Age", index=0), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Sex", index=0), pd.DataFrame) def test_pos_label(self): self.explainer.pos_label = 1 self.explainer.pos_label = "Not survived" self.assertIsInstance(self.explainer.pos_label, int) self.assertIsInstance(self.explainer.pos_label_str, str) self.assertEqual(self.explainer.pos_label, 0) self.assertEqual(self.explainer.pos_label_str, "Not survived") def test_pred_probas(self): self.assertIsInstance(self.explainer.pred_probas(), np.ndarray) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics(cutoff=0.9), dict) def test_precision_df(self): self.assertIsInstance(self.explainer.get_precision_df(), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(multiclass=True), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(quantiles=4), pd.DataFrame) def test_lift_curve_df(self): self.assertIsInstance(self.explainer.get_liftcurve_df(), pd.DataFrame)
class MultiClassClassifierBunchTests(unittest.TestCase): def setUp(self): X_train, y_train, X_test, y_test = titanic_embarked() train_names, test_names = titanic_names() model = RandomForestClassifier(n_estimators=5, max_depth=2) model.fit(X_train, y_train) self.explainer = ClassifierExplainer(model, X_test, y_test, cats=[{'Gender': ['Sex_female', 'Sex_male', 'Sex_nan']}, 'Deck'], idxs=test_names, labels=['Queenstown', 'Southampton', 'Cherbourg']) def test_preds(self): self.assertIsInstance(self.explainer.preds, np.ndarray) def test_pred_percentiles(self): self.assertIsInstance(self.explainer.pred_percentiles(), np.ndarray) def test_columns_ranked_by_shap(self): self.assertIsInstance(self.explainer.columns_ranked_by_shap(), list) def test_permutation_importances(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) def test_X_cats(self): self.assertIsInstance(self.explainer.X_cats, pd.DataFrame) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics_descriptions(), dict) def test_mean_abs_shap_df(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_top_interactions(self): self.assertIsInstance(self.explainer.top_shap_interactions("Age"), list) self.assertIsInstance(self.explainer.top_shap_interactions("Age", topx=4), list) def test_permutation_importances_df(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) self.assertIsInstance(self.explainer.get_permutation_importances_df(topx=3), pd.DataFrame) self.assertIsInstance(self.explainer.get_permutation_importances_df(cutoff=0.01), pd.DataFrame) def test_contrib_df(self): self.assertIsInstance(self.explainer.get_contrib_df(0), pd.DataFrame) self.assertIsInstance(self.explainer.get_contrib_df(0, topx=3), pd.DataFrame) def test_contrib_summary_df(self): self.assertIsInstance(self.explainer.get_contrib_summary_df(0), pd.DataFrame) self.assertIsInstance(self.explainer.get_contrib_summary_df(0, topx=3), pd.DataFrame) self.assertIsInstance(self.explainer.get_contrib_summary_df(0, round=3), pd.DataFrame) def test_shap_base_value(self): self.assertIsInstance(self.explainer.shap_base_value(), (np.floating, float)) def test_shap_values_shape(self): self.assertTrue(self.explainer.get_shap_values_df().shape == (len(self.explainer), len(self.explainer.merged_cols))) def test_shap_values(self): self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_shap_interaction_values(self): self.assertIsInstance(self.explainer.shap_interaction_values(), np.ndarray) def test_mean_abs_shap(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_calculate_properties(self): self.explainer.calculate_properties() def test_shap_interaction_values_by_col(self): self.assertIsInstance(self.explainer.shap_interaction_values_for_col("Age"), np.ndarray) self.assertEqual(self.explainer.shap_interaction_values_for_col("Age").shape, self.explainer.get_shap_values_df().shape) def test_pdp_df(self): self.assertIsInstance(self.explainer.pdp_df("Age"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Gender"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Deck"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Age", index=0), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Gender", index=0), pd.DataFrame) def test_plot_importances(self): fig = self.explainer.plot_importances() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_importances(kind='permutation') self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_importances(topx=3) self.assertIsInstance(fig, go.Figure) def test_plot_interactions(self): fig = self.explainer.plot_interactions_importance("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_interactions_importance("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_interactions_importance("Gender") self.assertIsInstance(fig, go.Figure) def test_plot_contributions(self): fig = self.explainer.plot_contributions(0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_contributions(0, topx=3) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_contributions(0, cutoff=0.05) self.assertIsInstance(fig, go.Figure) def test_plot_shap_detailed(self): fig = self.explainer.plot_importances_detailed() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_importances_detailed(topx=3) self.assertIsInstance(fig, go.Figure) def test_plot_interactions_detailed(self): fig = self.explainer.plot_interactions_detailed("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_interactions_detailed("Age", topx=3) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_interactions_detailed("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_interactions_detailed("Gender") self.assertIsInstance(fig, go.Figure) def test_plot_dependence(self): fig = self.explainer.plot_dependence("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_dependence("Age", "Gender") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_dependence("Age", highlight_index=0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_dependence("Gender", highlight_index=0) self.assertIsInstance(fig, go.Figure) def test_plot_interaction(self): fig = self.explainer.plot_dependence("Gender", "Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_dependence("Age", "Gender") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_dependence("Age", "Gender", highlight_index=0) self.assertIsInstance(fig, go.Figure) def test_plot_pdp(self): fig = self.explainer.plot_pdp("Age") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_pdp("Gender") self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_pdp("Gender", index=0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_pdp("Age", index=0) self.assertIsInstance(fig, go.Figure) def test_pos_label(self): self.explainer.pos_label = 1 self.explainer.pos_label = "Southampton" self.assertIsInstance(self.explainer.pos_label, int) self.assertIsInstance(self.explainer.pos_label_str, str) self.assertEqual(self.explainer.pos_label, 1) self.assertEqual(self.explainer.pos_label_str, "Southampton") def test_pred_probas(self): self.assertIsInstance(self.explainer.pred_probas(), np.ndarray) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics(cutoff=0.9), dict) def test_precision_df(self): self.assertIsInstance(self.explainer.get_precision_df(), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(multiclass=True), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(quantiles=4), pd.DataFrame) def test_lift_curve_df(self): self.assertIsInstance(self.explainer.get_liftcurve_df(), pd.DataFrame) def test_keep_shap_pos_label_only(self): self.explainer.keep_shap_pos_label_only() self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_calculate_properties(self): self.explainer.calculate_properties() def test_plot_precision(self): fig = self.explainer.plot_precision() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_precision(multiclass=True) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_precision(quantiles=10, cutoff=0.5) self.assertIsInstance(fig, go.Figure) def test_plot_cumulative_precision(self): fig = self.explainer.plot_cumulative_precision() self.assertIsInstance(fig, go.Figure) def test_plot_confusion_matrix(self): fig = self.explainer.plot_confusion_matrix(normalized=False, binary=False) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_confusion_matrix(normalized=False, binary=True) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_confusion_matrix(normalized=True, binary=False) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_confusion_matrix(normalized=True, binary=True) self.assertIsInstance(fig, go.Figure) def test_plot_lift_curve(self): fig = self.explainer.plot_lift_curve() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_lift_curve(percentage=True) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_lift_curve(cutoff=0.5) self.assertIsInstance(fig, go.Figure) def test_plot_lift_curve(self): fig = self.explainer.plot_lift_curve() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_lift_curve(percentage=True) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_lift_curve(cutoff=0.5) self.assertIsInstance(fig, go.Figure) def test_plot_classification(self): fig = self.explainer.plot_classification() self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_classification(percentage=True) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_classification(cutoff=0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_classification(cutoff=1) self.assertIsInstance(fig, go.Figure) def test_plot_roc_auc(self): fig = self.explainer.plot_roc_auc(0.5) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_roc_auc(0.0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_roc_auc(1.0) self.assertIsInstance(fig, go.Figure) def test_plot_pr_auc(self): fig = self.explainer.plot_pr_auc(0.5) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_pr_auc(0.0) self.assertIsInstance(fig, go.Figure) fig = self.explainer.plot_pr_auc(1.0) self.assertIsInstance(fig, go.Figure)
class CatBoostClassifierTests(unittest.TestCase): def setUp(self): X_train, y_train, X_test, y_test = titanic_survive() train_names, test_names = titanic_names() model = CatBoostClassifier(iterations=100, learning_rate=0.1, verbose=0) model.fit(X_train, y_train) self.explainer = ClassifierExplainer( model, X_test, y_test, cats=[{'Gender': ['Sex_female', 'Sex_male', 'Sex_nan']}, 'Deck', 'Embarked'], labels=['Not survived', 'Survived'], idxs=test_names) def test_preds(self): self.assertIsInstance(self.explainer.preds, np.ndarray) def test_pred_probas(self): self.assertIsInstance(self.explainer.pred_probas(), np.ndarray) def test_permutation_importances(self): self.assertIsInstance(self.explainer.get_permutation_importances_df(), pd.DataFrame) def test_shap_base_value(self): self.assertIsInstance(self.explainer.shap_base_value(), (np.floating, float)) def test_shap_values_shape(self): self.assertTrue(self.explainer.get_shap_values_df().shape == (len(self.explainer), len(self.explainer.merged_cols))) def test_shap_values(self): self.assertIsInstance(self.explainer.get_shap_values_df(), pd.DataFrame) def test_shap_values_all_probabilities(self): self.assertTrue(self.explainer.shap_base_value() >= 0) self.assertTrue(self.explainer.shap_base_value() <= 1) self.assertTrue(np.all(self.explainer.get_shap_values_df().sum(axis=1) + self.explainer.shap_base_value() >= 0)) self.assertTrue(np.all(self.explainer.get_shap_values_df().sum(axis=1) + self.explainer.shap_base_value() <= 1)) def test_mean_abs_shap(self): self.assertIsInstance(self.explainer.get_mean_abs_shap_df(), pd.DataFrame) def test_calculate_properties(self): self.explainer.calculate_properties(include_interactions=False) def test_pdp_df(self): self.assertIsInstance(self.explainer.pdp_df("Age"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Gender"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Deck"), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Age", index=0), pd.DataFrame) self.assertIsInstance(self.explainer.pdp_df("Gender", index=0), pd.DataFrame) def test_metrics(self): self.assertIsInstance(self.explainer.metrics(), dict) self.assertIsInstance(self.explainer.metrics(cutoff=0.9), dict) def test_precision_df(self): self.assertIsInstance(self.explainer.get_precision_df(), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(multiclass=True), pd.DataFrame) self.assertIsInstance(self.explainer.get_precision_df(quantiles=4), pd.DataFrame) def test_lift_curve_df(self): self.assertIsInstance(self.explainer.get_liftcurve_df(), pd.DataFrame) def test_prediction_result_df(self): self.assertIsInstance(self.explainer.prediction_result_df(0), pd.DataFrame)