def test_ale_variance(self):
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
ale = explainer.ale(features=self.X.columns, n_bins=10)
ale_var_results = explainer.ale_variance(ale)
with self.assertRaises(Exception) as ex_1:
ale_var_results = explainer.ale_variance(ale=np.array([0, 0]))
except_msg_1 = """
ale must be an xarray.Dataset,
perferably generated by InterpretToolkit.ale
to be formatted correctly
"""
self.assertEqual(ex_1.exception.args[0], except_msg_1)
with self.assertRaises(Exception) as ex_2:
ale_var_results = explainer.ale_variance(
ale, estimator_names=[self.lr_estimator_name, 'Fake'])
except_msg_2 = 'ale does not contain values for all the estimator names given!'
self.assertEqual(ex_2.exception.args[0], except_msg_2)
def test_correct_rankings(self):
# rankings are correct for simple case (for multi-pass, single-pass, and ale_variance)
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
results = explainer.permutation_importance(n_vars=len(self.X.columns),
evaluation_fn='mse',
n_permute=10)
ale = explainer.ale(features=self.X.columns, n_bins=10)
ale_var_results = explainer.ale_variance(
ale, estimator_names=self.lr_estimator_name)
true_rankings = np.array(['X_1', 'X_2', 'X_3', 'X_4', 'X_5'])
np.testing.assert_array_equal(
results[f'multipass_rankings__{self.lr_estimator_name}'].values,
true_rankings)
np.testing.assert_array_equal(
results[f'singlepass_rankings__{self.lr_estimator_name}'].values,
true_rankings)
np.testing.assert_array_equal(
ale_var_results[f'ale_variance_rankings__{self.lr_estimator_name}']
.values, true_rankings)
def test_estimator_has_been_fit(self):
# estimators must be fit!
with self.assertRaises(Exception) as ex:
pymint.InterpretToolkit(estimators=('Random Forest',
RandomForestRegressor()),
X=self.X,
y=self.y)
except_msg = "One or more of the estimators given has NOT been fit!"
self.assertEqual(ex.exception.args[0], except_msg)
def test_bad_feature_names_exception(self):
feature = 'bad_feature'
explainer = pymint.InterpretToolkit(estimators=self.estimators[0],
X=self.X_clf,
y=self.y_clf)
with self.assertRaises(KeyError) as ex:
explainer.ale(features=feature)
except_msg = f"'{feature}' is not a valid feature."
self.assertEqual(ex.exception.args[0], except_msg)
def test_results_shape(self):
# Bootstrap has correct shape
feature = 'X_1'
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
results = explainer.ale(features=feature, n_bins=10, n_bootstrap=5)
ydata = results[f'{feature}__{self.lr_estimator_name}__ale'].values
self.assertEqual(ydata.shape, (5, 10))
def test_X_and_feature_names(self):
# Feature names must be provided if X is an numpy.array.
with self.assertRaises(Exception) as ex:
pymint.InterpretToolkit(
estimators=self.estimators[0],
X=self.X.values,
y=self.y,
feature_names=None,
)
except_msg = "Feature names must be specified if using NumPy array."
self.assertEqual(ex.exception.args[0], except_msg)
def test_xdata(self):
# Bin values are correct.
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
feature = 'X_1'
results = explainer.ale(features=feature, n_bins=5)
xdata = results[f'{feature}__bin_values'].values
self.assertCountEqual(
np.round(xdata, 8),
[0.09082125, 0.27714732, 0.48378955, 0.6950751, 0.89978646])
def test_bad_evaluation_fn(self):
# Make sure the metrics are correct
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
available_scores = ["auc", "auprc", "bss", "mse", "norm_aupdc"]
with self.assertRaises(Exception) as ex:
explainer.permutation_importance(n_vars=len(self.X.columns),
evaluation_fn='bad')
except_msg = f"evaluation_fn is not set! Available options are {available_scores}"
self.assertEqual(ex.exception.args[0], except_msg)
def test_pd_simple(self):
# PD is correct for a simple case
# The coefficient of the PD curves must
# match that of the actual coefficient.
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
feature = 'X_1'
results = explainer.pd(features=feature, n_bins=5)
lr = LinearRegression()
lr.fit(
results[f'{feature}__bin_values'].values.reshape(-1, 1),
results[f'{feature}__{self.lr_estimator_name}__pd'].values[0, :])
self.assertAlmostEqual(lr.coef_[0], self.weights[0])
def test_estimator_output(self):
estimator_output = 'regression'
available_options = ["raw", "probability"]
with self.assertRaises(Exception) as ex:
pymint.InterpretToolkit(
estimators=self.estimators[0],
X=self.X,
y=self.y,
estimator_output=estimator_output,
)
except_msg = f"""
{estimator_output} is not an accepted options.
The available options are {available_options}.
Check for syntax errors!
"""
self.assertEqual(ex.exception.args[0], except_msg)
def test_too_many_bins(self):
explainer = pymint.InterpretToolkit(estimators=self.estimators[0],
X=self.X_clf,
y=self.y_clf)
n_bins = 100
with self.assertRaises(ValueError) as ex:
explainer.ale(
features=['temp2m'],
subsample=100,
n_bins=n_bins,
)
except_msg = f"""
The value of n_bins ({n_bins}) is likely too
high relative to the sample size of the data. Either increase
the data size (if using subsample) or use less bins.
"""
self.assertEqual(ex.exception.args[0], except_msg)
def test_shape(self):
# Shape is correct (with bootstrapping)
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
n_vars = 3
n_permute = 8
results = explainer.permutation_importance(n_vars=n_vars,
evaluation_fn='mse',
n_permute=n_permute)
# shape should be (n_vars_multipass, n_permute)
self.assertEqual(
results[f'multipass_scores__{self.lr_estimator_name}'].values.
shape, (n_vars, n_permute))
# shape should be (n_vars_singlepass, n_permute)
self.assertEqual(
results[f'singlepass_scores__{self.lr_estimator_name}'].values.
shape, (len(self.X.columns), n_permute))
def test_1d_plot(self):
# Make sure the plot data is correct.
feature = 'X_1'
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
results = explainer.ale(features=feature, n_bins=30, n_bootstrap=1)
ydata = results[f'{feature}__{self.lr_estimator_name}__ale'].values[
0, :]
xdata = results[f'{feature}__bin_values'].values
fig, ax = explainer.plot_ale(ale=results, features=feature)
## effect line
eff_plt_data = ax.lines[0].get_xydata()
# the x values should be the bins
np.testing.assert_array_equal(eff_plt_data[:, 0], xdata)
# the y values should be the effect
np.testing.assert_array_equal(eff_plt_data[:, 1], ydata)
def test_custom_evaluation_fn(self):
# scoring_strategy exception for custom evaluation funcs
explainer = pymint.InterpretToolkit(estimators=(self.lr_estimator_name,
self.lr),
X=self.X,
y=self.y)
available_scores = ["auc", "auprc", "bss", "mse", "norm_aupdc"]
with self.assertRaises(Exception) as ex:
explainer.permutation_importance(
n_vars=len(self.X.columns),
evaluation_fn=roc_auc_score,
)
except_msg = """
The scoring_strategy argument is None! If you are using a non-default evaluation_fn
then scoring_strategy must be set! If the metric is positively-oriented (a higher value is better),
then set scoring_strategy = "argmin_of_mean" and if it is negatively-oriented-
(a lower value is better), then set scoring_strategy = "argmax_of_mean"
"""
self.assertEqual(ex.exception.args[0], except_msg)
