def test_regression_datasets(exact_tests_dataset, model):
# in general permutation shap does not behave as predictable as
# kernel shap, even when comparing permutation against kernel SHAP of the
# mainline SHAP package. So these tests assure us that we're doing the
# correct calculations, even if we can't compare directly.
X_train, X_test, y_train, y_test = exact_tests_dataset
mod = model().fit(X_train, y_train)
explainer = cuml.experimental.explainer.PermutationExplainer(
model=mod.predict,
data=X_train)
cu_shap_values = explainer.shap_values(X_test)
exp_v = float(explainer.expected_value)
fx = mod.predict(X_test)
assert (np.sum(cp.asnumpy(cu_shap_values)) - abs(fx - exp_v)) <= 1e-5
skmod = cuml_skl_class_dict[model]().fit(X_train, y_train)
explainer = cuml.experimental.explainer.PermutationExplainer(
model=skmod.predict,
data=X_train)
skl_shap_values = explainer.shap_values(X_test)
exp_v = float(explainer.expected_value)
fx = mod.predict(X_test)
assert (np.sum(cp.asnumpy(skl_shap_values)) - abs(fx - exp_v)) <= 1e-5
def test_exact_regression_datasets(exact_tests_dataset, model):
X_train, X_test, y_train, y_test = exact_tests_dataset
mod = model().fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(model=mod.predict,
data=X_train)
cu_shap_values = explainer.shap_values(X_test)
assert np.allclose(cu_shap_values,
golden_regression_results[model],
rtol=1e-02,
atol=1e-02)
skmod = cuml_skl_class_dict[model]().fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(
model=skmod.predict, data=X_train)
cu_shap_values = explainer.shap_values(X_test)
# since the values were calculated with the cuml models, a little
# looser tolerance in the comparison is expected
assert np.allclose(cu_shap_values,
golden_regression_results[model],
rtol=1e-02,
atol=1e-02)
def test_exact_regression_datasets(exact_tests_dataset, model):
# todo (dd): idx parameter is for repeating the test for a few CI runs
# will be removed before merging
X_train, X_test, y_train, y_test = exact_tests_dataset
mod = model().fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(model=mod.predict,
data=X_train)
cu_shap_values = explainer.shap_values(X_test)
experimental_test_and_log(cu_shap_values, golden_regression_results[model],
mod.predict(X_test),
float(explainer.expected_value))
skmod = cuml_skl_class_dict[model]().fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(
model=skmod.predict, data=X_train)
cu_shap_values = explainer.shap_values(X_test)
# since the values were calculated with the cuml models, a little
# looser tolerance in the comparison is expected
experimental_test_and_log(cu_shap_values, golden_regression_results[model],
mod.predict(X_test),
float(explainer.expected_value))
def test_exact_classification_datasets():
X, y = make_classification(n_samples=101,
n_features=11,
random_state=42,
n_informative=2,
n_classes=2)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=1,
random_state=42)
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
y_train = y_train.astype(np.float32)
y_test = y_test.astype(np.float32)
mod = cuml.SVC(probability=True).fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(
model=mod.predict_proba, data=X_train)
cu_shap_values = explainer.shap_values(X_test)
experimental_test_and_log(cu_shap_values[0],
golden_classification_result[0],
float(mod.predict_proba(X_test)[0][0]),
float(explainer.expected_value[0]),
tolerance=1e-01)
experimental_test_and_log(cu_shap_values[1],
golden_classification_result[1],
float(mod.predict_proba(X_test)[0][1]),
float(explainer.expected_value[1]),
tolerance=1e-01)
mod = sklearn.svm.SVC(probability=True).fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(
model=mod.predict_proba, data=X_train)
cu_shap_values = explainer.shap_values(X_test)
# Some values are very small, which mean our tolerance here needs to be
# a little looser to avoid false positives from comparisons like
# 0.00348627 - 0.00247397. The loose tolerance still tests that the
# distribution of the values matches.
experimental_test_and_log(cu_shap_values[0],
golden_classification_result[0],
float(mod.predict_proba(X_test)[0][0]),
float(explainer.expected_value[0]),
tolerance=1e-01)
experimental_test_and_log(cu_shap_values[1],
golden_classification_result[1],
float(mod.predict_proba(X_test)[0][1]),
float(explainer.expected_value[1]),
tolerance=1e-01)
def test_kernel_housing_dataset(housing_dataset):
X, y, _ = housing_dataset
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25,
random_state=42)
# making all float32 to use gpu predict on random forest
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
y_train = y_train.astype(np.float32)
y_test = y_test.astype(np.float32)
cumodel = cuml.RandomForestRegressor().fit(X_train, y_train)
explainer = cuml.experimental.explainer.KernelExplainer(
model=cumodel.predict, data=X_train[:100], output_type='numpy')
cu_shap_values = explainer.shap_values(X_test[:2])
assert np.allclose(cu_shap_values,
housing_regression_result,
rtol=1e-01,
atol=1e-01)
assert True
def test_exact_classification_datasets():
X, y = make_classification(n_samples=101,
n_features=11,
random_state=42,
n_informative=2,
n_classes=2)
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=1, random_state=42)
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
y_train = y_train.astype(np.float32)
y_test = y_test.astype(np.float32)
mod = cuml.SVC(probability=True).fit(X_train, y_train)
explainer = cuml.experimental.explainer.PermutationExplainer(
model=mod.predict_proba,
data=X_train)
cu_shap_values = explainer.shap_values(X_test)
exp_v = explainer.expected_value
fx = mod.predict_proba(X_test)[0]
assert (np.sum(cp.asnumpy(
cu_shap_values[0])) - abs(fx[0] - exp_v[0])) <= 1e-5
assert (np.sum(cp.asnumpy(
cu_shap_values[1])) - abs(fx[1] - exp_v[1])) <= 1e-5
mod = sklearn.svm.SVC(probability=True).fit(X_train, y_train)
explainer = cuml.experimental.explainer.PermutationExplainer(
model=mod.predict_proba,
data=X_train)
skl_shap_values = explainer.shap_values(X_test)
exp_v = explainer.expected_value
fx = mod.predict_proba(X_test)[0]
assert (np.sum(cp.asnumpy(
skl_shap_values[0])) - abs(fx[0] - exp_v[0])) <= 1e-5
assert (np.sum(cp.asnumpy(
skl_shap_values[1])) - abs(fx[1] - exp_v[1])) <= 1e-5
def test_permutation(exact_tests_dataset):
X_train, X_test, y_train, y_test = exact_tests_dataset
# Train arbitrary model to get some coefficients
mod = cuml.LinearRegression().fit(X_train, y_train)
# Single background and foreground instance
# Gives zero effect to features when they are 'off'
# and the effect of the regression coefficient when they are 'on'
X_background = np.zeros((1, X_train.shape[1]))
X_foreground = np.ones((1, X_train.shape[1]))
explainer = cuml.experimental.explainer.PermutationExplainer(
model=mod.predict,
data=X_background)
shap_values = explainer.shap_values(
X_foreground,
npermutations=5,
)
assert np.allclose(mod.coef_, shap_values, rtol=1e-04, atol=1e-04)
def test_not_shuffled_explanation(exact_tests_dataset):
# in general permutation shap does not behave as predictable as
# kernel shap, even when comparing permutation against kernel SHAP of the
# mainline SHAP package. So these tests assure us that we're doing the
# correct calculations, even if we can't compare directly.
X_train, X_test, y_train, y_test = exact_tests_dataset
mod = cuml.LinearRegression().fit(X_train, y_train)
explainer = cuml.experimental.explainer.PermutationExplainer(
model=mod.predict,
data=X_train)
shap_values = explainer.shap_values(
X_test,
npermutations=1,
testing=True
)
assert np.allclose(shap_values, not_shuffled_shap_values,
rtol=1e-04, atol=1e-04)
def test_kernel_gpu_cpu_shap(dtype, nfeatures, nbackground, model):
X, y = cuml.datasets.make_regression(n_samples=nbackground + 5,
n_features=nfeatures,
noise=0.1)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=5,
random_state=42)
X_train = X_train.astype(dtype)
X_test = X_test.astype(dtype)
y_train = y_train.astype(dtype)
y_test = y_test.astype(dtype)
mod = model().fit(X_train, y_train)
cu_explainer = \
cuml.experimental.explainer.KernelExplainer(model=mod.predict,
data=X_train,
is_gpu_model=True)
cu_shap_values = cu_explainer.shap_values(X_test)
exp_v = cu_explainer.expected_value
fx = mod.predict(X_test)
for test_idx in range(5):
assert (np.sum(cu_shap_values[test_idx]) -
abs(fx[test_idx] - exp_v)) <= 1e-5
if has_shap("0.37"):
import shap
explainer = shap.KernelExplainer(mod.predict, cp.asnumpy(X_train))
shap_values = explainer.shap_values(cp.asnumpy(X_test))
# note that small variances in the l1_regression with larger
# n_features, even among runs of the same explainer can cause this
# test to be flaky, better testing strategy in process.
assert np.allclose(cu_shap_values, shap_values, rtol=1e-01, atol=1e-01)
