def test_explainer(n_explainer_runs, at_defaults, rf_classifier, explainer,
test_instance_idx):
"""
Convergence test on Adult and Iris datasets.
"""
# fixture returns a fitted AnchorTabular explainer
X_test, explainer, predict_fn, predict_type = explainer
if predict_type == 'proba':
instance_label = np.argmax(predict_fn(
X_test[test_instance_idx, :].reshape(1, -1)),
axis=1)
else:
instance_label = predict_fn(X_test[test_instance_idx, :].reshape(
1, -1))[0]
explainer.instance_label = instance_label
explain_defaults = at_defaults
threshold = explain_defaults['desired_confidence']
n_covered_ex = explain_defaults['n_covered_ex']
for _ in range(n_explainer_runs):
explanation = explainer.explain(X_test[test_instance_idx],
threshold=threshold,
**explain_defaults)
assert explainer.instance_label == instance_label
assert explanation.precision >= threshold
assert explanation.coverage >= 0.05
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
sampler = explainer.samplers[0]
assert sampler.instance_label == instance_label
assert sampler.n_covered_ex == n_covered_ex
def test_explainer(n_explainer_runs, at_defaults, rf_classifier, explainer,
test_instance_idx, caplog):
"""
Convergence test on Adult and Iris datasets.
"""
# fixture returns a fitted AnchorTabular explainer
X_test, explainer, predict_fn, predict_type = explainer
if predict_type == 'proba':
instance_label = np.argmax(predict_fn(
X_test[test_instance_idx, :].reshape(1, -1)),
axis=1)
else:
instance_label = predict_fn(X_test[test_instance_idx, :].reshape(
1, -1))[0]
explainer.instance_label = instance_label
explain_defaults = at_defaults
threshold = explain_defaults['desired_confidence']
n_covered_ex = explain_defaults['n_covered_ex']
run_precisions = []
for _ in range(n_explainer_runs):
explanation = explainer.explain(X_test[test_instance_idx],
threshold=threshold,
**explain_defaults)
assert explainer.instance_label == instance_label
if not "Could not find" in caplog.text:
assert explanation.precision >= threshold
assert explanation.coverage >= 0.01
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
run_precisions.append(explanation.precision)
# check that 80% of runs returned a valid anchor
assert ((np.asarray(run_precisions) >
threshold).sum()) / n_explainer_runs >= 0.80
sampler = explainer.samplers[0]
assert sampler.instance_label == instance_label
assert sampler.n_covered_ex == n_covered_ex
def test_anchor_text(lr_classifier, text, n_punctuation_marks, n_unique_words,
predict_type, anchor, use_similarity_proba, use_unk,
threshold):
# test parameters
num_samples = 100
sample_proba = .5
top_n = 500
temperature = 1.
n_covered_ex = 5 # number of examples where the anchor applies to be returned
# fit and initialise predictor
clf, preprocessor = lr_classifier
predictor = predict_fcn(predict_type, clf, preproc=preprocessor)
# test explainer initialization
explainer = AnchorText(nlp, predictor)
assert explainer.predictor(['book']).shape == (1, )
# setup explainer
perturb_opts = {
'use_similarity_proba': use_similarity_proba,
'sample_proba': sample_proba,
'temperature': temperature,
}
explainer.n_covered_ex = n_covered_ex
explainer.set_words_and_pos(text)
explainer.set_sampler_perturbation(use_unk, perturb_opts, top_n)
explainer.set_data_type(use_unk)
if predict_type == 'proba':
label = np.argmax(predictor([text])[0])
elif predict_type == 'class':
label = predictor([text])[0]
explainer.instance_label = label
assert isinstance(explainer.dtype, str)
assert len(explainer.punctuation) == n_punctuation_marks
assert len(explainer.words) == len(explainer.positions)
# test sampler
cov_true, cov_false, labels, data, coverage, _ = explainer.sampler(
(0, anchor), num_samples)
if not anchor:
assert coverage == -1
if use_similarity_proba and len(
anchor
) > 0: # check that words in present are in the proposed anchor
assert len(anchor) * data.shape[0] == data[:, anchor].sum()
if use_unk:
# get list of unique words
all_words = explainer.words
# unique words = words in text + UNK
assert len(np.unique(all_words)) == n_unique_words
# test explanation
explanation = explainer.explain(
text,
use_unk=use_unk,
threshold=threshold,
use_similarity_proba=use_similarity_proba,
)
assert explanation.precision >= threshold
assert explanation.raw['prediction'].item() == label
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
# check if sampled sentences are not cut short
keys = ['covered_true', 'covered_false']
for i in range(len(explanation.raw['feature'])):
example_dict = explanation.raw['examples'][i]
for k in keys:
for example in example_dict[k]:
# check that we have perturbed the sentences
if use_unk:
assert 'UNK' in example or example.replace(
' ', '') == text.replace(' ', '')
else:
assert 'UNK' not in example
assert example[-1] in ['.', 'K']