def test_explainer(n_explainer_runs, at_defaults, rf_classifier, explainer,
test_instance_idx):
"""
Convergence test on Adult and Iris datasets.
"""
# fixture returns a fitted AnchorTabular explainer
X_test, explainer, predict_fn, predict_type = explainer
if predict_type == 'proba':
instance_label = np.argmax(predict_fn(
X_test[test_instance_idx, :].reshape(1, -1)),
axis=1)
else:
instance_label = predict_fn(X_test[test_instance_idx, :].reshape(
1, -1))[0]
explainer.instance_label = instance_label
explain_defaults = at_defaults
threshold = explain_defaults['desired_confidence']
n_covered_ex = explain_defaults['n_covered_ex']
for _ in range(n_explainer_runs):
explanation = explainer.explain(X_test[test_instance_idx],
threshold=threshold,
**explain_defaults)
assert explainer.instance_label == instance_label
assert explanation.precision >= threshold
assert explanation.coverage >= 0.05
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
sampler = explainer.samplers[0]
assert sampler.instance_label == instance_label
assert sampler.n_covered_ex == n_covered_ex
def test_explainer(n_explainer_runs, at_defaults, rf_classifier, explainer,
test_instance_idx, caplog):
"""
Convergence test on Adult and Iris datasets.
"""
# fixture returns a fitted AnchorTabular explainer
X_test, explainer, predict_fn, predict_type = explainer
if predict_type == 'proba':
instance_label = np.argmax(predict_fn(
X_test[test_instance_idx, :].reshape(1, -1)),
axis=1)
else:
instance_label = predict_fn(X_test[test_instance_idx, :].reshape(
1, -1))[0]
explainer.instance_label = instance_label
explain_defaults = at_defaults
threshold = explain_defaults['desired_confidence']
n_covered_ex = explain_defaults['n_covered_ex']
run_precisions = []
for _ in range(n_explainer_runs):
explanation = explainer.explain(X_test[test_instance_idx],
threshold=threshold,
**explain_defaults)
assert explainer.instance_label == instance_label
if not "Could not find" in caplog.text:
assert explanation.precision >= threshold
assert explanation.coverage >= 0.01
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
run_precisions.append(explanation.precision)
# check that 80% of runs returned a valid anchor
assert ((np.asarray(run_precisions) >
threshold).sum()) / n_explainer_runs >= 0.80
sampler = explainer.samplers[0]
assert sampler.instance_label == instance_label
assert sampler.n_covered_ex == n_covered_ex
def test_anchor_text(lr_classifier, text, n_punctuation_marks, n_unique_words,
predict_type, anchor, use_similarity_proba, use_unk,
threshold):
# test parameters
num_samples = 100
sample_proba = .5
top_n = 500
temperature = 1.
n_covered_ex = 5 # number of examples where the anchor applies to be returned
# fit and initialise predictor
clf, preprocessor = lr_classifier
predictor = predict_fcn(predict_type, clf, preproc=preprocessor)
# test explainer initialization
explainer = AnchorText(nlp, predictor)
assert explainer.predictor(['book']).shape == (1, )
# setup explainer
perturb_opts = {
'use_similarity_proba': use_similarity_proba,
'sample_proba': sample_proba,
'temperature': temperature,
}
explainer.n_covered_ex = n_covered_ex
explainer.set_words_and_pos(text)
explainer.set_sampler_perturbation(use_unk, perturb_opts, top_n)
explainer.set_data_type(use_unk)
if predict_type == 'proba':
label = np.argmax(predictor([text])[0])
elif predict_type == 'class':
label = predictor([text])[0]
explainer.instance_label = label
assert isinstance(explainer.dtype, str)
assert len(explainer.punctuation) == n_punctuation_marks
assert len(explainer.words) == len(explainer.positions)
# test sampler
cov_true, cov_false, labels, data, coverage, _ = explainer.sampler(
(0, anchor), num_samples)
if not anchor:
assert coverage == -1
if use_similarity_proba and len(
anchor
) > 0: # check that words in present are in the proposed anchor
assert len(anchor) * data.shape[0] == data[:, anchor].sum()
if use_unk:
# get list of unique words
all_words = explainer.words
# unique words = words in text + UNK
assert len(np.unique(all_words)) == n_unique_words
# test explanation
explanation = explainer.explain(
text,
use_unk=use_unk,
threshold=threshold,
use_similarity_proba=use_similarity_proba,
)
assert explanation.precision >= threshold
assert explanation.raw['prediction'].item() == label
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR.keys()
# check if sampled sentences are not cut short
keys = ['covered_true', 'covered_false']
for i in range(len(explanation.raw['feature'])):
example_dict = explanation.raw['examples'][i]
for k in keys:
for example in example_dict[k]:
# check that we have perturbed the sentences
if use_unk:
assert 'UNK' in example or example.replace(
' ', '') == text.replace(' ', '')
else:
assert 'UNK' not in example
assert example[-1] in ['.', 'K']
def test_anchor_image(conv_net):
segmentation_fn = 'slic'
segmentation_kwargs = {'n_segments': 10, 'compactness': 10, 'sigma': .5}
image_shape = (28, 28, 1)
p_sample = 0.5 # probability of perturbing a superpixel
num_samples = 10
# img scaling settings
scaling_offset = 260
min_val = 0
max_val = 255
eps = 0.0001 # tolerance for tensor comparisons
n_covered_ex = 3 # nb of examples where the anchor applies that are saved
# define and train model
clf = conv_net
predict_fn = lambda x: clf.predict(x)
explainer = AnchorImage(
predict_fn,
image_shape,
segmentation_fn=segmentation_fn,
segmentation_kwargs=segmentation_kwargs,
)
# test explainer initialization
assert explainer.predictor(np.zeros((1,) + image_shape)).shape == (1,)
assert explainer.custom_segmentation == False
# test sampling and segmentation functions
image = x_train[0]
explainer.instance_label = predict_fn(image[np.newaxis, ...])[0]
explainer.image = image
explainer.n_covered_ex = n_covered_ex
explainer.p_sample = p_sample
segments = explainer.generate_superpixels(image)
explainer.segments = segments
image_preproc = explainer._preprocess_img(image)
explainer.segment_labels = list(np.unique(segments))
superpixels_mask = explainer._choose_superpixels(num_samples=num_samples)
# grayscale image should be replicated across channel dim before segmentation
assert image_preproc.shape[-1] == 3
for channel in range(image_preproc.shape[-1]):
assert (image.squeeze() - image_preproc[..., channel] <= eps).all() == True
# check superpixels mask
assert superpixels_mask.shape[0] == num_samples
assert superpixels_mask.shape[1] == len(list(np.unique(segments)))
assert superpixels_mask.sum(axis=1).any() <= segmentation_kwargs['n_segments']
assert superpixels_mask.any() <= 1
cov_true, cov_false, labels, data, coverage, _ = explainer.sampler((0, ()), num_samples)
assert data.shape[0] == labels.shape[0]
assert data.shape[1] == len(np.unique(segments))
assert coverage == -1
# test explanation
threshold = .95
explanation = explainer.explain(image, threshold=threshold)
if explanation.raw['feature']:
assert explanation.raw['examples'][-1]['covered_true'].shape[0] <= explainer.n_covered_ex
assert explanation.raw['examples'][-1]['covered_false'].shape[0] <= explainer.n_covered_ex
else:
assert not explanation.raw['examples']
assert explanation.anchor.shape == image_shape
assert explanation.precision >= threshold
assert len(np.unique(explanation.segments)) == len(np.unique(segments))
assert explanation.meta.keys() == DEFAULT_META_ANCHOR.keys()
assert explanation.data.keys() == DEFAULT_DATA_ANCHOR_IMG.keys()
# test scaling
fake_img = np.random.random(size=image_shape) + scaling_offset
scaled_img = explainer._scale(fake_img, scale=(min_val, max_val))
assert (scaled_img <= max_val).all()
assert (scaled_img >= min_val).all()