def get_decision_path_explanation(estimator, doc, vec, vectorized,
x, feature_names,
feature_filter, feature_re, top,
original_display_names,
target_names, targets, top_targets,
is_regression, is_multiclass, proba,
get_score_weights):
# type: (...) -> Explanation
display_names = get_target_display_names(
original_display_names, target_names, targets, top_targets, proba)
flt_feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
def get_top_features(weights, scale=1.0):
return get_top_features_filtered(x, flt_feature_names, flt_indices,
weights, top, scale)
explanation = Explanation(
estimator=repr(estimator),
method='decision paths',
description={
(False, False): DESCRIPTION_CLF_BINARY,
(False, True): DESCRIPTION_CLF_MULTICLASS,
(True, False): DESCRIPTION_REGRESSION,
}[is_regression, is_multiclass],
is_regression=is_regression,
targets=[],
)
assert explanation.targets is not None
if is_multiclass:
for label_id, label in display_names:
score, all_feature_weights = get_score_weights(label_id)
target_expl = TargetExplanation(
target=label,
feature_weights=get_top_features(all_feature_weights),
score=score,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
explanation.targets.append(target_expl)
else:
score, all_feature_weights = get_score_weights(0)
if is_regression:
target, scale, label_id = display_names[-1][1], 1.0, 1
else:
target, scale, label_id = get_binary_target_scale_label_id(
score, display_names, proba)
target_expl = TargetExplanation(
target=target,
feature_weights=get_top_features(all_feature_weights, scale),
score=score,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
explanation.targets.append(target_expl)
return explanation
def test_transition_features():
expl = Explanation(
estimator='some estimator',
targets=[
TargetExplanation('class1',
feature_weights=FeatureWeights(
pos=[FeatureWeight('pos', 13, value=1)],
neg=[],
)),
TargetExplanation('class2',
feature_weights=FeatureWeights(
pos=[FeatureWeight('pos', 13, value=1)],
neg=[],
)),
],
transition_features=TransitionFeatureWeights(
class_names=['class2', 'class1'], # reverse on purpose
coef=np.array([[1.5, 2.5], [3.5, 4.5]]),
))
df_dict = format_as_dataframes(expl)
assert isinstance(df_dict, dict)
assert set(df_dict) == {'targets', 'transition_features'}
assert df_dict['targets'].equals(format_as_dataframe(expl.targets))
df = df_dict['transition_features']
print(df)
print(format_as_text(expl))
assert str(df) == ('to class2 class1\n'
'from \n'
'class2 1.5 2.5\n'
'class1 3.5 4.5')
with pytest.warns(UserWarning):
single_df = format_as_dataframe(expl)
assert single_df.equals(df)
def test_targets_with_value():
expl = Explanation(
estimator='some estimator',
targets=[
TargetExplanation('y',
feature_weights=FeatureWeights(
pos=[
FeatureWeight('a', 13, value=1),
FeatureWeight('b', 5, value=2)
],
neg=[
FeatureWeight('neg1', -10, value=3),
FeatureWeight('neg2', -1, value=4)
],
)),
TargetExplanation('y2',
feature_weights=FeatureWeights(
pos=[FeatureWeight('f', 1, value=5)],
neg=[],
)),
],
)
df = format_as_dataframe(expl)
expected_df = pd.DataFrame(
{
'weight': [13, 5, -1, -10, 1],
'value': [1, 2, 4, 3, 5]
},
columns=['weight', 'value'],
index=pd.MultiIndex.from_tuples([('y', 'a'), ('y', 'b'), ('y', 'neg2'),
('y', 'neg1'), ('y2', 'f')],
names=['target', 'feature']))
print(df, expected_df, sep='\n')
assert expected_df.equals(df)
def explain_prediction_linear_classifier(clf,
doc,
vec=None,
top=None,
target_names=None,
targets=None,
feature_names=None,
vectorized=False):
""" Explain prediction of a linear classifier. """
vec, feature_names = _handle_vec(clf, doc, vec, vectorized, feature_names)
X = _get_X(doc, vec=vec, vectorized=vectorized)
if is_probabilistic_classifier(clf):
try:
proba, = clf.predict_proba(X)
except NotImplementedError:
proba = None
else:
proba = None
score, = clf.decision_function(X)
if has_intercept(clf):
X = _add_intercept(X)
x, = X
res = Explanation(
estimator=repr(clf),
method='linear model',
targets=[],
)
def _weights(label_id):
coef = get_coef(clf, label_id)
scores = _multiply(x, coef)
return get_top_features(feature_names, scores, top)
display_names = get_display_names(clf.classes_, target_names, targets)
if is_multiclass_classifier(clf):
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id],
proba=proba[label_id] if proba is not None else None,
)
_add_weighted_spans(doc, vec, target_expl)
res.targets.append(target_expl)
else:
target_expl = TargetExplanation(
target=display_names[1][1],
feature_weights=_weights(0),
score=score,
proba=proba[1] if proba is not None else None,
)
_add_weighted_spans(doc, vec, target_expl)
res.targets.append(target_expl)
return res
def test_targets(with_std, with_value):
expl = Explanation(
estimator='some estimator',
targets=[
TargetExplanation(
'y',
feature_weights=FeatureWeights(
pos=[
FeatureWeight('a',
13,
std=0.13 if with_std else None,
value=2 if with_value else None),
FeatureWeight('b',
5,
std=0.5 if with_std else None,
value=1 if with_value else None)
],
neg=[
FeatureWeight('neg1',
-10,
std=0.2 if with_std else None,
value=5 if with_value else None),
FeatureWeight('neg2',
-1,
std=0.3 if with_std else None,
value=4 if with_value else None)
],
)),
TargetExplanation('y2',
feature_weights=FeatureWeights(
pos=[FeatureWeight('f', 1)],
neg=[],
)),
],
)
df_dict = format_as_dataframes(expl)
assert isinstance(df_dict, dict)
assert list(df_dict) == ['targets']
df = df_dict['targets']
expected_df = pd.DataFrame(
{
'target': ['y', 'y', 'y', 'y', 'y2'],
'feature': ['a', 'b', 'neg2', 'neg1', 'f'],
'weight': [13, 5, -1, -10, 1]
},
columns=['target', 'feature', 'weight'])
if with_std:
expected_df['std'] = [0.13, 0.5, 0.3, 0.2, None]
if with_value:
expected_df['value'] = [2, 1, 4, 5, None]
print(df, expected_df, sep='\n')
assert expected_df.equals(df)
single_df = format_as_dataframe(expl)
assert expected_df.equals(single_df)
def test_transition_features():
expl = Explanation(
estimator='some estimator',
targets=[
TargetExplanation('class1',
feature_weights=FeatureWeights(
pos=[FeatureWeight('pos', 13, value=1)],
neg=[],
)),
TargetExplanation('class2',
feature_weights=FeatureWeights(
pos=[FeatureWeight('pos', 13, value=1)],
neg=[],
)),
],
transition_features=TransitionFeatureWeights(
class_names=['class2', 'class1'], # reverse on purpose
coef=np.array([[1.5, 2.5], [3.5, 4.5]]),
))
df_dict = format_as_dataframes(expl)
assert isinstance(df_dict, dict)
assert set(df_dict) == {'targets', 'transition_features'}
assert df_dict['targets'].equals(format_as_dataframe(expl.targets))
df = df_dict['transition_features']
print(df)
print(format_as_text(expl))
expected = pd.DataFrame([
{
'from': 'class2',
'to': 'class2',
'coef': 1.5
},
{
'from': 'class2',
'to': 'class1',
'coef': 2.5
},
{
'from': 'class1',
'to': 'class2',
'coef': 3.5
},
{
'from': 'class1',
'to': 'class1',
'coef': 4.5
},
],
columns=['from', 'to', 'coef'])
assert df.equals(expected)
with pytest.warns(UserWarning):
single_df = format_as_dataframe(expl)
assert single_df.equals(df)
def explain_prediction_linear_regressor(reg,
doc,
vec=None,
top=None,
target_names=None,
targets=None,
feature_names=None,
vectorized=False):
""" Explain prediction of a linear regressor. """
vec, feature_names = _handle_vec(reg, doc, vec, vectorized, feature_names)
X = _get_X(doc, vec=vec, vectorized=vectorized)
score, = reg.predict(X)
if has_intercept(reg):
X = _add_intercept(X)
x, = X
res = Explanation(
estimator=repr(reg),
method='linear model',
targets=[],
is_regression=True,
)
def _weights(label_id):
coef = get_coef(reg, label_id)
scores = _multiply(x, coef)
return get_top_features(feature_names, scores, top)
names = get_default_target_names(reg)
display_names = get_display_names(names, target_names, targets)
if is_multitarget_regressor(reg):
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id],
)
_add_weighted_spans(doc, vec, target_expl)
res.targets.append(target_expl)
else:
target_expl = TargetExplanation(
target=display_names[0][1],
feature_weights=_weights(0),
score=score,
)
_add_weighted_spans(doc, vec, target_expl)
res.targets.append(target_expl)
return res
def get_decision_path_explanation(estimator, doc, vec, vectorized,
original_display_names,
target_names, targets, top_targets,
is_regression, is_multiclass, proba,
get_score_feature_weights):
display_names = get_target_display_names(
original_display_names, target_names, targets, top_targets, proba)
explanation = Explanation(
estimator=repr(estimator),
method='decision paths',
description={
(False, False): DESCRIPTION_CLF_BINARY,
(False, True): DESCRIPTION_CLF_MULTICLASS,
(True, False): DESCRIPTION_REGRESSION,
}[is_regression, is_multiclass],
is_regression=is_regression,
targets=[],
)
if is_multiclass:
for label_id, label in display_names:
score, feature_weights = get_score_feature_weights(label_id)
target_expl = TargetExplanation(
target=label,
feature_weights=feature_weights,
score=score,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
explanation.targets.append(target_expl)
else:
score, feature_weights = get_score_feature_weights(0)
target_expl = TargetExplanation(
target=display_names[-1][1],
feature_weights=feature_weights,
score=score,
proba=proba[1] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
explanation.targets.append(target_expl)
return explanation
def test_format_as_dict():
assert format_as_dict(
Explanation(
estimator='some estimator',
targets=[
TargetExplanation('y',
feature_weights=FeatureWeights(pos=[
FeatureWeight('a', np.float32(13.0))
],
neg=[])),
],
)) == {
'estimator':
'some estimator',
'targets': [
{
'target': 'y',
'feature_weights': {
'pos': [{
'feature': 'a',
'weight': 13.0,
'std': None,
'value': None
}],
'pos_remaining':
0,
'neg': [],
'neg_remaining':
0,
},
'score': None,
'proba': None,
'weighted_spans': None,
'heatmap': None,
},
],
'decision_tree':
None,
'description':
None,
'error':
None,
'feature_importances':
None,
'highlight_spaces':
None,
'is_regression':
False,
'method':
None,
'transition_features':
None,
'image':
None,
}
def explain_weights_sklearn_crfsuite(crf,
top=20,
target_names=None,
targets=None,
feature_re=None,
feature_filter=None):
""" Explain sklearn_crfsuite.CRF weights.
See :func:`eli5.explain_weights` for description of
``top``, ``target_names``, ``targets``,
``feature_re`` and ``feature_filter`` parameters.
"""
feature_names = np.array(crf.attributes_)
state_coef = crf_state_coef(crf).todense().A
transition_coef = crf_transition_coef(crf)
if feature_filter is not None or feature_re is not None:
state_feature_names, flt_indices = (
FeatureNames(feature_names).handle_filter(feature_filter, feature_re))
state_feature_names = np.array(state_feature_names.feature_names)
state_coef = state_coef[:, flt_indices]
else:
state_feature_names = feature_names
def _features(label_id):
return get_top_features(state_feature_names, state_coef[label_id], top)
if targets is None:
targets = sorted_for_ner(crf.classes_)
display_names = get_target_display_names(crf.classes_, target_names,
targets)
indices, names = zip(*display_names)
transition_coef = filter_transition_coefs(transition_coef, indices)
return Explanation(
targets=[
TargetExplanation(
target=label,
feature_weights=_features(label_id)
)
for label_id, label in zip(indices, names)
],
transition_features=TransitionFeatureWeights(
class_names=names,
coef=transition_coef,
),
estimator=repr(crf),
method='CRF',
)
def explain_predictions(self, docs, top=30):
if not isinstance(self.clf, XGBClassifier):
raise NotImplementedError
booster = self.clf.booster()
xgb_feature_names = {f: i for i, f in enumerate(booster.feature_names)}
feature_names = get_feature_names(self.clf,
self.vec,
num_features=len(xgb_feature_names))
feature_names.bias_name = '<BIAS>'
X = self.vec.transform(docs)
X = X.tocsc()
dmatrix = DMatrix(X, missing=self.clf.missing)
leaf_ids = booster.predict(dmatrix, pred_leaf=True)
tree_dumps = booster.get_dump(with_stats=True)
docs_weights = []
for i, _leaf_ids in enumerate(leaf_ids):
all_weights = _target_feature_weights(
_leaf_ids,
tree_dumps,
feature_names=feature_names,
xgb_feature_names=xgb_feature_names)[1]
weights = np.zeros_like(all_weights)
idx = X[i].nonzero()[1]
bias_idx = feature_names.bias_idx
weights[idx] = all_weights[idx]
weights[bias_idx] = all_weights[bias_idx]
docs_weights.append(weights)
weights = np.mean(docs_weights, axis=0)
feature_weights = get_top_features(feature_names=np.array(
[_prettify_feature(f) for f in feature_names]),
coef=weights,
top=top)
return Explanation(
estimator=type(self.clf).__name__,
targets=[TargetExplanation('y', feature_weights=feature_weights)],
)
def explain_prediction_xgboost(
xgb, doc,
vec=None,
top=None,
top_targets=None,
target_names=None,
targets=None,
feature_names=None,
feature_re=None,
feature_filter=None,
vectorized=False,
):
""" Return an explanation of XGBoost prediction (via scikit-learn wrapper
XGBClassifier or XGBRegressor) as feature weights.
See :func:`eli5.explain_prediction` for description of
``top``, ``top_targets``, ``target_names``, ``targets``,
``feature_names``, ``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the estimator ``xgb``
(e.g. a fitted CountVectorizer instance); you can pass it
instead of ``feature_names``.
``vectorized`` is a flag which tells eli5 if ``doc`` should be
passed through ``vec`` or not. By default it is False, meaning that
if ``vec`` is not None, ``vec.transform([doc])`` is passed to the
estimator. Set it to False if you're passing ``vec``,
but ``doc`` is already vectorized.
Method for determining feature importances follows an idea from
http://blog.datadive.net/interpreting-random-forests/.
Feature weights are calculated by following decision paths in trees
of an ensemble.
Each leaf has an output score, and expected scores can also be assigned
to parent nodes.
Contribution of one feature on the decision path is how much expected score
changes from parent to child.
Weights of all features sum to the output score of the estimator.
"""
num_features = len(xgb.booster().feature_names)
vec, feature_names = handle_vec(
xgb, doc, vec, vectorized, feature_names, num_features=num_features)
if feature_names.bias_name is None:
# XGBoost estimators do not have an intercept, but here we interpret
# them as having an intercept
feature_names.bias_name = '<BIAS>'
X = get_X(doc, vec, vectorized=vectorized)
if sp.issparse(X):
# Work around XGBoost issue:
# https://github.com/dmlc/xgboost/issues/1238#issuecomment-243872543
X = X.tocsc()
proba = predict_proba(xgb, X)
scores_weights = _prediction_feature_weights(xgb, X, feature_names)
x, = add_intercept(X)
x = _missing_values_set_to_nan(x, xgb.missing, sparse_missing=True)
feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
is_multiclass = _xgb_n_targets(xgb) > 1
is_regression = isinstance(xgb, XGBRegressor)
names = xgb.classes_ if not is_regression else ['y']
display_names = get_target_display_names(names, target_names, targets,
top_targets, proba)
res = Explanation(
estimator=repr(xgb),
method='decision paths',
description={
(False, False): DESCRIPTION_CLF_BINARY,
(False, True): DESCRIPTION_CLF_MULTICLASS,
(True, False): DESCRIPTION_REGRESSION,
}[is_regression, is_multiclass],
is_regression=is_regression,
targets=[],
)
def get_score_feature_weights(_label_id):
_score, _feature_weights = scores_weights[_label_id]
_x = x
if flt_indices is not None:
_x = mask(_x, flt_indices)
_feature_weights = mask(_feature_weights, flt_indices)
return _score, get_top_features(
feature_names, _feature_weights, top, _x)
if is_multiclass:
for label_id, label in display_names:
score, feature_weights = get_score_feature_weights(label_id)
target_expl = TargetExplanation(
target=label,
feature_weights=feature_weights,
score=score,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
else:
score, feature_weights = get_score_feature_weights(0)
target_expl = TargetExplanation(
target=display_names[-1][1],
feature_weights=feature_weights,
score=score,
proba=proba[1] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
return res
def explain_prediction_keras(
estimator, # type: Model
doc, # type: np.ndarray
target_names=None,
targets=None, # type: Optional[list]
layer=None, # type: Optional[Union[int, str, Layer]]
):
# type: (...) -> Explanation
"""
Explain the prediction of a Keras image classifier.
We make two explicit assumptions
* The input is images.
* The model's task is classification, i.e. final output is class scores.
See :func:`eli5.explain_prediction` for more information about the ``estimator``,
``doc``, ``target_names``, and ``targets`` parameters.
:param keras.models.Model estimator:
Instance of a Keras neural network model,
whose predictions are to be explained.
:param numpy.ndarray doc:
An input image as a tensor to ``estimator``,
from which prediction will be done and explained.
Currently only numpy arrays are supported.
The tensor must be of suitable shape for the ``estimator``.
For example, some models require input images to be
rank 4 in format `(batch_size, dims, ..., channels)` (channels last)
or `(batch_size, channels, dims, ...)` (channels first),
where `dims` is usually in order `height, width`
and `batch_size` is 1 for a single image.
Check ``estimator.input_shape`` to confirm the required dimensions of the input tensor.
:raises TypeError: if ``doc`` is not a numpy array.
:raises ValueError: if ``doc`` shape does not match.
:param target_names:
*Not Implemented*.
Names for classes in the final output layer.
:type target_names: list, optional
:param targets:
Prediction ID's to focus on.
*Currently only the first prediction from the list is explained*.
The list must be length one.
If None, the model is fed the input image and its top prediction
is taken as the target automatically.
:raises ValueError: if ``targets`` is a list with more than one item.
:raises TypeError: if ``targets`` is not list or None.
:type targets: list[int], optional
:param layer:
The activation layer in the model to perform Grad-CAM on:
a valid keras layer name, layer index, or an instance of a Keras layer.
If None, a suitable layer is attempted to be retrieved.
The layer is searched for going backwards from the output layer,
checking that the rank of the layer's output
equals to the rank of the input.
:raises TypeError: if ``layer`` is not None, str, int, or keras.layers.Layer instance.
:raises ValueError: if suitable layer can not be found.
:raises ValueError: if differentiation fails with respect to retrieved ``layer``.
:type layer: int or str or keras.layers.Layer, optional
Returns
-------
expl : :class:`eli5.base.Explanation`
An :class:`eli5.base.Explanation` object with the following attributes:
* ``image`` a Pillow image with mode RGBA.
* ``targets`` a list of :class:`eli5.base.TargetExplanation` objects \
for each target. Currently only 1 target is supported.
The :class:`eli5.base.TargetExplanation` objects will have the following attributes:
* ``heatmap`` a rank 2 numpy array with floats in interval [0, 1] \
with the localization map values.
* ``target`` ID of target class.
* ``score`` value for predicted class.
"""
_validate_doc(estimator, doc)
activation_layer = _get_activation_layer(estimator, layer)
# TODO: maybe do the sum / loss calculation in this function and pass it to gradcam.
# This would be consistent with what is done in
# https://github.com/ramprs/grad-cam/blob/master/misc/utils.lua
# and https://github.com/ramprs/grad-cam/blob/master/classification.lua
values = gradcam_backend(estimator, doc, targets, activation_layer)
weights, activations, grads, predicted_idx, predicted_val = values
heatmap = gradcam(weights, activations)
doc, = doc # rank 4 batch -> rank 3 single image
image = keras.preprocessing.image.array_to_img(doc) # -> RGB Pillow image
image = image.convert(mode='RGBA')
return Explanation(
estimator.name,
description=DESCRIPTION_KERAS,
error='',
method='Grad-CAM',
image=image, # RGBA Pillow image
targets=[
TargetExplanation(
predicted_idx,
score=
predicted_val, # for now we keep the prediction in the .score field (not .proba)
heatmap=heatmap, # 2D [0, 1] numpy array
)
],
is_regression=
False, # might be relevant later when explaining for regression tasks
highlight_spaces=
None, # might be relevant later when explaining text models
)
def explain_linear_classifier_weights(
clf,
vec=None,
top=_TOP,
target_names=None,
targets=None,
feature_names=None,
coef_scale=None,
feature_re=None,
feature_filter=None,
):
"""
Return an explanation of a linear classifier weights.
See :func:`eli5.explain_weights` for description of
``top``, ``target_names``, ``targets``, ``feature_names``,
``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the classifier ``clf``
(e.g. a fitted CountVectorizer instance); you can pass it
instead of ``feature_names``.
``coef_scale`` is a 1D np.ndarray with a scaling coefficient
for each feature; coef[i] = coef[i] * coef_scale[i] if
coef_scale[i] is not nan. Use it if you want to scale coefficients
before displaying them, to take input feature sign or scale in account.
"""
feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
coef_scale)
feature_names = get_feature_names(clf, vec, feature_names=feature_names)
feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re)
_extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''
def _features(label_id):
coef = get_coef(clf, label_id, scale=coef_scale)
if flt_indices is not None:
coef = coef[flt_indices]
return get_top_features(feature_names, coef, top)
display_names = get_target_display_names(clf.classes_, target_names,
targets)
if is_multiclass_classifier(clf):
return Explanation(
targets=[
TargetExplanation(target=label,
feature_weights=_features(label_id))
for label_id, label in display_names
],
description=DESCRIPTION_CLF_MULTICLASS + _extra_caveats,
estimator=repr(clf),
method='linear model',
)
else:
# for binary classifiers scikit-learn stores a single coefficient
# vector, which corresponds to clf.classes_[1].
return Explanation(
targets=[
TargetExplanation(
target=display_names[1][1],
feature_weights=_features(0),
)
],
description=DESCRIPTION_CLF_BINARY + _extra_caveats,
estimator=repr(clf),
method='linear model',
)
def explain_linear_regressor_weights(
reg,
vec=None,
top=_TOP,
target_names=None,
targets=None,
feature_names=None,
coef_scale=None,
feature_re=None,
feature_filter=None,
):
"""
Return an explanation of a linear regressor weights.
See :func:`eli5.explain_weights` for description of
``top``, ``target_names``, ``targets``, ``feature_names``,
``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the regressor ``reg``; you can
pass it instead of ``feature_names``.
``coef_scale`` is a 1D np.ndarray with a scaling coefficient
for each feature; coef[i] = coef[i] * coef_scale[i] if
coef_scale[i] is not nan. Use it if you want to scale coefficients
before displaying them, to take input feature sign or scale in account.
"""
feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
coef_scale)
feature_names = get_feature_names(reg, vec, feature_names=feature_names)
feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re)
_extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''
def _features(target_id):
coef = get_coef(reg, target_id, scale=coef_scale)
if flt_indices is not None:
coef = coef[flt_indices]
return get_top_features(feature_names, coef, top)
display_names = get_target_display_names(get_default_target_names(reg),
target_names, targets)
if is_multitarget_regressor(reg):
return Explanation(
targets=[
TargetExplanation(target=target_name,
feature_weights=_features(target_id))
for target_id, target_name in display_names
],
description=DESCRIPTION_REGRESSION_MULTITARGET + _extra_caveats,
estimator=repr(reg),
method='linear model',
is_regression=True,
)
else:
return Explanation(
targets=[
TargetExplanation(
target=display_names[0][1],
feature_weights=_features(0),
)
],
description=DESCRIPTION_REGRESSION + _extra_caveats,
estimator=repr(reg),
method='linear model',
is_regression=True,
)
def test_prepare_weighted_spans():
targets = [
TargetExplanation(target='one',
feature_weights=FeatureWeights(pos=[], neg=[]),
weighted_spans=WeightedSpans(docs_weighted_spans=[
DocWeightedSpans(
document='ab',
spans=[
('a', [(0, 1)], 1.5),
('b', [(1, 2)], 2.5),
],
),
DocWeightedSpans(
document='xy',
spans=[
('xy', [(0, 2)], -4.5),
],
)
])),
TargetExplanation(
target='two',
feature_weights=FeatureWeights(pos=[], neg=[]),
weighted_spans=WeightedSpans(
docs_weighted_spans=[
DocWeightedSpans(
document='abc',
spans=[
('a', [(0, 1)], 0.5),
('c', [(2, 3)], 3.5),
],
),
DocWeightedSpans(
document='xz',
spans=[
# char_wb at the start of the document
(' xz', [(-1, 2)], 1.5),
],
)
], )),
]
assert prepare_weighted_spans(targets, preserve_density=False) == [
[
PreparedWeightedSpans(
targets[0].weighted_spans.docs_weighted_spans[0],
char_weights=np.array([1.5, 2.5]),
weight_range=3.5),
PreparedWeightedSpans(
targets[0].weighted_spans.docs_weighted_spans[1],
char_weights=np.array([-4.5, -4.5]),
weight_range=4.5),
],
[
PreparedWeightedSpans(
targets[1].weighted_spans.docs_weighted_spans[0],
char_weights=np.array([0.5, 0, 3.5]),
weight_range=3.5),
PreparedWeightedSpans(
targets[1].weighted_spans.docs_weighted_spans[1],
char_weights=np.array([1.5, 1.5]),
weight_range=4.5),
],
]
def mock_expl(catdog_rgba):
return Explanation(
'mock estimator',
image=catdog_rgba,
targets=[TargetExplanation(-1, heatmap=np.zeros((7, 7)))])
def explain_linear_regressor_weights(reg,
vec=None,
top=_TOP,
target_names=None,
targets=None,
feature_names=None,
coef_scale=None,
feature_re=None):
"""
Return an explanation of a linear regressor weights in the following
format::
Explanation(
estimator="<regressor repr>",
method="<interpretation method>",
description="<human readable description>",
targets=[
TargetExplanation(
target="<target name>",
feature_weights=FeatureWeights(
# positive weights
pos=[
(feature_name, coefficient),
...
],
# negative weights
neg=[
(feature_name, coefficient),
...
],
# A number of features not shown
pos_remaining = <int>,
neg_remaining = <int>,
# Sum of feature weights not shown
# pos_remaining_sum = <float>,
# neg_remaining_sum = <float>,
),
),
...
]
)
To print it use utilities from eli5.formatters.
"""
feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
coef_scale)
feature_names = get_feature_names(reg, vec, feature_names=feature_names)
if feature_re is not None:
feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
_extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''
def _features(target_id):
coef = get_coef(reg, target_id, scale=coef_scale)
if feature_re is not None:
coef = coef[flt_indices]
return get_top_features(feature_names, coef, top)
display_names = get_display_names(get_default_target_names(reg),
target_names, targets)
if is_multitarget_regressor(reg):
return Explanation(
targets=[
TargetExplanation(target=target_name,
feature_weights=_features(target_id))
for target_id, target_name in display_names
],
description=DESCRIPTION_REGRESSION_MULTITARGET + _extra_caveats,
estimator=repr(reg),
method='linear model',
is_regression=True,
)
else:
return Explanation(
targets=[
TargetExplanation(
target=display_names[0][1],
feature_weights=_features(0),
)
],
description=DESCRIPTION_REGRESSION + _extra_caveats,
estimator=repr(reg),
method='linear model',
is_regression=True,
)
def explain_prediction_linear_classifier(
clf,
doc,
vec=None,
top=None,
top_targets=None,
target_names=None,
targets=None,
feature_names=None,
feature_re=None,
feature_filter=None,
vectorized=False,
):
"""
Explain prediction of a linear classifier.
See :func:`eli5.explain_prediction` for description of
``top``, ``top_targets``, ``target_names``, ``targets``,
``feature_names``, ``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the classifier ``clf``
(e.g. a fitted CountVectorizer instance); you can pass it
instead of ``feature_names``.
``vectorized`` is a flag which tells eli5 if ``doc`` should be
passed through ``vec`` or not. By default it is False, meaning that
if ``vec`` is not None, ``vec.transform([doc])`` is passed to the
classifier. Set it to True if you're passing ``vec``, but ``doc``
is already vectorized.
"""
vec, feature_names = handle_vec(clf, doc, vec, vectorized, feature_names)
X = get_X(doc, vec=vec, vectorized=vectorized, to_dense=True)
proba = predict_proba(clf, X)
score, = clf.decision_function(X)
if has_intercept(clf):
X = add_intercept(X)
x = get_X0(X)
feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
res = Explanation(
estimator=repr(clf),
method='linear model',
targets=[],
)
assert res.targets is not None
_weights = _linear_weights(clf, x, top, feature_names, flt_indices)
classes = getattr(clf, "classes_", ["-1", "1"]) # OneClassSVM support
display_names = get_target_display_names(classes, target_names, targets,
top_targets, score)
if is_multiclass_classifier(clf):
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id],
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
else:
if len(display_names) == 1: # target is passed explicitly
label_id, target = display_names[0]
else:
label_id = 1 if score >= 0 else 0
target = display_names[label_id][1]
scale = -1 if label_id == 0 else 1
target_expl = TargetExplanation(
target=target,
feature_weights=_weights(0, scale=scale),
score=score,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
return res
def explain_prediction_tree_classifier(clf,
doc,
vec=None,
top=None,
top_targets=None,
target_names=None,
targets=None,
feature_names=None,
feature_re=None,
feature_filter=None,
vectorized=False):
""" Explain prediction of a tree classifier.
See :func:`eli5.explain_prediction` for description of
``top``, ``top_targets``, ``target_names``, ``targets``,
``feature_names``, ``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the classifier ``clf``
(e.g. a fitted CountVectorizer instance); you can pass it
instead of ``feature_names``.
``vectorized`` is a flag which tells eli5 if ``doc`` should be
passed through ``vec`` or not. By default it is False, meaning that
if ``vec`` is not None, ``vec.transform([doc])`` is passed to the
classifier. Set it to True if you're passing ``vec``,
but ``doc`` is already vectorized.
Method for determining feature importances follows an idea from
http://blog.datadive.net/interpreting-random-forests/.
Feature weights are calculated by following decision paths in trees
of an ensemble (or a single tree for DecisionTreeClassifier).
Each node of the tree has an output score, and contribution of a feature
on the decision path is how much the score changes from parent to child.
Weights of all features sum to the output score or proba of the estimator.
"""
vec, feature_names = handle_vec(clf, doc, vec, vectorized, feature_names)
X = get_X(doc, vec=vec, vectorized=vectorized)
if feature_names.bias_name is None:
# Tree estimators do not have an intercept, but here we interpret
# them as having an intercept
feature_names.bias_name = '<BIAS>'
proba = predict_proba(clf, X)
if hasattr(clf, 'decision_function'):
score, = clf.decision_function(X)
else:
score = None
is_multiclass = clf.n_classes_ > 2
feature_weights = _trees_feature_weights(clf, X, feature_names,
clf.n_classes_)
x = get_X0(add_intercept(X))
flt_feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
def _weights(label_id, scale=1.0):
weights = feature_weights[:, label_id]
return get_top_features_filtered(x, flt_feature_names, flt_indices,
weights, top, scale)
res = Explanation(
estimator=repr(clf),
method='decision path',
targets=[],
description=(DESCRIPTION_TREE_CLF_MULTICLASS
if is_multiclass else DESCRIPTION_TREE_CLF_BINARY),
)
assert res.targets is not None
display_names = get_target_display_names(
clf.classes_,
target_names,
targets,
top_targets,
score=score if score is not None else proba)
if is_multiclass:
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id] if score is not None else None,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
else:
target, scale, label_id = get_binary_target_scale_label_id(
score, display_names, proba)
target_expl = TargetExplanation(
target=target,
feature_weights=_weights(label_id, scale=scale),
score=score if score is not None else None,
proba=proba[label_id] if proba is not None else None,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
return res
def explain_prediction_linear_regressor(reg,
doc,
vec=None,
top=None,
top_targets=None,
target_names=None,
targets=None,
feature_names=None,
feature_re=None,
feature_filter=None,
vectorized=False):
"""
Explain prediction of a linear regressor.
See :func:`eli5.explain_prediction` for description of
``top``, ``top_targets``, ``target_names``, ``targets``,
``feature_names``, ``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the classifier ``clf``;
you can pass it instead of ``feature_names``.
``vectorized`` is a flag which tells eli5 if ``doc`` should be
passed through ``vec`` or not. By default it is False, meaning that
if ``vec`` is not None, ``vec.transform([doc])`` is passed to the
regressor ``reg``. Set it to True if you're passing ``vec``,
but ``doc`` is already vectorized.
"""
if isinstance(reg, (SVR, NuSVR)) and reg.kernel != 'linear':
return explain_prediction_sklearn_not_supported(reg, doc)
vec, feature_names = handle_vec(reg, doc, vec, vectorized, feature_names)
X = get_X(doc, vec=vec, vectorized=vectorized, to_dense=True)
score, = reg.predict(X)
if has_intercept(reg):
X = add_intercept(X)
x = get_X0(X)
feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
res = Explanation(
estimator=repr(reg),
method='linear model',
targets=[],
is_regression=True,
)
assert res.targets is not None
_weights = _linear_weights(reg, x, top, feature_names, flt_indices)
names = get_default_target_names(reg)
display_names = get_target_display_names(names, target_names, targets,
top_targets, score)
if is_multitarget_regressor(reg):
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id],
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
else:
target_expl = TargetExplanation(
target=display_names[0][1],
feature_weights=_weights(0),
score=score,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
return res
def explain_linear_classifier_weights(clf,
vec=None,
top=_TOP,
target_names=None,
targets=None,
feature_names=None,
coef_scale=None,
feature_re=None):
"""
Return an explanation of a linear classifier weights in the following
format::
Explanation(
estimator="<classifier repr>",
method="<interpretation method>",
description="<human readable description>",
targets=[
TargetExplanation(
target="<class name>",
feature_weights=FeatureWeights(
# positive weights
pos=[
(feature_name, coefficient),
...
],
# negative weights
neg=[
(feature_name, coefficient),
...
],
# A number of features not shown
pos_remaining = <int>,
neg_remaining = <int>,
# Sum of feature weights not shown
# pos_remaining_sum = <float>,
# neg_remaining_sum = <float>,
),
),
...
]
)
To print it use utilities from eli5.formatters.
"""
feature_names, coef_scale = handle_hashing_vec(vec, feature_names,
coef_scale)
feature_names = get_feature_names(clf, vec, feature_names=feature_names)
if feature_re is not None:
feature_names, flt_indices = feature_names.filtered_by_re(feature_re)
_extra_caveats = "\n" + HASHING_CAVEATS if is_invhashing(vec) else ''
def _features(label_id):
coef = get_coef(clf, label_id, scale=coef_scale)
if feature_re is not None:
coef = coef[flt_indices]
return get_top_features(feature_names, coef, top)
display_names = get_display_names(clf.classes_, target_names, targets)
if is_multiclass_classifier(clf):
return Explanation(
targets=[
TargetExplanation(target=label,
feature_weights=_features(label_id))
for label_id, label in display_names
],
description=DESCRIPTION_CLF_MULTICLASS + _extra_caveats,
estimator=repr(clf),
method='linear model',
)
else:
# for binary classifiers scikit-learn stores a single coefficient
# vector, which corresponds to clf.classes_[1].
return Explanation(
targets=[
TargetExplanation(
target=display_names[1][1],
feature_weights=_features(0),
)
],
description=DESCRIPTION_CLF_BINARY + _extra_caveats,
estimator=repr(clf),
method='linear model',
)
def explain_prediction_tree_regressor(reg,
doc,
vec=None,
top=None,
top_targets=None,
target_names=None,
targets=None,
feature_names=None,
feature_re=None,
feature_filter=None,
vectorized=False):
""" Explain prediction of a tree regressor.
See :func:`eli5.explain_prediction` for description of
``top``, ``top_targets``, ``target_names``, ``targets``,
``feature_names``, ``feature_re`` and ``feature_filter`` parameters.
``vec`` is a vectorizer instance used to transform
raw features to the input of the regressor ``reg``
(e.g. a fitted CountVectorizer instance); you can pass it
instead of ``feature_names``.
``vectorized`` is a flag which tells eli5 if ``doc`` should be
passed through ``vec`` or not. By default it is False, meaning that
if ``vec`` is not None, ``vec.transform([doc])`` is passed to the
regressor. Set it to True if you're passing ``vec``,
but ``doc`` is already vectorized.
Method for determining feature importances follows an idea from
http://blog.datadive.net/interpreting-random-forests/.
Feature weights are calculated by following decision paths in trees
of an ensemble (or a single tree for DecisionTreeRegressor).
Each node of the tree has an output score, and contribution of a feature
on the decision path is how much the score changes from parent to child.
Weights of all features sum to the output score of the estimator.
"""
vec, feature_names = handle_vec(reg, doc, vec, vectorized, feature_names)
X = get_X(doc, vec=vec, vectorized=vectorized)
if feature_names.bias_name is None:
# Tree estimators do not have an intercept, but here we interpret
# them as having an intercept
feature_names.bias_name = '<BIAS>'
score, = reg.predict(X)
num_targets = getattr(reg, 'n_outputs_', 1)
is_multitarget = num_targets > 1
feature_weights = _trees_feature_weights(reg, X, feature_names,
num_targets)
x = get_X0(add_intercept(X))
flt_feature_names, flt_indices = feature_names.handle_filter(
feature_filter, feature_re, x)
def _weights(label_id, scale=1.0):
weights = feature_weights[:, label_id]
return get_top_features_filtered(x, flt_feature_names, flt_indices,
weights, top, scale)
res = Explanation(
estimator=repr(reg),
method='decision path',
description=(DESCRIPTION_TREE_REG_MULTITARGET
if is_multitarget else DESCRIPTION_TREE_REG),
targets=[],
is_regression=True,
)
assert res.targets is not None
names = get_default_target_names(reg, num_targets=num_targets)
display_names = get_target_display_names(names, target_names, targets,
top_targets, score)
if is_multitarget:
for label_id, label in display_names:
target_expl = TargetExplanation(
target=label,
feature_weights=_weights(label_id),
score=score[label_id],
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
else:
target_expl = TargetExplanation(
target=display_names[0][1],
feature_weights=_weights(0),
score=score,
)
add_weighted_spans(doc, vec, vectorized, target_expl)
res.targets.append(target_expl)
return res
def explain_prediction_keras_image(model,
doc,
image=None, # type: Optional['PIL.Image.Image']
targets=None,
layer=None,
):
"""
Explain an image-based model, highlighting what contributed in the image.
:param numpy.ndarray doc:
Input representing an image.
Must have suitable format. Some models require tensors to be
rank 4 in format `(batch_size, dims, ..., channels)` (channels last)
or `(batch_size, channels, dims, ...)` (channels first),
where `dims` is usually in order `height, width`
and `batch_size` is 1 for a single image.
If ``image`` argument is not given, an image will be created
from ``doc``, where possible.
:param image:
Pillow image over which to overlay the heatmap.
Corresponds to the input ``doc``.
:type image: PIL.Image.Image, optional
See :func:`eli5.keras.explain_prediction.explain_prediction_keras`
for a description of ``model``, ``doc``, ``targets``, and ``layer`` parameters.
Returns
-------
expl : eli5.base.Explanation
An :class:`eli5.base.Explanation` object with the following attributes:
* ``image`` a Pillow image representing the input.
* ``targets`` a list of :class:`eli5.base.TargetExplanation` objects \
for each target. Currently only 1 target is supported.
The :class:`eli5.base.TargetExplanation` objects will have the following attributes:
* ``heatmap`` a rank 2 numpy array with the localization map \
values as floats.
* ``target`` ID of target class.
* ``score`` value for predicted class.
"""
if image is None:
image = _extract_image(doc)
_validate_doc(model, doc)
activation_layer = _get_activation_layer(model, layer)
# TODO: maybe do the sum / loss calculation in this function and pass it to gradcam.
# This would be consistent with what is done in
# https://github.com/ramprs/grad-cam/blob/master/misc/utils.lua
# and https://github.com/ramprs/grad-cam/blob/master/classification.lua
values = gradcam_backend(model, doc, targets, activation_layer)
weights, activations, grads, predicted_idx, predicted_val = values
heatmap = gradcam(weights, activations)
return Explanation(
model.name,
description=DESCRIPTION_KERAS,
error='',
method='Grad-CAM',
image=image,
targets=[TargetExplanation(
predicted_idx,
score=predicted_val, # for now we keep the prediction in the .score field (not .proba)
heatmap=heatmap, # 2D [0, 1] numpy array
)],
is_regression=False, # might be relevant later when explaining for regression tasks
highlight_spaces=None, # might be relevant later when explaining text models
)