class BaseGradient(Initializer):
"""
Base class for gradient-based relevance computation
Reference
- https://github.com/marcoancona/DeepExplain/blob/master/deepexplain/tensorflow/methods.py
"""
__name__ = "BaseGradient"
logger = build_logger(_INFO, __name__)
def _default_relevance_score(self):
BaseGradient.logger.debug("Computing default relevance score...")
return tf.gradients(self.output_tensor, self.input_tensor)
def _run(self):
BaseGradient.logger.info("Executing operations ...")
relevance_scores = self._default_relevance_score()
results = self._session_run(relevance_scores, self.samples)
return results[0]
@classmethod
def _non_linear_grad(cls, op, grad):
BaseGradient.logger.debug(
"Computing gradient with activation type {}".format(op.type))
return cls._original_grad(op, grad)
class Initializer(object):
"""
"""
__name__ = "Initializer"
# Currently supported Activation ops
activation_ops = ['Relu', 'Elu', 'Softplus', 'Tanh', 'Sigmoid']
_enabled_method_class = None
_grad_override_checkflag = 0
logger = build_logger(_INFO, __name__)
def __init__(self, output_tensor, input_tensor, samples, session):
self.output_tensor = output_tensor
self.input_tensor = input_tensor
self.samples = samples
self.session = session
def _session_run(self, output_tensor, samples):
feed_dict = {}
feed_dict[self.input_tensor] = samples
return self.session.run(output_tensor, feed_dict)
@classmethod
def _original_grad(cls, op, grad):
if op.type not in cls.activation_ops:
warnings.warn('Selected Activation Ops({}) is currently not supported.'.format(op.type))
op_name = '_{}Grad'.format(op.type)
Initializer.logger.debug("Operation name : {}".format(op_name))
ops_func = getattr(nn_grad, op_name) if hasattr(nn_grad, op_name) else getattr(math_grad, op_name)
return ops_func(op, grad)
def __init__(self,
graph=None,
session=tf.compat.v1.get_default_session,
log_level=_WARNING):
self.logger = build_logger(log_level, __name__)
self.relevance_type = None
self.use_case_str = None
self.batch_size = None
self.session = session
if self.session is None:
raise RuntimeError('Relevant session not retrieved')
else:
self.logger.info("Current session: {}".format(session.__dict__))
self.graph = session.graph if graph is None else graph
# request for the default graph
self.graph_context = self.graph.as_default()
self.override_context = self.graph.gradient_override_map(
self._get_gradient_override_map())
self.context_on = False
self.__supported_relevance_type_dict = OrderedDict({
'elrp': {
'use_case_type': ['image'],
'method': LRP
},
'ig': {
'use_case_type': ['image', 'txt'],
'method': IntegratedGradients
},
'occlusion': {
'use_case_type': ['image'],
'method': Occlusion
}
})
def __init__(self,
training_data=None,
training_labels=None,
class_names=None,
feature_names=None,
index=None,
log_level=30):
"""
Attaches local and global interpretations
to Interpretation object.
Parameters
-----------
log_level: int
Logger Verbosity, see https://docs.python.org/2/library/logging.html
for details.
"""
self._log_level = log_level
self.logger = build_logger(log_level, __name__)
self.data_set = None
self.feature_names = feature_names
self.class_names = class_names
self.load_data(training_data,
training_labels=training_labels,
feature_names=feature_names,
index=index)
self.partial_dependence = PartialDependence(self)
self.feature_importance = FeatureImportance(self)
self.tree_surrogate = TreeSurrogate
class IntegratedGradients(BaseGradient):
""" Integrated Gradient is a relevance scoring algorithm for Deep network based on final predictions to its input
features. The algorithm statisfies two fundamental axioms related to relevance/attribution computation,
1.Sensitivity : For every input and baseline, if the change in one feature causes the prediction to change,
then the that feature should have non-zero relevance score
2.Implementation Invariance : Compute relevance(attribution) should be identical for functionally equivalent
networks.
References
----------
.. [1] Sundararajan, Mukund, Taly, Ankur, Yan, Qiqi (ICML, 2017).
.. Axiomatic Attribution for Deep Networks (http://arxiv.org/abs/1703.01365)
.. [2] Ancona M, Ceolini E, Öztireli C, Gross M:
.. Towards better understanding of gradient-based attribution methods for Deep Neural Networks. ICLR, 2018
.. [3] Taly, Ankur(2017) http://theory.stanford.edu/~ataly/Talks/sri_attribution_talk_jun_2017.pdf
"""
__name__ = "IntegratedGradients"
logger = build_logger(_INFO, __name__)
def __init__(self,
output_tensor,
input_tensor,
samples,
session,
steps=100,
baseline=None):
super(IntegratedGradients, self).__init__(output_tensor, input_tensor,
samples, session)
self.steps = steps
# Using black image or zero embedding vector for text as a default baseline, as suggested in the paper
# Mukund Sundararajan, Ankir Taly, Qibi Yan. Axiomatic Attribution for Deep Networks(ICML2017)
self.baseline = np.zeros(
(1, ) + self.samples.shape[1:]) if baseline is None else baseline
def _run(self):
IntegratedGradients.logger.info(
"Executing operations to compute relevance using Integrated Gradient"
)
t_grad = self._default_relevance_score()
gradient = None
alpha_list = list(
np.linspace(start=1. / self.steps, stop=1.0, num=self.steps))
for alpha in alpha_list:
xs_scaled = (self.samples - self.baseline) * alpha
# compute the gradient for each alpha value
_scores = self._session_run(t_grad, xs_scaled)
gradient = _scores if gradient is None else [
g + a for g, a in zip(gradient, _scores)
]
results = [
(x - b) * (g / self.steps)
for g, x, b in zip(gradient, [self.samples], [self.baseline])
]
return results[0]
class BasePerturbationMethod(Initializer):
"""
Base class for perturbation-based relevance/attribution computation
"""
__name__ = "BasePerturbationMethod"
logger = build_logger(_INFO, __name__)
def __init__(self, output_tensor, input_tensor, samples, current_session):
super(BasePerturbationMethod, self).__init__(output_tensor, input_tensor, samples, current_session)
class LRP(BaseGradient):
""" LRP is technique to decompose the prediction(output) of a deep neural networks(DNNs) by computing relevance at
each layer in a backward pass. Current implementation is computed using backpropagation by applying change rule on
a modified gradient function. LRP could be implemented in different ways.
This version implements the epsilon-LRP(Eq (58) as stated in [1] or Eq (2) in [2].
Epsilon acts as a numerical stabilizer.
References
----------
.. [1] Bach S, Binder A, Montavon G, Klauschen F, Müller K-R, Samek W (2015)
On Pixel-Wise Explanations for Non-Linear Classifier Decisions by Layer-Wise Relevance Propagation.
PLoS ONE 10(7): e0130140. https://doi.org/10.1371/journal.pone.0130140
.. [2] Ancona M, Ceolini E, Öztireli C, Gross M:
Towards better understanding of gradient-based attribution methods for Deep Neural Networks. ICLR, 2018
"""
__name__ = "LRP"
_eps = None
logger = build_logger(_INFO, __name__)
def __init__(self,
output_tensor,
input_tensor,
samples,
session,
epsilon=1e-4):
super(LRP, self).__init__(output_tensor, input_tensor, samples,
session)
assert epsilon > 0.0, 'LRP epsilon must be > 0'
LRP._eps = epsilon
LRP.logger.info("Epsilon value: {}".format(LRP._eps))
def _default_relevance_score(self):
# computing dot product of the feature wts of the input data and the gradients of the prediction label
return [
g * x for g, x in zip(
tf.gradients(self.output_tensor, self.input_tensor),
[self.input_tensor])
]
@classmethod
def _non_linear_grad(cls, op, grad):
LRP.logger.debug(
"Computing non-linear gradient with activation type {}".format(
op.type))
op_out = op.outputs[0]
op_in = op.inputs[0]
stabilizer_epsilon = cls._eps * tf.sign(op_in)
op_in += stabilizer_epsilon
return grad * op_out / op_in
class Initializer(object):
"""
"""
__name__ = "Initializer"
# Currently supported Activation ops
activation_ops = ['Relu', 'Elu', 'Softplus', 'Tanh', 'Sigmoid']
_enabled_method_class = None
_grad_override_checkflag = 0
logger = build_logger(_INFO, __name__)
def __init__(self, output_tensor, input_tensor, samples, session):
self.output_tensor = output_tensor
self.input_tensor = input_tensor
self.samples = samples
self.session = session
def _session_run(self, output_tensor, samples):
feed_dict = {}
feed_dict[self.input_tensor] = samples
return self.session.run(output_tensor, feed_dict)
def _validate_baseline(self, baseline):
if baseline is not None and baseline.shape != ((1,) + self.samples.shape[1:]):
if baseline.shape == self.samples.shape[1:]:
baseline = np.expand_dims(baseline, 0)
else:
raise RuntimeError('Baseline input shape {} does not match expected input shape {}'
.format(baseline.shape, self.samples.shape[1:]))
elif baseline is None:
baseline = np.zeros((1,) + self.samples.shape[1:])
return baseline
@classmethod
def _original_grad(cls, op, grad):
if op.type not in cls.activation_ops:
warnings.warn('Selected Activation Ops({}) is currently not supported.'.format(op.type))
op_name = '_{}Grad'.format(op.type)
Initializer.logger.debug("Operation name : {}".format(op_name))
ops_func = getattr(nn_grad, op_name) if hasattr(nn_grad, op_name) else getattr(math_grad, op_name)
return ops_func(op, grad)
def __init__(self, graph=None, session=tf.get_default_session(), log_level=_WARNING):
self.logger = build_logger(log_level, __name__)
self.relevance_type = None
self.use_case_str = None
self.batch_size = None
self.session = session
if self.session is None:
raise RuntimeError('Relevant session not retrieved')
else:
self.logger.info("Current session: {}".format(session.__dict__))
self.graph = session.graph if graph is None else graph
# request for the default graph
self.graph_context = self.graph.as_default()
self.override_context = self.graph.gradient_override_map(self._get_gradient_override_map())
self.context_on = False
self.__supported_relevance_type_dict = OrderedDict({
'elrp': {'use_case_type': ['image'], 'method': LRP},
'ig': {'use_case_type': ['image', 'txt'], 'method': IntegratedGradients}
})
def __init__(self,
estimator_type='classifier',
splitter='best',
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=None,
seed=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
class_weight="balanced",
class_names=None,
presort=False,
feature_names=None,
impurity_threshold=0.01,
log_level=_WARNING):
self.logger = build_logger(log_level, __name__)
self.__model = None
self.__model_type = None
self.feature_names = feature_names
self.class_names = class_names
self.impurity_threshold = impurity_threshold
self.criterion_types = {
'classifier': {
'criterion': ['gini', 'entropy']
},
'regressor': {
'criterion': ['mse', 'friedman_mse', 'mae']
}
}
self.splitter_types = ['best', 'random']
self.splitter = splitter if any(
splitter in item for item in self.splitter_types) else 'best'
self.seed = seed
# TODO validate the parameters based on estimator type
if estimator_type == 'classifier':
self.__model_type = estimator_type
self.__model = DecisionTreeClassifier(
splitter=self.splitter,
max_depth=max_depth,
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_features=max_features,
random_state=seed,
max_leaf_nodes=max_leaf_nodes,
min_impurity_decrease=min_impurity_decrease,
min_impurity_split=min_impurity_split,
class_weight=class_weight,
presort=presort)
elif estimator_type == 'regressor':
self.__model_type = estimator_type
self.__model = DecisionTreeRegressor(
splitter=self.splitter,
max_depth=None,
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_features=max_features,
random_state=seed,
max_leaf_nodes=max_leaf_nodes,
min_impurity_decrease=min_impurity_decrease,
min_impurity_split=min_impurity_split,
presort=presort)
else:
raise exceptions.ModelError(
"Model type not supported. Supported options types{'classifier', 'regressor'}"
)
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
from sklearn.model_selection import RandomizedSearchCV
import numpy as np
from skater.model.base import ModelType
from skater.core.visualizer.tree_visualizer import plot_tree, tree_to_text
from skater.util.logger import build_logger
from skater.util.logger import _WARNING
from skater.util.logger import _INFO
from skater.util import exceptions
logger = build_logger(_INFO, __name__)
class TreeSurrogate(object):
""" :: Experimental :: The implementation is currently experimental and might change in future.
The idea of using TreeSurrogates as means for explaining a model's(Oracle or the base model)
learned decision policies(for inductive learning tasks) is inspired by the work of Mark W. Craven
described as the TREPAN algorithm. In this explanation learning hypothesis, the base estimator(Oracle)
could be any form of supervised learning predictive models. The explanations are approximated using
DecisionTrees(both for Classification/Regression) by learning decision boundaries similar to that learned by
the Oracle(predictions from the base model are used for learning the DecisionTree representation).
The implementation also generates a fidelity score to quantify tree based surrogate model's approximation to the Oracle.
Ideally, the score should be 0 for truthful explanation both globally and locally.
Parameters
----------
estimator_type='classifier'
splitter='best'
max_depth=None
def __init__(self,
oracle=None,
splitter='best',
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=None,
seed=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
class_weight="balanced",
presort=False,
impurity_threshold=0.01):
if not isinstance(oracle, ModelType):
raise exceptions.ModelError(
"Incorrect estimator used, create one with skater.model.local.InMemoryModel"
)
self.oracle = oracle
self.logger = build_logger(oracle.logger.level, __name__)
self.__model_type = None
self.feature_names = oracle.feature_names
self.class_names = oracle.target_names
self.impurity_threshold = impurity_threshold
self.criterion_types = {
'classifier': {
'criterion': ['gini', 'entropy']
},
'regressor': {
'criterion': ['mse', 'friedman_mse', 'mae']
}
}
self.splitter_types = ['best', 'random']
self.splitter = splitter if any(
splitter in item for item in self.splitter_types) else 'best'
self.seed = seed
self.__model_type = oracle.model_type
self.__scorer_name = None
self.__best_score = None
# TODO validate the parameters based on estimator type
if self.__model_type == 'classifier':
est = DecisionTreeClassifier(
splitter=self.splitter,
max_depth=max_depth,
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_features=max_features,
random_state=seed,
max_leaf_nodes=max_leaf_nodes,
min_impurity_decrease=min_impurity_decrease,
class_weight=class_weight,
presort=presort)
elif self.__model_type == 'regressor':
est = DecisionTreeRegressor(
splitter=self.splitter,
max_depth=None,
min_samples_split=min_samples_split,
min_samples_leaf=min_samples_leaf,
min_weight_fraction_leaf=min_weight_fraction_leaf,
max_features=max_features,
random_state=seed,
max_leaf_nodes=max_leaf_nodes,
min_impurity_split=min_impurity_split,
presort=presort)
else:
raise exceptions.ModelError(
"Model type not supported. Supported options types{'classifier', 'regressor'}"
)
self.__model = est
self.__pred_func = lambda X, prob: self.__model.predict(
X) if prob is False else self.__model.predict_proba(X)
from matplotlib.cm import get_cmap
import matplotlib as mpl
from matplotlib.patches import Patch
import pandas as pd
from skater.data.datamanager import DataManager as DM
from skater.util.exceptions import MatplotlibUnavailableError
from skater.util.logger import build_logger
from skater.util.logger import _INFO
from skater.core.local_interpretation.text_interpreter import relevance_wt_assigner
from skater.util.dataops import convert_dataframe_to_dict
from skater.util.text_ops import generate_word_list
logger = build_logger(_INFO, __name__)
def __set_plot_feature_relevance_keyword(**plot_kw):
plot_name = plot_kw['plot_name'] if 'plot_name' in plot_kw.keys() else 'feature_relevance.png'
top_k = plot_kw['top_k'] if 'top_k' in plot_kw.keys() else 10
color_map = plot_kw['color_map'] if 'color_map' in plot_kw.keys() else ('Red', 'Blue')
fig_size = plot_kw['fig_size'] if 'fig_size' in plot_kw.keys() else (20, 10)
font_name = plot_kw['font_name'] if 'font_name' in plot_kw.keys() else "Avenir Black"
txt_font_size = plot_kw['txt_font_size'] if 'txt_font_size' in plot_kw.keys() else '14'
return plot_name, top_k, color_map, fig_size, font_name, txt_font_size
# Reference: https://stackoverflow.com/questions/30618002/static-variable-in-a-function-with-python-decorator
def static_var(varname, value):
def decorate(func):
setattr(func, varname, value)
return func
class Occlusion(BasePerturbationMethod):
""" Occlusion is a perturbation based inference algorithm. Such forms of algorithm direcly computes the
relevance/attribution of the input features :math:`(X_{i})` by systematically occluding different
portions of the image (by removing, masking or altering them), then running a forward pass on the new input to
produce a new output, and then measuring and monitoring the difference between the original output and new output.
Perturbation based interpretation helps one to compute direct estimation of the marginal effect of a feature but
the inference might be computationally expensive depending on the cardinatlity of the feature space.
The choice of the baseline value while perturbing through the feature space could be set to 0,
as explained in detail by Zeiler & Fergus, 2014[2].
References
----------
.. [1] Ancona M, Ceolini E, Oztireli C, Gross M (ICLR, 2018).
.. Towards better understanding of gradient-based attribution methods for Deep Neural Networks.
.. [2] Zeiler, M and Fergus, R (Springer, 2014). Visualizing and understanding convolutional networks.
.. In European conference on computer vision, pp. 818–833.
.. [3] https://github.com/marcoancona/DeepExplain/blob/master/deepexplain/tensorflow/methods.py
"""
__name__ = "Occlusion"
logger = build_logger(_INFO, __name__)
def __init__(self, output_tensor, input_tensor, samples, current_session,
**kwargs):
super(Occlusion, self).__init__(output_tensor, input_tensor, samples,
current_session)
self.input_shape = samples.shape[1:]
self.replace_value = kwargs[
'replace_value'] if 'replace_value' in kwargs.keys() else 0
self.window_size = kwargs[
'window_size'] if 'window_size' in kwargs.keys() else 1
self.step = kwargs['step'] if 'step' in kwargs.keys() else 1
# the input samples are expected to be of the shape,
# (1, 150, 150, 3) <batch_size, image_width, image_height, no_of_channels>
self.batch_size = self.samples.shape[0]
Occlusion.logger.info(
'Input shape: {}; window_size/step: ({}/{}); replace value: {}; batch size: {}'
.format(self.input_shape, self.window_size, self.step,
self.replace_value, self.batch_size))
def _create_masked_input(self, row_value, col_value):
masked_input = np.array(self.samples)
# mask the region as set by the window size by replacing the pixel values with the specified value(default:0)
masked_input[:, row_value:(row_value + self.window_size),
col_value:(col_value +
self.window_size), :] = self.replace_value
return masked_input
def _run(self):
mask = np.array([
self.batch_size, self.window_size, self.window_size,
self.samples[0].shape[2]
])
mask.fill(self.replace_value)
Occlusion.logger.info('Shape of the mask patch: {}'.format(mask.shape))
relevance_score = np.zeros_like(self.samples, dtype=np.float32)
# normalizer matrix is set to 1 default; as matrix cell gets used atleast once
normalizer = np.ones_like(relevance_score)
# Compute original output
default_eval = self._session_run(self.output_tensor, self.samples)
Occlusion.logger.info("shape of the default eval value :{}".format(
default_eval.shape))
count = 1 # to keep track of the number of times a matrix cell is used while perturbing through the feature space
# Perturb through the feature space by replacing and masking
for row in range(0, self.samples[0].shape[0] - self.window_size,
self.step):
for col in range(0, self.samples[0].shape[1] - self.window_size,
self.step):
# create masked input while rolling through the input matrix
new_input = self._create_masked_input(row, col)
# compute entropy when compared to the original eval value
delta = default_eval - self._session_run(
self.output_tensor, new_input)
delta_aggregated = np.sum(delta.reshape((self.batch_size, -1)),
-1,
keepdims=True)
relevance_score[:, row:(row + self.window_size),
col:(col +
self.window_size), :] += delta_aggregated
# keeping track of the number of time a matrix cell is used while perturbing feature space based
# on window size
normalizer[:, row:(row + self.window_size),
col:(col + self.window_size), :] += (count - 1)
Occlusion.logger.info("Min/Max normalizer weight: {}/{}".format(
np.min(normalizer.shape), np.max(normalizer.shape)))
relevance_score_norm = relevance_score / normalizer
Occlusion.logger.info("relevance score matrix shape :{}".format(
relevance_score_norm.shape))
return relevance_score_norm
