class BaseGradient(Initializer): """ Base class for gradient-based relevance computation Reference - https://github.com/marcoancona/DeepExplain/blob/master/deepexplain/tensorflow/methods.py """ __name__ = "BaseGradient" logger = build_logger(_INFO, __name__) def _default_relevance_score(self): BaseGradient.logger.debug("Computing default relevance score...") return tf.gradients(self.output_tensor, self.input_tensor) def _run(self): BaseGradient.logger.info("Executing operations ...") relevance_scores = self._default_relevance_score() results = self._session_run(relevance_scores, self.samples) return results[0] @classmethod def _non_linear_grad(cls, op, grad): BaseGradient.logger.debug( "Computing gradient with activation type {}".format(op.type)) return cls._original_grad(op, grad)
class Initializer(object): """ """ __name__ = "Initializer" # Currently supported Activation ops activation_ops = ['Relu', 'Elu', 'Softplus', 'Tanh', 'Sigmoid'] _enabled_method_class = None _grad_override_checkflag = 0 logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, session): self.output_tensor = output_tensor self.input_tensor = input_tensor self.samples = samples self.session = session def _session_run(self, output_tensor, samples): feed_dict = {} feed_dict[self.input_tensor] = samples return self.session.run(output_tensor, feed_dict) @classmethod def _original_grad(cls, op, grad): if op.type not in cls.activation_ops: warnings.warn('Selected Activation Ops({}) is currently not supported.'.format(op.type)) op_name = '_{}Grad'.format(op.type) Initializer.logger.debug("Operation name : {}".format(op_name)) ops_func = getattr(nn_grad, op_name) if hasattr(nn_grad, op_name) else getattr(math_grad, op_name) return ops_func(op, grad)
def __init__(self, graph=None, session=tf.compat.v1.get_default_session, log_level=_WARNING): self.logger = build_logger(log_level, __name__) self.relevance_type = None self.use_case_str = None self.batch_size = None self.session = session if self.session is None: raise RuntimeError('Relevant session not retrieved') else: self.logger.info("Current session: {}".format(session.__dict__)) self.graph = session.graph if graph is None else graph # request for the default graph self.graph_context = self.graph.as_default() self.override_context = self.graph.gradient_override_map( self._get_gradient_override_map()) self.context_on = False self.__supported_relevance_type_dict = OrderedDict({ 'elrp': { 'use_case_type': ['image'], 'method': LRP }, 'ig': { 'use_case_type': ['image', 'txt'], 'method': IntegratedGradients }, 'occlusion': { 'use_case_type': ['image'], 'method': Occlusion } })
def __init__(self, training_data=None, training_labels=None, class_names=None, feature_names=None, index=None, log_level=30): """ Attaches local and global interpretations to Interpretation object. Parameters ----------- log_level: int Logger Verbosity, see https://docs.python.org/2/library/logging.html for details. """ self._log_level = log_level self.logger = build_logger(log_level, __name__) self.data_set = None self.feature_names = feature_names self.class_names = class_names self.load_data(training_data, training_labels=training_labels, feature_names=feature_names, index=index) self.partial_dependence = PartialDependence(self) self.feature_importance = FeatureImportance(self) self.tree_surrogate = TreeSurrogate
class IntegratedGradients(BaseGradient): """ Integrated Gradient is a relevance scoring algorithm for Deep network based on final predictions to its input features. The algorithm statisfies two fundamental axioms related to relevance/attribution computation, 1.Sensitivity : For every input and baseline, if the change in one feature causes the prediction to change, then the that feature should have non-zero relevance score 2.Implementation Invariance : Compute relevance(attribution) should be identical for functionally equivalent networks. References ---------- .. [1] Sundararajan, Mukund, Taly, Ankur, Yan, Qiqi (ICML, 2017). .. Axiomatic Attribution for Deep Networks (http://arxiv.org/abs/1703.01365) .. [2] Ancona M, Ceolini E, Öztireli C, Gross M: .. Towards better understanding of gradient-based attribution methods for Deep Neural Networks. ICLR, 2018 .. [3] Taly, Ankur(2017) http://theory.stanford.edu/~ataly/Talks/sri_attribution_talk_jun_2017.pdf """ __name__ = "IntegratedGradients" logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, session, steps=100, baseline=None): super(IntegratedGradients, self).__init__(output_tensor, input_tensor, samples, session) self.steps = steps # Using black image or zero embedding vector for text as a default baseline, as suggested in the paper # Mukund Sundararajan, Ankir Taly, Qibi Yan. Axiomatic Attribution for Deep Networks(ICML2017) self.baseline = np.zeros( (1, ) + self.samples.shape[1:]) if baseline is None else baseline def _run(self): IntegratedGradients.logger.info( "Executing operations to compute relevance using Integrated Gradient" ) t_grad = self._default_relevance_score() gradient = None alpha_list = list( np.linspace(start=1. / self.steps, stop=1.0, num=self.steps)) for alpha in alpha_list: xs_scaled = (self.samples - self.baseline) * alpha # compute the gradient for each alpha value _scores = self._session_run(t_grad, xs_scaled) gradient = _scores if gradient is None else [ g + a for g, a in zip(gradient, _scores) ] results = [ (x - b) * (g / self.steps) for g, x, b in zip(gradient, [self.samples], [self.baseline]) ] return results[0]
class BasePerturbationMethod(Initializer): """ Base class for perturbation-based relevance/attribution computation """ __name__ = "BasePerturbationMethod" logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, current_session): super(BasePerturbationMethod, self).__init__(output_tensor, input_tensor, samples, current_session)
class LRP(BaseGradient): """ LRP is technique to decompose the prediction(output) of a deep neural networks(DNNs) by computing relevance at each layer in a backward pass. Current implementation is computed using backpropagation by applying change rule on a modified gradient function. LRP could be implemented in different ways. This version implements the epsilon-LRP(Eq (58) as stated in [1] or Eq (2) in [2]. Epsilon acts as a numerical stabilizer. References ---------- .. [1] Bach S, Binder A, Montavon G, Klauschen F, Müller K-R, Samek W (2015) On Pixel-Wise Explanations for Non-Linear Classifier Decisions by Layer-Wise Relevance Propagation. PLoS ONE 10(7): e0130140. https://doi.org/10.1371/journal.pone.0130140 .. [2] Ancona M, Ceolini E, Öztireli C, Gross M: Towards better understanding of gradient-based attribution methods for Deep Neural Networks. ICLR, 2018 """ __name__ = "LRP" _eps = None logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, session, epsilon=1e-4): super(LRP, self).__init__(output_tensor, input_tensor, samples, session) assert epsilon > 0.0, 'LRP epsilon must be > 0' LRP._eps = epsilon LRP.logger.info("Epsilon value: {}".format(LRP._eps)) def _default_relevance_score(self): # computing dot product of the feature wts of the input data and the gradients of the prediction label return [ g * x for g, x in zip( tf.gradients(self.output_tensor, self.input_tensor), [self.input_tensor]) ] @classmethod def _non_linear_grad(cls, op, grad): LRP.logger.debug( "Computing non-linear gradient with activation type {}".format( op.type)) op_out = op.outputs[0] op_in = op.inputs[0] stabilizer_epsilon = cls._eps * tf.sign(op_in) op_in += stabilizer_epsilon return grad * op_out / op_in
class Initializer(object): """ """ __name__ = "Initializer" # Currently supported Activation ops activation_ops = ['Relu', 'Elu', 'Softplus', 'Tanh', 'Sigmoid'] _enabled_method_class = None _grad_override_checkflag = 0 logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, session): self.output_tensor = output_tensor self.input_tensor = input_tensor self.samples = samples self.session = session def _session_run(self, output_tensor, samples): feed_dict = {} feed_dict[self.input_tensor] = samples return self.session.run(output_tensor, feed_dict) def _validate_baseline(self, baseline): if baseline is not None and baseline.shape != ((1,) + self.samples.shape[1:]): if baseline.shape == self.samples.shape[1:]: baseline = np.expand_dims(baseline, 0) else: raise RuntimeError('Baseline input shape {} does not match expected input shape {}' .format(baseline.shape, self.samples.shape[1:])) elif baseline is None: baseline = np.zeros((1,) + self.samples.shape[1:]) return baseline @classmethod def _original_grad(cls, op, grad): if op.type not in cls.activation_ops: warnings.warn('Selected Activation Ops({}) is currently not supported.'.format(op.type)) op_name = '_{}Grad'.format(op.type) Initializer.logger.debug("Operation name : {}".format(op_name)) ops_func = getattr(nn_grad, op_name) if hasattr(nn_grad, op_name) else getattr(math_grad, op_name) return ops_func(op, grad)
def __init__(self, graph=None, session=tf.get_default_session(), log_level=_WARNING): self.logger = build_logger(log_level, __name__) self.relevance_type = None self.use_case_str = None self.batch_size = None self.session = session if self.session is None: raise RuntimeError('Relevant session not retrieved') else: self.logger.info("Current session: {}".format(session.__dict__)) self.graph = session.graph if graph is None else graph # request for the default graph self.graph_context = self.graph.as_default() self.override_context = self.graph.gradient_override_map(self._get_gradient_override_map()) self.context_on = False self.__supported_relevance_type_dict = OrderedDict({ 'elrp': {'use_case_type': ['image'], 'method': LRP}, 'ig': {'use_case_type': ['image', 'txt'], 'method': IntegratedGradients} })
def __init__(self, estimator_type='classifier', splitter='best', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, seed=None, max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, class_weight="balanced", class_names=None, presort=False, feature_names=None, impurity_threshold=0.01, log_level=_WARNING): self.logger = build_logger(log_level, __name__) self.__model = None self.__model_type = None self.feature_names = feature_names self.class_names = class_names self.impurity_threshold = impurity_threshold self.criterion_types = { 'classifier': { 'criterion': ['gini', 'entropy'] }, 'regressor': { 'criterion': ['mse', 'friedman_mse', 'mae'] } } self.splitter_types = ['best', 'random'] self.splitter = splitter if any( splitter in item for item in self.splitter_types) else 'best' self.seed = seed # TODO validate the parameters based on estimator type if estimator_type == 'classifier': self.__model_type = estimator_type self.__model = DecisionTreeClassifier( splitter=self.splitter, max_depth=max_depth, min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, max_features=max_features, random_state=seed, max_leaf_nodes=max_leaf_nodes, min_impurity_decrease=min_impurity_decrease, min_impurity_split=min_impurity_split, class_weight=class_weight, presort=presort) elif estimator_type == 'regressor': self.__model_type = estimator_type self.__model = DecisionTreeRegressor( splitter=self.splitter, max_depth=None, min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, max_features=max_features, random_state=seed, max_leaf_nodes=max_leaf_nodes, min_impurity_decrease=min_impurity_decrease, min_impurity_split=min_impurity_split, presort=presort) else: raise exceptions.ModelError( "Model type not supported. Supported options types{'classifier', 'regressor'}" )
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor from sklearn.model_selection import RandomizedSearchCV import numpy as np from skater.model.base import ModelType from skater.core.visualizer.tree_visualizer import plot_tree, tree_to_text from skater.util.logger import build_logger from skater.util.logger import _WARNING from skater.util.logger import _INFO from skater.util import exceptions logger = build_logger(_INFO, __name__) class TreeSurrogate(object): """ :: Experimental :: The implementation is currently experimental and might change in future. The idea of using TreeSurrogates as means for explaining a model's(Oracle or the base model) learned decision policies(for inductive learning tasks) is inspired by the work of Mark W. Craven described as the TREPAN algorithm. In this explanation learning hypothesis, the base estimator(Oracle) could be any form of supervised learning predictive models. The explanations are approximated using DecisionTrees(both for Classification/Regression) by learning decision boundaries similar to that learned by the Oracle(predictions from the base model are used for learning the DecisionTree representation). The implementation also generates a fidelity score to quantify tree based surrogate model's approximation to the Oracle. Ideally, the score should be 0 for truthful explanation both globally and locally. Parameters ---------- estimator_type='classifier' splitter='best' max_depth=None
def __init__(self, oracle=None, splitter='best', max_depth=None, min_samples_split=2, min_samples_leaf=1, min_weight_fraction_leaf=0.0, max_features=None, seed=None, max_leaf_nodes=None, min_impurity_decrease=0.0, min_impurity_split=None, class_weight="balanced", presort=False, impurity_threshold=0.01): if not isinstance(oracle, ModelType): raise exceptions.ModelError( "Incorrect estimator used, create one with skater.model.local.InMemoryModel" ) self.oracle = oracle self.logger = build_logger(oracle.logger.level, __name__) self.__model_type = None self.feature_names = oracle.feature_names self.class_names = oracle.target_names self.impurity_threshold = impurity_threshold self.criterion_types = { 'classifier': { 'criterion': ['gini', 'entropy'] }, 'regressor': { 'criterion': ['mse', 'friedman_mse', 'mae'] } } self.splitter_types = ['best', 'random'] self.splitter = splitter if any( splitter in item for item in self.splitter_types) else 'best' self.seed = seed self.__model_type = oracle.model_type self.__scorer_name = None self.__best_score = None # TODO validate the parameters based on estimator type if self.__model_type == 'classifier': est = DecisionTreeClassifier( splitter=self.splitter, max_depth=max_depth, min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, max_features=max_features, random_state=seed, max_leaf_nodes=max_leaf_nodes, min_impurity_decrease=min_impurity_decrease, class_weight=class_weight, presort=presort) elif self.__model_type == 'regressor': est = DecisionTreeRegressor( splitter=self.splitter, max_depth=None, min_samples_split=min_samples_split, min_samples_leaf=min_samples_leaf, min_weight_fraction_leaf=min_weight_fraction_leaf, max_features=max_features, random_state=seed, max_leaf_nodes=max_leaf_nodes, min_impurity_split=min_impurity_split, presort=presort) else: raise exceptions.ModelError( "Model type not supported. Supported options types{'classifier', 'regressor'}" ) self.__model = est self.__pred_func = lambda X, prob: self.__model.predict( X) if prob is False else self.__model.predict_proba(X)
from matplotlib.cm import get_cmap import matplotlib as mpl from matplotlib.patches import Patch import pandas as pd from skater.data.datamanager import DataManager as DM from skater.util.exceptions import MatplotlibUnavailableError from skater.util.logger import build_logger from skater.util.logger import _INFO from skater.core.local_interpretation.text_interpreter import relevance_wt_assigner from skater.util.dataops import convert_dataframe_to_dict from skater.util.text_ops import generate_word_list logger = build_logger(_INFO, __name__) def __set_plot_feature_relevance_keyword(**plot_kw): plot_name = plot_kw['plot_name'] if 'plot_name' in plot_kw.keys() else 'feature_relevance.png' top_k = plot_kw['top_k'] if 'top_k' in plot_kw.keys() else 10 color_map = plot_kw['color_map'] if 'color_map' in plot_kw.keys() else ('Red', 'Blue') fig_size = plot_kw['fig_size'] if 'fig_size' in plot_kw.keys() else (20, 10) font_name = plot_kw['font_name'] if 'font_name' in plot_kw.keys() else "Avenir Black" txt_font_size = plot_kw['txt_font_size'] if 'txt_font_size' in plot_kw.keys() else '14' return plot_name, top_k, color_map, fig_size, font_name, txt_font_size # Reference: https://stackoverflow.com/questions/30618002/static-variable-in-a-function-with-python-decorator def static_var(varname, value): def decorate(func): setattr(func, varname, value) return func
class Occlusion(BasePerturbationMethod): """ Occlusion is a perturbation based inference algorithm. Such forms of algorithm direcly computes the relevance/attribution of the input features :math:`(X_{i})` by systematically occluding different portions of the image (by removing, masking or altering them), then running a forward pass on the new input to produce a new output, and then measuring and monitoring the difference between the original output and new output. Perturbation based interpretation helps one to compute direct estimation of the marginal effect of a feature but the inference might be computationally expensive depending on the cardinatlity of the feature space. The choice of the baseline value while perturbing through the feature space could be set to 0, as explained in detail by Zeiler & Fergus, 2014[2]. References ---------- .. [1] Ancona M, Ceolini E, Oztireli C, Gross M (ICLR, 2018). .. Towards better understanding of gradient-based attribution methods for Deep Neural Networks. .. [2] Zeiler, M and Fergus, R (Springer, 2014). Visualizing and understanding convolutional networks. .. In European conference on computer vision, pp. 818–833. .. [3] https://github.com/marcoancona/DeepExplain/blob/master/deepexplain/tensorflow/methods.py """ __name__ = "Occlusion" logger = build_logger(_INFO, __name__) def __init__(self, output_tensor, input_tensor, samples, current_session, **kwargs): super(Occlusion, self).__init__(output_tensor, input_tensor, samples, current_session) self.input_shape = samples.shape[1:] self.replace_value = kwargs[ 'replace_value'] if 'replace_value' in kwargs.keys() else 0 self.window_size = kwargs[ 'window_size'] if 'window_size' in kwargs.keys() else 1 self.step = kwargs['step'] if 'step' in kwargs.keys() else 1 # the input samples are expected to be of the shape, # (1, 150, 150, 3) <batch_size, image_width, image_height, no_of_channels> self.batch_size = self.samples.shape[0] Occlusion.logger.info( 'Input shape: {}; window_size/step: ({}/{}); replace value: {}; batch size: {}' .format(self.input_shape, self.window_size, self.step, self.replace_value, self.batch_size)) def _create_masked_input(self, row_value, col_value): masked_input = np.array(self.samples) # mask the region as set by the window size by replacing the pixel values with the specified value(default:0) masked_input[:, row_value:(row_value + self.window_size), col_value:(col_value + self.window_size), :] = self.replace_value return masked_input def _run(self): mask = np.array([ self.batch_size, self.window_size, self.window_size, self.samples[0].shape[2] ]) mask.fill(self.replace_value) Occlusion.logger.info('Shape of the mask patch: {}'.format(mask.shape)) relevance_score = np.zeros_like(self.samples, dtype=np.float32) # normalizer matrix is set to 1 default; as matrix cell gets used atleast once normalizer = np.ones_like(relevance_score) # Compute original output default_eval = self._session_run(self.output_tensor, self.samples) Occlusion.logger.info("shape of the default eval value :{}".format( default_eval.shape)) count = 1 # to keep track of the number of times a matrix cell is used while perturbing through the feature space # Perturb through the feature space by replacing and masking for row in range(0, self.samples[0].shape[0] - self.window_size, self.step): for col in range(0, self.samples[0].shape[1] - self.window_size, self.step): # create masked input while rolling through the input matrix new_input = self._create_masked_input(row, col) # compute entropy when compared to the original eval value delta = default_eval - self._session_run( self.output_tensor, new_input) delta_aggregated = np.sum(delta.reshape((self.batch_size, -1)), -1, keepdims=True) relevance_score[:, row:(row + self.window_size), col:(col + self.window_size), :] += delta_aggregated # keeping track of the number of time a matrix cell is used while perturbing feature space based # on window size normalizer[:, row:(row + self.window_size), col:(col + self.window_size), :] += (count - 1) Occlusion.logger.info("Min/Max normalizer weight: {}/{}".format( np.min(normalizer.shape), np.max(normalizer.shape))) relevance_score_norm = relevance_score / normalizer Occlusion.logger.info("relevance score matrix shape :{}".format( relevance_score_norm.shape)) return relevance_score_norm