def __init__(self,
model,
data,
session=None,
batch_size=50,
local_smoothing=0):
# try and import keras and tensorflow
global tf, keras
if tf is None:
import tensorflow as tf
if LooseVersion(tf.__version__) < LooseVersion("1.4.0"):
warnings.warn(
"Your TensorFlow version is older than 1.4.0 and not supported."
)
if keras is None:
try:
import keras
if LooseVersion(keras.__version__) < LooseVersion("2.1.0"):
warnings.warn(
"Your Keras version is older than 2.1.0 and not supported."
)
except:
pass
# determine the model inputs and outputs
self.model = model
self.model_inputs = _get_model_inputs(model)
self.model_output = _get_model_output(model)
assert type(
self.model_output
) != list, "The model output to be explained must be a single tensor!"
assert len(
self.model_output.shape
) < 3, "The model output must be a vector or a single value!"
self.multi_output = True
if len(self.model_output.shape) == 1:
self.multi_output = False
# check if we have multiple inputs
self.multi_input = True
if type(self.model_inputs) != list:
self.model_inputs = [self.model_inputs]
self.multi_input = len(self.model_inputs) > 1
if type(data) != list:
data = [data]
self.data = data
self._num_vinputs = {}
self.batch_size = batch_size
self.local_smoothing = local_smoothing
if not tf.executing_eagerly():
self.session = _get_session(session)
self.graph = _get_graph(self)
# see if there is a keras operation we need to save
self.keras_phase_placeholder = None
for op in self.graph.get_operations():
if 'keras_learning_phase' in op.name:
self.keras_phase_placeholder = op.outputs[0]
# save the expected output of the model
#self.expected_value = self.run(self.model_output, self.model_inputs, self.data).mean(0)
if not self.multi_output:
self.gradients = [None]
else:
self.gradients = [None for i in range(self.model_output.shape[1])]
def __init__(self, model, data, session=None, learning_phase_flags=None):
""" An explainer object for a deep model using a given background dataset.
Note that the complexity of the method scales linearly with the number of background data
samples. Passing the entire training dataset as `data` will give very accurate expected
values, but be unreasonably expensive. The variance of the expectation estimates scale by
roughly 1/sqrt(N) for N background data samples. So 100 samples will give a good estimate,
and 1000 samples a very good estimate of the expected values.
Parameters
----------
model : tf.keras.Model or (input : [tf.Operation], output : tf.Operation)
A keras model object or a pair of TensorFlow operations (or a list and an op) that
specifies the input and output of the model to be explained. Note that SHAP values
are specific to a single output value, so you get an explanation for each element of
the output tensor (which must be a flat rank one vector).
data : [numpy.array] or [pandas.DataFrame] or function
The background dataset to use for integrating out features. DeepExplainer integrates
over all these samples for each explanation. The data passed here must match the input
operations given to the model. If a function is supplied, it must be a function that
takes a particular input example and generates the background dataset for that example
session : None or tensorflow.Session
The TensorFlow session that has the model we are explaining. If None is passed then
we do our best to find the right session, first looking for a keras session, then
falling back to the default TensorFlow session.
learning_phase_flags : None or list of tensors
If you have your own custom learning phase flags pass them here. When explaining a prediction
we need to ensure we are not in training mode, since this changes the behavior of ops like
batch norm or dropout. If None is passed then we look for tensors in the graph that look like
learning phase flags (this works for Keras models). Note that we assume all the flags should
have a value of False during predictions (and hence explanations).
"""
# try and import keras and tensorflow
global tf, tf_ops, tf_backprop, tf_execute, tf_gradients_impl
if tf is None:
from tensorflow.python.framework import ops as tf_ops # pylint: disable=E0611
from tensorflow.python.ops import gradients_impl as tf_gradients_impl # pylint: disable=E0611
from tensorflow.python.eager import backprop as tf_backprop
from tensorflow.python.eager import execute as tf_execute
if not hasattr(tf_gradients_impl, "_IsBackpropagatable"):
from tensorflow.python.ops import gradients_util as tf_gradients_impl
import tensorflow as tf
if LooseVersion(tf.__version__) < LooseVersion("1.4.0"):
warnings.warn(
"Your TensorFlow version is older than 1.4.0 and not supported."
)
global keras
if keras is None:
try:
import keras
warnings.warn(
"keras is no longer supported, please use tf.keras instead."
)
except:
pass
# determine the model inputs and outputs
self.model_inputs = _get_model_inputs(model)
self.model_output = _get_model_output(model)
assert type(
self.model_output
) != list, "The model output to be explained must be a single tensor!"
assert len(
self.model_output.shape
) < 3, "The model output must be a vector or a single value!"
self.multi_output = True
if len(self.model_output.shape) == 1:
self.multi_output = False
if tf.executing_eagerly():
if type(model) is tuple or type(model) is list:
assert len(
model
) == 2, "When a tuple is passed it must be of the form (inputs, outputs)"
self.model = Model(model[0], model[1])
else:
self.model = model
# check if we have multiple inputs
self.multi_input = True
if type(self.model_inputs) != list or len(self.model_inputs) == 1:
self.multi_input = False
if type(self.model_inputs) != list:
self.model_inputs = [self.model_inputs]
if type(data) != list and (hasattr(data, '__call__') == False):
data = [data]
self.data = data
self._vinputs = {
} # used to track what op inputs depends on the model inputs
self.orig_grads = {}
if not tf.executing_eagerly():
self.session = _get_session(session)
self.graph = _get_graph(self)
# if no learning phase flags were given we go looking for them
# ...this will catch the one that keras uses
# we need to find them since we want to make sure learning phase flags are set to False
if learning_phase_flags is None:
self.learning_phase_ops = []
for op in self.graph.get_operations():
if 'learning_phase' in op.name and op.type == "Const" and len(
op.outputs[0].shape) == 0:
if op.outputs[0].dtype == tf.bool:
self.learning_phase_ops.append(op)
self.learning_phase_flags = [
op.outputs[0] for op in self.learning_phase_ops
]
else:
self.learning_phase_ops = [t.op for t in learning_phase_flags]
# save the expected output of the model
# if self.data is a function, set self.expected_value to None
if (hasattr(self.data, '__call__')):
self.expected_value = None
else:
if self.data[0].shape[0] > 5000:
warnings.warn(
"You have provided over 5k background samples! For better performance consider using smaller random sample."
)
if not tf.executing_eagerly():
self.expected_value = self.run(self.model_output,
self.model_inputs,
self.data).mean(0)
else:
if type(self.model) is tuple:
self.fModel(cnn.inputs,
cnn.get_layer(theNameYouWant).outputs)
self.expected_value = tf.reduce_mean(self.model(self.data[0]),
0)
# PJT
# data_tensor = tf.convert_to_tensor(self.data[0])
# self.expected_value = tf.reduce_mean(self.model(data_tensor), 0)
# self.expected_value = tf.reduce_mean(self.model.predict(self.data[0]), 0).numpy()
if not tf.executing_eagerly():
self._init_between_tensors(self.model_output.op, self.model_inputs)
# make a blank array that will get lazily filled in with the SHAP value computation
# graphs for each output. Lazy is important since if there are 1000 outputs and we
# only explain the top 5 it would be a waste to build graphs for the other 995
if not self.multi_output:
self.phi_symbolics = [None]
else:
noutputs = self.model_output.shape.as_list()[1]
if noutputs is not None:
self.phi_symbolics = [None for i in range(noutputs)]
else:
raise Exception(
"The model output tensor to be explained cannot have a static shape in dim 1 of None!"
)