def create_analyzer_model(self):
self._subanalyzer.create_analyzer_model()
if self._subanalyzer._n_debug_output > 0:
raise NotImplementedError(
"No debug output at subanalyzer is supported.")
model = self._subanalyzer._analyzer_model
if None in model.input_shape[1:]:
raise ValueError("The input shape for the model needs "
"to be fully specified (except the batch axis). "
f"Model input shape is: {model.input_shape}")
inputs = model.inputs[:self._subanalyzer._n_data_input]
extra_inputs = model.inputs[self._subanalyzer._n_data_input:]
outputs = model.outputs[:self._subanalyzer._n_data_output]
extra_outputs = model.outputs[self._subanalyzer._n_data_output:]
if len(extra_outputs) > 0:
raise Exception("No extra output is allowed " "with this wrapper.")
new_inputs = iutils.to_list(self._augment(inputs))
# print(type(new_inputs), type(extra_inputs))
tmp = iutils.to_list(model(new_inputs + extra_inputs))
new_outputs = iutils.to_list(self._reduce(tmp))
new_constant_inputs = self._keras_get_constant_inputs()
new_model = keras.models.Model(
inputs=inputs + extra_inputs + new_constant_inputs,
outputs=new_outputs + extra_outputs,
)
self._subanalyzer._analyzer_model = new_model
def apply(self, Xs, Ys, Rs, reverse_state):
grad = ilayers.GradientWRT(len(Xs))
to_low = keras.layers.Lambda(lambda x: x * 0 + self._low)
to_high = keras.layers.Lambda(lambda x: x * 0 + self._high)
low = [to_low(x) for x in Xs]
high = [to_high(x) for x in Xs]
# Get values for the division.
A = kutils.apply(self._layer_wo_act, Xs)
B = kutils.apply(self._layer_wo_act_positive, low)
C = kutils.apply(self._layer_wo_act_negative, high)
Zs = [
keras.layers.Subtract()([a, keras.layers.Add()([b, c])])
for a, b, c in zip(A, B, C)
]
# Divide relevances with the value.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Distribute along the gradient.
tmpA = iutils.to_list(grad(Xs + A + tmp))
tmpB = iutils.to_list(grad(low + B + tmp))
tmpC = iutils.to_list(grad(high + C + tmp))
tmpA = [keras.layers.Multiply()([a, b]) for a, b in zip(Xs, tmpA)]
tmpB = [keras.layers.Multiply()([a, b]) for a, b in zip(low, tmpB)]
tmpC = [keras.layers.Multiply()([a, b]) for a, b in zip(high, tmpC)]
tmp = [
keras.layers.Subtract()([a, keras.layers.Add()([b, c])])
for a, b, c in zip(tmpA, tmpB, tmpC)
]
return tmp
def _create_analysis(self, model, stop_analysis_at_tensors=[]):
tensors_to_analyze = [x for x in iutils.to_list(model.inputs)
if x not in stop_analysis_at_tensors]
gradients = iutils.to_list(ilayers.Gradient()(
tensors_to_analyze+[model.outputs[0]]))
return [keras.layers.Multiply()([i, g])
for i, g in zip(tensors_to_analyze, gradients)]
def f(layer1, layer2, X1, X2):
# Get activations of full positive or negative part.
Z1 = kutils.apply(layer1, X1)
Z2 = kutils.apply(layer2, X2)
Zs = [
tensorflow.keras.layers.Add()([a, b]) for a, b in zip(Z1, Z2)
]
# Divide incoming relevance by the activations.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to the input neurons
# using the gradient
tmp1 = iutils.to_list(grad(X1 + Z1 + tmp))
tmp2 = iutils.to_list(grad(X2 + Z2 + tmp))
# Re-weight relevance with the input values.
tmp1 = [
tensorflow.keras.layers.Multiply()([a, b])
for a, b in zip(X1, tmp1)
]
tmp2 = [
tensorflow.keras.layers.Multiply()([a, b])
for a, b in zip(X2, tmp2)
]
#combine and return
return [
tensorflow.keras.layers.Add()([a, b])
for a, b in zip(tmp1, tmp2)
]
def run(self, inputs):
"""Runs the model given the inputs.
:return: Tuple with model output and analyzer output.
"""
return_list = True
if not isinstance(inputs, list):
return_list = False
inputs = iutils.to_list(inputs)
augmented = []
for i, inp in enumerate(inputs):
if len(inp.shape) == len(self._input_shapes[i]) - 1:
# Augment by batch axis.
augmented.append(i)
inputs[i] = inp.reshape((1, ) + inp.shape)
outputs = iutils.to_list(self._model.predict_on_batch(inputs))
analysis = iutils.to_list(self._analyzer.analyze(inputs))
for i in augmented:
# Remove batch axis.
outputs[i] = outputs[i][0]
analysis[i] = analysis[i][0]
if return_list:
return outputs, analysis
else:
return outputs[0], analysis[0]
def apply(self, Xs, Ys, Rs, reverse_state):
grad = ilayers.GradientWRT(len(Xs))
# Create dummy forward path to take the derivative below.
Ys = kutils.apply(self._layer_wo_act_b, Xs)
# Compute the sum of the weights.
ones = ilayers.OnesLike()(Xs)
Zs = iutils.to_list(self._layer_wo_act_b(ones))
# Weight the incoming relevance.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Redistribute the relevances along the gradient.
tmp = iutils.to_list(grad(Xs + Ys + tmp))
return tmp
def compute_output_shape(self, input_shape: ShapeTuple) -> ShapeTuple:
if self.axis is None:
if self.keepdims is False:
return (1, )
else:
return tuple(np.ones_like(input_shape)) # type: ignore
else:
axes = np.arange(len(input_shape))
if self.keepdims is False:
for i in iutils.to_list(self.axis):
axes = np.delete(axes, i, 0)
else:
for i in iutils.to_list(self.axis):
axes[i] = 1
return tuple(
[idx for i, idx in enumerate(input_shape) if i in axes])
def broadcast_np_tensors_to_keras_tensors(
keras_tensors: OptionalList[Tensor],
np_tensors: Union[float, np.ndarray, List[np.ndarray]],
) -> List[np.ndarray]:
"""Broadcasts numpy tensors to the shape of Keras tensors.
:param keras_tensors: The Keras tensors with the target shapes.
:type keras_tensors: OptionalList[Tensor]
:param np_tensors: Numpy tensors that should be broadcasted.
:type np_tensors: Union[float, np.ndarray, List[np.ndarray]]
:return: The broadcasted Numpy tensors.
:rtype: List[np.ndarray]
"""
def none_to_one(tmp):
return [1 if x is None else x for x in tmp]
keras_tensors = iutils.to_list(keras_tensors)
if isinstance(np_tensors, list):
return [
np.broadcast_to(ri, none_to_one(int_shape(x)))
for x, ri in zip(keras_tensors, np_tensors)
]
return [
np.broadcast_to(np_tensors, none_to_one(int_shape(x)))
for x in keras_tensors
]
def __init__(
self,
model,
pattern_type="linear",
# TODO: this options seems to be buggy,
# if it sequential tensorflow still pushes all models to gpus
compute_layers_in_parallel=True,
gpus=None,
):
self.model = model
supported_layers = (
innvestigate.analyzer.pattern_based.SUPPORTED_LAYER_PATTERNNET)
for layer in self.model.layers:
if not isinstance(layer, supported_layers):
raise Exception("Model contains not supported layer: %s" %
layer)
pattern_types = iutils.to_list(pattern_type)
self.pattern_types = {k: get_pattern_class(k) for k in pattern_types}
self.compute_layers_in_parallel = compute_layers_in_parallel
self.gpus = gpus
if self.compute_layers_in_parallel is False:
raise NotImplementedError("Not supported.")
def __init__(self, model, analyzer, weights=None):
"""Helper class for retrieving output and analysis in test cases.
:param model: A Keras layer object or a list of layer objects.
In this case a sequntial model will be build. The first layer
must have set input_shape or batch_input_shape.
Alternatively a tuple with input and output tensors, in which
case the keras modle api will be used.
:param analyzer: Either an analyzer class or a function
that takes a keras model and returns an analyzer.
:param weights: After creating the model set the given weights.
"""
if isinstance(model, keras.engine.topology.Layer):
model = [model]
if isinstance(model, list):
self._model = keras.models.Sequential(model)
else:
self._model = keras.models.Model(*model)
self._input_shapes = iutils.to_list(self._model.input_shape)
if weights is not None:
self._model.set_weights(weights)
self._analyzer = analyzer(self._model)
def apply(layer: Layer, inputs: OptionalList[Tensor]) -> List[Tensor]:
"""
Apply a layer to input[s].
A flexible apply that tries to fit input to layers expected input.
This is useful when one doesn't know if a layer expects a single tensor
or many.
:param layer: A Keras layer instance.
:type layer: Layer
:param inputs: A list of input tensors or a single tensor.
:type inputs: OptionalList[Tensor]
:return: Output from applying the layer to the input.
:rtype: List[Tensor]
"""
if isinstance(inputs, list) and len(inputs) > 1:
try:
ret = layer(inputs)
except (TypeError, AttributeError) as err:
# layer expects a single tensor.
if len(inputs) != 1:
raise ValueError("Layer expects only a single input!") from err
ret = layer(inputs[0])
else:
ret = layer(inputs[0])
return iutils.to_list(ret)
def _create_analysis(self, model):
gradients = iutils.to_list(ilayers.Gradient()(model.inputs + [
model.outputs[0],
]))
return [
keras.layers.Multiply()([i, g])
for i, g in zip(model.inputs, gradients)
]
def _create_analysis(self, model, stop_analysis_at_tensors=None):
if stop_analysis_at_tensors is None:
stop_analysis_at_tensors = []
tensors_to_analyze = [
x for x in iutils.to_list(model.inputs)
if x not in stop_analysis_at_tensors
]
ret = iutils.to_list(ilayers.Gradient()(tensors_to_analyze +
[model.outputs[0]]))
if self._postprocess == "abs":
ret = ilayers.Abs()(ret)
elif self._postprocess == "square":
ret = ilayers.Square()(ret)
return iutils.to_list(ret)
def create_analyzer_model(self) -> None:
"""
Creates the analyze functionality. If not called beforehand
it will be called by :func:`analyze`.
"""
model_inputs = self._model.inputs
model, analysis_inputs, stop_analysis_at_tensors = self._prepare_model(
self._model
)
self._analysis_inputs = analysis_inputs
self._prepared_model = model
tmp = self._create_analysis(
model, stop_analysis_at_tensors=stop_analysis_at_tensors
)
if isinstance(tmp, tuple):
if len(tmp) == 3:
analysis_outputs, debug_outputs, constant_inputs = tmp # type: ignore
elif len(tmp) == 2:
analysis_outputs, debug_outputs = tmp # type: ignore
constant_inputs = []
elif len(tmp) == 1:
analysis_outputs = tmp[0]
constant_inputs = []
debug_outputs = []
else:
raise Exception("Unexpected output from _create_analysis.")
else:
analysis_outputs = tmp
constant_inputs = []
debug_outputs = []
analysis_outputs = iutils.to_list(analysis_outputs)
debug_outputs = iutils.to_list(debug_outputs)
constant_inputs = iutils.to_list(constant_inputs)
self._n_data_input = len(model_inputs)
self._n_constant_input = len(constant_inputs)
self._n_data_output = len(analysis_outputs)
self._n_debug_output = len(debug_outputs)
self._analyzer_model = keras.models.Model(
inputs=model_inputs + analysis_inputs + constant_inputs,
outputs=analysis_outputs + debug_outputs,
)
self._analyzer_model_done = True
def _postprocess_analysis(self, X):
ret = super()._postprocess_analysis(X)
if self._postprocess == "abs":
ret = ilayers.Abs()(ret)
elif self._postprocess == "square":
ret = ilayers.Square()(ret)
return iutils.to_list(ret)
def _reduce(self, X):
X_shape = [kbackend.int_shape(x) for x in iutils.to_list(X)]
reshape = [
ilayers.Reshape((-1, self._augment_by_n) + shape[1:])
for shape in X_shape
]
mean = ilayers.Mean(axis=1)
return [mean(reshape_x(x)) for x, reshape_x in zip(X, reshape)]
def _create_analysis(self, model, stop_analysis_at_tensors=None):
if stop_analysis_at_tensors is None:
stop_analysis_at_tensors = []
tensors_to_analyze = [
x for x in iutils.to_list(model.inputs)
if x not in stop_analysis_at_tensors
]
return [ilayers.Identity()(x) for x in tensors_to_analyze]
def f(layer, X):
Zs = kutils.apply(layer, X)
# Divide incoming relevance by the activations.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to the input neurons
# using the gradient
tmp = iutils.to_list(grad(X + Zs + tmp))
# Re-weight relevance with the input values.
tmp = [keras.layers.Multiply()([a, b]) for a, b in zip(X, tmp)]
return tmp
def get_input_layers(layer: Layer) -> Set[Layer]:
"""Returns all layers that created this layer's inputs."""
ret = set()
for node_index in range(len(layer._inbound_nodes)):
Xs = iutils.to_list(layer.get_input_at(node_index))
for X in Xs:
ret.add(X._keras_history[0])
return ret
def _create_analysis(self, model, stop_analysis_at_tensors=None):
if stop_analysis_at_tensors is None:
stop_analysis_at_tensors = []
noise = ilayers.TestPhaseGaussianNoise(stddev=self._stddev)
tensors_to_analyze = [
x for x in iutils.to_list(model.inputs)
if x not in stop_analysis_at_tensors
]
return [noise(x) for x in tensors_to_analyze]
def apply(self, Xs, Ys, Rs, reverse_state):
grad = ilayers.GradientWRT(len(Xs))
# Get activations.
Zs = kutils.apply(self._layer_wo_act, Xs)
# Divide incoming relevance by the activations.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to input neurons
# using the gradient.
tmp = iutils.to_list(grad(Xs + Zs + tmp))
# Re-weight relevance with the input values.
return [keras.layers.Multiply()([a, b]) for a, b in zip(Xs, tmp)]
def _create_analysis(self, model, stop_analysis_at_tensors=None):
if stop_analysis_at_tensors is None:
stop_analysis_at_tensors = []
tensors_to_analyze = [
x for x in iutils.to_list(model.inputs)
if x not in stop_analysis_at_tensors
]
gradients = super()._create_analysis(
model, stop_analysis_at_tensors=stop_analysis_at_tensors)
return [
keras.layers.Multiply()([i, g])
for i, g in zip(tensors_to_analyze, gradients)
]
def apply(self, Xs, Ys, Rs, reverse_state):
grad = ilayers.GradientWRT(len(Xs))
#TODO: assert all inputs are positive, instead of only keeping the positives.
#keep_positives = keras.layers.Lambda(lambda x: x * K.cast(K.greater(x,0), K.floatx()))
#Xs = kutils.apply(keep_positives, Xs)
# Get activations.
Zs = kutils.apply(self._layer_wo_act_b_positive, Xs)
# Divide incoming relevance by the activations.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to input neurons
# using the gradient.
tmp = iutils.to_list(grad(Xs + Zs + tmp))
# Re-weight relevance with the input values.
return [keras.layers.Multiply()([a, b]) for a, b in zip(Xs, tmp)]
def apply(self, Xs, Ys, Rs, reverse_state):
grad = ilayers.GradientWRT(len(Xs))
# The epsilon rule aligns epsilon with the (extended) sign: 0 is considered to be positive
prepare_div = keras.layers.Lambda(lambda x: x + (K.cast(
K.greater_equal(x, 0), K.floatx()) * 2 - 1) * self._epsilon)
# Get activations.
Zs = kutils.apply(self._layer_wo_act, Xs)
# Divide incoming relevance by the activations.
tmp = [ilayers.Divide()([a, prepare_div(b)]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to input neurons
# using the gradient.
tmp = iutils.to_list(grad(Xs + Zs + tmp))
# Re-weight relevance with the input values.
return [keras.layers.Multiply()([a, b]) for a, b in zip(Xs, tmp)]
def apply(self, Xs, Ys, Rs, reverse_state):
# the outputs of the pooling operation at each location is the sum of its inputs.
# the forward message must be known in this case, and are the inputs for each pooling thing.
# the gradient is 1 for each output-to-input connection, which corresponds to the "weights"
# of the layer. It should thus be sufficient to reweight the relevances and and do a gradient_wrt
grad = ilayers.GradientWRT(len(Xs))
# Get activations.
Zs = kutils.apply(self._layer_wo_act, Xs)
# Divide incoming relevance by the activations.
tmp = [ilayers.SafeDivide()([a, b]) for a, b in zip(Rs, Zs)]
# Propagate the relevance to input neurons
# using the gradient.
tmp = iutils.to_list(grad(Xs + Zs + tmp))
# Re-weight relevance with the input values.
return [keras.layers.Multiply()([a, b]) for a, b in zip(Xs, tmp)]
def analyze(
self,
X: OptionalList[np.ndarray],
neuron_selection: Optional[int] = None,
) -> OptionalList[np.ndarray]:
"""
Same interface as :class:`Analyzer` besides
:param neuron_selection: If neuron_selection_mode is 'index' this
should be an integer with the index for the chosen neuron.
"""
# TODO: what does should mean in docstring?
if self._analyzer_model_done is False:
self.create_analyzer_model()
if neuron_selection is not None and self._neuron_selection_mode != "index":
raise ValueError(
f"neuron_selection_mode {self._neuron_selection_mode} doesn't support ",
"'neuron_selection' parameter.",
)
if neuron_selection is None and self._neuron_selection_mode == "index":
raise ValueError(
"neuron_selection_mode 'index' expects 'neuron_selection' parameter."
)
X = iutils.to_list(X)
ret: OptionalList[np.ndarray]
if self._neuron_selection_mode == "index":
if neuron_selection is not None:
# TODO: document how this works
selection = self._get_neuron_selection_array(
X, neuron_selection)
ret = self._analyzer_model.predict_on_batch(X + [selection])
else:
raise RuntimeError(
'neuron_selection_mode "index" requires neuron_selection.')
else:
ret = self._analyzer_model.predict_on_batch(X)
if self._n_debug_output > 0:
self._handle_debug_output(ret[-self._n_debug_output:])
ret = ret[:-self._n_debug_output]
return iutils.unpack_singleton(ret)
def _get_active_node_indices(self):
"""
A layer can be applied in several models.
This functions returns a list with all nodes of the given
layer that are active/used in the current model.
If no model_tensors are passed to the pattern,
it is assumed all nodes are active.
"""
n_nodes = kgraph.get_layer_inbound_count(self.layer)
if self.model_tensors is None:
return list(range(n_nodes))
else:
ret = []
for i in range(n_nodes):
output_tensors = iutils.to_list(self.layer.get_output_at(i))
# Check if output is used in the model.
if all([tmp in self.model_tensors for tmp in output_tensors]):
ret.append(i)
return ret
def apply(self, Xs, _Ys, reversed_Ys, _reverse_state: Dict):
# Reapply the prepared layers.
act_Xs = kutils.apply(self._filter_layer, Xs)
act_Ys = kutils.apply(self._act_layer, act_Xs)
pattern_Ys = kutils.apply(self._pattern_layer, Xs)
# Layers that apply the backward pass.
grad_act = ilayers.GradientWRT(len(act_Xs))
grad_pattern = ilayers.GradientWRT(len(Xs))
# First step: propagate through the activation layer.
# Workaround for linear activations.
linear_activations = [None, keras.activations.get("linear")]
if self._act_layer.activation in linear_activations:
tmp = reversed_Ys
else:
# if linear activation this behaves strange
tmp = iutils.to_list(grad_act(act_Xs + act_Ys + reversed_Ys))
# Second step: propagate through the pattern layer.
return grad_pattern(Xs + pattern_Ys + tmp)
def pre_softmax_tensors(Xs: Tensor,
should_find_softmax: bool = True) -> List[Tensor]:
"""Finds the tensors that were preceeding a potential softmax."""
softmax_found = False
Xs = iutils.to_list(Xs)
ret = []
for x in Xs:
layer, node_index, _tensor_index = x._keras_history
if kchecks.contains_activation(layer, activation="softmax"):
softmax_found = True
if isinstance(layer, keras.layers.Activation):
ret.append(layer.get_input_at(node_index))
else:
layer_wo_act = copy_layer_wo_activation(layer)
ret.append(layer_wo_act(layer.get_input_at(node_index)))
if should_find_softmax and not softmax_found:
raise Exception("No softmax found.")
return ret
def model_contains(
model: Model,
layer_condition: OptionalList[LayerCheck],
) -> List[List[Layer]]:
"""
Collect layers in model which satisfy `layer_condition`.
If multiple conditions are given in `layer_condition`,
the collected layers are returned for each condition.
:param model: A Keras model.
:type model: Model
:param layer_condition: A boolean function or list of functions that
check Keras layers.
:type layer_condition: Union[LayerCheck, List[LayerCheck]]
:return: List, which for each condition in layer_condition
contains a list of layers which satisfy that condition.
:rtype: List[List[Layer]]
"""
conditions = iutils.to_list(layer_condition)
layers = get_model_layers(model)
# return layers for which condition c holds true
return [[l for l in layers if c(l)] for c in conditions]