def _get_dict(self, values, keys):
if isinstance(values, dict):
_values = defaultdict(list, values)
for key in keys:
_values[key] = listify(_values[key])
else:
_values = defaultdict(list)
values = listify(values)
for k in keys:
_values[k] = values
return _values
def _get_seed_inputs(self, seed_inputs, input_ranges):
# Prepare input_ranges
input_ranges = ((low, 1.) if high is None else (low, high)
for (low, high) in input_ranges)
input_ranges = ((0., high) if low is None else (low, high)
for (low, high) in input_ranges)
input_ranges = ((low, high, (high - low) * 0.1)
for (low, high) in input_ranges)
# Prepare input_shape
input_shapes = (input_tensor.shape[1:]
for input_tensor in self.model.inputs)
# Prepare seed_inputs
if seed_inputs is None or len(seed_inputs) == 0:
seed_inputs = [None] * len(self.model.inputs)
seed_inputs = (tf.random.uniform(shape, low + margin,
high - margin)
for X, (low, high, margin), shape in zip(
seed_inputs, input_ranges, input_shapes))
else:
seed_inputs = listify(seed_inputs)
# Convert numpy to tf-tensor
seed_inputs = (
X if tf.is_tensor(X) else tf.Variable(X, dtype=input_tensor.dtype)
for X, input_tensor in zip(seed_inputs, self.model.inputs))
# Do expand_dims when tensor doesn't have the dim for samples
seed_inputs = (
tf.expand_dims(X, axis=0)
if len(X.shape) < len(input_tensor.shape) else X
for X, input_tensor in zip(seed_inputs, self.model.inputs))
return list(seed_inputs)
def _get_seed_inputs(self, seed_inputs, input_ranges):
# Prepare seed_inputs
if seed_inputs is None or len(seed_inputs) == 0:
# Replace None to 0.0-1.0 or any properly value
input_ranges = ((0., 1.) if low is None and high is None else
(low, high) for low, high in input_ranges)
input_ranges = ((high - np.abs(high / 2.0),
high) if low is None else (low, high)
for low, high in input_ranges)
input_ranges = ((low, low + np.abs(low * 2.0)) if high is None else
(low, high) for low, high in input_ranges)
input_ranges = list(input_ranges)
# Prepare input_shape
input_shapes = (input_tensor.shape[1:]
for input_tensor in self.model.inputs)
# Generae seed-inputs
seed_inputs = (tf.random.uniform(
shape, low,
high) for (low,
high), shape in zip(input_ranges, input_shapes))
else:
seed_inputs = listify(seed_inputs)
# Convert numpy to tf-tensor
seed_inputs = (
tf.constant(X, dtype=input_tensor.dtype)
for X, input_tensor in zip(seed_inputs, self.model.inputs))
# Do expand_dims when tensor doesn't have the dim for samples
seed_inputs = (
tf.expand_dims(X, axis=0)
if len(X.shape) < len(input_tensor.shape) else X
for X, input_tensor in zip(seed_inputs, self.model.inputs))
return list(seed_inputs)
def _get_losses_for_multiple_outputs(self, loss):
losses = listify(loss)
if len(losses) == 1 and len(losses) < len(self.model.outputs):
losses = losses * len(self.model.outputs)
if len(losses) != len(self.model.outputs):
raise ValueError(('The model has {} outputs, '
'but the number of loss-functions you passed is {}.').format(
len(self.model.outputs), len(losses)))
return losses
def _get_gradients(self, seed_inputs, losses, gradient_modifier):
with tf.GradientTape() as tape:
tape.watch(seed_inputs)
outputs = self.model(seed_inputs)
outputs = listify(outputs)
loss_values = [loss(output) for output, loss in zip(outputs, losses)]
grads = tape.gradient(loss_values, seed_inputs)
if gradient_modifier is not None:
grads = [gradient_modifier(g) for g in grads]
return grads
def _get_seed_inputs_for_multiple_inputs(self, seed_input):
seed_inputs = listify(seed_input)
if len(seed_inputs) != len(self.model.inputs):
raise ValueError(('The model has {} inputs, '
'but the number of seed-inputs tensors you passed is {}.').format(
len(self.model.inputs), len(seed_inputs)))
seed_inputs = (x if tf.is_tensor(x) else tf.constant(x) for x in seed_inputs)
seed_inputs = (tf.expand_dims(x, axis=0) if len(x.shape) == len(tensor.shape[1:]) else x
for x, tensor in zip(seed_inputs, self.model.inputs))
return list(seed_inputs)
def _prepare_list(self,
value,
list_length_if_created,
empty_list_if_none=True,
convert_tuple_to_list=True):
values = listify(value,
empty_list_if_none=empty_list_if_none,
convert_tuple_to_list=convert_tuple_to_list)
if len(values) == 1 and list_length_if_created > 1:
values = values * list_length_if_created
return values
def _prepare_inputmodifier_dictionary(self, input_modifier):
input_modifiers = listify(input_modifier)
input_modifiers = self._prepare_dictionary(
input_modifiers, [l.name for l in self.model.inputs])
if len(input_modifiers) != 0 and len(input_modifiers) != len(
self.model.inputs):
raise ValueError(
'The model has {} inputs, but you passed {} as input_modifiers. '
'When the model has multipul inputs, '
'you must pass a dictionary as input_modifiers.'.format(
len(self.model.inputs), input_modifier))
return input_modifiers
def _get_input_ranges(self, input_range):
input_ranges = listify(input_range,
return_empty_list_if_none=False,
convert_tuple_to_list=False)
if len(input_ranges) == 1 and len(self.model.inputs) > 1:
input_ranges = input_ranges * len(self.model.inputs)
input_ranges = [(None, None) if r is None else r for r in input_ranges]
for i, r in enumerate(input_ranges):
if len(r) != 2:
raise ValueError(
'The length of input range tuple must be 2 (Or it is just `None`, not tuple), '
'but you passed {} as `input_ranges[{}]`.'.format(r, i))
return input_ranges
def _get_gradients(self, seed_inputs, losses, gradient_modifier, training):
with tf.GradientTape(watch_accessed_variables=False,
persistent=True) as tape:
tape.watch(seed_inputs)
outputs = self.model(seed_inputs, training=training)
outputs = listify(outputs)
loss_values = [
loss(output) for output, loss in zip(outputs, losses)
]
grads = tape.gradient(
loss_values,
seed_inputs,
unconnected_gradients=tf.UnconnectedGradients.ZERO)
if gradient_modifier is not None:
grads = [gradient_modifier(g) for g in grads]
return grads
def _prepare_dictionary(self,
values,
keys,
default_value=list,
empty_list_if_none=True,
convert_tuple_to_list=True):
if isinstance(values, dict):
values = defaultdict(default_value, values)
else:
_values = defaultdict(default_value)
for k in keys:
_values[k] = values
values = _values
for key in values.keys():
values[key] = listify(values[key],
empty_list_if_none=empty_list_if_none,
convert_tuple_to_list=convert_tuple_to_list)
return values
def __call__(self,
loss,
seed_input=None,
input_range=(0, 255),
input_modifiers=[Jitter(2), Rotate(2)],
regularizers=[TotalVariation(10.),
L2Norm(10.)],
steps=200,
optimizer=tf.optimizers.RMSprop(1., 0.95),
normalize_gradient=True,
gradient_modifier=None,
callbacks=None,
training=False):
"""Generate the model inputs that maximize the output of the given `loss` functions.
# Arguments
loss: A loss function. If the model has multiple outputs, you can use a different
loss on each output by passing a list of losses. The loss value that will be
maximized will then be the sum of all individual losses
(and all regularization values).
seed_input: `None`(default) or An N-dim Numpy array. When `None`, the seed_input
value will be generated with randome noise. If the model has multiple inputs,
you have to pass a list of N-dim Numpy arrays.
input_range: A tuple that specifies the input range as a `(min, max)` tuple. If the
model has multiple inputs, you can use a different input range on each input by
passing as list of input ranges. When `None` or a `(None, None)` tuple, the range of
a input value (i.e., the result of this function) will be no applied any limitation.
input_modifiers: A input modifier function or a list of input modifier functions.
You can also use a instance of `tf_keras-vis.input_modifiers. InputModifier`'s
subclass, instead of a function. If the model has multiple inputs, you have to pass
a dictionary of input modifier functions or instances on each model inputs:
such as `input_modifiers={'input_a': [ input_modifier_a_1, input_modifier_a_2 ],
'input_b': input_modifier_b, ... }`.
regularizers: A regularization function or a list of regularization functions. You can
also use a instance of `tf_keras-vis.regularizers.Regularizer`'s subclass,
instead of a function. A regularization value will be calculated with
a corresponding model input will add to the loss value.
steps: The number of gradient descent iterations.
optimizer: A `tf.optimizers.Optimizer` instance.
normalize_gradient: This option is disabled and will be removed at version 0.6.0.
gradient_modifier: A function to modify gradients. This function is executed before
normalizing gradients.
callbacks: A `tf_keras_vis.callbacks.OptimizerCallback` instance or a list of them.
training: A bool whether the model's trainig-mode turn on or off.
# Returns
An Numpy arrays when the model has a single input and `seed_input` is None or An N-dim
Numpy Array, Or a list of Numpy arrays when otherwise. The Numpy is that the model
inputs that maximize the out of `loss`.
# Raises
ValueError: In case of invalid arguments for `loss`, `input_range`, `input_modifiers`
or `regularizers`.
"""
# losses
losses = self._get_losses_for_multiple_outputs(loss)
# Get initial seed-inputs
input_ranges = self._get_input_ranges(input_range)
seed_inputs = self._get_seed_inputs(seed_input, input_ranges)
# input_modifiers
input_modifiers = self._get_input_modifiers(input_modifiers)
# regularizers
regularizers = self._get_regularizers(regularizers)
callbacks = listify(callbacks)
for callback in callbacks:
callback.on_begin()
for i in range(check_steps(steps)):
# Apply input modifiers
for j, name in enumerate(self.model.input_names):
for modifier in input_modifiers[name]:
seed_inputs[j] = modifier(seed_inputs[j])
seed_inputs = [tf.Variable(X) for X in seed_inputs]
# Calculate gradients
with tf.GradientTape(watch_accessed_variables=False) as tape:
tape.watch(seed_inputs)
outputs = self.model(seed_inputs, training=training)
outputs = listify(outputs)
loss_values = (loss(output)
for output, loss in zip(outputs, losses))
loss_values = (tf.stack(loss_value, axis=0) if isinstance(
loss_value, (list, tuple)) else loss_value
for loss_value in loss_values)
loss_values = [
K.mean(tf.reshape(loss_value, (X.shape[0], -1)), axis=1)
for loss_value, X in zip(loss_values, seed_inputs)
]
# Calculate regularization values
regularizations = [(regularizer.name, regularizer(seed_inputs))
for regularizer in regularizers]
regularized_loss_values = [
(-1. * loss_value) + sum([v for _, v in regularizations])
for loss_value in loss_values
]
grads = tape.gradient(
regularized_loss_values,
seed_inputs,
unconnected_gradients=tf.UnconnectedGradients.ZERO)
grads = listify(grads)
if gradient_modifier is not None:
grads = (gradient_modifier(g) for g in grads)
if normalize_gradient:
# XXX `normalize_gradient` option is not working correctly.
# XXX So this option will be removed as of version 0.6.0.
# XXX For now, disable this option.
# grads = (K.l2_normalize(g, axis=tuple(range(len(g))[1:])) for g in grads)
pass
optimizer.apply_gradients(zip(grads, seed_inputs))
for callback in callbacks:
callback(i,
self._apply_clip(seed_inputs, input_ranges),
grads,
loss_values,
outputs,
regularizations=regularizations,
overall_loss=regularized_loss_values)
for callback in callbacks:
callback.on_end()
cliped_value = self._apply_clip(seed_inputs, input_ranges)
if len(self.model.inputs) == 1 and (seed_input is None or
not isinstance(seed_input, list)):
cliped_value = cliped_value[0]
return cliped_value
def __init__(self, indices, epsilon=0.05):
super().__init__('CategoricalSmoothedScore')
self.indices = listify(indices)
self.epsilon = epsilon
def __init__(self, indices, depth):
super().__init__('CategoricalScore')
self.indices = listify(indices)
self.depth = depth
def __call__(self,
loss,
seed_input,
penultimate_layer=-1,
activation_modifier=lambda cam: K.relu(cam),
normalize_gradient=True):
"""Generate a gradient based class activation map (CAM) by using positive gradient of
penultimate_layer with respect to loss.
For details on Grad-CAM, see the paper:
[Grad-CAM: Why did you say that? Visual Explanations from Deep Networks via
Gradient-based Localization](https://arxiv.org/pdf/1610.02391v1.pdf).
# Arguments
loss: A loss function. If the model has multipul outputs, you can use a different
loss on each output by passing a list of losses.
seed_input: An N-dim Numpy array. If the model has multipul inputs,
you have to pass a list of N-dim Numpy arrays.
penultimate_layer: A number of integer or a tf.keras.layers.Layer object.
normalize_gradient: True to normalize gradients.
activation_modifier: A function to modify gradients.
# Returns
The heatmap image or a list of their images that indicate the `seed_input` regions
whose change would most contribute the loss value,
# Raises
ValueError: In case of invalid arguments for `loss`, or `penultimate_layer`.
"""
losses = self._prepare_losses(loss)
seed_inputs = [
x if tf.is_tensor(x) else tf.constant(x)
for x in listify(seed_input)
]
seed_inputs = [
tf.expand_dims(seed_input, axis=0)
if X.shape == input_tensor.shape[1:] else X
for X, input_tensor in zip(seed_inputs, self.model.inputs)
]
if len(seed_inputs) != len(self.model.inputs):
raise ValueError('')
penultimate_output_tensor = self._find_penultimate_output(
self.model, penultimate_layer)
model = tf.keras.Model(inputs=self.model.inputs,
outputs=self.model.outputs +
[penultimate_output_tensor])
with tf.GradientTape() as tape:
tape.watch(seed_inputs)
outputs = model(seed_inputs)
outputs = listify(outputs)
loss_values = [loss(y) for y, loss in zip(outputs[:-1], losses)]
penultimate_outputs = outputs[-1]
grads = tape.gradient(loss_values, penultimate_outputs)
if normalize_gradient:
grads = K.l2_normalize(grads)
weights = K.mean(grads, axis=tuple(np.arange(len(grads.shape))[1:-1]))
cam = np.asarray([
np.sum(o * w, axis=-1)
for o, w in zip(penultimate_outputs, weights)
])
if activation_modifier is not None:
cam = activation_modifier(cam)
input_dims_list = (X.shape[1:-1] for X in seed_inputs)
output_dims = penultimate_outputs.shape[1:-1]
zoom_factors = ([
i / (j * 1.0) for i, j in iter(zip(input_dims, output_dims))
] for input_dims in input_dims_list)
cams = [
np.asarray([zoom(v, factor) for v in cam])
for factor in zoom_factors
]
if len(self.model.inputs) == 1 and not isinstance(seed_input, list):
cams = cams[0]
return cams
def __call__(self,
loss,
seed_input,
smooth_samples=0,
smooth_noise=0.20,
keepdims=False,
gradient_modifier=lambda grads: K.abs(grads)):
"""Generate an attention map that appears how output value changes with respect to a small
change in input image pixels.
See details: https://arxiv.org/pdf/1706.03825.pdf
# Arguments
loss: A loss function. If the model has multipul outputs, you can use a different
loss on each output by passing a list of losses.
seed_input: An N-dim Numpy array. If the model has multipul inputs,
you have to pass a list of N-dim Numpy arrays.
smooth_samples: The number of calculating gradients iterations. If set to zero,
the noise for smoothing won't be generated.
keepdims: A boolean that whether to keep the channels-dim or not.
smooth_noise: Noise level that is recommended no tweaking when there is no reason.
gradient_modifier: A function to modify gradients. By default, the function modify
gradients to `absolute` values.
# Returns
The heatmap image indicating the `seed_input` regions whose change would most contribute
towards maximizing the loss value, Or a list of their images.
A list of Numpy arrays that the model inputs that maximize the out of `loss`.
# Raises
ValueError: In case of invalid arguments for `loss`, or `seed_input`.
"""
losses = self._prepare_losses(loss)
seed_inputs = [
X if tf.is_tensor(X) else tf.constant(X)
for X in listify(seed_input)
]
seed_inputs = [
tf.expand_dims(seed_input, axis=0)
if X.shape == input_tensor.shape[1:] else X
for X, input_tensor in zip(seed_inputs, self.model.inputs)
]
if len(seed_inputs) != len(self.model.inputs):
raise ValueError('')
if smooth_samples > 0:
axes = [tuple(range(1, len(X.shape))) for X in seed_inputs]
sigmas = [
smooth_noise * (np.max(X, axis=axis) - np.min(X, axis=axis))
for X, axis in zip(seed_inputs, axes)
]
total_gradients = (np.zeros_like(X) for X in seed_inputs)
for i in range(check_steps(smooth_samples)):
seed_inputs_plus_noise = [
tf.constant(
np.concatenate([
x + np.random.normal(0., s, (1, ) + x.shape)
for x, s in zip(X, sigma)
])) for X, sigma in zip(seed_inputs, sigmas)
]
gradients, loss_values, outputs = self._get_gradients(
self.model, seed_inputs_plus_noise, losses,
gradient_modifier)
total_gradients = (
total + g for total, g in zip(total_gradients, gradients))
grads = [g / smooth_samples for g in total_gradients]
else:
grads, loss_values, outputs = self._get_gradients(
self.model, seed_inputs, losses, gradient_modifier)
if not keepdims:
grads = [np.max(g, axis=-1) for g in grads]
if len(self.model.inputs) == 1 and not isinstance(seed_input, list):
grads = grads[0]
return grads
def _get_regularizers(self, regularizer):
regularizers = listify(regularizer)
return regularizers
def __init__(self, indices, epsilon=0.05):
super().__init__('SmoothedLoss')
self.indices = listify(indices)
self.epsilon = epsilon
def __call__(self,
loss,
seed_input,
penultimate_layer=-1,
seek_penultimate_conv_layer=True,
activation_modifier=lambda cam: K.relu(cam),
expand_cam=True,
batch_size=32,
max_N=None,
training=False):
"""Generate score-weighted class activation maps (CAM) by using gradient-free visualization method.
For details on Score-CAM, see the paper:
[Score-CAM: Score-Weighted Visual Explanations for Convolutional Neural Networks ]
(https://arxiv.org/pdf/1910.01279.pdf).
# Arguments
loss: A loss function. If the model has multiple outputs, you can use a different
loss on each output by passing a list of losses.
seed_input: An N-dim Numpy array. If the model has multiple inputs,
you have to pass a list of N-dim Numpy arrays.
penultimate_layer: A number of integer or a tf.keras.layers.Layer object.
seek_penultimate_conv_layer: True to seek the penultimate layter that is a subtype of
`keras.layers.convolutional.Conv` class.
If False, the penultimate layer is that was elected by penultimate_layer index.
activation_modifier: A function to modify activations.
expand_cam: True to expand cam to same as input image size.
![Note] Even if the model has multiple inputs, this function return only one cam
value (That's, when `expand_cam` is True, multiple cam images are generated from
a model that has multiple inputs).
batch_size: Integer or None. Number of samples per batch.
If unspecified, batch_size will default to 32.
max_N: Integer or None. If None, we do NOT recommend, because it takes huge time.
If not None, that's setting Integer, run as Faster-ScoreCAM.
Set larger number, need more time to visualize CAM but to be able to get
clearer attention images.
(see for details: https://github.com/tabayashi0117/Score-CAM#faster-score-cam)
training: A bool whether the model's trainig-mode turn on or off.
# Returns
The heatmap image or a list of their images that indicate the `seed_input` regions
whose change would most contribute the loss value,
# Raises
ValueError: In case of invalid arguments for `loss`, or `penultimate_layer`.
"""
# Preparing
losses = self._get_losses_for_multiple_outputs(loss)
seed_inputs = self._get_seed_inputs_for_multiple_inputs(seed_input)
penultimate_output_tensor = self._find_penultimate_output(
penultimate_layer, seek_penultimate_conv_layer)
# Processing score-cam
penultimate_output = tf.keras.Model(inputs=self.model.inputs,
outputs=penultimate_output_tensor)(
seed_inputs, training=training)
# For efficiently visualizing, extract maps that has a large variance.
# This excellent idea is devised by tabayashi0117.
# (see for details: https://github.com/tabayashi0117/Score-CAM#faster-score-cam)
if max_N is not None and max_N > -1:
activation_map_std = tf.math.reduce_std(
penultimate_output,
axis=tuple(range(penultimate_output.ndim)[1:-1]),
keepdims=True)
_, top_k_indices = tf.math.top_k(activation_map_std, max_N)
top_k_indices, _ = tf.unique(tf.reshape(top_k_indices, (-1, )))
penultimate_output = tf.gather(penultimate_output,
top_k_indices,
axis=-1)
channels = penultimate_output.shape[-1]
# Upsampling activation-maps
penultimate_output = penultimate_output.numpy()
input_shapes = [seed_input.shape for seed_input in seed_inputs]
factors = (zoom_factor(penultimate_output.shape[:-1], input_shape[:-1])
for input_shape in input_shapes)
upsampled_activation_maps = [
zoom(penultimate_output, factor + (1, )) for factor in factors
]
map_shapes = [
activation_map.shape
for activation_map in upsampled_activation_maps
]
# Normalizing activation-maps
min_activation_maps = (np.min(activation_map,
axis=tuple(
range(activation_map.ndim)[1:-1]),
keepdims=True)
for activation_map in upsampled_activation_maps)
max_activation_maps = (np.max(activation_map,
axis=tuple(
range(activation_map.ndim)[1:-1]),
keepdims=True)
for activation_map in upsampled_activation_maps)
normalized_activation_maps = (
(activation_map - min_activation_map) /
(max_activation_map - min_activation_map)
for activation_map, min_activation_map, max_activation_map in zip(
upsampled_activation_maps, min_activation_maps,
max_activation_maps))
# Masking inputs
input_tile_axes = (
(map_shape[-1], ) + tuple(np.ones(len(input_shape), np.int))
for input_shape, map_shape in zip(input_shapes, map_shapes))
mask_templates = (np.tile(
seed_input,
axes) for seed_input, axes in zip(seed_inputs, input_tile_axes))
map_transpose_axes = ((len(map_shape) - 1, ) +
tuple(range(len(map_shape))[:-1])
for map_shape in map_shapes)
masks = (np.transpose(activation_map, transpose_axis)
for activation_map, transpose_axis in zip(
normalized_activation_maps, map_transpose_axes))
map_tile_axes = (
tuple(np.ones(len(map_shape), np.int)) + (input_shape[-1], )
for input_shape, map_shape in zip(input_shapes, map_shapes))
masks = (np.tile(np.expand_dims(activation_map, axis=-1), tile_axis)
for activation_map, tile_axis in zip(masks, map_tile_axes))
masked_seed_inputs = (
mask_template * mask
for mask_template, mask in zip(mask_templates, masks))
masked_seed_inputs = [
np.reshape(masked_seed_input, (-1, ) + masked_seed_input.shape[2:])
for masked_seed_input in masked_seed_inputs
]
# Predicting masked seed-inputs
preds = self.model.predict(masked_seed_inputs, batch_size=batch_size)
preds = (np.reshape(prediction, (channels, -1, prediction.shape[-1]))
for prediction in listify(preds))
# Calculating weights
weights = ([loss(p) for p in prediction]
for loss, prediction in zip(losses, preds))
weights = (np.array(w, dtype=np.float32) for w in weights)
weights = (np.reshape(w, (channels, -1)) for w in weights)
weights = np.array(list(weights), dtype=np.float32)
weights = np.sum(weights, axis=0)
weights = np.transpose(weights, (1, 0))
# Generate cam
cam = K.batch_dot(penultimate_output, weights)
if activation_modifier is not None:
cam = activation_modifier(cam)
if not expand_cam:
return cam
factors = (zoom_factor(cam.shape, X.shape) for X in seed_inputs)
cam = [zoom(cam, factor) for factor in factors]
if len(self.model.inputs) == 1 and not isinstance(seed_input, list):
cam = cam[0]
return cam
def __call__(self,
loss,
seed_input=None,
input_range=(0, 255),
input_modifiers=[Jitter(2), Rotate(2)],
regularizers=[TotalVariation(10.),
L2Norm(10.)],
steps=200,
optimizer=tf.optimizers.RMSprop(1., 0.95),
normalize_gradient=True,
gradient_modifier=None,
callbacks=None):
"""Generate the model inputs that maximize the output of the given `loss` functions.
# Arguments
loss: A loss function. If the model has multipul outputs, you can use a different
loss on each output by passing a list of losses. The loss value that will be
maximized will then be the sum of all individual losses
(and all regularization values).
seed_input: `None`(default) or An N-dim Numpy array. When `None`, the seed_input
value will be generated with randome noise. If the model has multipul inputs,
you have to pass a list of N-dim Numpy arrays.
input_range: A tuple that specifies the input range as a `(min, max)` tuple. If the
model has multipul inputs, you can use a different input range on each input by
passing as list of input ranges. When `None` or a `(None, None)` tuple, the range of
a input value (i.e., the result of this function) will be no applied any limitation.
input_modifiers: A input modifier function or a list of input modifier functions.
You can also use a instance of `tf_keras-vis.input_modifiers. InputModifier`'s
subclass, instead of a function. If the model has multipul inputs, you have to pass
a dictionary of input modifier functions or instances on each model inputs:
such as `input_modifiers={'input_a': [ input_modifier_a_1, input_modifier_a_2 ],
'input_b': input_modifier_b, ... }`.
regularizers: A regularization function or a list of regularization functions. You can
also use a instance of `tf_keras-vis.regularizers.Regularizer`'s subclass,
instead of a function. If the model has multipul outputs, you can use
a dictionary of regularization functions or instances on each model outputs:
such as `regularizers={'output_a': [ regularizer_a_1, regularizer_a_2 ],
'output_b': regularizer_b, ... }`. A regularization value will be calculated with
a corresponding model input will add to the loss value.
steps: The number of gradient descent iterations.
optimizer: A `tf.optimizers.Optimizer` instance.
normalize_gradient: True to normalize gradients. Normalization avoids too small or
large gradients and ensures a smooth gradient descent process.
gradient_modifier: A function to modify gradients. This function is executed before
normalizing gradients.
callbacks: A `tf_keras_vis.callbacks.OptimizerCallback` instance or a list of them.
# Returns
An Numpy arrays when the model has a single input and `seed_input` is None or An N-dim
Numpy Array, Or a list of Numpy arrays when otherwise. The Numpy is that the model
inputs that maximize the out of `loss`.
# Raises
ValueError: In case of invalid arguments for `loss`, `input_range`, `input_modifiers`
or `regularizers`.
"""
# losses
losses = self._prepare_losses(loss)
# Get initial seed-inputs
input_ranges = self._prepare_input_ranges(input_range)
seed_inputs = self._get_seed_inputs(seed_input, input_ranges)
# input_modifiers
input_modifiers = self._prepare_inputmodifier_dictionary(
input_modifiers)
# regularizers
regularizers = self._prepare_regularizer_dictionary(regularizers)
callbacks = listify(callbacks)
for callback in callbacks:
callback.on_begin()
for i in range(check_steps(steps)):
# Apply input modifiers
for j, input_layer in enumerate(self.model.inputs):
for modifier in input_modifiers[input_layer.name]:
seed_inputs[j] = modifier(seed_inputs[j])
seed_inputs = [tf.Variable(X) for X in seed_inputs]
# Calculate gradients
with tf.GradientTape() as tape:
tape.watch(seed_inputs)
outputs = self.model(seed_inputs)
outputs = listify(outputs)
loss_values = [
loss(output) for output, loss in zip(outputs, losses)
]
# Calculate regularization values
regularization_values = [[
(regularizer.name, regularizer(seed_inputs))
for regularizer in regularizers[output_layer.name]
] for output_layer in self.model.outputs]
ys = [(-1. * loss_value) +
sum([rv for (_, rv) in regularization_value])
for loss_value, regularization_value in zip(
loss_values, regularization_values)]
grads = tape.gradient(ys, seed_inputs)
grads = listify(grads)
if gradient_modifier is not None:
grads = [gradient_modifier(g) for g in grads]
if normalize_gradient:
grads = [K.l2_normalize(g) for g in grads]
optimizer.apply_gradients(zip(grads, seed_inputs))
for callback in callbacks:
callback(i,
self._apply_clip(seed_inputs, input_ranges),
grads,
loss_values,
outputs,
regularizations=regularization_values,
overall_loss=ys)
for callback in callbacks:
callback.on_end()
cliped_value = self._apply_clip(seed_inputs, input_ranges)
if len(self.model.inputs) == 1 and (seed_input is None or
not isinstance(seed_input, list)):
cliped_value = cliped_value[0]
return cliped_value
