def test_internal_multiple_inputs(self):
class ConcatenateLayer(Module):
def forward(this, x1, x2):
return cat((x1, x2), 1)
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(5, 6)
this.concat = ConcatenateLayer()
this.z3 = Linear(7, 7)
this.y = Linear(7, 3)
def forward(this, x1, x2):
x1 = this.z1(x1)
z = this.concat(x1, x2)
z = this.z3(z)
return this.y(z)
model = ModelWrapper(M(), [(5,), (1,)])
infl = InternalInfluence(
model, Cut('concat', anchor='in'), ClassQoI(1), PointDoi())
res = infl.attributions(
np.array([[1., 2., 3., 4., 5.]]).astype('float32'),
np.array([[1.]]).astype('float32'))
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 1))
def test_internal_slice_multiple_layers(self):
graph = Graph()
with graph.as_default():
x1 = tf.placeholder('float32', (None, 5))
z1 = x1 @ tf.random.normal((5, 6))
x2 = tf.placeholder('float32', (None, 1))
z2 = x2 @ tf.random.normal((1, 2))
z3 = z2 @ tf.random.normal((2, 4))
z4 = tf.concat([z1, z3], axis=1)
z5 = z4 @ tf.random.normal((10, 7))
y = z5 @ tf.random.normal((7, 3))
model = ModelWrapper(
graph, [x1, x2], y, dict(cut_layer1=z1, cut_layer2=z2))
infl = InternalInfluence(
model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def test_internal_slice_multiple_layers(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.cut_layer1 = Linear(5, 6)
this.cut_layer2 = Linear(1, 2)
this.z3 = Linear(2, 4)
this.z5 = Linear(10, 7)
this.y = Linear(7, 3)
def forward(this, x1, x2):
z1 = this.cut_layer1(x1)
z2 = this.cut_layer2(x2)
z3 = this.z3(z2)
z4 = cat((z1, z3), 1)
z5 = this.z5(z4)
return this.y(z5)
model = ModelWrapper(M(), [(5,), (1,)])
infl = InternalInfluence(
model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi())
res = infl.attributions(
np.array([[1., 2., 3., 4., 5.]]).astype('float32'),
np.array([[1.]]).astype('float32'))
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def test_batch_processing_deep(self):
infl = InternalInfluence(self.model_deep, InputCut(), MaxClassQoI(),
LinearDoi())
r1 = np.concatenate([infl.attributions(x[None]) for x in self.batch_x])
r2 = infl.attributions(self.batch_x)
self.assertTrue(np.allclose(r1, r2))
def test_catch_cut_index_error(self):
x = Input((2, ))
z1 = Dense(2)(x)
z2 = Activation('relu')(z1)
y = Dense(1)(z2)
model = ModelWrapper(Model(x, y))
with self.assertRaises(ValueError):
infl = InternalInfluence(model, Cut(4), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]))
def test_linear_agreement_linear_slice(self):
c = 4
infl = InternalInfluence(
self.model_deep, (Cut(self.layer2), Cut(self.layer3)),
InternalChannelQoI(c),
PointDoi(),
multiply_activation=False)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.internal1_size))
self.assertTrue(np.allclose(res[0], self.model_deep_weights_2[:, c]))
self.assertTrue(np.allclose(res[1], self.model_deep_weights_2[:, c]))
def test_linear_agreement_multiply_activation(self):
c = 1
infl = InternalInfluence(
self.model_lin,
InputCut(),
ClassQoI(c),
PointDoi(),
multiply_activation=True)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.input_size))
self.assertTrue(np.allclose(res, self.model_lin_weights[:, c] * self.x))
def test_linear_agreement_linear_slice_multiply_activation(self):
c = 4
infl = InternalInfluence(
self.model_deep, (Cut(self.layer2), Cut(self.layer3)),
InternalChannelQoI(c),
PointDoi(),
multiply_activation=True)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.internal1_size))
z = self.model_deep.fprop((self.x,), to_cut=Cut(self.layer2))[0]
self.assertTrue(np.allclose(res, self.model_deep_weights_2[:, c] * z))
def test_completeness_zero_baseline(self):
c = 2
infl = InternalInfluence(
self.model_deep,
InputCut(),
ClassQoI(c),
LinearDoi(resolution=100),
multiply_activation=True)
out_x = self.model_deep.fprop((self.x,))[0][:, c]
out_baseline = self.model_deep.fprop((self.baseline * 0,))[0][:, c]
res = infl.attributions(self.x)
self.assertTrue(
np.allclose(res.sum(axis=1), out_x - out_baseline, atol=5e-2))
def test_catch_cut_name_error(self):
graph = Graph()
with graph.as_default():
x = tf.placeholder('float32', (None, 2))
z1 = x @ tf.random.normal((2, 2))
z2 = relu(z1)
y = z2 @ tf.random.normal((2, 1))
model = ModelWrapper(graph, x, y)
with self.assertRaises(ValueError):
infl = InternalInfluence(
model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]))
def test_sensitivity(self):
c = 2
infl = InternalInfluence(
self.model_deep,
InputCut(),
ClassQoI(c),
LinearDoi(self.baseline),
multiply_activation=False)
out_x = self.model_deep.fprop((self.x[0:1],))[0][:, c]
out_baseline = self.model_deep.fprop((self.baseline,))[0][:, c]
if not np.allclose(out_x, out_baseline):
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.input_size))
self.assertNotEqual(res[0, 3], 0.)
def test_multiple_inputs(self):
x1 = Input((5, ))
z1 = Dense(6)(x1)
x2 = Input((1, ))
z2 = Concatenate()([z1, x2])
z3 = Dense(7)(z2)
y = Dense(3)(z3)
model = ModelWrapper(Model([x1, x2], y))
infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 5))
self.assertEqual(res[1].shape, (1, 1))
def per_timestep_qoi(self, model_wrapper, num_classes, num_features,
num_timesteps, batch_size):
cuts = (Cut('rnn', 'in', None), Cut('dense', 'out', None))
infl = InternalInfluence(model_wrapper, cuts, PerTimestepQoI(),
RNNLinearDoi())
input_attrs = infl.attributions(
np.ones(
(batch_size, num_timesteps, num_features)).astype('float32'))
original_output_shape = (num_classes * num_timesteps, batch_size,
num_timesteps, num_features)
self.assertEqual(np.stack(input_attrs).shape, original_output_shape)
rotated = np.stack(input_attrs, axis=-1)
attr_shape = list(rotated.shape)[:-1]
attr_shape.append(int(rotated.shape[-1] / num_classes))
attr_shape.append(num_classes)
attr_shape = tuple(attr_shape)
self.assertEqual(attr_shape, (batch_size, num_timesteps, num_features,
num_timesteps, num_classes))
input_attrs = np.reshape(rotated, attr_shape)
def test_completeness_internal_zero_baseline(self):
c = 2
infl = InternalInfluence(
self.model_deep,
Cut(self.layer2),
ClassQoI(c),
LinearDoi(resolution=100, cut=Cut(self.layer2)),
multiply_activation=True)
g = partial(
self.model_deep.fprop,
doi_cut=Cut(self.layer2),
intervention=np.zeros((2, 10)))
out_x = self.model_deep.fprop((self.x,))[0][:, c]
out_baseline = g((self.x,))[0][:, c]
res = infl.attributions(self.x)
self.assertTrue(
np.allclose(res.sum(axis=1), out_x - out_baseline, atol=5e-2))
def test_anchors(self):
x = Input((2, ))
z1 = Dense(2)(x)
z2 = Activation('relu')(z1)
y = Dense(1)(z2)
k_model = Model(x, y)
k_model.set_weights([
np.array([[1., 0.], [0., -1.]]),
np.array([0., 0.]),
np.array([[1.], [1.]]),
np.array([0.])
])
model = ModelWrapper(k_model)
infl_out = InternalInfluence(model,
Cut(2, anchor='out'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
infl_in = InternalInfluence(model,
Cut(2, anchor='in'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
res_out = infl_out.attributions(np.array([[1., 1.]]))
res_in = infl_in.attributions(np.array([[1., 1.]]))
self.assertEqual(res_out.shape, (1, 2))
self.assertEqual(res_in.shape, (1, 2))
self.assertTrue(np.allclose(res_out, np.array([[1., 1.]])))
self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_catch_cut_name_error(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(2, 2)
this.z2 = ReLU()
this.y = Linear(2, 1)
def forward(this, x):
z1 = this.z1(x)
z2 = this.z2(z1)
return this.y(z2)
model = ModelWrapper(M(), (2,))
with self.assertRaises(ValueError):
infl = InternalInfluence(
model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]).astype('float32'))
def test_multiple_inputs(self):
graph = Graph()
with graph.as_default():
x1 = tf.placeholder('float32', (None, 5))
z1 = x1 @ tf.random.normal((5, 6))
x2 = tf.placeholder('float32', (None, 1))
z2 = tf.concat([z1, x2], axis=1)
z3 = z2 @ tf.random.normal((7, 7))
y = z3 @ tf.random.normal((7, 3))
model = ModelWrapper(graph, [x1, x2], y)
infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 5))
self.assertEqual(res[1].shape, (1, 1))
def test_internal_slice_multiple_layers(self):
x1 = Input((5, ))
z1 = Dense(6, name='cut_layer1')(x1)
x2 = Input((1, ))
z2 = Dense(2, name='cut_layer2')(x2)
z3 = Dense(4)(z2)
z4 = Concatenate()([z1, z3])
z5 = Dense(7)(z4)
y = Dense(3)(z5)
model = ModelWrapper(Model([x1, x2], y))
infl = InternalInfluence(model, Cut(['cut_layer1', 'cut_layer2']),
ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def test_anchors(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(2, 2)
this.z2 = ReLU()
this.y = Linear(2, 1)
this.z1.weight.data = B.as_tensor(
np.array([[1., 0.], [0., -1.]]).T)
this.z1.bias.data = B.as_tensor(np.array([0., 0.]))
this.y.weight.data = B.as_tensor(np.array([[1.], [1.]]).T)
this.y.bias.data = B.as_tensor(np.array([0.]))
def forward(this, x):
z1 = this.z1(x)
z2 = this.z2(z1)
return this.y(z2)
model = ModelWrapper(M(), (2,))
infl_out = InternalInfluence(
model,
Cut('z2', anchor='out'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
infl_in = InternalInfluence(
model,
Cut('z2', anchor='in'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
res_out = infl_out.attributions(np.array([[1., 1.]]))
res_in = infl_in.attributions(np.array([[1., 1.]]))
self.assertEqual(res_out.shape, (1, 2))
self.assertEqual(res_in.shape, (1, 2))
self.assertTrue(np.allclose(res_out, np.array([[1., 1.]])))
self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_distributional_linearity_internal_influence(self):
x1, x2 = self.x[0:1], self.x[1:]
p1, p2 = 0.25, 0.75
class DistLinDoI(DoI):
'''
Represents the distribution of interest that weights `z` with
probability 1/4 and `z + diff` with probability 3/4.
'''
def __init__(self, diff):
super(DistLinDoI, self).__init__()
self.diff = diff
def __call__(self, z):
return [z, z + self.diff, z + self.diff, z + self.diff]
infl_pt = InternalInfluence(
self.model_deep,
Cut(self.layer2),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
attr1 = infl_pt.attributions(x1)
attr2 = infl_pt.attributions(x2)
infl_dl = InternalInfluence(
self.model_deep,
Cut(self.layer2),
ClassQoI(0),
DistLinDoI(x2 - x1),
multiply_activation=False)
attr12 = infl_dl.attributions(x1)
self.assertTrue(np.allclose(attr12, p1 * attr1 + p2 * attr2))
def test_idempotence(self):
infl = InternalInfluence(
self.model_lin,
InputCut(),
MaxClassQoI(),
PointDoi(),
multiply_activation=False)
res1 = infl.attributions(self.x)
res2 = infl.attributions(self.x)
self.assertTrue(np.allclose(res1, res2))
infl_act = InternalInfluence(
self.model_lin,
InputCut(),
MaxClassQoI(),
PointDoi(),
multiply_activation=True)
res1 = infl_act.attributions(self.x)
res2 = infl_act.attributions(self.x)
self.assertTrue(np.allclose(res1, res2))
def __init__(self,
model,
layer,
channel,
channel_axis=B.channel_axis,
agg_fn=None,
doi=None,
blur=None,
threshold=0.5,
masked_opacity=0.2,
combine_channels=True,
use_attr_as_opacity=None,
positive_only=None):
"""
Configures the default parameters for the `__call__` method (these can
be overridden by passing in values to `__call__`).
Parameters:
model:
The wrapped model whose channel we're visualizing.
layer:
The identifier (either index or name) of the layer in which the
channel we're visualizing resides.
channel:
Index of the channel (for convolutional layers) or internal
neuron (for fully-connected layers) that we'd like to visualize.
channel_axis:
If different from the channel axis specified by the backend, the
supplied `channel_axis` will be used if operating on a
convolutional layer with 4-D image format.
agg_fn:
Function with which to aggregate the remaining dimensions
(except the batch dimension) in order to get a single scalar
value for each channel; If `None`, a sum over each neuron in the
channel will be taken. This argument is not used when the
channels are scalars, e.g., for dense layers.
doi:
The distribution of interest to use when computing the input
attributions towards the specified channel. If `None`,
`PointDoI` will be used.
blur:
Gives the radius of a Gaussian blur to be applied to the
attributions before visualizing. This can be used to help focus
on salient regions rather than specific salient pixels.
threshold:
Value in the range [0, 1]. Attribution values at or below the
percentile given by `threshold` (after normalization, blurring,
etc.) will be masked.
masked_opacity:
Value in the range [0, 1] specifying the opacity for the parts
of the image that are masked.
combine_channels:
If `True`, the attributions will be averaged across the channel
dimension, resulting in a 1-channel attribution map.
use_attr_as_opacity:
If `True`, instead of using `threshold` and `masked_opacity`,
the opacity of each pixel is given by the 0-1-normalized
attribution value.
positive_only:
If `True`, only pixels with positive attribution will be
unmasked (or given nonzero opacity when `use_attr_as_opacity` is
true).
"""
self.mask_visualizer = MaskVisualizer(blur, threshold, masked_opacity,
combine_channels,
use_attr_as_opacity,
positive_only)
self.infl_input = InternalInfluence(
model, (InputCut(), Cut(layer)),
InternalChannelQoI(channel, channel_axis, agg_fn),
PointDoi() if doi is None else doi)
class ChannelMaskVisualizer(object):
"""
Uses internal influence to visualize the pixels that are most salient
towards a particular internal channel or neuron.
"""
def __init__(self,
model,
layer,
channel,
channel_axis=B.channel_axis,
agg_fn=None,
doi=None,
blur=None,
threshold=0.5,
masked_opacity=0.2,
combine_channels=True,
use_attr_as_opacity=None,
positive_only=None):
"""
Configures the default parameters for the `__call__` method (these can
be overridden by passing in values to `__call__`).
Parameters:
model:
The wrapped model whose channel we're visualizing.
layer:
The identifier (either index or name) of the layer in which the
channel we're visualizing resides.
channel:
Index of the channel (for convolutional layers) or internal
neuron (for fully-connected layers) that we'd like to visualize.
channel_axis:
If different from the channel axis specified by the backend, the
supplied `channel_axis` will be used if operating on a
convolutional layer with 4-D image format.
agg_fn:
Function with which to aggregate the remaining dimensions
(except the batch dimension) in order to get a single scalar
value for each channel; If `None`, a sum over each neuron in the
channel will be taken. This argument is not used when the
channels are scalars, e.g., for dense layers.
doi:
The distribution of interest to use when computing the input
attributions towards the specified channel. If `None`,
`PointDoI` will be used.
blur:
Gives the radius of a Gaussian blur to be applied to the
attributions before visualizing. This can be used to help focus
on salient regions rather than specific salient pixels.
threshold:
Value in the range [0, 1]. Attribution values at or below the
percentile given by `threshold` (after normalization, blurring,
etc.) will be masked.
masked_opacity:
Value in the range [0, 1] specifying the opacity for the parts
of the image that are masked.
combine_channels:
If `True`, the attributions will be averaged across the channel
dimension, resulting in a 1-channel attribution map.
use_attr_as_opacity:
If `True`, instead of using `threshold` and `masked_opacity`,
the opacity of each pixel is given by the 0-1-normalized
attribution value.
positive_only:
If `True`, only pixels with positive attribution will be
unmasked (or given nonzero opacity when `use_attr_as_opacity` is
true).
"""
self.mask_visualizer = MaskVisualizer(blur, threshold, masked_opacity,
combine_channels,
use_attr_as_opacity,
positive_only)
self.infl_input = InternalInfluence(
model, (InputCut(), Cut(layer)),
InternalChannelQoI(channel, channel_axis, agg_fn),
PointDoi() if doi is None else doi)
def __call__(self,
x,
x_preprocessed=None,
output_file=None,
blur=None,
threshold=None,
masked_opacity=None,
combine_channels=None):
"""
Visualizes the given attributions by overlaying an attribution heatmap
over the given image.
Parameters
----------
attributions : numpy.ndarray
The attributions to visualize. Expected to be in 4-D image format.
x : numpy.ndarray
The original image(s) over which the attributions are calculated.
Must be the same shape as expected by the model used with this
visualizer.
x_preprocessed : numpy.ndarray, optional
If the model requires a preprocessed input (e.g., with the mean
subtracted) that is different from how the image should be
visualized, ``x_preprocessed`` should be specified. In this case
``x`` will be used for visualization, and ``x_preprocessed`` will be
passed to the model when calculating attributions. Must be the same
shape as ``x``.
output_file : str, optional
If specified, the resulting visualization will be saved to a file
with the name given by ``output_file``.
blur : float, optional
If specified, gives the radius of a Gaussian blur to be applied to
the attributions before visualizing. This can be used to help focus
on salient regions rather than specific salient pixels. If None,
defaults to the value supplied to the constructor. Default None.
threshold : float
Value in the range [0, 1]. Attribution values at or below the
percentile given by ``threshold`` will be masked. If None, defaults
to the value supplied to the constructor. Default None.
masked_opacity: float
Value in the range [0, 1] specifying the opacity for the parts of
the image that are masked. Default 0.2. If None, defaults to the
value supplied to the constructor. Default None.
combine_channels : bool
If True, the attributions will be averaged across the channel
dimension, resulting in a 1-channel attribution map. If None,
defaults to the value supplied to the constructor. Default None.
"""
attrs_input = self.infl_input.attributions(
x if x_preprocessed is None else x_preprocessed)
return self.mask_visualizer(attrs_input, x, output_file, blur,
threshold, masked_opacity,
combine_channels)
