def test_anchors(self):
x = Input((2, ))
z1 = Dense(2)(x)
z2 = Activation('relu')(z1)
y = Dense(1)(z2)
k_model = Model(x, y)
k_model.set_weights([
np.array([[1., 0.], [0., -1.]]),
np.array([0., 0.]),
np.array([[1.], [1.]]),
np.array([0.])
])
model = ModelWrapper(k_model)
infl_out = InternalInfluence(model,
Cut(2, anchor='out'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
infl_in = InternalInfluence(model,
Cut(2, anchor='in'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
res_out = infl_out.attributions(np.array([[1., 1.]]))
res_in = infl_in.attributions(np.array([[1., 1.]]))
self.assertEqual(res_out.shape, (1, 2))
self.assertEqual(res_in.shape, (1, 2))
self.assertTrue(np.allclose(res_out, np.array([[1., 1.]])))
self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_internal_slice_multiple_layers(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.cut_layer1 = Linear(5, 6)
this.cut_layer2 = Linear(1, 2)
this.z3 = Linear(2, 4)
this.z5 = Linear(10, 7)
this.y = Linear(7, 3)
def forward(this, x1, x2):
z1 = this.cut_layer1(x1)
z2 = this.cut_layer2(x2)
z3 = this.z3(z2)
z4 = cat((z1, z3), 1)
z5 = this.z5(z4)
return this.y(z5)
model = ModelWrapper(M(), [(5,), (1,)])
infl = InternalInfluence(
model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi())
res = infl.attributions(
np.array([[1., 2., 3., 4., 5.]]).astype('float32'),
np.array([[1.]]).astype('float32'))
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def test_internal_slice_multiple_layers(self):
graph = Graph()
with graph.as_default():
x1 = tf.placeholder('float32', (None, 5))
z1 = x1 @ tf.random.normal((5, 6))
x2 = tf.placeholder('float32', (None, 1))
z2 = x2 @ tf.random.normal((1, 2))
z3 = z2 @ tf.random.normal((2, 4))
z4 = tf.concat([z1, z3], axis=1)
z5 = z4 @ tf.random.normal((10, 7))
y = z5 @ tf.random.normal((7, 3))
model = ModelWrapper(
graph, [x1, x2], y, dict(cut_layer1=z1, cut_layer2=z2))
infl = InternalInfluence(
model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def __get_doi(doi_arg):
"""
Helper function to get a `DoI` object from more user-friendly primitive
arguments.
"""
if isinstance(doi_arg, DoI):
# We were already given a DoI, so return it.
return doi_arg
elif isinstance(doi_arg, str):
# We can specify `PointDoi` via the string 'point', or `LinearDoi`
# via the string 'linear'.
if doi_arg == 'point':
return PointDoi()
elif doi_arg == 'linear':
return LinearDoi()
else:
raise ValueError(
'String argument for `doi` must be one of the following:\n'
' - "point"\n'
' - "linear"')
else:
raise ValueError('Unrecognized argument type for `doi`')
def test_internal_multiple_inputs(self):
class ConcatenateLayer(Module):
def forward(this, x1, x2):
return cat((x1, x2), 1)
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(5, 6)
this.concat = ConcatenateLayer()
this.z3 = Linear(7, 7)
this.y = Linear(7, 3)
def forward(this, x1, x2):
x1 = this.z1(x1)
z = this.concat(x1, x2)
z = this.z3(z)
return this.y(z)
model = ModelWrapper(M(), [(5,), (1,)])
infl = InternalInfluence(
model, Cut('concat', anchor='in'), ClassQoI(1), PointDoi())
res = infl.attributions(
np.array([[1., 2., 3., 4., 5.]]).astype('float32'),
np.array([[1.]]).astype('float32'))
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 1))
def test_point(self):
res = PointDoi()(self.z)
self.assertEqual(len(res), 1, 'PointDoi should return a single point')
self.assertTrue(np.array_equal(B.as_array(res[0]), B.as_array(self.z)),
'Value of point should not change')
def test_anchors(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(2, 2)
this.z2 = ReLU()
this.y = Linear(2, 1)
this.z1.weight.data = B.as_tensor(
np.array([[1., 0.], [0., -1.]]).T)
this.z1.bias.data = B.as_tensor(np.array([0., 0.]))
this.y.weight.data = B.as_tensor(np.array([[1.], [1.]]).T)
this.y.bias.data = B.as_tensor(np.array([0.]))
def forward(this, x):
z1 = this.z1(x)
z2 = this.z2(z1)
return this.y(z2)
model = ModelWrapper(M(), (2,))
infl_out = InternalInfluence(
model,
Cut('z2', anchor='out'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
infl_in = InternalInfluence(
model,
Cut('z2', anchor='in'),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
res_out = infl_out.attributions(np.array([[1., 1.]]))
res_in = infl_in.attributions(np.array([[1., 1.]]))
self.assertEqual(res_out.shape, (1, 2))
self.assertEqual(res_in.shape, (1, 2))
self.assertTrue(np.allclose(res_out, np.array([[1., 1.]])))
self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_catch_cut_index_error(self):
x = Input((2, ))
z1 = Dense(2)(x)
z2 = Activation('relu')(z1)
y = Dense(1)(z2)
model = ModelWrapper(Model(x, y))
with self.assertRaises(ValueError):
infl = InternalInfluence(model, Cut(4), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]))
def test_linear_agreement_linear_slice(self):
c = 4
infl = InternalInfluence(
self.model_deep, (Cut(self.layer2), Cut(self.layer3)),
InternalChannelQoI(c),
PointDoi(),
multiply_activation=False)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.internal1_size))
self.assertTrue(np.allclose(res[0], self.model_deep_weights_2[:, c]))
self.assertTrue(np.allclose(res[1], self.model_deep_weights_2[:, c]))
def test_linear_agreement_multiply_activation(self):
c = 1
infl = InternalInfluence(
self.model_lin,
InputCut(),
ClassQoI(c),
PointDoi(),
multiply_activation=True)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.input_size))
self.assertTrue(np.allclose(res, self.model_lin_weights[:, c] * self.x))
def test_idempotence(self):
infl = InternalInfluence(
self.model_lin,
InputCut(),
MaxClassQoI(),
PointDoi(),
multiply_activation=False)
res1 = infl.attributions(self.x)
res2 = infl.attributions(self.x)
self.assertTrue(np.allclose(res1, res2))
infl_act = InternalInfluence(
self.model_lin,
InputCut(),
MaxClassQoI(),
PointDoi(),
multiply_activation=True)
res1 = infl_act.attributions(self.x)
res2 = infl_act.attributions(self.x)
self.assertTrue(np.allclose(res1, res2))
def test_linear_agreement_linear_slice_multiply_activation(self):
c = 4
infl = InternalInfluence(
self.model_deep, (Cut(self.layer2), Cut(self.layer3)),
InternalChannelQoI(c),
PointDoi(),
multiply_activation=True)
res = infl.attributions(self.x)
self.assertEqual(res.shape, (2, self.internal1_size))
z = self.model_deep.fprop((self.x,), to_cut=Cut(self.layer2))[0]
self.assertTrue(np.allclose(res, self.model_deep_weights_2[:, c] * z))
def test_catch_cut_name_error(self):
graph = Graph()
with graph.as_default():
x = tf.placeholder('float32', (None, 2))
z1 = x @ tf.random.normal((2, 2))
z2 = relu(z1)
y = z2 @ tf.random.normal((2, 1))
model = ModelWrapper(graph, x, y)
with self.assertRaises(ValueError):
infl = InternalInfluence(
model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]))
def test_multiple_inputs(self):
x1 = Input((5, ))
z1 = Dense(6)(x1)
x2 = Input((1, ))
z2 = Concatenate()([z1, x2])
z3 = Dense(7)(z2)
y = Dense(3)(z3)
model = ModelWrapper(Model([x1, x2], y))
infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 5))
self.assertEqual(res[1].shape, (1, 1))
def test_multiple_inputs(self):
graph = Graph()
with graph.as_default():
x1 = tf.placeholder('float32', (None, 5))
z1 = x1 @ tf.random.normal((5, 6))
x2 = tf.placeholder('float32', (None, 1))
z2 = tf.concat([z1, x2], axis=1)
z3 = z2 @ tf.random.normal((7, 7))
y = z3 @ tf.random.normal((7, 3))
model = ModelWrapper(graph, [x1, x2], y)
infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 5))
self.assertEqual(res[1].shape, (1, 1))
def test_internal_slice_multiple_layers(self):
x1 = Input((5, ))
z1 = Dense(6, name='cut_layer1')(x1)
x2 = Input((1, ))
z2 = Dense(2, name='cut_layer2')(x2)
z3 = Dense(4)(z2)
z4 = Concatenate()([z1, z3])
z5 = Dense(7)(z4)
y = Dense(3)(z5)
model = ModelWrapper(Model([x1, x2], y))
infl = InternalInfluence(model, Cut(['cut_layer1', 'cut_layer2']),
ClassQoI(1), PointDoi())
res = infl.attributions(
[np.array([[1., 2., 3., 4., 5.]]),
np.array([[1.]])])
self.assertEqual(len(res), 2)
self.assertEqual(res[0].shape, (1, 6))
self.assertEqual(res[1].shape, (1, 2))
def test_catch_cut_name_error(self):
class M(Module):
def __init__(this):
super(M, this).__init__()
this.z1 = Linear(2, 2)
this.z2 = ReLU()
this.y = Linear(2, 1)
def forward(this, x):
z1 = this.z1(x)
z2 = this.z2(z1)
return this.y(z2)
model = ModelWrapper(M(), (2,))
with self.assertRaises(ValueError):
infl = InternalInfluence(
model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi())
infl.attributions(np.array([[1., 1.]]).astype('float32'))
def test_distributional_linearity_internal_influence(self):
x1, x2 = self.x[0:1], self.x[1:]
p1, p2 = 0.25, 0.75
class DistLinDoI(DoI):
'''
Represents the distribution of interest that weights `z` with
probability 1/4 and `z + diff` with probability 3/4.
'''
def __init__(self, diff):
super(DistLinDoI, self).__init__()
self.diff = diff
def __call__(self, z):
return [z, z + self.diff, z + self.diff, z + self.diff]
infl_pt = InternalInfluence(
self.model_deep,
Cut(self.layer2),
ClassQoI(0),
PointDoi(),
multiply_activation=False)
attr1 = infl_pt.attributions(x1)
attr2 = infl_pt.attributions(x2)
infl_dl = InternalInfluence(
self.model_deep,
Cut(self.layer2),
ClassQoI(0),
DistLinDoI(x2 - x1),
multiply_activation=False)
attr12 = infl_dl.attributions(x1)
self.assertTrue(np.allclose(attr12, p1 * attr1 + p2 * attr2))
def __init__(self,
model,
layer,
channel,
channel_axis=B.channel_axis,
agg_fn=None,
doi=None,
blur=None,
threshold=0.5,
masked_opacity=0.2,
combine_channels=True,
use_attr_as_opacity=None,
positive_only=None):
"""
Configures the default parameters for the `__call__` method (these can
be overridden by passing in values to `__call__`).
Parameters:
model:
The wrapped model whose channel we're visualizing.
layer:
The identifier (either index or name) of the layer in which the
channel we're visualizing resides.
channel:
Index of the channel (for convolutional layers) or internal
neuron (for fully-connected layers) that we'd like to visualize.
channel_axis:
If different from the channel axis specified by the backend, the
supplied `channel_axis` will be used if operating on a
convolutional layer with 4-D image format.
agg_fn:
Function with which to aggregate the remaining dimensions
(except the batch dimension) in order to get a single scalar
value for each channel; If `None`, a sum over each neuron in the
channel will be taken. This argument is not used when the
channels are scalars, e.g., for dense layers.
doi:
The distribution of interest to use when computing the input
attributions towards the specified channel. If `None`,
`PointDoI` will be used.
blur:
Gives the radius of a Gaussian blur to be applied to the
attributions before visualizing. This can be used to help focus
on salient regions rather than specific salient pixels.
threshold:
Value in the range [0, 1]. Attribution values at or below the
percentile given by `threshold` (after normalization, blurring,
etc.) will be masked.
masked_opacity:
Value in the range [0, 1] specifying the opacity for the parts
of the image that are masked.
combine_channels:
If `True`, the attributions will be averaged across the channel
dimension, resulting in a 1-channel attribution map.
use_attr_as_opacity:
If `True`, instead of using `threshold` and `masked_opacity`,
the opacity of each pixel is given by the 0-1-normalized
attribution value.
positive_only:
If `True`, only pixels with positive attribution will be
unmasked (or given nonzero opacity when `use_attr_as_opacity` is
true).
"""
self.mask_visualizer = MaskVisualizer(blur, threshold, masked_opacity,
combine_channels,
use_attr_as_opacity,
positive_only)
self.infl_input = InternalInfluence(
model, (InputCut(), Cut(layer)),
InternalChannelQoI(channel, channel_axis, agg_fn),
PointDoi() if doi is None else doi)