def test_anchors(self): x = Input((2, )) z1 = Dense(2)(x) z2 = Activation('relu')(z1) y = Dense(1)(z2) k_model = Model(x, y) k_model.set_weights([ np.array([[1., 0.], [0., -1.]]), np.array([0., 0.]), np.array([[1.], [1.]]), np.array([0.]) ]) model = ModelWrapper(k_model) infl_out = InternalInfluence(model, Cut(2, anchor='out'), ClassQoI(0), PointDoi(), multiply_activation=False) infl_in = InternalInfluence(model, Cut(2, anchor='in'), ClassQoI(0), PointDoi(), multiply_activation=False) res_out = infl_out.attributions(np.array([[1., 1.]])) res_in = infl_in.attributions(np.array([[1., 1.]])) self.assertEqual(res_out.shape, (1, 2)) self.assertEqual(res_in.shape, (1, 2)) self.assertTrue(np.allclose(res_out, np.array([[1., 1.]]))) self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_batch_processing_deep(self): infl = InternalInfluence(self.model_deep, InputCut(), MaxClassQoI(), LinearDoi()) r1 = np.concatenate([infl.attributions(x[None]) for x in self.batch_x]) r2 = infl.attributions(self.batch_x) self.assertTrue(np.allclose(r1, r2))
def test_catch_cut_index_error(self): x = Input((2, )) z1 = Dense(2)(x) z2 = Activation('relu')(z1) y = Dense(1)(z2) model = ModelWrapper(Model(x, y)) with self.assertRaises(ValueError): infl = InternalInfluence(model, Cut(4), ClassQoI(0), PointDoi()) infl.attributions(np.array([[1., 1.]]))
def test_internal_slice_multiple_layers(self): class M(Module): def __init__(this): super(M, this).__init__() this.cut_layer1 = Linear(5, 6) this.cut_layer2 = Linear(1, 2) this.z3 = Linear(2, 4) this.z5 = Linear(10, 7) this.y = Linear(7, 3) def forward(this, x1, x2): z1 = this.cut_layer1(x1) z2 = this.cut_layer2(x2) z3 = this.z3(z2) z4 = cat((z1, z3), 1) z5 = this.z5(z4) return this.y(z5) model = ModelWrapper(M(), [(5,), (1,)]) infl = InternalInfluence( model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi()) res = infl.attributions( np.array([[1., 2., 3., 4., 5.]]).astype('float32'), np.array([[1.]]).astype('float32')) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 6)) self.assertEqual(res[1].shape, (1, 2))
def test_internal_multiple_inputs(self): class ConcatenateLayer(Module): def forward(this, x1, x2): return cat((x1, x2), 1) class M(Module): def __init__(this): super(M, this).__init__() this.z1 = Linear(5, 6) this.concat = ConcatenateLayer() this.z3 = Linear(7, 7) this.y = Linear(7, 3) def forward(this, x1, x2): x1 = this.z1(x1) z = this.concat(x1, x2) z = this.z3(z) return this.y(z) model = ModelWrapper(M(), [(5,), (1,)]) infl = InternalInfluence( model, Cut('concat', anchor='in'), ClassQoI(1), PointDoi()) res = infl.attributions( np.array([[1., 2., 3., 4., 5.]]).astype('float32'), np.array([[1.]]).astype('float32')) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 6)) self.assertEqual(res[1].shape, (1, 1))
def test_catch_cut_name_error(self): graph = Graph() with graph.as_default(): x = tf.placeholder('float32', (None, 2)) z1 = x @ tf.random.normal((2, 2)) z2 = relu(z1) y = z2 @ tf.random.normal((2, 1)) model = ModelWrapper(graph, x, y) with self.assertRaises(ValueError): infl = InternalInfluence( model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi()) infl.attributions(np.array([[1., 1.]]))
def test_internal_slice_multiple_layers(self): graph = Graph() with graph.as_default(): x1 = tf.placeholder('float32', (None, 5)) z1 = x1 @ tf.random.normal((5, 6)) x2 = tf.placeholder('float32', (None, 1)) z2 = x2 @ tf.random.normal((1, 2)) z3 = z2 @ tf.random.normal((2, 4)) z4 = tf.concat([z1, z3], axis=1) z5 = z4 @ tf.random.normal((10, 7)) y = z5 @ tf.random.normal((7, 3)) model = ModelWrapper( graph, [x1, x2], y, dict(cut_layer1=z1, cut_layer2=z2)) infl = InternalInfluence( model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi()) res = infl.attributions( [np.array([[1., 2., 3., 4., 5.]]), np.array([[1.]])]) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 6)) self.assertEqual(res[1].shape, (1, 2))
def test_anchors(self): class M(Module): def __init__(this): super(M, this).__init__() this.z1 = Linear(2, 2) this.z2 = ReLU() this.y = Linear(2, 1) this.z1.weight.data = B.as_tensor( np.array([[1., 0.], [0., -1.]]).T) this.z1.bias.data = B.as_tensor(np.array([0., 0.])) this.y.weight.data = B.as_tensor(np.array([[1.], [1.]]).T) this.y.bias.data = B.as_tensor(np.array([0.])) def forward(this, x): z1 = this.z1(x) z2 = this.z2(z1) return this.y(z2) model = ModelWrapper(M(), (2,)) infl_out = InternalInfluence( model, Cut('z2', anchor='out'), ClassQoI(0), PointDoi(), multiply_activation=False) infl_in = InternalInfluence( model, Cut('z2', anchor='in'), ClassQoI(0), PointDoi(), multiply_activation=False) res_out = infl_out.attributions(np.array([[1., 1.]])) res_in = infl_in.attributions(np.array([[1., 1.]])) self.assertEqual(res_out.shape, (1, 2)) self.assertEqual(res_in.shape, (1, 2)) self.assertTrue(np.allclose(res_out, np.array([[1., 1.]]))) self.assertTrue(np.allclose(res_in, np.array([[1., 0.]])))
def test_catch_cut_name_error(self): class M(Module): def __init__(this): super(M, this).__init__() this.z1 = Linear(2, 2) this.z2 = ReLU() this.y = Linear(2, 1) def forward(this, x): z1 = this.z1(x) z2 = this.z2(z1) return this.y(z2) model = ModelWrapper(M(), (2,)) with self.assertRaises(ValueError): infl = InternalInfluence( model, Cut('not_a_real_layer'), ClassQoI(0), PointDoi()) infl.attributions(np.array([[1., 1.]]).astype('float32'))
def test_linear_agreement_multiply_activation(self): c = 1 infl = InternalInfluence( self.model_lin, InputCut(), ClassQoI(c), PointDoi(), multiply_activation=True) res = infl.attributions(self.x) self.assertEqual(res.shape, (2, self.input_size)) self.assertTrue(np.allclose(res, self.model_lin_weights[:, c] * self.x))
def test_linear_agreement_linear_slice(self): c = 4 infl = InternalInfluence( self.model_deep, (Cut(self.layer2), Cut(self.layer3)), InternalChannelQoI(c), PointDoi(), multiply_activation=False) res = infl.attributions(self.x) self.assertEqual(res.shape, (2, self.internal1_size)) self.assertTrue(np.allclose(res[0], self.model_deep_weights_2[:, c])) self.assertTrue(np.allclose(res[1], self.model_deep_weights_2[:, c]))
def test_distributional_linearity_internal_influence(self): x1, x2 = self.x[0:1], self.x[1:] p1, p2 = 0.25, 0.75 class DistLinDoI(DoI): ''' Represents the distribution of interest that weights `z` with probability 1/4 and `z + diff` with probability 3/4. ''' def __init__(self, diff): super(DistLinDoI, self).__init__() self.diff = diff def __call__(self, z): return [z, z + self.diff, z + self.diff, z + self.diff] infl_pt = InternalInfluence( self.model_deep, Cut(self.layer2), ClassQoI(0), PointDoi(), multiply_activation=False) attr1 = infl_pt.attributions(x1) attr2 = infl_pt.attributions(x2) infl_dl = InternalInfluence( self.model_deep, Cut(self.layer2), ClassQoI(0), DistLinDoI(x2 - x1), multiply_activation=False) attr12 = infl_dl.attributions(x1) self.assertTrue(np.allclose(attr12, p1 * attr1 + p2 * attr2))
def test_idempotence(self): infl = InternalInfluence( self.model_lin, InputCut(), MaxClassQoI(), PointDoi(), multiply_activation=False) res1 = infl.attributions(self.x) res2 = infl.attributions(self.x) self.assertTrue(np.allclose(res1, res2)) infl_act = InternalInfluence( self.model_lin, InputCut(), MaxClassQoI(), PointDoi(), multiply_activation=True) res1 = infl_act.attributions(self.x) res2 = infl_act.attributions(self.x) self.assertTrue(np.allclose(res1, res2))
def test_linear_agreement_linear_slice_multiply_activation(self): c = 4 infl = InternalInfluence( self.model_deep, (Cut(self.layer2), Cut(self.layer3)), InternalChannelQoI(c), PointDoi(), multiply_activation=True) res = infl.attributions(self.x) self.assertEqual(res.shape, (2, self.internal1_size)) z = self.model_deep.fprop((self.x,), to_cut=Cut(self.layer2))[0] self.assertTrue(np.allclose(res, self.model_deep_weights_2[:, c] * z))
def test_completeness_zero_baseline(self): c = 2 infl = InternalInfluence( self.model_deep, InputCut(), ClassQoI(c), LinearDoi(resolution=100), multiply_activation=True) out_x = self.model_deep.fprop((self.x,))[0][:, c] out_baseline = self.model_deep.fprop((self.baseline * 0,))[0][:, c] res = infl.attributions(self.x) self.assertTrue( np.allclose(res.sum(axis=1), out_x - out_baseline, atol=5e-2))
def test_sensitivity(self): c = 2 infl = InternalInfluence( self.model_deep, InputCut(), ClassQoI(c), LinearDoi(self.baseline), multiply_activation=False) out_x = self.model_deep.fprop((self.x[0:1],))[0][:, c] out_baseline = self.model_deep.fprop((self.baseline,))[0][:, c] if not np.allclose(out_x, out_baseline): res = infl.attributions(self.x) self.assertEqual(res.shape, (2, self.input_size)) self.assertNotEqual(res[0, 3], 0.)
def test_multiple_inputs(self): x1 = Input((5, )) z1 = Dense(6)(x1) x2 = Input((1, )) z2 = Concatenate()([z1, x2]) z3 = Dense(7)(z2) y = Dense(3)(z3) model = ModelWrapper(Model([x1, x2], y)) infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi()) res = infl.attributions( [np.array([[1., 2., 3., 4., 5.]]), np.array([[1.]])]) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 5)) self.assertEqual(res[1].shape, (1, 1))
def per_timestep_qoi(self, model_wrapper, num_classes, num_features, num_timesteps, batch_size): cuts = (Cut('rnn', 'in', None), Cut('dense', 'out', None)) infl = InternalInfluence(model_wrapper, cuts, PerTimestepQoI(), RNNLinearDoi()) input_attrs = infl.attributions( np.ones( (batch_size, num_timesteps, num_features)).astype('float32')) original_output_shape = (num_classes * num_timesteps, batch_size, num_timesteps, num_features) self.assertEqual(np.stack(input_attrs).shape, original_output_shape) rotated = np.stack(input_attrs, axis=-1) attr_shape = list(rotated.shape)[:-1] attr_shape.append(int(rotated.shape[-1] / num_classes)) attr_shape.append(num_classes) attr_shape = tuple(attr_shape) self.assertEqual(attr_shape, (batch_size, num_timesteps, num_features, num_timesteps, num_classes)) input_attrs = np.reshape(rotated, attr_shape)
def test_completeness_internal_zero_baseline(self): c = 2 infl = InternalInfluence( self.model_deep, Cut(self.layer2), ClassQoI(c), LinearDoi(resolution=100, cut=Cut(self.layer2)), multiply_activation=True) g = partial( self.model_deep.fprop, doi_cut=Cut(self.layer2), intervention=np.zeros((2, 10))) out_x = self.model_deep.fprop((self.x,))[0][:, c] out_baseline = g((self.x,))[0][:, c] res = infl.attributions(self.x) self.assertTrue( np.allclose(res.sum(axis=1), out_x - out_baseline, atol=5e-2))
def test_multiple_inputs(self): graph = Graph() with graph.as_default(): x1 = tf.placeholder('float32', (None, 5)) z1 = x1 @ tf.random.normal((5, 6)) x2 = tf.placeholder('float32', (None, 1)) z2 = tf.concat([z1, x2], axis=1) z3 = z2 @ tf.random.normal((7, 7)) y = z3 @ tf.random.normal((7, 3)) model = ModelWrapper(graph, [x1, x2], y) infl = InternalInfluence(model, InputCut(), ClassQoI(1), PointDoi()) res = infl.attributions( [np.array([[1., 2., 3., 4., 5.]]), np.array([[1.]])]) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 5)) self.assertEqual(res[1].shape, (1, 1))
def test_internal_slice_multiple_layers(self): x1 = Input((5, )) z1 = Dense(6, name='cut_layer1')(x1) x2 = Input((1, )) z2 = Dense(2, name='cut_layer2')(x2) z3 = Dense(4)(z2) z4 = Concatenate()([z1, z3]) z5 = Dense(7)(z4) y = Dense(3)(z5) model = ModelWrapper(Model([x1, x2], y)) infl = InternalInfluence(model, Cut(['cut_layer1', 'cut_layer2']), ClassQoI(1), PointDoi()) res = infl.attributions( [np.array([[1., 2., 3., 4., 5.]]), np.array([[1.]])]) self.assertEqual(len(res), 2) self.assertEqual(res[0].shape, (1, 6)) self.assertEqual(res[1].shape, (1, 2))
class ChannelMaskVisualizer(object): """ Uses internal influence to visualize the pixels that are most salient towards a particular internal channel or neuron. """ def __init__(self, model, layer, channel, channel_axis=B.channel_axis, agg_fn=None, doi=None, blur=None, threshold=0.5, masked_opacity=0.2, combine_channels=True, use_attr_as_opacity=None, positive_only=None): """ Configures the default parameters for the `__call__` method (these can be overridden by passing in values to `__call__`). Parameters: model: The wrapped model whose channel we're visualizing. layer: The identifier (either index or name) of the layer in which the channel we're visualizing resides. channel: Index of the channel (for convolutional layers) or internal neuron (for fully-connected layers) that we'd like to visualize. channel_axis: If different from the channel axis specified by the backend, the supplied `channel_axis` will be used if operating on a convolutional layer with 4-D image format. agg_fn: Function with which to aggregate the remaining dimensions (except the batch dimension) in order to get a single scalar value for each channel; If `None`, a sum over each neuron in the channel will be taken. This argument is not used when the channels are scalars, e.g., for dense layers. doi: The distribution of interest to use when computing the input attributions towards the specified channel. If `None`, `PointDoI` will be used. blur: Gives the radius of a Gaussian blur to be applied to the attributions before visualizing. This can be used to help focus on salient regions rather than specific salient pixels. threshold: Value in the range [0, 1]. Attribution values at or below the percentile given by `threshold` (after normalization, blurring, etc.) will be masked. masked_opacity: Value in the range [0, 1] specifying the opacity for the parts of the image that are masked. combine_channels: If `True`, the attributions will be averaged across the channel dimension, resulting in a 1-channel attribution map. use_attr_as_opacity: If `True`, instead of using `threshold` and `masked_opacity`, the opacity of each pixel is given by the 0-1-normalized attribution value. positive_only: If `True`, only pixels with positive attribution will be unmasked (or given nonzero opacity when `use_attr_as_opacity` is true). """ self.mask_visualizer = MaskVisualizer(blur, threshold, masked_opacity, combine_channels, use_attr_as_opacity, positive_only) self.infl_input = InternalInfluence( model, (InputCut(), Cut(layer)), InternalChannelQoI(channel, channel_axis, agg_fn), PointDoi() if doi is None else doi) def __call__(self, x, x_preprocessed=None, output_file=None, blur=None, threshold=None, masked_opacity=None, combine_channels=None): """ Visualizes the given attributions by overlaying an attribution heatmap over the given image. Parameters ---------- attributions : numpy.ndarray The attributions to visualize. Expected to be in 4-D image format. x : numpy.ndarray The original image(s) over which the attributions are calculated. Must be the same shape as expected by the model used with this visualizer. x_preprocessed : numpy.ndarray, optional If the model requires a preprocessed input (e.g., with the mean subtracted) that is different from how the image should be visualized, ``x_preprocessed`` should be specified. In this case ``x`` will be used for visualization, and ``x_preprocessed`` will be passed to the model when calculating attributions. Must be the same shape as ``x``. output_file : str, optional If specified, the resulting visualization will be saved to a file with the name given by ``output_file``. blur : float, optional If specified, gives the radius of a Gaussian blur to be applied to the attributions before visualizing. This can be used to help focus on salient regions rather than specific salient pixels. If None, defaults to the value supplied to the constructor. Default None. threshold : float Value in the range [0, 1]. Attribution values at or below the percentile given by ``threshold`` will be masked. If None, defaults to the value supplied to the constructor. Default None. masked_opacity: float Value in the range [0, 1] specifying the opacity for the parts of the image that are masked. Default 0.2. If None, defaults to the value supplied to the constructor. Default None. combine_channels : bool If True, the attributions will be averaged across the channel dimension, resulting in a 1-channel attribution map. If None, defaults to the value supplied to the constructor. Default None. """ attrs_input = self.infl_input.attributions( x if x_preprocessed is None else x_preprocessed) return self.mask_visualizer(attrs_input, x, output_file, blur, threshold, masked_opacity, combine_channels)