def runTest(self):
with tf.Graph().as_default():
# Input shape: (2,3,4)
inp = tf.constant(
[[[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]],
[[-1, -2, -3, -4], [-5, -6, -7, -8], [-9, -10, -11, -12]]],
dtype=tf.float32)
# Output shape: (2, 12)
out = tf.reshape(inp, [2, 12])
# Calculate relevances
expl = lrp.lrp(inp, out)
with tf.Session() as s:
output = s.run(out)
explanation = s.run(expl)
# Shape: (2,3,4)
expected_explanation = np.array([[[0., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 12.]],
[[-1., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]]])
self.assertTrue(
np.allclose(expected_explanation, explanation),
"Explanation does not match expected explanation")
def test_softmax(self):
with tf.Graph().as_default():
inp = tf.constant([[[1, 2, 3], [1, 2, 3]], [[1, 2, 3], [1, 2, 3]]], dtype=tf.float32)
out = tf.nn.softmax(inp)
config = LRPConfiguration()
config.set(LAYER.SOFTMAX, AlphaBetaConfiguration(alpha=2, beta=-1))
expl = lrp.lrp(inp, out, config)
with tf.Session() as s:
explanation = s.run(expl)
expected = np.array([[[[-0.05989202454, -0.162803402, 0.8879363823],
[0, 0, 0]],
[[0, 0, 0],
[-0.05989202454, -0.162803402, 0.8879363823]]],
[[[-0.05989202454, -0.162803402, 0.8879363823],
[0, 0, 0]],
[[0, 0, 0],
[-0.05989202454, -0.162803402, 0.8879363823]]]])
# Check if the relevance scores are correct (the correct values are found by
# calculating the example by hand)
self.assertTrue(
np.allclose(expected, explanation, rtol=1e-03, atol=1e-03),
msg="Should be a good explanation")
def _do_test(self, expected_result, config=None):
# Make sure that expected_result is an np array
if not type(expected_result).__module__ == np.__name__:
expected_result = np.array(expected_result)
with tf.Graph().as_default():
inputs, indices, _ = dense_to_sparse(self.input)
sparse_tensor_reordered = tf.sparse_reorder(inputs)
sparse_tensor_reshaped = tf.sparse_reshape(sparse_tensor_reordered, self.input.shape)
W = tf.constant(self.W1, dtype=tf.float32)
b = tf.constant(self.b1, dtype=tf.float32)
# Sparse layer
logits = tf.sparse_tensor_dense_matmul(sparse_tensor_reshaped, W) + b
# Dense layer
logits = logits @ tf.constant(self.W2, tf.float32) + tf.constant(self.b2, tf.float32)
explanation = lrp.lrp(inputs, logits, config)
with tf.Session() as s:
expl = s.run(explanation)
self.assertTrue(np.all(np.equal(indices, expl.indices)),
"expected indices did not equal actual indices")
self.assertTrue(np.allclose(expl.values, expected_result.reshape((-1)), rtol=1.e-3, atol=1.e-3),
"expected indices did not equal actual indices")
def test_different_consumers_for_each_output(self):
with tf.Graph().as_default() as g:
inp = tf.placeholder(tf.float32, shape=(3, 4))
# Split input to construct path in lrp with two paths from
# input to output.
in1, in2 = tf.split(inp, 2, 1)
# Do additional work on one of the paths
w = tf.constant([[1, 2], [3, 4]], dtype=tf.float32)
in1 = tf.matmul(in1, w)
# Concatenate the paths again in order to end up with one output.
out = tf.concat([in1, in2], 1, 'wtf')
expl = lrp.lrp(inp, out)
# Do some testing
with tf.Session() as s:
explanation = s.run(expl,
feed_dict={
inp: [[1, 1, 0, 1], [0, 0, 0, 0],
[1, 0, 0, 0]]
})
self.assertTrue(
np.allclose(
np.array([[2, 4, 0, 0], [0, 0, 0, 0], [2, 0, 0, 0]]),
explanation), "Relevances should match")
def runTest(self):
with tf.Graph().as_default():
# Create the input
# Shape: (1,2,3)
inp = tf.constant([[[1, 2, 3], [4, 5, 6]]], dtype=tf.float32)
# Shape: (1, 4, 6)
pred = tf.tile(inp, (1, 2, 2))
# Calculate the explanation
expl = lrp.lrp(inp, pred)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Calculate the explanation
explanation = sess.run(expl)
# Create the expected explanation
# Shape: (1,4,2,3)
expected_explanation = np.array([[[[0, 0, 3], [0, 0, 0]],
[[0, 0, 0], [0, 0, 6]],
[[0, 0, 3], [0, 0, 0]],
[[0, 0, 0], [0, 0, 6]]]])
# Check if the calculated explanation matches the expected explanation
self.assertTrue(
np.allclose(expected_explanation,
explanation,
rtol=1e-03,
atol=1e-03),
msg=
"The explanation does not match the expected explanation")
def _do_linear_test(self, config, expected_result):
if not type(expected_result).__module__ == np.__name__:
expected_result = np.array(expected_result)
with tf.Graph().as_default():
inp = tf.constant([[
0.61447761, -0.47432536, -0.29292757, -0.78589278, -0.86108047
], [
0.28479454, -0.60827365, 0.86519678, -0.65091976, -0.6819959
], [-0.4422958, 0.55866813, -0.88997564, -0.87868751, -0.0389981]],
dtype=tf.float32)
W1 = tf.constant(
[[-0.70950127, -0.15957509, -0.607047, 0.13172],
[-0.9520821, -0.79133917, -0.03131101, -0.00217408],
[-0.35051205, 0.84566609, 0.22297791, 0.39139763],
[-0.05067179, 0.07747386, -0.89703108, 0.22393099],
[-0.43415774, 0.44243544, -0.17682024, -0.31072929]],
dtype=tf.float32)
b1 = tf.constant(
[0.10282315, -0.07288911, -0.53922754, -0.3299993],
dtype=tf.float32)
out1 = tf.nn.relu(inp @ W1 + b1)
W2 = tf.constant(
[[-0.3378281, -0.03719562, -0.05190714, 0.3983907],
[-0.92650528, -0.97646332, 0.08498075, 0.37901429],
[-0.36540267, -0.26421945, -0.79152602, 0.73636482],
[0.59652669, 0.89863044, 0.02424345, 0.09883726]],
dtype=tf.float32)
b2 = tf.constant(
[-0.26253957, 0.91930372, 0.11791677, -0.28088199],
dtype=tf.float32)
out2 = out1 @ W2 + b2
out3 = tf.reshape(out2, (3, 2, 2))
out = tf.nn.softmax(out3)
expl = lrp.lrp(inp, out, config)
with tf.Session() as s:
explanation = s.run(expl)
# Check if the explanation has the right shape
self.assertEqual(explanation.shape,
expected_result.shape,
msg="Should be a wellformed explanation")
# Check if the relevance scores are correct (the correct values are found by
# calculating the example by hand)
self.assertTrue(np.allclose(explanation,
expected_result,
rtol=1e-03,
atol=1e-03),
msg="Should be a good explanation")
def runTest(self):
# Get a tensorflow graph
g = tf.Graph()
# Set the graph as default
with g.as_default():
# Create a placeholder for the input
inp = tf.placeholder(tf.float32, shape=(1, 1, 4, 6))
# Create max pooling layer
activation = tf.nn.max_pool(inp, [1, 1, 2, 1], [1, 1, 2, 1],
"SAME")
# Reshape predictions to shape (batch_size, predictions_per_sample, classes) so they can be used
# as input for the lrp framework
pred = tf.reshape(activation, (1, 2, 6))
# Calculate the relevance scores using lrp
expl = lrp.lrp(inp, pred)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Run the operations of interest and feed an input to the network
prediction, explanation = sess.run(
[pred, expl],
feed_dict={
inp: [[[[1, 2, 3, 4, 5, 6], [3, 4, 3, 5, 6, 7],
[5, 6, -1, -5, 0, -1], [-1, -2, 1, 1, 1, 1]]]]
})
# Check if the predictions has the right shape
self.assertEqual(
prediction.shape, (1, 2, 6),
msg="Should be able to do a linear forward pass")
# Check if the explanation has the right shape
self.assertEqual((1, 2, 1, 4, 6),
explanation.shape,
msg="Should be a wellformed explanation")
# Expected explanation
expected = np.array([[[[[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 7],
[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0,
0]]],
[[[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0],
[0, 6, 0, 0, 0, 0], [0, 0, 0, 0, 0,
0]]]]])
# Check if the relevance scores are correct (the correct values
# are found by calculating the example by hand)
self.assertTrue(np.allclose(expected,
explanation,
rtol=1e-03,
atol=1e-03),
msg="Should be a good explanation")
def runTest(self):
# Get a tensorflow graph
g = tf.Graph()
# Set the graph as default
with g.as_default():
# Create a placeholder for the input
inp = tf.placeholder(tf.float32, shape=(1, 2, 2, 2))
# Create the convolutional layer
# Set the filters and biases
filters = tf.constant(
[[[[2., 1, 1], [1., 1, 0]], [[2., 2, 1], [1., 1, 1]]],
[[[1., -1, 1], [0., 1, 0]], [[2., 0, 0], [0., -1, 1]]]],
dtype=tf.float32)
bias = tf.constant([1, 2, -2], dtype=tf.float32)
# Perform the convolution
activation = tf.nn.conv2d(inp, filters, [1, 1, 1, 1], "SAME")
# Add bias
activation = tf.nn.bias_add(activation, bias)
# Reshape predictions to (batch_size, pred. pr. sample, classes) to fit the required input shape of
# the framework
pred = tf.reshape(activation, (1, 2, 6))
# Calculate the relevance scores using lrp
expl = lrp.lrp(inp, pred)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Run the operations of interest and feed an input to the network
prediction, explanation = sess.run([pred, expl],
feed_dict={
inp: [[[[1., 0.],
[-1., 2.]],
[[2., -1.],
[3., 0.]]]]
})
# Check if the relevance scores are correct
self.assertTrue(
np.allclose([[[[[1.692307692, 0], [0, 1.692307692]],
[[1.692307692, 0], [5.076923077, 0]]],
[[[0, 0], [0, 0]],
[[3.636363636, 0], [5.454545455, 0]]]]],
explanation,
rtol=1e-03,
atol=1e-03),
msg=
"The calculated relevances do not match the expected relevances"
)
def test_flat(self):
g = tf.Graph()
with g.as_default():
inp = tf.placeholder(tf.float32, (3, 5))
W = tf.constant([[1.493394546, 0.5987773779],
[0.7321155851, 1.23063763],
[2.488971816, 0.9885881838],
[0.9965223115, 0.8397688134],
[2.089138346, 0.8398492639]]
, dtype=tf.float32)
b = [[2.665864718, 0.8793648172]]
to_normalize = inp @ W + b
x = tf.contrib.layers.batch_norm(to_normalize, is_training=False, scale=True)
vars = tf.global_variables()
beta = next(i for i in vars if 'beta' in i.name)
gamma = next(i for i in vars if 'gamma' in i.name)
mean = next(i for i in vars if 'mean' in i.name)
variance = next(i for i in vars if 'variance' in i.name)
b = tf.constant([0.8481817169, -1.118752611], dtype=tf.float32)
assign_beta = tf.assign(beta, b)
g = tf.constant([0.1005506696, 0.308355701], dtype=tf.float32)
assign_gamma = tf.assign(gamma, g)
m = tf.constant([-0.8224766215, 0.9257031289], dtype=tf.float32)
assign_mean = tf.assign(mean, m)
v = tf.constant([0.7134228722, 1.065337135], dtype=tf.float32)
assign_variance = tf.assign(variance, v)
# Get the explanation
config = LRPConfiguration()
config.set(LAYER.LINEAR, EpsilonConfiguration(bias_strategy=BIAS_STRATEGY.NONE))
config.set(LAYER.ELEMENTWISE_LINEAR, BaseConfiguration(RULE.IDENTITY))
explanation = lrp.lrp(inp, x, config)
with tf.Session() as s:
s.run(tf.global_variables_initializer())
s.run([assign_beta, assign_gamma, assign_mean, assign_variance])
expected_relevances = np.array([[0.3331443396, 0.5080297851, 0.2213932963, 0.1821642756, 0.832728544],
[0.3800239444, 0.1179592254, 0.5207348458, 0.5280974564, 0.3505242006],
[0.5231591662, 0.0895191051, 0.6016423127, 0.423324388, -0.1957729137]]
)
out, relevances = s.run([x, explanation],
feed_dict={inp: [[1.187187323, 3.692928471, 0.4733755909, 0.9728313491, 2.121278175],
[1.302276662, 0.8245540266, 1.070689437, 2.712026357, 0.8586529453],
[1.57555465, 0.5499336234, 1.087157802, 1.910559011,
-0.4214631393]]})
self.assertTrue(np.allclose(expected_relevances, relevances, rtol=1e-03, atol=1e-03),
msg="The relevances do not match the expected")
def runTest(self):
with tf.Graph().as_default() as g:
# shape: (6,6)
values = tf.constant([1], dtype=tf.float32, name="vals")
inp = tf.SparseTensor([[5, 5]], values, [6, 6])
inp_reordered = tf.sparse_reorder(inp)
out = tf.sparse_reshape(inp_reordered, (9, 4))
out_as_dense = tf.sparse_tensor_dense_matmul(
out, tf.eye(4, dtype=tf.float32))
writer = tf.summary.FileWriter("./tmp")
writer.add_graph(g)
# Calculate the explanation
expl = lrp.lrp(inp, out_as_dense)
# Run a tensorflow session to evaluate the graph
with tf.Session() as s:
s.run(inp)
s.run(out)
# Initialize the variables
s.run(tf.global_variables_initializer())
s.run(out)
# Calculate the explanation
explanation = s.run(expl)
# Extract the indices of non-zero elements, the values of the non-zero elements and the dense shape of
# the expected explanation
calculated_indices = explanation[0]
calculated_values = explanation[1]
calculated_shape = explanation[2]
# Create the expected explanation by creating an array that holds the indices of non-zero
# elements, an array that holds the values of the non-zero elements and an array that holds
# the dense shape of the expected explanation
expected_indices = np.array([[5, 5]])
expected_values = np.array([1])
expected_shape = np.array([6, 6])
# Check if the explanation matches the expected explanation
self.assertTrue(np.array_equal(expected_indices,
calculated_indices),
msg="The indicies do not match")
self.assertTrue(np.allclose(expected_values,
calculated_values,
atol=1e-03,
rtol=1e-03),
msg="The values do not match")
self.assertTrue(np.array_equal(expected_shape,
calculated_shape),
msg="The shapes do not match")
def do_lrp_pertubation_tests(configs, selected_model, model_file, destination,
**kwargs):
result_writer = ResultWriter(destination)
feed = DataFeed(False)
iterations = kwargs['test_size'] // kwargs['batch_size']
start = kwargs['start']
end = kwargs['end']
if end == -1:
end = len(configs)
for config_idx, config in enumerate(configs[start:end]):
graph = tf.Graph()
feed.reset_permutation()
with graph.as_default():
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
is_training = False
y, _ = selected_model['nn'](x, y_, is_training)
if isinstance(config, str):
if config == 'random':
print("Testing random relevance")
explanation = get_random_relevance(x)
else:
print("Testing sensitivity analysis")
explanation = get_sensitivity_relevance(x, y)
else:
print("Testing ({}/{}) {}".format(config_idx, end - start,
config))
with tf.name_scope("LRP"):
explanation = lrp.lrp(x, y, config)
init = get_initializer(False)
important_variables = tf.trainable_variables()
important_variables.extend(
[v for v in tf.global_variables() if 'moving_' in v.name])
saver = tf.train.Saver(important_variables)
with tf.Session() as s:
# Initialize stuff and restore model
s.run(init)
saver.restore(s, model_file)
do_pertubations(config, s, feed, explanation, iterations,
kwargs['batch_size'], kwargs['pertubations'],
result_writer, x, y)
def do_nan_searching(configs, selected_model, model_file, model_name,
**kwargs):
iterations = kwargs['test_size'] // kwargs['batch_size']
feed = DataFeed()
start = kwargs['start']
end = kwargs['end']
if end == -1:
end = len(configs)
found_nans = 0
for config_idx, config in enumerate(configs[start:end]):
graph = tf.Graph()
feed.reset_permutation()
with graph.as_default():
x = tf.placeholder(tf.float32, shape=[None, 784])
y_ = tf.placeholder(tf.float32, shape=[None, 10])
is_training = False
y, _ = selected_model['nn'](x, y_, is_training)
print("Testing ({}/{}), Nan-cnt {}: {}".format(
config_idx, end - start, found_nans, config))
explanation = lrp.lrp(x, y, config)
init = get_initializer(False)
important_variables = tf.trainable_variables()
important_variables.extend(
[v for v in tf.global_variables() if 'moving_' in v.name])
saver = tf.train.Saver(important_variables)
with tf.Session() as s:
# Initialize stuff and restore model
s.run(init)
saver.restore(s, model_file)
found_nans += do_search(config, s, feed, explanation,
iterations, kwargs['batch_size'], x)
logger.info(
"Found nans for {} configurations in total for model {}".format(
found_nans, model_name))
print("Found nans for {} configurations in total for model {}".format(
found_nans, model_name))
return model_name, found_nans
def runTest(self):
with tf.Graph().as_default():
inp = tf.constant([[1, 2, 3], [4, 5, 6]], dtype=tf.float32)
w1 = tf.constant([[1, 1], [1, 1], [1, 1]], dtype=tf.float32)
b1 = tf.constant([2, 4], dtype=tf.float32)
activation = tf.matmul(inp, w1) + b1
# Calculate relevances using lrp
explanation = lrp.lrp(inp, activation)
with tf.Session() as s:
act, expl = s.run([activation, explanation])
self.assertTrue(
np.allclose(np.array([[1, 2, 3], [4, 5, 6]]), expl))
def runTest(self):
with tf.Graph().as_default():
inp = tf.constant([[[1, 2, 3, 4], [5, 6, 7, 8], [9, 1, 2, 3]],
[[11, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]],
[[1, 2, 2, 2], [3, 3, 3, 3], [4, 4, 4, 4]]],
dtype=tf.float32)
out = tf.slice(inp, [0, 0, 2], [-1, 3, 2])
R = lrp.lrp(inp, out)
with tf.Session() as s:
expected_explanation = np.array([[[[0., 0., 0., 4.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 0., 8.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 3.]]],
[[[0., 0., 1., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 1., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 1., 0.]]],
[[[0., 0., 2., 0.],
[0., 0., 0., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 3., 0.],
[0., 0., 0., 0.]],
[[0., 0., 0., 0.],
[0., 0., 0., 0.],
[0., 0., 4., 0.]]]])
explanation = s.run(R)
self.assertTrue(
np.allclose(expected_explanation,
explanation,
atol=1e-03,
rtol=1e-03),
"The explanation does not match the expected explanation")
def test_distribute(self):
with tf.Graph().as_default():
input = tf.constant([[0.3315922947, 1.053559579, 0.7477053648, 1.22290369,
0.3730588596, -1.034354431, 0.9187013371, 1.478589349,
-0.7325915066, -0.3569675024, -1.136600512, 0.5516666285,
0.4834049101, -1.613833301, 0.1520745652, 0.117390006]],
dtype=tf.float32)
i = tf.reshape(input, (1, 4, 4, 1))
# Create max pooling layer
# (1, 2, 2, 1)
activation = tf.nn.max_pool(i, [1, 2, 2, 1], [1, 2, 2, 1], "SAME")
# Reshape to get 1 p/s
output = tf.reshape(activation, (1, 4))
config = LRPConfiguration()
config.set(LAYER.MAX_POOLING, EpsilonConfiguration())
explanation = lrp.lrp(input, output, config)
with tf.Session() as s:
expl = s.run(explanation)
# Check if the explanation has the right shape
self.assertEqual((1, 16), expl.shape,
msg="Should be a wellformed explanation")
# Expected explanation
expected = np.array(
[[0, 0, 0.2531077301, 0.4139683781, 0, 0, 0.3109920311, 0.50052121, 0, 0, 0, 0, 0, 0, 0, 0]])
# Check if the relevance scores are correct (the correct values
# are found by calculating the example by hand)
self.assertTrue(
np.allclose(expected,
expl,
rtol=1e-03,
atol=1e-03),
msg="Should be a good explanation")
self.assertTrue(True)
def test_multiple_output_uses(self):
with tf.Graph().as_default() as g:
inp = tf.placeholder(tf.float32, shape=(3, 4))
# Split input to construct path in lrp with two paths from
# input to output.
in1, in2 = tf.split(inp, 2, 1)
# Concatenate the paths again in order to end up with one output
out = tf.concat([in1, in2], 1)
expl = lrp.lrp(inp, out)
with tf.Session() as s:
input_values = [[1, 1, 0, 1], [0, 0, 0, 0], [1, 0, 0, 0]]
explanation = s.run(expl, feed_dict={inp: input_values})
expected_output = [[1, 0, 0, 0], [0, 0, 0, 0], [1, 0, 0, 0]]
self.assertTrue(np.allclose(expected_output, explanation),
"Relevances should match")
def test_distribute(self):
with tf.Graph().as_default():
input = tf.constant([[1., 2., 2., 3., 3., 3., 3., 2., 1]],
dtype=tf.float32)
i = tf.reshape(input, (1, 3, 3, 1))
# Create max pooling layer
# (1, 2, 2, 1)
activation = tf.nn.max_pool(i, [1, 2, 2, 1], [1, 2, 2, 1], "SAME")
# Reshape to get 1 p/s
output = tf.reshape(activation, (2, 2))
config = LRPConfiguration()
config.set(LAYER.MAX_POOLING,
BaseConfiguration(RULE.WINNER_TAKES_ALL))
explanation = lrp.lrp(input, output, config)
with tf.Session() as s:
expl = s.run(explanation)
# Check if the explanation has the right shape
self.assertEqual((1, 9),
expl.shape,
msg="Should be a wellformed explanation")
# Expected explanation
expected = np.array([[0., 0., 0., 3., 0., 0., 3., 0., 0]])
# Check if the relevance scores are correct (the correct values
# are found by calculating the example by hand)
self.assertTrue(np.allclose(expected,
expl,
rtol=1e-03,
atol=1e-03),
msg="Should be a good explanation")
self.assertTrue(True)
def test_one_prediction_per_sample(self):
g = tf.Graph()
with g.as_default():
inp = tf.placeholder(tf.float32, (2, 1, 5))
x = tf.contrib.layers.batch_norm(inp, is_training=False, scale=True)
vars = tf.global_variables()
beta = next(i for i in vars if 'beta' in i.name)
gamma = next(i for i in vars if 'gamma' in i.name)
mean = next(i for i in vars if 'mean' in i.name)
variance = next(i for i in vars if 'variance' in i.name)
b = tf.constant([0, 1, 0, 1, 0], dtype=tf.float32)
assign_beta = tf.assign(beta, b)
g = tf.constant([0.1, 0.2, 0.3, 0.4, 0.5], dtype=tf.float32)
assign_gamma = tf.assign(gamma, g)
m = tf.constant([1, 2, 3, 4, 4.5], dtype=tf.float32)
assign_mean = tf.assign(mean, m)
v = tf.constant([0.2, 0.2, 0.2, 0.2, 0.2], dtype=tf.float32)
assign_variance = tf.assign(variance, v)
# Get the explanation
config = LRPConfiguration()
config.set(LAYER.ELEMENTWISE_LINEAR, AlphaBetaConfiguration())
explanation = lrp.lrp(inp, x, config)
with tf.Session() as s:
s.run(tf.global_variables_initializer())
s.run([assign_beta, assign_gamma, assign_mean, assign_variance])
# Shape: (2, 1, 1, 5)
expected_relevances = np.array(
[[[[0, 0, 0, 1, 0]]],
[[[0, 0.8921994513, 0, 0, 0]]]])
relevances = s.run(explanation, feed_dict={inp: [[[1, 0, 3, 4, 5]],
[[1, 2, 3, 0, 4]]]})
self.assertTrue(np.allclose(expected_relevances, relevances, rtol=1e-03, atol=1e-03),
msg="The relevances do not match the expected")
def test_ww_rule_sparse(self):
with tf.Graph().as_default():
indices = [[0, 2], [0, 5], [1, 0], [1, 6]]
values = tf.constant([2, 1, 1, 2.], dtype=tf.float32)
inp = tf.SparseTensor(indices, values, (3, 7))
sparse_tensor_reordered = tf.sparse_reorder(inp)
sparse_tensor_reshaped = tf.sparse_reshape(sparse_tensor_reordered,
(3, 7))
W = tf.constant([[0.48237237, -2.17243375, -0.97473115],
[2.35669847, 3.11619017, 0.84322384],
[5.01346225, 1.69588809, -1.47801861],
[2.28595446, 0.24175502, -0.23067427],
[0.41892012, -1.44306815, 2.21516808],
[1.88990215, 1.46110879, 2.89949934],
[2.01381318, 2.12360494, 2.34057405]],
dtype=tf.float32)
b = tf.constant([1.47436833, -0.27795767, 4.28945125],
dtype=tf.float32)
out = tf.sparse_tensor_dense_matmul(sparse_tensor_reshaped, W) + b
config = LRPConfiguration()
config.set(LAYER.SPARSE_LINEAR, WWConfiguration())
expl = lrp.lrp(inp, out, config)
with tf.Session() as s:
output, explanation = s.run([out, expl])
expected_indices = indices
expected_values = np.array(
[11.72503303, 1.666161958, 1.181770802, 6.814097394])
self.assertTrue(
np.allclose(expected_indices, explanation.indices))
self.assertTrue(
np.allclose(expected_values, explanation.values))
def runTest(self):
# Get a tensorflow graph
g = tf.Graph()
# Set the graph as default
with g.as_default():
# Create a placeholder for the input
inp = tf.placeholder(tf.float32, shape=(1, 4, 4, 1))
# Create max pooling layer
activation = tf.nn.max_pool(inp, [1, 2, 2, 1], [1, 2, 2, 1],
"VALID")
# Reshape predictions to (batch_size, pred. pr. sample, classes)
pred = tf.reshape(activation, (1, 4, 1))
# Get the explanation tensor
expl = lrp.lrp(inp, pred)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Run the operations of interest and feed an input to the network
prediction, explanation = sess.run(
[pred, expl],
feed_dict={
inp: [[[[1], [2], [3], [4]], [[5], [6], [7], [8]],
[[9], [10], [11], [12]], [[13], [14], [15],
[16]]]]
})
# Check if the predictions has the right shape
self.assertEqual(
prediction.shape, (1, 4, 1),
msg="Should be able to do a linear forward pass")
# Check if the explanation has the right shape
self.assertEqual((1, 4, 4, 4, 1),
explanation.shape,
msg="Should be a wellformed explanation")
# Expected output has shape
# (batch_size, pred. pr. sample, input_size ...)
expected = np.array([[[[[0], [0], [0], [0]],
[[0], [6], [0], [0]],
[[0], [0], [0], [0]],
[[0], [0], [0], [0]]],
[[[0], [0], [0], [0]],
[[0], [0], [0], [8]],
[[0], [0], [0], [0]],
[[0], [0], [0], [0]]],
[[[0], [0], [0], [0]],
[[0], [0], [0], [0]],
[[0], [0], [0], [0]],
[[0], [14], [0], [0]]],
[[[0], [0], [0], [0]],
[[0], [0], [0], [0]],
[[0], [0], [0], [0]],
[[0], [0], [0], [16]]]]])
# Check if the relevance scores are correct
# (the correct values are found by calculating the example by hand)
self.assertTrue(np.allclose(expected,
explanation,
rtol=1e-03,
atol=1e-03),
msg="Should be a good explanation")
def runTest(self):
with tf.Graph().as_default():
lstm_units = 2
max_time_step = 3
input_depth = 4
TYPES_OF_GATES_IN_A_LSTM = 4
# Create input
inp = tf.constant(
[[[-5, -4, -3, -2], [-1, 0, 1, 2], [3, 4, 5, 6]],
[[7, 8, 9, 10], [11, 12, 13, 14], [15, 16, 17, 18]]],
dtype=tf.float32)
# Create mock weights
LSTM_WEIGHTS = tf.constant(
[[
0.19517923, 0.24883463, 0.65906681, 0.43171532, 0.30894309,
0.18143875, 0.9064917, 0.34376469
],
[
0.36688612, 0.41893102, 0.68622539, 0.92279857,
0.18437027, 0.7582207, 0.70674838, 0.8861974
],
[
0.60149935, 0.84269909, 0.3129998, 0.75019745, 0.75946505,
0.76014145, 0.012957, 0.06685569
],
[
0.09053277, 0.91693017, 0.11203575, 0.85798137,
0.14988363, 0.96619787, 0.63018615, 0.77663712
],
[
0.18449367, 0.61801985, 0.07125719, 0.02529254,
0.42940272, 0.96136843, 0.95259111, 0.33910939
],
[
0.88669326, 0.58888385, 0.11549774, 0.63704878,
0.85553019, 0.39069136, 0.56481662, 0.27301619
]],
dtype=tf.float32)
# Create bias weights
LSTM_BIAS = tf.constant([
0.37282269, 0.16629956, 0.92285417, 0.86485604, -0.13370907,
0.75214074, 0.72669859, -0.261183
],
dtype=tf.float32)
# Create lstm layer
lstm = tf.contrib.rnn.LSTMCell(lstm_units, forget_bias=0.)
# Put it into Multi RNN Cell
lstm = tf.contrib.rnn.MultiRNNCell([lstm])
# Let dynamic rnn setup the control flow (making while loops and stuff)
lstm_output, _ = tf.nn.dynamic_rnn(lstm, inp, dtype=tf.float32)
# Construct operation for assigning mock weights
kernel = next(i for i in tf.global_variables()
if i.shape == (input_depth + lstm_units, lstm_units *
TYPES_OF_GATES_IN_A_LSTM))
assign_kernel = kernel.assign(LSTM_WEIGHTS)
# Construct operation for assigning mock bias
bias = next(i for i in tf.global_variables()
if i.shape == (lstm_units *
TYPES_OF_GATES_IN_A_LSTM, ))
assign_bias = bias.assign(LSTM_BIAS)
# Get the explanation from the LRP framework
R = lrp.lrp(inp, lstm_output)
R_sum = tf.reduce_sum(R, [2, 3])
with tf.Session() as s:
# Initialize variables
s.run(tf.global_variables_initializer())
# Assign mock weights and mock bias
s.run([assign_kernel, assign_bias])
# Calculate the relevances
explanation = s.run(R)
# Create array with expected relevances
expected_explanation = np.array(
[[[[
-0.00000000329187, -0.00000000579639,
-0.00000000344224, -0.00000000256858
], [0, 0, 0, 0], [0, 0, 0, 0]],
[[
0.01917747822, 0.02132167498, 0.009472542897,
0.004885821672
],
[
-0.05303456479, 0.02658300098, 0.1649355033,
0.3430433395
], [0, 0, 0, 0]],
[[
0.01775032921, 0.01975440857, 0.008784149807,
0.004537977098
],
[
-0.05051601016, 0.02468158083, 0.1500054205,
0.3081801763
],
[
0.04747183272, 0.1265003211, 0.1284724633,
0.1745462303
]]],
[[[
0.09064830212495, 0.21548798884060, 0.19743329647079,
0.24977081145921
],
[
0.00000000000000, 0.00000000000000,
0.00000000000000, 0.00000000000000
],
[
0.00000000000000, 0.00000000000000,
0.00000000000000, 0.00000000000000
]],
[[
0.05894237044, 0.1393483447, 0.127375767,
0.1608060994
],
[
0.06054255911, 0.1387999706, 0.1224546803,
0.1504080344
], [0, 0, 0, 0]],
[[
0.04103863682, 0.09681307209, 0.08841408434,
0.1115271072
],
[
0.04215389263, 0.09642979223, 0.0849954929,
0.1043121762
],
[
0.04310312299, 0.09707452916, 0.08397782613,
0.1014948822
]]]])
self.assertTrue(
np.allclose(expected_explanation,
explanation,
rtol=1e-3,
atol=1e-3),
"The eplanation should match the expected explanation")
def runTest(self):
with tf.Graph().as_default():
batch_size = 4
sequence_length = 3
units = 2
# Shape: (4, 3, 2)
inp = tf.constant(
[[[0.25946831, 0.73117677], [-0.86958342, 0.71637971],
[1.37765705, -0.94898418]],
[[-1.58566558, 0.81390618], [-0.80477955, 0.93804462],
[-0.21492174, -0.84860344]],
[[1.14450824, 0.76930967], [0.05394846, 0.26253664],
[1.47991874, 1.5336967]],
[[0.75599386, 0.04976039], [1.37701431, 1.33116121],
[0.19442138, -0.2920729]]],
dtype=tf.float32)
# Make a reshape that changes the 'batch_size'
# New shape: (12, 2)
inp_reshaped = tf.reshape(inp, (-1, 2))
# Shape back to the original batch_size: (4, 3, 2)
out = tf.reshape(inp_reshaped,
(batch_size, sequence_length, units))
# Perform lrp
R = lrp.lrp(inp, out)
with tf.Session() as s:
explanation = s.run(R)
expected_explanation = np.array([[[[0, 0.73117679], [0, 0],
[0, 0]],
[[0, 0], [0, 0.7163797],
[0, 0]],
[[0, 0], [0, 0],
[1.37765706, 0]]],
[[[0, 0.81390619], [0, 0],
[0, 0]],
[[0, 0], [0, 0.93804461],
[0, 0]],
[[0, 0], [0, 0],
[-0.21492174, 0]]],
[[[1.14450824, 0], [0, 0],
[0, 0]],
[[0, 0], [0, 0.26253664],
[0, 0]],
[[0, 0], [0, 0],
[0, 1.53369665]]],
[[[0.75599384, 0], [0, 0],
[0, 0]],
[[0, 0], [1.37701428, 0],
[0, 0]],
[[0, 0], [0, 0],
[0.19442138, 0]]]])
self.assertTrue(
np.allclose(expected_explanation,
explanation,
atol=1e-03,
rtol=1e-03),
"Explanation does not match expected explanation")
def runTest(self):
# Construct tensorflow graph
g = tf.Graph()
# Use tensorflows default graph
with g.as_default():
# Create a placeholder for the input
inp = tf.placeholder(tf.float32, shape=(1, 2, 2, 2))
# Create first convolutional layer (simple layer that copies the
# input to the next layer and copies relevances backwards in the
# same manner (so we were able to reuse calculations from earlier
# test case, `test_convolution_with_bias.py`)
with tf.name_scope('conv1'):
weights = tf.constant([[[[9, 5], [0, -2]], [[2, 0], [4, 0]]],
[[[4, 6], [1, 0]], [[-8, 7], [2, 3]]]],
dtype=tf.float32)
activation = tf.nn.conv2d(inp, weights, [1, 1, 1, 1], "SAME")
# Create the second convolutional layer equal to `test_convolution_with_bias.py`
with tf.name_scope('conv2'):
# Set the weights and biases
weights = tf.constant(
[[[[2., 1, 1], [1., 1, 0]], [[2., 2, 1], [1., 1, 1]]],
[[[1., -1, 1], [0., 1, 0]], [[2., 0, 0], [0., -1, 1]]]],
dtype=tf.float32)
bias = tf.constant([1, 2, -2], dtype=tf.float32)
# Perform the convolution
activation = tf.nn.conv2d(activation, weights, [1, 1, 1, 1],
"SAME")
# Add bias
activation = tf.nn.bias_add(activation, bias)
print(activation)
# Flatten the activations to get something of the shape (batch_size, predictions_per_sample, classes)
# as required by the lrp framework
final_output = tf.reshape(activation, [1, 2, 6])
# Calculate the relevance scores using lrp
expl = lrp.lrp(inp, final_output)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Run the operations of interest and feed an input to the network
prediction, explanation = sess.run([final_output, expl],
feed_dict={
inp: [[[[-1., 0.],
[1., 2.]],
[[12., 9.],
[13., 6.]]]]
})
# The expected relevances (calculated in Google sheet)
expected_explanation = np.array([[[[[0, 0], [20.77998952, 0]],
[[162.531828, 0],
[480.9247907,
48.85034831]]],
[[[0, 0], [0, 0]],
[[258, 0], [339, 48]]]]])
# Check if the relevance scores are correct
self.assertTrue(
np.allclose(expected_explanation,
explanation,
rtol=1e-03,
atol=1e-03),
msg="Should be a good convolutional explanation")
def test_four_predictions_per_sample(self):
g = tf.Graph()
with g.as_default():
inp = tf.placeholder(tf.float32, (2, 4, 5))
x = tf.contrib.layers.batch_norm(inp, is_training=False, scale=True)
vars = tf.global_variables()
beta = next(i for i in vars if 'beta' in i.name)
gamma = next(i for i in vars if 'gamma' in i.name)
mean = next(i for i in vars if 'mean' in i.name)
variance = next(i for i in vars if 'variance' in i.name)
b = tf.constant([0, 1, 0, 1, 0], dtype=tf.float32)
assign_beta = tf.assign(beta, b)
g = tf.constant([0.1, 0.2, 0.3, 0.4, 0.5], dtype=tf.float32)
assign_gamma = tf.assign(gamma, g)
m = tf.constant([1, 2, 3, 4, 4.5], dtype=tf.float32)
assign_mean = tf.assign(mean, m)
v = tf.constant([0.2, 0.2, 0.2, 0.2, 0.2], dtype=tf.float32)
assign_variance = tf.assign(variance, v)
# Get the explanation
config = LRPConfiguration()
config.set(LAYER.ELEMENTWISE_LINEAR, AlphaBetaConfiguration())
explanation = lrp.lrp(inp, x, config)
with tf.Session() as s:
s.run(tf.global_variables_initializer())
s.run([assign_beta, assign_gamma, assign_mean, assign_variance])
input = np.array([[[-0.3597203655, 2.416366089, -0.7543762749, 0.8718006654, -2.221761776],
[1.099448308, 0.4108163696, 1.798067039, -0.6544576652, -0.6968107745],
[-0.5612699962, -0.2597267932, 0.06325442832, -1.236885473, 0.9369620591],
[-0.06784464057, -0.004403155247, -1.195337879, -0.528265092, -0.1020843691]],
[[0.8766405077, 0.522839272, 0.4197016166, -1.497174712, 0.05348117451],
[0.08739119149, -0.9997059536, -0.6212993685, 0.04027413639, 0.3979749684],
[0.06180908495, -0.5322826252, -1.585670194, -0.5220654844, -1.096597863],
[1.003261811, -1.865129316, -1.134796217, -0.1038509194, -0.8933464003]]])
# Shape: (2, 4, 4, 5)
expected_relevances = np.array(
[[[[0.0, 1.07794025, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.1832650698, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]]],
[[[0.0, 0.2332384558, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[-0.2035572695, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0]],
[[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0],
[0.0007275465407, 0.0, 0.0, 0.0, 0.0]]]])
relevances = s.run(explanation, feed_dict={inp: input})
self.assertTrue(np.allclose(expected_relevances, relevances, rtol=1e-03, atol=1e-03),
msg="The relevances do not match the expected")
def runTest(self):
# Get a tensorflow graph
g = tf.Graph()
# Set the graph as default
with g.as_default():
# Create a placeholder for the input
inp = tf.placeholder(tf.float32, shape=(1, 4))
# --------------------- Linear layer -----------------------------
# Setup the linear layer
weights_1 = tf.constant(
[[10, 1, 0], [-10, 1, 1], [5, 1, 0], [17, 1, -1]],
dtype=tf.float32)
bias_1 = tf.constant([-20, 13, 2], dtype=tf.float32)
# Calculate the activations
activation_1 = tf.matmul(inp, weights_1, name="MUL1") + bias_1
# --------------------- Conv layer -----------------------------
# Create the convolutional layer
# Set the filters and biases
filters = tf.constant([[[-3, 3]], [[3, 10]]], dtype=tf.float32)
bias_2 = tf.constant([-9, 19], dtype=tf.float32)
# Expand the dimensions of the output from the first layer,
# since the 1dconv takes rank 3 tensors as input
activation_1 = tf.expand_dims(activation_1, -1)
# Perform the convolution and add bias
activation_2 = tf.nn.conv1d(activation_1, filters, 1,
"SAME") + bias_2
# --------------------- Linear layer -----------------------------
# Flatten the output from the conv layer so it can serve as input
# to a linear layer
activation_2_flattened = tf.reshape(activation_2, (1, 6))
# Setup the linear layer
weights_3 = tf.constant([[14], [14], [1], [3], [5], [-1]],
dtype=tf.float32)
bias_3 = tf.constant([0], dtype=tf.float32)
# Calculate the activations
activation_3 = tf.matmul(
activation_2_flattened, weights_3, name="MUL3") + bias_3
# Set the prediction to be equal to the activations of the last layer
# (there is no softmax in this network)
pred = activation_3
# --------------------- Calculate relevances -----------------------------
# Calculate the relevance scores using lrp
expl = lrp.lrp(inp, pred)
# Run a tensorflow session to evaluate the graph
with tf.Session() as sess:
# Initialize the variables
sess.run(tf.global_variables_initializer())
# Run the operations of interest and feed an input to the network
prediction, explanation = sess.run(
[pred, expl], feed_dict={inp: [[-1, 3, 55, 0]]})
# Check if the predictions has the right shape
self.assertEqual(prediction.shape, (1, 1),
msg="Should be able to do a forward pass")
# Check if the explanation has the right shape
self.assertEqual(explanation.shape,
inp.shape,
msg="Should be a wellformed explanation")
# Check if the relevance scores are correct (the correct values are found by
# calculating the example by hand)
self.assertTrue(np.allclose(explanation[0],
[[0, 353.58, 11466.76, 0]],
rtol=1e-01,
atol=1e-01),
msg="Should be a good explanation")
def _test_configuration(self, config):
# Start new graph for this configuration
graph = tf.Graph()
with graph.as_default():
logger.debug(
"Start of new test graph with config {}".format(config))
X, X_reordered, self.features_batch, feed_dict = _construct_X_and_feed_dict(
self.features_read)
# Prepare template (that uses parameter sharing across calls to sirs_template)
sirs_template = tf.make_template('', self.create_model)
logger.debug("Building graph for forward pass")
logger.debug(sirs_template)
# Compute the DRN graph
model = sirs_template(X_reordered)
should_write_input = False
if isinstance(config, str):
# The config is either random or SA
if config == 'random':
# Compute random relevances
logger.debug("Building random graph")
R = get_random_relevance(X)
should_write_input = True
else:
# Compute sensitivity analysis
logger.debug("Building SA graph")
R = get_sensitivity_analysis(X, model['y_hat'])
else:
logger.debug('Building lrp graph')
R = lrp.lrp(X, model['y_hat'], config)
logger.debug('Done building lrp graph')
logger.debug("Instantiating pertubation class")
# Make pertuber for X and R that prepares a number of pertubations of X
pertuber = Pertuber(X, R, self.batch_size, **self.confs)
# Build the pertubation graph
benchmark = pertuber.build_pertubation_graph(sirs_template)
# Create a tf Saver that can be used to restore a pre-trained model below
saver = tf.train.Saver()
with tf.Session(graph=graph) as s:
logger.debug("Initializing vars and restoring model")
# Initialize the local variables and restore the model that was trained earlier
s.run([tf.local_variables_initializer()])
self.restore_checkpoint(s, saver)
logger.debug("Restored model. Now starting threads.")
# Create the threads that run the model
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=s)
try:
# Run the benchmarks. Shapes:
# Benchmark_result: batch_size, pertubations, num_classes
# y batch_size, 1
# y_hat batch_size, num_classes
logger.debug("Starting session for benchmarks")
benchmark_result, expl, y, y_hat = self.run_model(
[benchmark, R, model['y'], model['y_hat']],
model,
feed_dict=feed_dict,
session=s)
logger.debug("Session done")
# Remove extra dimension from y
# y shape: (batch_size,)
y = y[:, 0]
# Find argmax for y_hat
# y_hat shape: (batch_size,)
y_hat = np.argmax(y_hat, axis=1)
# Write results to file
logger.debug("Writing result to file")
self.writer.write_result(config, y, y_hat,
benchmark_result)
logger.debug("Writing explanation to file")
self.writer.write_explanation(config, expl)
if should_write_input:
logger.debug("Writing input to file")
self.writer.write_input(self.features_read['features'])
except tf.errors.OutOfRangeError:
logger.debug("Done with the testing")
except KeyboardInterrupt:
logger.debug("Process interrupted by user. Wrapping up.")
finally:
coord.request_stop()
logger.debug("Joining threads")
coord.join(threads)
logger.info("Done with test")
def _do_test_with_config_and_expected_result(self, config,
expected_result):
with tf.Graph().as_default():
lstm_units = 2
# Make static input shape: (1, 2, 3)
input = [[[0.42624769, -0.24526631, 2.6827445],
[1.50864387, 2.01764531, 0.49280675]]]
inp = tf.constant(input, dtype=tf.float32)
# Create lstm layer
lstm = tf.contrib.rnn.LSTMCell(
lstm_units,
# initializer=tf.constant_initializer(1., dtype=tf.float32),
forget_bias=0.)
# Put it into Multi RNN Cell
lstm = tf.contrib.rnn.MultiRNNCell([lstm])
# Let dynamic rnn setup the control flow (making while loops and stuff)
lstm_output, _ = tf.nn.dynamic_rnn(lstm, inp, dtype=tf.float32)
# Construct operation for assigning mock weights
kernel = next(i for i in tf.global_variables()
if i.shape == (5, 8))
assign_kernel = kernel.assign(
[[
1.48840206, -0.948746, 0.77125305, 2.55016428, 1.18691298,
-0.24476569, 2.56425766, 0.13880421
],
[
2.08840452, -0.46632275, 0.84440069, 0.98795753,
0.61527844, 2.7123294, 0.33261274, 1.86915179
],
[
1.42373006, 1.23778513, 0.63839003, 0.68332758,
0.82368828, -0.11620465, 0.11787995, 1.58372134
],
[
2.35450518, -0.41308389, 1.31977204, 0.91312955,
-0.13488139, 0.93544023, 0.0894083, 0.12383227
],
[
0.0330369, 2.63521215, 1.48256475, -0.28661456,
1.70166103, 1.80855782, 1.35295711, 0.58774797
]])
# Construct operation for assigning mock bias
bias = next(i for i in tf.global_variables() if i.shape == (8, ))
assign_bias = bias.assign([
1.39120539, -0.91791735, -1.02699767, 1.34115046, 0.19859183,
0.73738726, 1.6626231, 2.31063315
])
output = lstm_output
# Get the explanation from the LRP framework.
R = lrp.lrp(inp, output, config)
with tf.Session() as s:
# Initialize variables
s.run(tf.global_variables_initializer())
# Assign mock kernel and bias
s.run([assign_kernel, assign_bias])
# Calculate relevance
relevances = s.run(R)
# Check for shape and actual result
self.assertEqual(
expected_result.shape, relevances.shape,
"Shapes of expected relevance and relevance should be equal"
)
self.assertTrue(
np.allclose(relevances,
expected_result,
rtol=1e-03,
atol=1e-03), "The relevances do not match")
def runTest(self):
with tf.Graph().as_default():
lstm_units = 5
# Make static input
np_input = np.reshape(np.arange(-10, 14), (1, 8, 3))
inp = tf.constant(np_input, dtype=tf.float32)
# Create lstm layer
lstm = tf.contrib.rnn.LSTMCell(lstm_units, forget_bias=0.)
# Put it into Multi RNN Cell
lstm = tf.contrib.rnn.MultiRNNCell([lstm])
# Let dynamic rnn setup the control flow (making while loops and stuff)
lstm_output, _ = tf.nn.dynamic_rnn(lstm, inp, dtype=tf.float32)
# Construct operation for assigning mock weights
kernel = next(i for i in tf.global_variables()
if i.shape == (8, 20))
assign_kernel = kernel.assign(LSTM_WEIGHTS)
# Slice the output to get shape (batch_size, 1, lstm_units) and then squueze the 2nd dimension to
# get shape (batch_size, lstm_units) so we test if the framework if capable of handling starting point
# predictions without the predictions_per_sample dimension
output = tf.squeeze(
tf.slice(lstm_output, [0, 7, 0], [1, 1, lstm_units]), 1)
# Get the explanation from the LRP framework.
# TODO The output of the lstm is not quite right.
explanation = lrp.lrp(inp, output)
with tf.Session() as s:
# Initialize variables
s.run(tf.global_variables_initializer())
# Assign mock kernel
s.run(assign_kernel)
# Calculate relevance
relevances = s.run(explanation)
# Expected result calculated in
# https://docs.google.com/spreadsheets/d/1_bmSEBSWVOkpdlZYEUckgrnUtxhEfnR84LZy1cU5fIw/edit?usp=sharing
expected_result = np.array(
[[[-0.001298088358, 0.01145384055, 0.0006777067672],
[0.02191306626, 0.008657823488, -0.01480988454],
[0.05461008726, 0.0005180473113, -0.02527708783],
[0.02971554238, 0.0, 0.02727937854],
[-0.08685950242, 0.02556374875, 0.1572219068],
[-0.2561188815, 0.06659819392, 0.3236008156],
[-0.4212428569, 0.1028748125, 0.4771196328],
[-0.569597743, 0.1599460364, 0.5879939284]]])
# Check for shape and actual result
self.assertEqual(
expected_result.shape, relevances.shape,
"Shapes of expected relevance and actual relevance should be the same"
)
self.assertTrue(
np.allclose(relevances,
expected_result,
rtol=1e-03,
atol=1e-03), "The relevances do not match")
def _do_convolutional_test(self, config, expected_result):
if not type(expected_result).__module__ == np.__name__:
expected_result = np.array(expected_result)
with tf.Graph().as_default():
# Shape (1, 3, 3, 2)
inp = tf.constant(
[[[[0.26327863, 1.34089666], [1.10478271, 1.52725762],
[-0.16542361, 0.9857486]],
[[0.59388839, 0.89302082], [-0.12325813, 1.37334976],
[1.70439274, 1.37814247]],
[[0.37915105, 0.89512656], [-0.12834121, 0.72336895],
[1.35641106, 1.76352144]]]],
dtype=tf.float32)
# Shape: (2, 2, 2, 1)
kernel = tf.constant([[[[0.28530154], [-0.095688446]],
[[-0.45116284], [0.87342771]]],
[[[0.51163061], [-0.86481167]],
[[-0.78022886], [-0.86212577]]]],
dtype=tf.float32)
bias = tf.constant([0.484489966914], dtype=tf.float32)
# Shape: (1, 2, 2, 1)
out1 = tf.nn.relu(
tf.nn.bias_add(tf.nn.conv2d(inp, kernel, [1, 2, 2, 1], 'SAME'),
bias))
# Shape: (1, 4)
out2 = tf.reshape(out1, (1, 4))
# Shape: (4, 2)
weights = tf.constant(
[[-0.93597473, -0.26599479], [-0.58296088, -0.05599708],
[-0.21809592, 0.93448646], [-0.36266413, -0.00806697]],
dtype=tf.float32)
# Shape: (2,)
bias2 = tf.constant([0.0794559, -0.48629082], dtype=tf.float32)
# Shape: (1, 2)
out3 = out2 @ weights + bias2
# Shape: (1, 2)
out4 = tf.nn.softmax(out3)
# Explanation shape: (1, 3, 3, 2)
expl = lrp.lrp(inp, out4, config)
with tf.Session() as s:
explanation = s.run(expl)
# Check if the explanation has the right shape
self.assertEqual(explanation.shape,
expected_result.shape,
msg="Should be a wellformed explanation")
print(expected_result)
print()
print(explanation)
# Check if the relevance scores are correct (the correct values are found by
# calculating the example by hand)
self.assertTrue(np.allclose(explanation,
expected_result,
rtol=1e-06,
atol=1e-06),
msg="Should be a good explanation")
def runTest(self):
# Build the computational graph
with tf.Graph().as_default():
# Make static input shape: (4, 6)
inp = tf.constant([[0.49529851, -0.64648792, 0.18508197, -0.14201359, 0.29480708,
-0.23168202],
[0.03458613, -0.11823616, -0.67511888, -0.17805438, 0.7495242,
-0.29286811],
[-1.51681199, -1.05214575, -0.31338711, -0.14845488, 0.32269641,
2.08227179],
[0.18766482, 0.10273498, 0.93645419, -0.71804516, -0.92730127,
0.11013126]]
, dtype=tf.float32)
# -------------------------------------------- FC 1--------------------------------------------
# shape: (6, 10)
weights_1 = tf.constant([[0.63023564, 0.72418477, -0.03827982, 0.48678625, 1.04275436,
0.35443291, 0.92035296, -0.89705308, -0.90312034, 0.51454559],
[-0.40643197, -0.19538514, 0.52029634, -0.43955133, -1.19634436,
0.33259718, -0.21127183, 0.6793771, -0.72406187, 1.54054022],
[-0.67484287, -1.14928279, 1.28560718, -0.02242465, 0.3377433,
0.74823952, 2.18620002, 0.18857024, 0.6531554, 2.72489568],
[-0.15728904, 0.28771674, -1.18420233, 2.17638949, -0.47370135,
-0.02005775, 0.41663315, 0.60860928, 0.57529257, -0.104214],
[-0.4055075, 0.8596076, -0.89655813, -1.39272219, 0.73927064,
1.44179635, -1.01808758, 1.20547704, -1.30409662, 0.02295059],
[0.80911711, 0.99273549, 0.31395664, 2.29630583, 0.58090097,
-1.05635963, -0.90120138, 1.63487712, 2.27660196, 0.51776111]]
,
dtype=tf.float32)
# shape: (10,)
bias_1 = tf.constant([-1.58876296, 1.44444094, 0.73600305, 0.99948251, -0.25653983,
0.54555058, 0.80193129, -0.46552793, 0.30203156, -0.28628351]
, dtype=tf.float32)
# shape: (4, 10)
output_1 = tf.matmul(inp, weights_1) + bias_1
# Prepare the output for the convolutional layer
# New shape: (4, 5, 2)
output_1 = tf.reshape(output_1, (4, 5, 2))
# -------------------------------------------- Convolution 1 --------------------------------------------
# Create the filter which has shape [filter_width, in_channels, out_channels]
# Shape: (2,2,1)
filter_2 = tf.constant(
[[[-0.41445586],
[1.26795033]],
[[-1.61688659],
[1.50628238]]]
, dtype=tf.float32)
# Shape: (1,)
bias_2 = tf.constant([0.84705889], dtype=tf.float32)
# Perform the convolution
# Shape of output_2: (4, 5, 1)
output_2 = tf.nn.conv1d(output_1, filter_2, 1, "SAME") + bias_2
# Prepare output for the max pooling
# Pooling is defined for 2d, so add dim of 1 (height)
# New shape of output_2_reshaped: (4,1,5,1)
output_2 = tf.expand_dims(output_2, 1)
# -------------------------------------------- Max pooling --------------------------------------------
# Kernel looks at 1 sample, 1 height, 2 width, and 1 depth
ksize = [1, 1, 2, 1]
# Move 1 sample, 1 height, 2 width, and 1 depth at a time
strides = [1, 1, 2, 1]
# Perform the max pooling
# Shape of pool: (4, 1, 3, 1)
pool = tf.nn.max_pool(output_2, ksize, strides, padding='SAME')
# Remove the "height" dimension again
# New shape: (4, 3, 1)
output_3 = tf.squeeze(pool, 1)
# -------------------------------------------- Convolution 2--------------------------------------------
# Create the filter which has shape [filter_width, in_channels, out_channels]
# Shape: (2,1,2)
filter_4 = tf.constant(
[[[0.19205464, -0.90562985]],
[[1.0016198, 0.89356491]]],
dtype=tf.float32)
# Shape: (2, )
bias_4 = tf.constant([0.07450981, -0.14901961], dtype=tf.float32)
# Perform the convolution
# Shape: (4, 3, 2)
output_4 = tf.nn.conv1d(output_3, filter_4, 1, "SAME") + bias_4
# Prepare output for the max pooling
# Pooling is defined for 2d, so add dim of 1 (height)
# New shape of output_2_reshaped: (4,1,3,2)
output_4 = tf.expand_dims(output_4, 1)
# -------------------------------------------- Max pooling --------------------------------------------
# Kernel looks at 1 sample, 1 height, 2 width, and 1 depth
ksize = [1, 1, 2, 1]
# Move 1 sample, 1 height, 2 width, and 1 depth at a time
strides = [1, 1, 2, 1]
# Perform the max pooling
# Shape of pool: (4, 1, 2, 2)
pool = tf.nn.max_pool(output_4, ksize, strides, padding='SAME')
# Remove the "height" dimension again
# New shape: (4, 2, 2)
output_5 = tf.squeeze(pool, 1)
# -------------------------------------------- LSTM --------------------------------------------
lstm_units = 3
# Shape of weights: (5, 12)
LSTM_weights = [[-0.6774261, -1.77336803, 0.37789944, -1.47642675, -0.77326061,
-0.41624885, -0.80737161, -1.00830384, 0.80501084, -0.10506079,
-0.42341706, 1.61561637],
[0.65131449, -1.25813521, -1.01188983, 1.58355103, -0.55863594,
0.59259386, -1.15333092, 1.31657508, -0.3582473, -0.91620798,
1.30231276, 0.32319264],
[1.11120673, 0.60646556, -1.11294626, -0.26202266, -1.53741017,
-0.09405062, -0.82200596, -0.41727707, 0.69017403, -2.67866443,
1.08780302, -0.53820959],
[0.12222124, 0.17716194, -0.96654223, 0.64953949, 1.55478632,
-1.22787184, 0.67456202, 0.34234439, -2.42116309, 0.22752669,
-0.40613203, -0.42356035],
[0.9004432, 1.9286521, 1.04199918, 1.17486178, -1.30394625,
0.60571671, 1.30499515, 2.12358405, -1.82775648, 0.81019163,
0.20284197, 0.72304922]]
# Shape: (12,)
LSTM_bias = [-9.46940060e-01, 5.69888879e-02, -4.06483928e-05,
-9.60644436e-01, -1.18161660e+00, -2.04222054e+00,
-2.27343882e-02, 2.39842965e-01, -4.42784509e-01,
2.86647829e+00, 2.92904572e-02, -1.45679881e+00]
# Create lstm layer
lstm = tf.contrib.rnn.LSTMCell(lstm_units,
forget_bias=0.)
# Put it into Multi RNN Cell
lstm = tf.contrib.rnn.MultiRNNCell([lstm])
# Let dynamic rnn setup the control flow (making while loops and stuff)
# Shape of output_6: (4, 2, 3)
output_6, _ = tf.nn.dynamic_rnn(lstm, output_5, dtype=tf.float32)
# Construct operation for assigning mock weights
kernel = next(i for i in tf.global_variables() if i.shape == (5, 12))
assign_kernel = kernel.assign(LSTM_weights)
# Construct operation for assigning mock bias
bias = next(i for i in tf.global_variables() if i.shape == (12,))
assign_bias = bias.assign(LSTM_bias)
# Prepare output for the linear layer by flattening the batch and timesteps
# to be able to use linear layer on all predictions and all samples in batch at the same time
# New shape of output_6: (8, 3)
output_6 = tf.reshape(output_6, (-1, lstm_units))
# -------------------------------------------- FC 2--------------------------------------------
# Shape of weights_7: (3,2)
weights_7 = tf.constant([[1.6295322, 1.54609607],
[1.04818304, -0.98323105],
[-1.35106161, -1.20737747]],
dtype=tf.float32)
bias_7 = tf.constant([0.85856544, 0.75856544], dtype=tf.float32)
# Perform the matmul
output_7 = tf.matmul(output_6, weights_7) + bias_7
# Reshape to shape (4, 2, 2)
output_7 = tf.reshape(output_7, (4, 2, 2))
# -------------------------------------------- Softmax -------------------------------------------
output_final = tf.nn.softmax(output_7)
# -------------------------------------------- LRP -------------------------------------------
# Get the explanation from the LRP framework.
R = lrp.lrp(inp, output_final)
# Run the computations
with tf.Session() as s:
# Initialize variables
s.run(tf.global_variables_initializer())
# Assign mock bias
s.run([assign_kernel, assign_bias])
relevances = s.run(R)
# Expected result calculated in
# https://docs.google.com/spreadsheets/d/1_bmSEBSWVOkpdlZYEUckgrnUtxhEfnR84LZy1cU5fIw/edit?usp=sharing
expected_result = np.array([[0, 0.07404811231, 0.01912311052, 0, 0, 0],
[0, 0.02956543224, 0.007635346947, 0, 0, 0],
[0.002886748521, 0.002103478273, 0.00662006448, 0.00155415063, 0.042944841,
0.02943967311],
[-0.002340507274, 0.03042895035, -0.005880763452, -0.001369479592,
-0.03478716069, -0.02364958506],
[0.02778684373, 0.0235402653, 0.001067970707, 4.045956959e-06,
0.008719875287, 0.09191818869],
[-0.0004585283713, -0.0004082257816, -2.710138429e-05, -1.02672324e-07,
-0.0001507774977, -0.001715578912],
[0.00195132106, 0.003198257203, 0.05156540372, 0.001512167255, 0,
0.001152290654],
[0.008284077171, 0.01357778073, 0.2189141462, 0.006419707395, 0,
0.00489189857]]
).reshape((4, 2, 6))
self.assertTrue(np.allclose(relevances, expected_result, rtol=1e-03, atol=1e-03),
"The relevances do not match")
