def _test_for_multiple_io(self, model):
cam = Xcam(model)
result = cam([CategoricalScore(0), CategoricalScore(0)],
[dummy_sample(
(1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))])
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__(self, model):
assert model.get_layer(name='output_1').activation != tf.keras.activations.linear
if len(model.outputs) > 1:
assert model.get_layer(name='output_2').activation != tf.keras.activations.linear
instance = ActivationMaximization(model, model_modifier=ReplaceToLinear())
assert instance.model != model
assert instance.model.get_layer(name='output_1').activation == tf.keras.activations.linear
if len(model.outputs) > 1:
assert instance.model.get_layer(
name='output_2').activation == tf.keras.activations.linear
instance([CategoricalScore(0), CategoricalScore(0)])
else:
instance([CategoricalScore(0)])
class TestXcamWithMultipleOutputsModel():
@pytest.mark.parametrize("scores,expected_error", [
(None, ValueError),
([None], ValueError),
(CategoricalScore(0), ValueError),
([CategoricalScore(0)], ValueError),
([None, None], ValueError),
([CategoricalScore(0), None], ValueError),
([None, CategoricalScore(0)], ValueError),
([CategoricalScore(0), BinaryScore(0)], NO_ERROR),
([score_with_tuple, score_with_tuple], NO_ERROR),
([score_with_list, score_with_list], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_score_is_(self, scores, expected_error, multiple_outputs_model):
cam = Xcam(multiple_outputs_model)
with assert_raises(expected_error):
result = cam(scores, dummy_sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
@pytest.mark.parametrize("seed_input,expected,expected_error", [
(None, None, ValueError),
(dummy_sample((8, )), None, ValueError),
(dummy_sample((8, 8, 3)), (1, 8, 8), NO_ERROR),
([dummy_sample((8, 8, 3))], [(1, 8, 8)], NO_ERROR),
(dummy_sample((1, 8, 8, 3)), (1, 8, 8), NO_ERROR),
([dummy_sample((1, 8, 8, 3))], [(1, 8, 8)], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_seed_input_is_(self, seed_input, expected, expected_error,
multiple_outputs_model):
cam = Xcam(multiple_outputs_model)
with assert_raises(expected_error):
result = cam([CategoricalScore(0), BinaryScore(0)], seed_input)
if type(expected) is list:
assert type(result) is list
expected = expected[0]
result = result[0]
assert result.shape == expected
@pytest.mark.parametrize("expand_cam", [False, True])
@pytest.mark.usefixtures("xcam", "mixed_precision")
def test__call__with_expand_cam(self, expand_cam, multiple_outputs_model):
cam = Xcam(multiple_outputs_model)
result = cam([CategoricalScore(0), BinaryScore(0)], [dummy_sample((1, 8, 8, 3))],
expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
else:
assert result.shape == (1, 6, 6)
def test__call__with_expand_cam(self, expand_cam, conv_model):
cam = Xcam(conv_model)
result = cam(CategoricalScore(0), [dummy_sample((1, 8, 8, 3))], expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
else:
assert result.shape == (1, 6, 6)
def test__call__without_regularization(self, conv_model):
activation_maximization = ActivationMaximization(conv_model)
result = activation_maximization(CategoricalScore(1),
steps=1,
regularizers=None,
callbacks=PrintLogger(1))
assert result.shape == (1, 8, 8, 3)
def test__call__if_max_N_is_(self, max_N, expected_error, conv_model):
with assert_raises(expected_error):
cam = Scorecam(conv_model)
result = cam(CategoricalScore(0),
dummy_sample((2, 8, 8, 3)),
max_N=max_N)
assert result.shape == (2, 8, 8)
def test__call__if_activation_modifier_is_(self, activation_modifier,
conv_model):
cam = GradcamPlusPlus(conv_model)
result = cam(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
activation_modifier=activation_modifier)
assert result.shape == (1, 8, 8)
class TestXcamWithMultipleInputsModel():
@pytest.mark.parametrize("scores,expected_error", [
(None, ValueError),
(CategoricalScore(0), NO_ERROR),
(score_with_tuple, NO_ERROR),
(score_with_list, NO_ERROR),
([None], ValueError),
([CategoricalScore(0)], NO_ERROR),
([score_with_tuple], NO_ERROR),
([score_with_list], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_score_is_(self, scores, expected_error, multiple_inputs_model):
cam = Xcam(multiple_inputs_model)
with assert_raises(expected_error):
result = cam(scores, [dummy_sample((1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
@pytest.mark.parametrize("seed_input,expected_error", [
(None, ValueError),
(dummy_sample((1, 8, 8, 3)), ValueError),
([dummy_sample((1, 8, 8, 3))], ValueError),
([dummy_sample((1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_seed_input_is_(self, seed_input, expected_error, multiple_inputs_model):
cam = Xcam(multiple_inputs_model)
with assert_raises(expected_error):
result = cam(CategoricalScore(0), seed_input)
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
@pytest.mark.parametrize("expand_cam", [False, True])
@pytest.mark.usefixtures("xcam", "mixed_precision")
def test__call__with_expand_cam(self, expand_cam, multiple_inputs_model):
cam = Xcam(multiple_inputs_model)
result = cam(CategoricalScore(0),
[dummy_sample(
(1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))],
expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
else:
assert result.shape == (1, 8, 8)
def test__call__(self, tmpdir, conv_model):
path = tmpdir.mkdir("tf-keras-vis").join("test.gif")
activation_maximization = ActivationMaximization(conv_model)
assert not os.path.isfile(path)
result = activation_maximization(CategoricalScore(0),
callbacks=GifGenerator2D(str(path)))
assert os.path.isfile(path)
assert result.shape == (1, 8, 8, 3)
def test__call__with_expand_cam(self, expand_cam, multiple_outputs_model):
cam = Xcam(multiple_outputs_model)
result = cam([CategoricalScore(0), BinaryScore(0)], [dummy_sample((1, 8, 8, 3))],
expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
else:
assert result.shape == (1, 6, 6)
def test__call__(self, model, layer, expected_error):
assert model.outputs[0].shape.as_list() == [None, 2]
with assert_raises(expected_error):
instance = ActivationMaximization(model,
model_modifier=ExtractIntermediateLayer(layer))
assert instance.model != model
assert instance.model.outputs[0].shape.as_list() == [None, 6, 6, 6]
instance([CategoricalScore(0)])
def test__call__(self, conv_model):
instance = Saliency(conv_model, model_modifier=GuidedBackpropagation())
guided_model = instance.model
assert guided_model != conv_model
assert guided_model.get_layer('conv_1').activation != conv_model.get_layer(
'conv_1').activation
assert guided_model.get_layer('dense_1').activation == conv_model.get_layer(
'dense_1').activation
instance(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
def _test_for_multiple_io(self, model):
saliency = Saliency(model)
result = saliency(
[CategoricalScore(0), BinaryScore(0)],
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))])
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__if_seed_input_is_(self, seed_input, expected, expected_error, conv_model):
cam = Xcam(conv_model)
with assert_raises(expected_error):
result = cam(CategoricalScore(0), seed_input)
if type(expected) is list:
assert type(result) is list
expected = expected[0]
result = result[0]
assert result.shape == expected
def test__call__if_penultimate_layer_is(self, penultimate_layer, seek_penultimate_conv_layer,
expected_error, conv_model):
cam = Xcam(conv_model)
with assert_raises(expected_error):
result = cam(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
penultimate_layer=penultimate_layer,
seek_penultimate_conv_layer=seek_penultimate_conv_layer)
assert result.shape == (1, 8, 8)
def test__call__if_seed_input_is_(self, seed_input, expected, expected_error,
multiple_outputs_model):
cam = Xcam(multiple_outputs_model)
with assert_raises(expected_error):
result = cam([CategoricalScore(0), BinaryScore(0)], seed_input)
if type(expected) is list:
assert type(result) is list
expected = expected[0]
result = result[0]
assert result.shape == expected
def test__call__if_keepdims_is_(self, keepdims, expected,
multiple_inputs_model):
saliency = Saliency(multiple_inputs_model)
result = saliency(
CategoricalScore(0),
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))],
keepdims=keepdims)
assert len(result) == 2
assert result[0].shape == expected[0]
assert result[1].shape == expected[1]
def test__call__if_smoothing_is_active(self, smooth_samples,
multiple_inputs_model):
saliency = Saliency(multiple_inputs_model)
result = saliency(
CategoricalScore(0),
[dummy_sample((1, 8, 8, 3)),
dummy_sample((1, 10, 10, 3))],
smooth_samples=smooth_samples)
assert len(result) == 2
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__with_expand_cam(self, expand_cam, multiple_inputs_model):
cam = Xcam(multiple_inputs_model)
result = cam(CategoricalScore(0),
[dummy_sample(
(1, 8, 8, 3)), dummy_sample((1, 10, 10, 3))],
expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
else:
assert result.shape == (1, 8, 8)
def run(self):
"""Run Worker Thread."""
# This is the code executing in the new thread.
with open("Model_definition.pkl", 'rb') as f:
model_config = pickle.load(f)
model = tf.keras.models.Model.from_config(model_config)
model.load_weights("Model_weights.h5")
wx.CallAfter(pub.sendMessage, "update", msg="Preparing Image")
# El callback aun no funciona, pero para cuando este disponible. De momento hay 3 saltos, uno que carga el
# modelo, otro procesa las imagenes y el ultimo ordena y copia.
my_call = ProgressBar2()
img_prep = transform_img_fn(self.file)
img_prep = np.concatenate([img_prep, img_prep])
if self.info_type == 'SmoothGrad Saliency':
wx.CallAfter(pub.sendMessage,
"update",
msg="Obtaining SmoothGrad Saliency")
replace2linear = ReplaceToLinear()
score = CategoricalScore([0, 1])
saliency = Saliency(model,
model_modifier=replace2linear,
clone=True)
self.saliency_maps = saliency(score,
img_prep,
smooth_samples=50,
smooth_noise=0.10)
else:
wx.CallAfter(pub.sendMessage, "update", msg="Obtaining GradCAM++")
replace2linear = ReplaceToLinear()
score = CategoricalScore([0, 1])
gradcam = GradcamPlusPlus(model,
model_modifier=replace2linear,
clone=True)
self.gradcam_maps = gradcam(score, img_prep, penultimate_layer=-1)
wx.CallAfter(pub.sendMessage, "update", msg="")
class TestScorecam():
@pytest.mark.parametrize("max_N,expected_error", [
(-100, NO_ERROR),
(-1, NO_ERROR),
(0, NO_ERROR),
(1, NO_ERROR),
(3, NO_ERROR),
(100, ValueError),
])
@pytest.mark.usefixtures("mixed_precision")
def test__call__if_max_N_is_(self, max_N, expected_error, conv_model):
with assert_raises(expected_error):
cam = Scorecam(conv_model)
result = cam(CategoricalScore(0),
dummy_sample((2, 8, 8, 3)),
max_N=max_N)
assert result.shape == (2, 8, 8)
@pytest.mark.parametrize("scores,expected_error", [
(None, ValueError),
(CategoricalScore(0), NO_ERROR),
(score_with_tuple, NO_ERROR),
(score_with_list, NO_ERROR),
(score_with_tensor, NO_ERROR),
(lambda x: np.mean(x), ValueError),
(lambda x: tf.reshape(x, (-1, )), ValueError),
([None], ValueError),
([CategoricalScore(0)], NO_ERROR),
([score_with_tuple], NO_ERROR),
([score_with_list], NO_ERROR),
([score_with_tensor], NO_ERROR),
([lambda x: np.mean(x)], ValueError),
([lambda x: tf.reshape(x, (-1, ))], ValueError),
])
@pytest.mark.usefixtures("mixed_precision")
def test__call__if_score_is_(self, scores, expected_error, conv_model):
cam = Scorecam(conv_model)
with assert_raises(expected_error):
result = cam(scores, dummy_sample((2, 8, 8, 3)))
assert result.shape == (2, 8, 8)
def test__init__(self, indices, expected, expected_error):
with assert_raises(expected_error):
score = CategoricalScore(indices)
assert score.indices == expected
def test__call__(self, indices, output_shape, expected_error):
output = tf.constant(dummy_sample(output_shape), tf.float32)
score = CategoricalScore(indices)
with assert_raises(expected_error):
score_value = score(output)
assert score_value.shape == output_shape[0:1]
def test__call__without_regularizers(self, conv_model):
activation_maximization = ActivationMaximization(conv_model)
result = activation_maximization(CategoricalScore(0),
regularizers=None,
callbacks=Progress())
assert result.shape == (1, 8, 8, 3)
def saliency(img_path, multiclass_model, name):
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import scipy.ndimage as ndimage
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.applications.densenet import preprocess_input
import numpy as np
# Preparing input data for VGG16
# X = preprocess_input(images)
# X = 'https://firebasestorage.googleapis.com/v0/b/intelrad-a680a.appspot.com/o/images%2Fxrayyyy.jpg?alt=media&token=63a8460f-5c79-422c-91d6-c5a2704d2383'
X = img_path
X = preprocess_image(X)
# Rendering
import matplotlib.pyplot as plt
from tf_keras_vis.scorecam import Scorecam
from matplotlib import cm
from tf_keras_vis.gradcam import Gradcam
from tf_keras_vis.utils.model_modifiers import ReplaceToLinear
replace2linear = ReplaceToLinear()
# Create Gradcam object
gradcam = Gradcam(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Instead of using CategoricalScore object,
from tf_keras_vis.utils.scores import CategoricalScore
# 1 is the imagenet index corresponding to Goldfish, 294 to Bear and 413 to Assault Rifle.
score = CategoricalScore(4)
from tensorflow.keras import backend as K
from tf_keras_vis.saliency import Saliency
# from tf_keras_vis.utils import normalize
# Create Saliency object.
saliency = Saliency(multiclass_model,
model_modifier=replace2linear,
clone=True)
# Generate saliency map
X = X.tolist()
X = np.asarray(X, dtype=np.float)
# saliency_map = saliency(score, X)
# Generate saliency map with smoothing that reduce noise by adding noise
saliency_map = saliency(
score,
X,
smooth_samples=20, # The number of calculating gradients iterations.
smooth_noise=0.20) # noise spread level.
# Generate saliency map
saliency_map = saliency(score, X)
# Render
# f, ax = plt.subplots(nrows=1, ncols=3, figsize=(12, 4))
plt.figure(figsize=(8, 8))
plt.imshow(saliency_map[0], cmap='jet', alpha=0.9)
plt.axis('off')
import os
img_dir = './static/img/saliency'
if (not os.path.isdir(img_dir)):
os.mkdir(img_dir)
plt.tight_layout()
plt.savefig(f'./static/img/saliency/{name}.svg',
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.close()
return "Done"
def test__call__if_expand_cam_is_False(self, conv_model):
cam = Xcam(conv_model)
result = cam(CategoricalScore(0), dummy_sample((1, 8, 8, 3)), expand_cam=False)
assert result.shape == (1, 6, 6)
def test__call__if_seed_input_is_(self, seed_input, expected_error, multiple_io_model):
cam = Xcam(multiple_io_model)
with assert_raises(expected_error):
result = cam([CategoricalScore(0), BinaryScore(0)], seed_input)
assert result[0].shape == (1, 8, 8)
assert result[1].shape == (1, 10, 10)
def test__call__(self, dense_model):
cam = Xcam(dense_model)
with assert_raises(ValueError):
result = cam(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
def _test_for_single_io(self, model):
cam = Xcam(model)
result = cam(CategoricalScore(0), dummy_sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
class TestXcam():
@pytest.mark.parametrize("scores,expected_error", [
(None, ValueError),
(CategoricalScore(0), NO_ERROR),
(score_with_tuple, NO_ERROR),
(score_with_list, NO_ERROR),
(score_with_tensor, NO_ERROR),
([None], ValueError),
([CategoricalScore(0)], NO_ERROR),
([score_with_tuple], NO_ERROR),
([score_with_list], NO_ERROR),
([score_with_tensor], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_score_is_(self, scores, expected_error, conv_model):
cam = Xcam(conv_model)
with assert_raises(expected_error):
result = cam(scores, dummy_sample((1, 8, 8, 3)))
assert result.shape == (1, 8, 8)
@pytest.mark.parametrize("seed_input,expected,expected_error", [
(None, None, ValueError),
(dummy_sample((8, )), None, ValueError),
(dummy_sample((8, 8, 3)), (1, 8, 8), NO_ERROR),
([dummy_sample((8, 8, 3))], [(1, 8, 8)], NO_ERROR),
(dummy_sample((1, 8, 8, 3)), (1, 8, 8), NO_ERROR),
([dummy_sample((1, 8, 8, 3))], [(1, 8, 8)], NO_ERROR),
])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__if_seed_input_is_(self, seed_input, expected, expected_error, conv_model):
cam = Xcam(conv_model)
with assert_raises(expected_error):
result = cam(CategoricalScore(0), seed_input)
if type(expected) is list:
assert type(result) is list
expected = expected[0]
result = result[0]
assert result.shape == expected
@pytest.mark.parametrize("penultimate_layer,seek_penultimate_conv_layer,expected_error", [
(None, True, NO_ERROR),
(-1, True, NO_ERROR),
(-1.0, True, ValueError),
('dense_1', True, NO_ERROR),
('dense_1', False, ValueError),
(1, False, NO_ERROR),
(1, True, NO_ERROR),
('conv_1', True, NO_ERROR),
(0, True, ValueError),
('input_1', True, ValueError),
(CategoricalScore(0), True, ValueError),
(mock_conv_model().layers[-1], False, ValueError),
])
@pytest.mark.usefixtures("xcam", "mixed_precision")
def test__call__if_penultimate_layer_is(self, penultimate_layer, seek_penultimate_conv_layer,
expected_error, conv_model):
cam = Xcam(conv_model)
with assert_raises(expected_error):
result = cam(CategoricalScore(0),
dummy_sample((1, 8, 8, 3)),
penultimate_layer=penultimate_layer,
seek_penultimate_conv_layer=seek_penultimate_conv_layer)
assert result.shape == (1, 8, 8)
@pytest.mark.usefixtures("xcam", "mixed_precision")
def test__call__if_expand_cam_is_False(self, conv_model):
cam = Xcam(conv_model)
result = cam(CategoricalScore(0), dummy_sample((1, 8, 8, 3)), expand_cam=False)
assert result.shape == (1, 6, 6)
@pytest.mark.parametrize("score_class", [BinaryScore, CategoricalScore])
@pytest.mark.parametrize("modifier_enabled", [False, True])
@pytest.mark.parametrize("clone_enabled", [False, True])
@pytest.mark.parametrize("batch_size", [0, 1, 5])
@pytest.mark.usefixtures("xcam", "saliency", "mixed_precision")
def test__call__with_categorical_score(self, score_class, modifier_enabled, clone_enabled,
batch_size, conv_model, conv_sigmoid_model):
# Release v.0.6.0@dev(May 22 2021):
# Add this case to test Scorecam with CAM class.
def model_modifier(model):
model.layers[-1].activation = tf.keras.activations.linear
if score_class is BinaryScore:
model = conv_sigmoid_model
else:
model = conv_model
score_targets = np.random.randint(0, 1, max(batch_size, 1))
score = score_class(list(score_targets))
seed_input_shape = (8, 8, 3)
if batch_size > 0:
seed_input_shape = (batch_size, ) + seed_input_shape
seed_input = dummy_sample(seed_input_shape)
cam = Xcam(model,
model_modifier=model_modifier if modifier_enabled else None,
clone=clone_enabled)
result = cam(score, seed_input=seed_input)
if modifier_enabled and clone_enabled:
assert model is not cam.model
else:
assert model is cam.model
assert result.shape == (max(batch_size, 1), 8, 8)
@pytest.mark.parametrize("expand_cam", [False, True])
@pytest.mark.usefixtures("xcam", "mixed_precision")
def test__call__with_expand_cam(self, expand_cam, conv_model):
cam = Xcam(conv_model)
result = cam(CategoricalScore(0), [dummy_sample((1, 8, 8, 3))], expand_cam=expand_cam)
if expand_cam:
assert result[0].shape == (1, 8, 8)
else:
assert result.shape == (1, 6, 6)
