def vanilla_gradients(self, value, index):
for classe in self.labels:
vanilla_grad = VanillaGradients()
grid = vanilla_grad.explain((value, self.setY[index]),
self.model,
class_index=classe)
name = "n_{}_".format(index) + str(classe) + "_vanilla_grad.png"
path = sg.PATH_VANILLA + name
vanilla_grad.save(grid, ".", path)
def test_should_explain_output(convolutional_model, random_data, mocker):
mocker.patch("tf_explain.core.vanilla_gradients.grid_display",
side_effect=lambda x: x)
images, labels = random_data
explainer = VanillaGradients()
grid = explainer.explain((images, labels), convolutional_model, 0)
# Outputs is in grayscale format
assert grid.shape == images.shape[:-1]
def test_get_score_model_returns_suitable_model(
convolutional_model_with_separate_activation_layer, ):
explainer = VanillaGradients()
# The last two layers of the original model are a Dense layer with no activation (i.e. linear)
# followed by a Softmax layer.
score_layer = convolutional_model_with_separate_activation_layer.layers[-2]
softmax_layer = convolutional_model_with_separate_activation_layer.layers[
-1]
score_model = explainer.get_score_model(
convolutional_model_with_separate_activation_layer)
# The score model should exclude the final activation layer.
assert score_model.layers[-1] == score_layer
assert softmax_layer not in score_model.layers
def test_get_averaged_gradients(random_data,
convolutional_model_without_final_activation):
images, _ = random_data
gradients = VanillaGradients.compute_gradients(
images, convolutional_model_without_final_activation, 0)
assert gradients.shape == images.shape
class VanillaGradientsCallback(Callback):
"""
Perform gradients backpropagation for a given input
Paper: [Deep Inside Convolutional Networks: Visualising Image Classification
Models and Saliency Maps](https://arxiv.org/abs/1312.6034)
"""
explainer = VanillaGradients()
default_output_subdir = "vanilla_gradients"
def __init__(self, validation_data, class_index, output_dir=None):
"""
Constructor.
Args:
validation_data (Tuple[np.ndarray, Optional[np.ndarray]]): Validation data
to perform the method on. Tuple containing (x, y).
class_index (int): Index of targeted class
num_samples (int): Number of noisy samples to generate for each input image
"""
super().__init__()
self.validation_data = validation_data
self.class_index = class_index
if not output_dir:
output_dir = Path("./logs") / self.default_output_subdir
self.output_dir = Path(output_dir) / datetime.now().strftime(
"%Y%m%d-%H%M%S.%f")
Path.mkdir(Path(self.output_dir), parents=True, exist_ok=True)
self.file_writer = tf.summary.create_file_writer(str(self.output_dir))
self.score_model = None
def set_model(self, model):
super().set_model(model)
self.score_model = self.explainer.get_score_model(model)
def on_epoch_end(self, epoch, logs=None):
"""
Draw VanillaGradients outputs at each epoch end to Tensorboard.
Args:
epoch (int): Epoch index
logs (dict): Additional information on epoch
"""
grid = self.explainer.explain_score_model(self.validation_data,
self.score_model,
self.class_index)
# Using the file writer, log the reshaped image.
with self.file_writer.as_default():
tf.summary.image(
self.explainer.__class__.__name__,
np.expand_dims([grid], axis=-1),
step=epoch,
)
def compute_gradients(images, model, class_index):
"""
Compute gradients ponderated by input values for target class.
Args:
images (numpy.ndarray): 4D-Tensor of images with shape (batch_size, H, W, 3)
model (tf.keras.Model): tf.keras model to inspect
class_index (int): Index of targeted class
Returns:
tf.Tensor: 4D-Tensor
"""
gradients = VanillaGradients.compute_gradients(images, model,
class_index)
inputs = tf.cast(images, tf.float32)
return tf.multiply(inputs, gradients)
import numpy as np
import tensorflow as tf
from tf_explain.core.vanilla_gradients import VanillaGradients
IMAGE_PATH = "./cat.jpg"
if __name__ == "__main__":
model = tf.keras.applications.vgg16.VGG16(weights="imagenet",
include_top=True)
img = tf.keras.preprocessing.image.load_img(IMAGE_PATH,
target_size=(224, 224))
img = tf.keras.preprocessing.image.img_to_array(img)
model.summary()
data = (np.array([img]), None)
tabby_cat_class_index = 281
explainer = VanillaGradients()
# Compute VanillaGradients on VGG16
grid = explainer.explain(data, model, tabby_cat_class_index)
explainer.save(grid, ".", "vanilla_gradients.png")
def test_get_averaged_gradients(random_data, convolutional_model):
images, _ = random_data
gradients = VanillaGradients.compute_gradients(images, convolutional_model,
0)
assert gradients.shape == images.shape
def explain_tfexplain():
global graph
data = {"success": "failed"}
# # ensure an image was properly uploaded to our endpoint
if flask.request.method == "POST":
if flask.request.form.get("image"):
explainer = request.args.get("explainer")
#with graph.as_default():
model_path = flask.request.form.get("model_path")
model = load_model(model_path)
# # read the image in PIL format
image64 = flask.request.form.get("image")
image = base64.b64decode(image64)
image = Image.open(io.BytesIO(image))
image = prepare_image(image, target=(224, 224))
image = image*(1./255)
#img = tf.keras.preprocessing.image.img_to_array(image)
prediction = model.predict(image)
topClass = getTopXpredictions(prediction, 1)
print(topClass[0])
image = np.squeeze(image)
if explainer == "GRADCAM":
im = ([image], None)
from tf_explain.core.grad_cam import GradCAM
exp = GradCAM()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "grad_cam.png")
elif explainer == "OCCLUSIONSENSITIVITY":
im = ([image], None)
from tf_explain.core.occlusion_sensitivity import OcclusionSensitivity
exp = OcclusionSensitivity()
imgFinal = exp.explain(im, model,class_index=topClass[0][0], patch_size=10)
#exp.save(imgFinal, ".", "grad_cam.png")
elif explainer == "GRADIENTSINPUTS":
im = (np.array([image]), None)
from tf_explain.core.gradients_inputs import GradientsInputs
exp = GradientsInputs()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "VANILLAGRADIENTS":
im = (np.array([image]), None)
from tf_explain.core.vanilla_gradients import VanillaGradients
exp = VanillaGradients()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "SMOOTHGRAD":
im = (np.array([image]), None)
from tf_explain.core.smoothgrad import SmoothGrad
exp = SmoothGrad()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "INTEGRATEDGRADIENTS":
im = (np.array([image]), None)
from tf_explain.core.integrated_gradients import IntegratedGradients
exp = IntegratedGradients()
imgFinal = exp.explain(im, model, class_index=topClass[0][0])
#exp.save(imgFinal, ".", "gradients_inputs.png")
elif explainer == "ACTIVATIONVISUALIZATION":
#need some solution to find out and submit layers name
im = (np.array([image]), None)
from tf_explain.core.activations import ExtractActivations
exp = ExtractActivations()
imgFinal = exp.explain(im, model, layers_name=["activation_1"])
#exp.save(imgFinal, ".", "gradients_inputs.png")
img = pilimage.fromarray(imgFinal)
imgByteArr = inputoutput.BytesIO()
img.save(imgByteArr, format='JPEG')
imgByteArr = imgByteArr.getvalue()
img64 = base64.b64encode(imgByteArr)
img64_string = img64.decode("utf-8")
data["explanation"] = img64_string
data["prediction"] = str(topClass[0][0])
data["prediction_score"] = str(topClass[0][1])
data["success"] = "success"
return flask.Response(json.dumps(data), mimetype="text/plain")
def test_get_score_model_logs_warnings_when_model_not_suitable(
convolutional_model):
explainer = VanillaGradients()
with pytest.warns(UserWarning, match=UNSUPPORTED_ARCHITECTURE_WARNING):
explainer.get_score_model(convolutional_model)