def attribute(self,
ih: ImageHandler,
method: str,
layer_no: int = None,
take_absolute: bool = False,
take_threshold: bool = False,
sigma_multiple: int = 0,
visualise: bool = False,
save: bool = True):
if layer_no is None:
layer_no = LAYER_TARGETS[method][self.curr_model_name]
self.initialise_for_method(method_name=method, layer_no=layer_no)
# get the 2D numpy array which represents the attribution
attribution = self.collect_attribution(ih,
method=method,
layer_no=layer_no)
# check if applying any thresholds / adjustments based on +ve / -ve evidence
if take_threshold or take_absolute:
attribution = apply_threshold(attribution, sigma_multiple,
take_absolute)
if check_invalid_attribution(attribution, ih):
return
if save:
ih.save_figure(attribution, method)
if visualise:
show_figure(attribution)
return attribution
def attribute(self, ih: ImageHandler):
# get outputs for top prediction count "ranked_outputs"
input_to_layer_n = self.map2layer(ih.get_expanded_img())
shap_values, indexes = self.explainer.shap_values(X=input_to_layer_n,
nsamples=200,
ranked_outputs=1)
# plot the explanations (SHAP value matrices) and save to file
# print(len(shap_values))
if type(shap_values) != list:
shap_values = [shap_values]
sh = shap_values[0]
# aggregate along third axis (the RGB axis), resize and normalise to (-1, 1)
sv = sh[0].sum(-1)
# resize into input shape (~4x rescale for some models)
sv = cv2.resize(sv, ih.get_size(), cv2.INTER_LINEAR)
sv /= np.max(np.abs(sv))
gb = self.guided_backprop(ih)
guided_shap = gb * sv[..., np.newaxis]
guided_shap = guided_shap.sum(axis=np.argmax(
np.asarray(guided_shap.shape) == 3))
guided_shap /= np.max(np.abs(guided_shap))
return guided_shap[0]
def demo_attribute(img_nos: list = None, att: Attributer = None):
if att is None:
model_name = VGG
att = Attributer(model_name=model_name)
if img_nos is None:
img_nos = [11, 13, 15]
#img_nos = [6, 97, 278]
for img_no in img_nos:
# image handler for later (method attributions)
ih = ImageHandler(img_no=img_no, model_name=VGG)
# predictions
max_pred, max_p = att.predict_for_model(ih)
plt.figure(figsize=(15, 10))
plt.suptitle(
'Attributions for example {}, prediction = `{}`, probability = {:.2f}'
.format(img_no, max_pred, max_p))
# original image
plt.subplot(2, 4, 1)
plt.axis('off')
plt.title('ImageNet Example {}'.format(img_no))
plt.imshow(
plt.imread(get_image_file_name(IMG_BASE_PATH, img_no) + '.JPEG'))
# annotated image
plt.subplot(2, 4, 2)
plt.title('Annotated Example {}'.format(img_no))
plt.imshow(
plt.imread(
get_image_file_name(ANNOTATE_BASE_PATH, img_no) + '.JPEG'))
# processed image
plt.subplot(2, 4, 3)
plt.title('Reshaped Example')
plt.imshow(demo_resizer(img_no=img_no, target_size=ih.get_size()))
# processed image
plt.subplot(2, 4, 4)
plt.title('Annotation Mask')
plt.imshow(get_mask_for_eval(img_no=img_no, target_size=ih.get_size()),
cmap='seismic',
clim=(-1, 1))
attributions = att.attribute_panel(ih=ih,
methods=METHODS,
save=False,
visualise=False,
take_threshold=False,
take_absolute=False,
sigma_multiple=1)
# show attributions
for i, a in enumerate(attributions.keys()):
plt.subplot(2, 4, 5 + i)
plt.title(a)
plt.axis('off')
plt.imshow(ih.get_original_img(), cmap='gray', alpha=0.75)
plt.imshow(attributions[a],
cmap='seismic',
clim=(-1, 1),
alpha=0.8)
plt.show()
plt.clf()
plt.close()
def get_image_handler_and_mask(self, img_no):
# this gets the image wrapped in the ImageHandler object, and the
# bounding box annotation mask for the image,
# ImageHandler is used to calculate attributions by each method, and the
# mask is used for evaluation
ih = ImageHandler(img_no=img_no, model_name=self.model_name)
# bounding box in the format of the model's input shape / attribution shape
annotation_mask = get_mask_for_eval(img_no=img_no,
target_size=ih.get_size(),
save=False,
visualise=False)
return ih, annotation_mask
def predict_for_model(self,
ih: ImageHandler,
top_n: int = 5,
print_to_stdout: bool = True) -> (str, float):
# returns a tuple with the top prediction, and the probability of the
# top prediction (i.e confidence)
logging.info('Classifying...')
predictions = self.curr_model.predict(ih.get_processed_img())
decoded_predictions = decode_predictions(predictions, top=top_n)
# print the top 5 predictions, labels and probabilities
if print_to_stdout:
print('Model predictions:')
max_p = 0.00
max_pred = ''
for (i, (img_net_ID, label, p)) in enumerate(decoded_predictions[0]):
if print_to_stdout:
print('{}: {}, Probability={:.2f}, ImageNet ID={}'.format(
i + 1, label, p, img_net_ID))
if p > max_p:
max_p = p
max_pred = label
if print_to_stdout:
print('')
return max_pred, max_p
def guided_backprop(self, ih: ImageHandler):
"""Guided Backpropagation method for visualizing input saliency."""
input_imgs = self.guided_model.input
layer_output = self.guided_model.layers[self.layer_no].output
grads = K.gradients(layer_output, input_imgs)[0]
backprop_fn = K.function([input_imgs, K.learning_phase()], [grads])
grads_val = backprop_fn([ih.get_processed_img(), 0])[0]
return grads_val
def get_good_examples(self, cap: int = 1001):
good_examples = []
for i in range(1, cap):
ih = ImageHandler(i, self.curr_model_name)
max_pred, p = self.predict_for_model(ih,
top_n=1,
print_to_stdout=False)
if p > 0.9:
good_examples.append(i)
return good_examples
def attribute_panel_wrapper(model_name: str):
methods = [LIME, LIFT, GRAD]
att = Attributer(model_name)
for i in [11]: # range(6, 7):
ih = ImageHandler(img_no=i, model_name=model_name)
att.attribute_panel(ih=ih,
methods=methods,
save=True,
visualise=True,
take_threshold=True,
take_absolute=True,
sigma_multiple=1)
def grad_cam(self, ih: ImageHandler, cls):
"""GradCAM method for visualizing input saliency."""
y_c = self.model.output[0, cls]
conv_output = self.model.layers[self.layer_no].output
grads = K.gradients(y_c, conv_output)[0]
# Normalize if necessary
# grads = normalize(grads)
gradient_function = K.function([self.model.input],
[conv_output, grads])
output, grads_val = gradient_function([ih.get_processed_img()])
output, grads_val = output[0, :], grads_val[0, :, :, :]
weights = np.mean(grads_val, axis=(0, 1))
cam = np.dot(output, weights)
# Process CAM
cam = cv2.resize(cam, ih.get_size(), cv2.INTER_LINEAR)
cam = np.maximum(cam, 0)
cam_max = cam.max()
if cam_max != 0:
cam = cam / cam_max
return cam
def attributer_wrapper(method: str, model: str):
# draw_annotations([i for i in range(16, 300)])
# run some attributions
att = Attributer(model)
start_time = time.time()
for i in range(1, 101):
ih = ImageHandler(img_no=i, model_name=model)
att.attribute(ih=ih,
method=method,
save=False,
visualise=False,
take_threshold=False,
take_absolute=False,
sigma_multiple=1)
print('{} total seconds for {}'.format(str(time.time() - start_time),
method))
def performance_timer():
# performance timing
for model in [INCEPT]:
att = Attributer(model)
for method in [GRAD]:
print('Performance test for {} and {}:'.format(method, model))
start_time = time.time()
for i in range(1, 101):
ih = ImageHandler(img_no=i, model_name=model)
att.attribute(ih=ih,
method=method,
save=False,
visualise=False,
take_threshold=False,
take_absolute=False,
sigma_multiple=1)
print('{} total seconds for {}'.format(
str(time.time() - start_time), method))
def attribute(self, ih: ImageHandler):
"""Compute saliency
"""
# get the class to localise if it's not available
predictions = self.model.predict(ih.get_processed_img())
cls = np.argmax(predictions)
gradcam = self.grad_cam(ih, cls)
gb = self.guided_backprop(ih)
guided_gradcam = gb * gradcam[..., np.newaxis]
# only interested in guided gradcam (the class discriminative "high-resolution" combination of guided-BP and GC.
# # normalise along color channels and normalise to (-1, 1)
guided_gradcam = guided_gradcam.sum(axis=np.argmax(
np.asarray(guided_gradcam.shape) == 3))
guided_gradcam /= np.max(np.abs(guided_gradcam))
# output attribution (numpy 2D array)
return guided_gradcam[0]