def attribute(self, ih: ImageHandler, method: str, layer_no: int = None, take_absolute: bool = False, take_threshold: bool = False, sigma_multiple: int = 0, visualise: bool = False, save: bool = True): if layer_no is None: layer_no = LAYER_TARGETS[method][self.curr_model_name] self.initialise_for_method(method_name=method, layer_no=layer_no) # get the 2D numpy array which represents the attribution attribution = self.collect_attribution(ih, method=method, layer_no=layer_no) # check if applying any thresholds / adjustments based on +ve / -ve evidence if take_threshold or take_absolute: attribution = apply_threshold(attribution, sigma_multiple, take_absolute) if check_invalid_attribution(attribution, ih): return if save: ih.save_figure(attribution, method) if visualise: show_figure(attribution) return attribution
def attribute(self, ih: ImageHandler): # get outputs for top prediction count "ranked_outputs" input_to_layer_n = self.map2layer(ih.get_expanded_img()) shap_values, indexes = self.explainer.shap_values(X=input_to_layer_n, nsamples=200, ranked_outputs=1) # plot the explanations (SHAP value matrices) and save to file # print(len(shap_values)) if type(shap_values) != list: shap_values = [shap_values] sh = shap_values[0] # aggregate along third axis (the RGB axis), resize and normalise to (-1, 1) sv = sh[0].sum(-1) # resize into input shape (~4x rescale for some models) sv = cv2.resize(sv, ih.get_size(), cv2.INTER_LINEAR) sv /= np.max(np.abs(sv)) gb = self.guided_backprop(ih) guided_shap = gb * sv[..., np.newaxis] guided_shap = guided_shap.sum(axis=np.argmax( np.asarray(guided_shap.shape) == 3)) guided_shap /= np.max(np.abs(guided_shap)) return guided_shap[0]
def demo_attribute(img_nos: list = None, att: Attributer = None): if att is None: model_name = VGG att = Attributer(model_name=model_name) if img_nos is None: img_nos = [11, 13, 15] #img_nos = [6, 97, 278] for img_no in img_nos: # image handler for later (method attributions) ih = ImageHandler(img_no=img_no, model_name=VGG) # predictions max_pred, max_p = att.predict_for_model(ih) plt.figure(figsize=(15, 10)) plt.suptitle( 'Attributions for example {}, prediction = `{}`, probability = {:.2f}' .format(img_no, max_pred, max_p)) # original image plt.subplot(2, 4, 1) plt.axis('off') plt.title('ImageNet Example {}'.format(img_no)) plt.imshow( plt.imread(get_image_file_name(IMG_BASE_PATH, img_no) + '.JPEG')) # annotated image plt.subplot(2, 4, 2) plt.title('Annotated Example {}'.format(img_no)) plt.imshow( plt.imread( get_image_file_name(ANNOTATE_BASE_PATH, img_no) + '.JPEG')) # processed image plt.subplot(2, 4, 3) plt.title('Reshaped Example') plt.imshow(demo_resizer(img_no=img_no, target_size=ih.get_size())) # processed image plt.subplot(2, 4, 4) plt.title('Annotation Mask') plt.imshow(get_mask_for_eval(img_no=img_no, target_size=ih.get_size()), cmap='seismic', clim=(-1, 1)) attributions = att.attribute_panel(ih=ih, methods=METHODS, save=False, visualise=False, take_threshold=False, take_absolute=False, sigma_multiple=1) # show attributions for i, a in enumerate(attributions.keys()): plt.subplot(2, 4, 5 + i) plt.title(a) plt.axis('off') plt.imshow(ih.get_original_img(), cmap='gray', alpha=0.75) plt.imshow(attributions[a], cmap='seismic', clim=(-1, 1), alpha=0.8) plt.show() plt.clf() plt.close()
def get_image_handler_and_mask(self, img_no): # this gets the image wrapped in the ImageHandler object, and the # bounding box annotation mask for the image, # ImageHandler is used to calculate attributions by each method, and the # mask is used for evaluation ih = ImageHandler(img_no=img_no, model_name=self.model_name) # bounding box in the format of the model's input shape / attribution shape annotation_mask = get_mask_for_eval(img_no=img_no, target_size=ih.get_size(), save=False, visualise=False) return ih, annotation_mask
def predict_for_model(self, ih: ImageHandler, top_n: int = 5, print_to_stdout: bool = True) -> (str, float): # returns a tuple with the top prediction, and the probability of the # top prediction (i.e confidence) logging.info('Classifying...') predictions = self.curr_model.predict(ih.get_processed_img()) decoded_predictions = decode_predictions(predictions, top=top_n) # print the top 5 predictions, labels and probabilities if print_to_stdout: print('Model predictions:') max_p = 0.00 max_pred = '' for (i, (img_net_ID, label, p)) in enumerate(decoded_predictions[0]): if print_to_stdout: print('{}: {}, Probability={:.2f}, ImageNet ID={}'.format( i + 1, label, p, img_net_ID)) if p > max_p: max_p = p max_pred = label if print_to_stdout: print('') return max_pred, max_p
def guided_backprop(self, ih: ImageHandler): """Guided Backpropagation method for visualizing input saliency.""" input_imgs = self.guided_model.input layer_output = self.guided_model.layers[self.layer_no].output grads = K.gradients(layer_output, input_imgs)[0] backprop_fn = K.function([input_imgs, K.learning_phase()], [grads]) grads_val = backprop_fn([ih.get_processed_img(), 0])[0] return grads_val
def get_good_examples(self, cap: int = 1001): good_examples = [] for i in range(1, cap): ih = ImageHandler(i, self.curr_model_name) max_pred, p = self.predict_for_model(ih, top_n=1, print_to_stdout=False) if p > 0.9: good_examples.append(i) return good_examples
def attribute_panel_wrapper(model_name: str): methods = [LIME, LIFT, GRAD] att = Attributer(model_name) for i in [11]: # range(6, 7): ih = ImageHandler(img_no=i, model_name=model_name) att.attribute_panel(ih=ih, methods=methods, save=True, visualise=True, take_threshold=True, take_absolute=True, sigma_multiple=1)
def grad_cam(self, ih: ImageHandler, cls): """GradCAM method for visualizing input saliency.""" y_c = self.model.output[0, cls] conv_output = self.model.layers[self.layer_no].output grads = K.gradients(y_c, conv_output)[0] # Normalize if necessary # grads = normalize(grads) gradient_function = K.function([self.model.input], [conv_output, grads]) output, grads_val = gradient_function([ih.get_processed_img()]) output, grads_val = output[0, :], grads_val[0, :, :, :] weights = np.mean(grads_val, axis=(0, 1)) cam = np.dot(output, weights) # Process CAM cam = cv2.resize(cam, ih.get_size(), cv2.INTER_LINEAR) cam = np.maximum(cam, 0) cam_max = cam.max() if cam_max != 0: cam = cam / cam_max return cam
def attributer_wrapper(method: str, model: str): # draw_annotations([i for i in range(16, 300)]) # run some attributions att = Attributer(model) start_time = time.time() for i in range(1, 101): ih = ImageHandler(img_no=i, model_name=model) att.attribute(ih=ih, method=method, save=False, visualise=False, take_threshold=False, take_absolute=False, sigma_multiple=1) print('{} total seconds for {}'.format(str(time.time() - start_time), method))
def performance_timer(): # performance timing for model in [INCEPT]: att = Attributer(model) for method in [GRAD]: print('Performance test for {} and {}:'.format(method, model)) start_time = time.time() for i in range(1, 101): ih = ImageHandler(img_no=i, model_name=model) att.attribute(ih=ih, method=method, save=False, visualise=False, take_threshold=False, take_absolute=False, sigma_multiple=1) print('{} total seconds for {}'.format( str(time.time() - start_time), method))
def attribute(self, ih: ImageHandler): """Compute saliency """ # get the class to localise if it's not available predictions = self.model.predict(ih.get_processed_img()) cls = np.argmax(predictions) gradcam = self.grad_cam(ih, cls) gb = self.guided_backprop(ih) guided_gradcam = gb * gradcam[..., np.newaxis] # only interested in guided gradcam (the class discriminative "high-resolution" combination of guided-BP and GC. # # normalise along color channels and normalise to (-1, 1) guided_gradcam = guided_gradcam.sum(axis=np.argmax( np.asarray(guided_gradcam.shape) == 3)) guided_gradcam /= np.max(np.abs(guided_gradcam)) # output attribution (numpy 2D array) return guided_gradcam[0]