background = x_train[np.random.choice(x_train.shape[0], 100, replace=False)] # explain predictions of the model on four images e = shap.DeepExplainer(model, background) # ...or pass tensors directly # mnist simple samps = [0,1,2,3,6] samps = [4, 29, 65] #samps = [3,5,6,9,12,13,15,23] # e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background) shap_values = e.shap_values(x_test[samps]) # plot the feature attributions plt.figure() shap.image_plot(shap_values, (x_test[samps] + 2) * 64, np.tile(np.array(class_names), (len(samps), 1)), sharetitles=True) X_s = x_test[0:50] samps = [3, 6] # mnist samps = [29, 65] # mnist(29,33) to_explain = x_test[samps] # explain how the input to the 7th layer of the model explains the top two classes def map2layer(x, layer): feed_dict = dict(zip([model.layers[0].input], [x])) return K.get_session().run(model.layers[layer].input, feed_dict) lay = 2 # last cnn layer e = shap.GradientExplainer( (model.layers[lay].input, model.layers[-1].output),
'mnist_data', train=False, transform=transforms.Compose([transforms.ToTensor()])), batch_size=batch_size, shuffle=True) model = Net().to(device) optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) for epoch in range(1, num_epochs + 1): train(model, device, train_loader, optimizer, epoch) test(model, device, test_loader) # since shuffle=True, this is a random sample of test data batch = next(iter(test_loader)) images, _ = batch background = images[:100] test_images = images[100:103] e = shap.DeepExplainer(model, background) print(type(test_images)) # test_images = torch.tensor(test_images) shap_values = e.shap_values(test_images) shap_numpy = [np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values] test_numpy = np.swapaxes(np.swapaxes(test_images.numpy(), 1, -1), 1, 2) # plot the feature attributions shap.image_plot(shap_numpy, -test_numpy)
def _plot_shap_values_for_images(self, test_data, shap_values): shap.image_plot(shap_values, test_data, show=False) plt.savefig( '{}/shap_value_plot_counter_{}_{}_{}.png'.format(self.output_folder, self.counter, self.problem_type, self.score_type))
print(len(test_images), "Batch of images is selected") X_image = Image.open('../' + folder_name + '/' + img_name) X = test_transforms(X_image) X = X.unsqueeze(0) # image for printing img = cv2.imread('../' + folder_name + '/' + img_name) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) print(img.shape) img = cv2.resize(img, shape, 2) img = img / 255 img = np.expand_dims(img, axis=0) print(img.shape) print("Training Gradient Explainer...") e_explainer = shap.GradientExplainer((model, layer), test_images.to(device)) print("Calculating Shap Values of given Images...") shap_values, indexes = e_explainer.shap_values(X.to(device), ranked_outputs=2, nsamples=200) shap_values = [np.swapaxes(np.swapaxes(s, 2, 3), 1, -1) for s in shap_values] # get the names for the classes # our classes classes = {0: 'Female', 1: 'Male'} index_names = np.vectorize(lambda i: classes[i])(indexes.cpu()) shap.image_plot(shap_values, img, index_names, show=False) plt.savefig('../image_results/' + save_img_name) print('The image ', save_img_name, 'has been Saved!')
def plot_shap(model, dataset_opts, transform_opts, batch_size, outputfilename, n_outputs=1, method='deep', local_smoothing=0.0, n_samples=20, pred_out=False): """Plot shapley attributions overlaid on images for classification tasks. Parameters ---------- model:nn.Module Pytorch model. dataset_opts:dict Options used to configure dataset transform_opts:dict Options used to configure transformers. batch_size:int Batch size for training. outputfilename:str Output filename. n_outputs:int Number of top outputs. method:str Gradient or deep explainer. local_smoothing:float How much to smooth shapley map. n_samples:int Number shapley samples to draw. pred_out:bool Label images with binary prediction score? """ import torch from torch.nn import functional as F import numpy as np from torch.utils.data import DataLoader import shap from pathflowai.datasets import DynamicImageDataset import matplotlib from matplotlib import pyplot as plt from pathflowai.sampler import ImbalancedDatasetSampler out_transform = dict(sigmoid=F.sigmoid, softmax=F.softmax, none=lambda x: x) binary_threshold = dataset_opts.pop('binary_threshold') num_targets = dataset_opts.pop('num_targets') dataset = DynamicImageDataset(**dataset_opts) if dataset_opts['classify_annotations']: binarizer = dataset.binarize_annotations( num_targets=num_targets, binary_threshold=binary_threshold) num_targets = len(dataset.targets) dataloader_val = DataLoader(dataset, batch_size=batch_size, num_workers=10, shuffle=True if num_targets > 1 else False, sampler=ImbalancedDatasetSampler(dataset) if num_targets == 1 else None) #dataloader_test = DataLoader(dataset,batch_size=batch_size,num_workers=10, shuffle=False) background, y_background = next(iter(dataloader_val)) if method == 'gradient': background = torch.cat([background, next(iter(dataloader_val))[0]], 0) X_test, y_test = next(iter(dataloader_val)) if torch.cuda.is_available(): background = background.cuda() X_test = X_test.cuda() if pred_out != 'none': if torch.cuda.is_available(): model2 = model.cuda() y_test = out_transform[pred_out](model2(X_test)).detach().cpu() y_test = y_test.numpy() if method == 'deep': e = shap.DeepExplainer(model, background) s = e.shap_values(X_test, ranked_outputs=n_outputs) elif method == 'gradient': e = shap.GradientExplainer(model, background, batch_size=batch_size, local_smoothing=local_smoothing) s = e.shap_values(X_test, ranked_outputs=n_outputs, nsamples=n_samples) if y_test.shape[1] > 1: y_test = y_test.argmax(axis=1) if n_outputs > 1: shap_values, idx = s else: shap_values, idx = s, y_test #print(shap_values) # .detach().cpu() if num_targets == 1: shap_numpy = [np.swapaxes(np.swapaxes(shap_values, 1, -1), 1, 2)] else: shap_numpy = [ np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values ] #print(shap_numpy.shape) X_test_numpy = X_test.detach().cpu().numpy() X_test_numpy = X_test_numpy.transpose((0, 2, 3, 1)) for i in range(X_test_numpy.shape[0]): X_test_numpy[i, ...] *= np.array(transform_opts['std']) X_test_numpy[i, ...] += np.array(transform_opts['mean']) X_test_numpy = X_test_numpy.transpose((0, 3, 1, 2)) test_numpy = np.swapaxes(np.swapaxes(X_test_numpy, 1, -1), 1, 2) if pred_out != 'none': labels = y_test.astype(str) else: labels = np.array([[(dataloader_val.dataset.targets[i[j]] if num_targets > 1 else str(i)) for j in range(n_outputs)] for i in idx]) #[:,np.newaxis] # y_test if 0 and (len(labels.shape) < 2 or labels.shape[1] == 1): labels = labels.flatten() #[:np.newaxis] #print(labels.shape,shap_numpy.shape[0]) plt.figure() shap.image_plot( shap_numpy, test_numpy, labels ) # if num_targets!=1 else shap_values -test_numpy , labels=dataloader_test.dataset.targets) plt.savefig(outputfilename, dpi=300)
back_images, _ = next(iter(train_loader)) explainer = shap.DeepExplainer(model, back_images.cuda()) shap_values = explainer.shap_values(images) for i, img in enumerate(images): convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR) images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1) # shape in shap list is (4,3,128,128) -> (4,3,128,128) for i, sv in enumerate(shap_values): sv = np.swapaxes(sv, 2, 1) sv = np.swapaxes(sv, 3, 2) shap_values[i] = sv # %% shap.image_plot(shap_values, images.data.permute(0, 2, 3, 1).numpy()) # %% img_indices = [2001, 2002, 2003, 2004] images, labels = train_set.getbatch(img_indices) # gather train background images back_images, _ = next(iter(train_loader)) explainer = shap.DeepExplainer(model, back_images.cuda()) shap_values = explainer.shap_values(images) for i, img in enumerate(images): convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR) images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1) # shape in shap list is (4,3,128,128) -> (4,3,128,128)
model.add(Dropout(0.5)) model.add(Dense(num_classes, activation='softmax')) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) model.fit(x_train, y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(x_test, y_test)) score = model.evaluate(x_test, y_test, verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1]) #### # select a set of background examples to take an expectation over background = x_train[np.random.choice(x_train.shape[0], 100, replace=False)] # explain predictions of the model on four images e = shap.DeepExplainer(model, background) # ...or pass tensors directly #e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background) shap_values = e.shap_values(x_test[1:5]) # plot the feature attributions shap.image_plot(shap_values, -x_test[1:5])
def plot_image(self, data, index): if self.expected_value is None: return if self.shap_values is None: return shap.image_plot(self.shap_values[index, :], data)
explainer = shap.DeepExplainer(model, back_images.cuda()) shap_values = explainer.shap_values(images) for i, img in enumerate(images): convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR) images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1) # shape in shap list is (4,3,128,128) -> (4,3,128,128) for i, sv in enumerate(shap_values): sv = np.swapaxes(sv, 2, 1) sv = np.swapaxes(sv, 3, 2) shap_values[i] = sv # %% shap.image_plot(shap_values, images.data.permute(0, 2, 3, 1).numpy(), show=False) print('saving fig 6') plt.savefig(os.path.join(output_path, '6.png')) # ========================= # report Q4 label 2 img_indices = [2001, 2002, 2003, 2004] # ========================= images, labels = train_set.getbatch(img_indices) # gather train background images back_images, _ = next(iter(train_loader)) explainer = shap.DeepExplainer(model, back_images.cuda()) shap_values = explainer.shap_values(images)
# Load the model to take expectation model = load_model('model_1.h5') model_explain = shap.DeepExplainer(model, background) # Generate shap values for validation data (0-Healthy, 1-Not Healthy) shap_values_0 = model_explain.shap_values(batches_val[0][0][[2, 24]]) shap_values_1 = model_explain.shap_values(batches_val[0][0][[3, 5]]) # Verify true postives/negatives print("Predictions for Validation Data") print(model.predict(batches_val[0][0][[2, 24, 3, 5]])) # Plot feature attributions print("\nHealthy - True Negatives") shap.image_plot(shap_values_0[1], -batches_val[0][0][[2, 24]]) print("Pneumonia - True Positives") shap.image_plot(shap_values_1[1], -batches_val[0][0][[3, 5]]) def visualize(layer_name): """Visualize layer output for Model 1""" # Define a function that gives layer output for a given input layer_idx = utils.find_layer_idx(model, layer_name) inputs = [backend.learning_phase()] + model.inputs _layer_outputs = backend.function(inputs, [model.layers[layer_idx].output]) # Format data to see layer outputs def layer_outputs(image_data): """Removes the training phase flag"""
if __name__ == "__main__": index = np.random.permutation(TOTAL_ID) print("\nLoading data...", flush=True) train_set = get_data(os.path.join(sys.argv[1], "training"), index[:B_SIZE].tolist() + IMG_ID) x = [] y = [] for i in range(B_SIZE + len(IMG_ID)): x_tmp, y_tmp = train_set.__getitem__(i) x.append(x_tmp) y.append(y_tmp) x = torch.stack(x).cuda() print("\nLoading model...", flush=True) model = CNN().cuda() model.load_state_dict(torch.load(sys.argv[2])) print("\nComputing...", flush=True) e = shap.DeepExplainer(model, x[:B_SIZE]) shap_values = e.shap_values(x[B_SIZE:]) shap_numpy = [ np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values ] shap.image_plot(shap_numpy, x[B_SIZE:][:, [2, 1, 0], :, :].permute(0, 2, 3, 1).cpu().numpy(), show=False) plt.savefig(os.path.join(sys.argv[3], "shap.jpg")) plt.close()
#import shap import numpy as np # select a set of background examples to take an expectation over background = x_train[np.random.choice(x_train.shape[0], 100, replace=False)] # explain predictions of the model on three images e = shap.DeepExplainer(model, background) # ...or pass tensors directly # e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background) shap_values = e.shap_values(x_test[1:10]) # COMMAND ---------- # plot the feature attributions shap_plot = shap.image_plot(shap_values, -x_test[1:5]) display(shap_plot) # COMMAND ---------- # MAGIC %md # MAGIC # MAGIC The plot above shows the explanations for each class on four predictions (of the four different images of 2, 1, 0, 4). Note that the explanations are ordered for the classes 0-9 going left to right along the rows, starting with the original image: # MAGIC # MAGIC * Red pixels increase the model's output # MAGIC * Blue pixels decrease the model's output. # MAGIC # MAGIC The input images are shown on the left, and as nearly transparent grayscale backings behind each of the explanations. # MAGIC # MAGIC The sum of the SHAP values equals the difference between the expected model output (averaged over the background dataset) and the current model output. # MAGIC
feed_dict = dict(zip([model.layers[0].input.experimental_ref()], [x.copy()])) graph = tf.compat.v1.get_default_graph() print(graph.get_operations()) with tf.compat.v1.Session() as sess: ret = sess.run(model.layers[layer], feed_dict) return ret preprocess_input = [] for i in range(100): img, label = shap_dict['TEST_GENERATOR'].next() img = np.squeeze(img, axis=0) preprocess_input.append(img) inference = [] for i in range(100): img, label = shap_dict['TEST_GENERATOR'].next() img = np.squeeze(img, axis=0) inference.append(img) preprocess_input = np.array(preprocess_input) inference = np.array(inference) e = shap.GradientExplainer((model.layers[0].input, model.layers[-1].output), map2layer(preprocess_input.copy(), 0, model)) shap_values, indexes = e.shap_values(map2layer(inference, 0, model), ranked_outputs=2) return shap_values, indexes if __name__ == '__main__': tf.compat.v1.disable_eager_execution() shap_dict = setup_shap() shap_values, indexes = shap_explain(shap_dict) shap.image_plot(shap_values, to_explain)
return [np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values] if __name__ == "__main__": model_path = './models/model.pth' data_path = '../data' batch_size = 10 #Number of images in dataset + test images cannot currently exceed batch size no_images = 6 no_test_images = 4 dataloaders = load_datasets(data_path, batch_size=batch_size) batch = next(iter(dataloaders['train'])) images, _ = batch model = get_model(model_path) background = images[:no_images] explainer = shap.DeepExplainer(model, background) test_images = images[no_images:no_images+no_test_images] shap_values = get_shap_values(explainer, test_images) shap_numpy = get_shap_numpy(shap_values) np_transformed_test_images = numpy_transform_test_images(test_images) labels = no_test_images*[['Salmon', 'Trout']] shap.image_plot(shap_numpy, np_transformed_test_images, labels=labels)
model = load_model('C:/.../model.h5', compile=False) back_img_path = 'C:/.../Background.png' base_img_path = 'C:/.../Base.png' ## load Image Function def loadImages(path): img_array = cv2.imread(path) img_array = np.float32(img_array) img_size = 40 new_array = cv2.resize(img_array, (img_size, img_size)) gray = cv2.cvtColor(new_array, cv2.COLOR_BGR2GRAY) return gray ## Load Test and Background Image base_img = loadImages(base_img_path) base_img = np.array(base_img) back_img = loadImages(back_img_path) back_img = np.array(back_img) ## Perform SHAP Explanations background = back_img.reshape(1, 40, 40, 1) e = shap.DeepExplainer(model, background) shap_values = e.shap_values(base_img.reshape(1, 40, 40, 1)) shap.image_plot(shap_values, back_img.reshape(1, 40, 40, 1), show=False) ## Save SHAP Explanations with open('shap_explanations.data', 'wb') as filehandle: pickle.dump(shap_values, filehandle)