background = x_train[np.random.choice(x_train.shape[0], 100,
replace=False)]
# explain predictions of the model on four images
e = shap.DeepExplainer(model, background)
# ...or pass tensors directly
# mnist simple samps = [0,1,2,3,6]
samps = [4, 29, 65]
#samps = [3,5,6,9,12,13,15,23]
# e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background)
shap_values = e.shap_values(x_test[samps])
# plot the feature attributions
plt.figure()
shap.image_plot(shap_values, (x_test[samps] + 2) * 64,
np.tile(np.array(class_names), (len(samps), 1)),
sharetitles=True)
X_s = x_test[0:50]
samps = [3, 6] # mnist
samps = [29, 65] # mnist(29,33)
to_explain = x_test[samps]
# explain how the input to the 7th layer of the model explains the top two classes
def map2layer(x, layer):
feed_dict = dict(zip([model.layers[0].input], [x]))
return K.get_session().run(model.layers[layer].input, feed_dict)
lay = 2 # last cnn layer
e = shap.GradientExplainer(
(model.layers[lay].input, model.layers[-1].output),
'mnist_data',
train=False,
transform=transforms.Compose([transforms.ToTensor()])),
batch_size=batch_size,
shuffle=True)
model = Net().to(device)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5)
for epoch in range(1, num_epochs + 1):
train(model, device, train_loader, optimizer, epoch)
test(model, device, test_loader)
# since shuffle=True, this is a random sample of test data
batch = next(iter(test_loader))
images, _ = batch
background = images[:100]
test_images = images[100:103]
e = shap.DeepExplainer(model, background)
print(type(test_images))
# test_images = torch.tensor(test_images)
shap_values = e.shap_values(test_images)
shap_numpy = [np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values]
test_numpy = np.swapaxes(np.swapaxes(test_images.numpy(), 1, -1), 1, 2)
# plot the feature attributions
shap.image_plot(shap_numpy, -test_numpy)
def _plot_shap_values_for_images(self, test_data, shap_values):
shap.image_plot(shap_values, test_data, show=False)
plt.savefig(
'{}/shap_value_plot_counter_{}_{}_{}.png'.format(self.output_folder, self.counter, self.problem_type,
self.score_type))
print(len(test_images), "Batch of images is selected")
X_image = Image.open('../' + folder_name + '/' + img_name)
X = test_transforms(X_image)
X = X.unsqueeze(0)
# image for printing
img = cv2.imread('../' + folder_name + '/' + img_name)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
print(img.shape)
img = cv2.resize(img, shape, 2)
img = img / 255
img = np.expand_dims(img, axis=0)
print(img.shape)
print("Training Gradient Explainer...")
e_explainer = shap.GradientExplainer((model, layer), test_images.to(device))
print("Calculating Shap Values of given Images...")
shap_values, indexes = e_explainer.shap_values(X.to(device), ranked_outputs=2, nsamples=200)
shap_values = [np.swapaxes(np.swapaxes(s, 2, 3), 1, -1) for s in shap_values]
# get the names for the classes
# our classes
classes = {0: 'Female', 1: 'Male'}
index_names = np.vectorize(lambda i: classes[i])(indexes.cpu())
shap.image_plot(shap_values, img, index_names, show=False)
plt.savefig('../image_results/' + save_img_name)
print('The image ', save_img_name, 'has been Saved!')
def plot_shap(model,
dataset_opts,
transform_opts,
batch_size,
outputfilename,
n_outputs=1,
method='deep',
local_smoothing=0.0,
n_samples=20,
pred_out=False):
"""Plot shapley attributions overlaid on images for classification tasks.
Parameters
----------
model:nn.Module
Pytorch model.
dataset_opts:dict
Options used to configure dataset
transform_opts:dict
Options used to configure transformers.
batch_size:int
Batch size for training.
outputfilename:str
Output filename.
n_outputs:int
Number of top outputs.
method:str
Gradient or deep explainer.
local_smoothing:float
How much to smooth shapley map.
n_samples:int
Number shapley samples to draw.
pred_out:bool
Label images with binary prediction score?
"""
import torch
from torch.nn import functional as F
import numpy as np
from torch.utils.data import DataLoader
import shap
from pathflowai.datasets import DynamicImageDataset
import matplotlib
from matplotlib import pyplot as plt
from pathflowai.sampler import ImbalancedDatasetSampler
out_transform = dict(sigmoid=F.sigmoid,
softmax=F.softmax,
none=lambda x: x)
binary_threshold = dataset_opts.pop('binary_threshold')
num_targets = dataset_opts.pop('num_targets')
dataset = DynamicImageDataset(**dataset_opts)
if dataset_opts['classify_annotations']:
binarizer = dataset.binarize_annotations(
num_targets=num_targets, binary_threshold=binary_threshold)
num_targets = len(dataset.targets)
dataloader_val = DataLoader(dataset,
batch_size=batch_size,
num_workers=10,
shuffle=True if num_targets > 1 else False,
sampler=ImbalancedDatasetSampler(dataset)
if num_targets == 1 else None)
#dataloader_test = DataLoader(dataset,batch_size=batch_size,num_workers=10, shuffle=False)
background, y_background = next(iter(dataloader_val))
if method == 'gradient':
background = torch.cat([background, next(iter(dataloader_val))[0]], 0)
X_test, y_test = next(iter(dataloader_val))
if torch.cuda.is_available():
background = background.cuda()
X_test = X_test.cuda()
if pred_out != 'none':
if torch.cuda.is_available():
model2 = model.cuda()
y_test = out_transform[pred_out](model2(X_test)).detach().cpu()
y_test = y_test.numpy()
if method == 'deep':
e = shap.DeepExplainer(model, background)
s = e.shap_values(X_test, ranked_outputs=n_outputs)
elif method == 'gradient':
e = shap.GradientExplainer(model,
background,
batch_size=batch_size,
local_smoothing=local_smoothing)
s = e.shap_values(X_test, ranked_outputs=n_outputs, nsamples=n_samples)
if y_test.shape[1] > 1:
y_test = y_test.argmax(axis=1)
if n_outputs > 1:
shap_values, idx = s
else:
shap_values, idx = s, y_test
#print(shap_values) # .detach().cpu()
if num_targets == 1:
shap_numpy = [np.swapaxes(np.swapaxes(shap_values, 1, -1), 1, 2)]
else:
shap_numpy = [
np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values
]
#print(shap_numpy.shape)
X_test_numpy = X_test.detach().cpu().numpy()
X_test_numpy = X_test_numpy.transpose((0, 2, 3, 1))
for i in range(X_test_numpy.shape[0]):
X_test_numpy[i, ...] *= np.array(transform_opts['std'])
X_test_numpy[i, ...] += np.array(transform_opts['mean'])
X_test_numpy = X_test_numpy.transpose((0, 3, 1, 2))
test_numpy = np.swapaxes(np.swapaxes(X_test_numpy, 1, -1), 1, 2)
if pred_out != 'none':
labels = y_test.astype(str)
else:
labels = np.array([[(dataloader_val.dataset.targets[i[j]]
if num_targets > 1 else str(i))
for j in range(n_outputs)]
for i in idx]) #[:,np.newaxis] # y_test
if 0 and (len(labels.shape) < 2 or labels.shape[1] == 1):
labels = labels.flatten() #[:np.newaxis]
#print(labels.shape,shap_numpy.shape[0])
plt.figure()
shap.image_plot(
shap_numpy, test_numpy, labels
) # if num_targets!=1 else shap_values -test_numpy , labels=dataloader_test.dataset.targets)
plt.savefig(outputfilename, dpi=300)
back_images, _ = next(iter(train_loader))
explainer = shap.DeepExplainer(model, back_images.cuda())
shap_values = explainer.shap_values(images)
for i, img in enumerate(images):
convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)
images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1)
# shape in shap list is (4,3,128,128) -> (4,3,128,128)
for i, sv in enumerate(shap_values):
sv = np.swapaxes(sv, 2, 1)
sv = np.swapaxes(sv, 3, 2)
shap_values[i] = sv
# %%
shap.image_plot(shap_values, images.data.permute(0, 2, 3, 1).numpy())
# %%
img_indices = [2001, 2002, 2003, 2004]
images, labels = train_set.getbatch(img_indices)
# gather train background images
back_images, _ = next(iter(train_loader))
explainer = shap.DeepExplainer(model, back_images.cuda())
shap_values = explainer.shap_values(images)
for i, img in enumerate(images):
convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)
images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1)
# shape in shap list is (4,3,128,128) -> (4,3,128,128)
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(x_test, y_test))
score = model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
####
# select a set of background examples to take an expectation over
background = x_train[np.random.choice(x_train.shape[0], 100, replace=False)]
# explain predictions of the model on four images
e = shap.DeepExplainer(model, background)
# ...or pass tensors directly
#e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background)
shap_values = e.shap_values(x_test[1:5])
# plot the feature attributions
shap.image_plot(shap_values, -x_test[1:5])
def plot_image(self, data, index):
if self.expected_value is None:
return
if self.shap_values is None:
return
shap.image_plot(self.shap_values[index, :], data)
explainer = shap.DeepExplainer(model, back_images.cuda())
shap_values = explainer.shap_values(images)
for i, img in enumerate(images):
convert_img = cv2.cvtColor(img.permute(1, 2, 0).numpy(), cv2.COLOR_RGB2BGR)
images[i] = torch.FloatTensor(convert_img).permute(2, 0, 1)
# shape in shap list is (4,3,128,128) -> (4,3,128,128)
for i, sv in enumerate(shap_values):
sv = np.swapaxes(sv, 2, 1)
sv = np.swapaxes(sv, 3, 2)
shap_values[i] = sv
# %%
shap.image_plot(shap_values,
images.data.permute(0, 2, 3, 1).numpy(),
show=False)
print('saving fig 6')
plt.savefig(os.path.join(output_path, '6.png'))
# =========================
# report Q4 label 2
img_indices = [2001, 2002, 2003, 2004]
# =========================
images, labels = train_set.getbatch(img_indices)
# gather train background images
back_images, _ = next(iter(train_loader))
explainer = shap.DeepExplainer(model, back_images.cuda())
shap_values = explainer.shap_values(images)
# Load the model to take expectation
model = load_model('model_1.h5')
model_explain = shap.DeepExplainer(model, background)
# Generate shap values for validation data (0-Healthy, 1-Not Healthy)
shap_values_0 = model_explain.shap_values(batches_val[0][0][[2, 24]])
shap_values_1 = model_explain.shap_values(batches_val[0][0][[3, 5]])
# Verify true postives/negatives
print("Predictions for Validation Data")
print(model.predict(batches_val[0][0][[2, 24, 3, 5]]))
# Plot feature attributions
print("\nHealthy - True Negatives")
shap.image_plot(shap_values_0[1], -batches_val[0][0][[2, 24]])
print("Pneumonia - True Positives")
shap.image_plot(shap_values_1[1], -batches_val[0][0][[3, 5]])
def visualize(layer_name):
"""Visualize layer output for Model 1"""
# Define a function that gives layer output for a given input
layer_idx = utils.find_layer_idx(model, layer_name)
inputs = [backend.learning_phase()] + model.inputs
_layer_outputs = backend.function(inputs, [model.layers[layer_idx].output])
# Format data to see layer outputs
def layer_outputs(image_data):
"""Removes the training phase flag"""
if __name__ == "__main__":
index = np.random.permutation(TOTAL_ID)
print("\nLoading data...", flush=True)
train_set = get_data(os.path.join(sys.argv[1], "training"),
index[:B_SIZE].tolist() + IMG_ID)
x = []
y = []
for i in range(B_SIZE + len(IMG_ID)):
x_tmp, y_tmp = train_set.__getitem__(i)
x.append(x_tmp)
y.append(y_tmp)
x = torch.stack(x).cuda()
print("\nLoading model...", flush=True)
model = CNN().cuda()
model.load_state_dict(torch.load(sys.argv[2]))
print("\nComputing...", flush=True)
e = shap.DeepExplainer(model, x[:B_SIZE])
shap_values = e.shap_values(x[B_SIZE:])
shap_numpy = [
np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values
]
shap.image_plot(shap_numpy,
x[B_SIZE:][:, [2, 1, 0], :, :].permute(0, 2, 3,
1).cpu().numpy(),
show=False)
plt.savefig(os.path.join(sys.argv[3], "shap.jpg"))
plt.close()
#import shap import numpy as np # select a set of background examples to take an expectation over background = x_train[np.random.choice(x_train.shape[0], 100, replace=False)] # explain predictions of the model on three images e = shap.DeepExplainer(model, background) # ...or pass tensors directly # e = shap.DeepExplainer((model.layers[0].input, model.layers[-1].output), background) shap_values = e.shap_values(x_test[1:10]) # COMMAND ---------- # plot the feature attributions shap_plot = shap.image_plot(shap_values, -x_test[1:5]) display(shap_plot) # COMMAND ---------- # MAGIC %md # MAGIC # MAGIC The plot above shows the explanations for each class on four predictions (of the four different images of 2, 1, 0, 4). Note that the explanations are ordered for the classes 0-9 going left to right along the rows, starting with the original image: # MAGIC # MAGIC * Red pixels increase the model's output # MAGIC * Blue pixels decrease the model's output. # MAGIC # MAGIC The input images are shown on the left, and as nearly transparent grayscale backings behind each of the explanations. # MAGIC # MAGIC The sum of the SHAP values equals the difference between the expected model output (averaged over the background dataset) and the current model output. # MAGIC
feed_dict = dict(zip([model.layers[0].input.experimental_ref()], [x.copy()]))
graph = tf.compat.v1.get_default_graph()
print(graph.get_operations())
with tf.compat.v1.Session() as sess:
ret = sess.run(model.layers[layer], feed_dict)
return ret
preprocess_input = []
for i in range(100):
img, label = shap_dict['TEST_GENERATOR'].next()
img = np.squeeze(img, axis=0)
preprocess_input.append(img)
inference = []
for i in range(100):
img, label = shap_dict['TEST_GENERATOR'].next()
img = np.squeeze(img, axis=0)
inference.append(img)
preprocess_input = np.array(preprocess_input)
inference = np.array(inference)
e = shap.GradientExplainer((model.layers[0].input, model.layers[-1].output),
map2layer(preprocess_input.copy(), 0, model))
shap_values, indexes = e.shap_values(map2layer(inference, 0, model), ranked_outputs=2)
return shap_values, indexes
if __name__ == '__main__':
tf.compat.v1.disable_eager_execution()
shap_dict = setup_shap()
shap_values, indexes = shap_explain(shap_dict)
shap.image_plot(shap_values, to_explain)
return [np.swapaxes(np.swapaxes(s, 1, -1), 1, 2) for s in shap_values]
if __name__ == "__main__":
model_path = './models/model.pth'
data_path = '../data'
batch_size = 10
#Number of images in dataset + test images cannot currently exceed batch size
no_images = 6
no_test_images = 4
dataloaders = load_datasets(data_path, batch_size=batch_size)
batch = next(iter(dataloaders['train']))
images, _ = batch
model = get_model(model_path)
background = images[:no_images]
explainer = shap.DeepExplainer(model, background)
test_images = images[no_images:no_images+no_test_images]
shap_values = get_shap_values(explainer, test_images)
shap_numpy = get_shap_numpy(shap_values)
np_transformed_test_images = numpy_transform_test_images(test_images)
labels = no_test_images*[['Salmon', 'Trout']]
shap.image_plot(shap_numpy, np_transformed_test_images, labels=labels)
model = load_model('C:/.../model.h5', compile=False)
back_img_path = 'C:/.../Background.png'
base_img_path = 'C:/.../Base.png'
## load Image Function
def loadImages(path):
img_array = cv2.imread(path)
img_array = np.float32(img_array)
img_size = 40
new_array = cv2.resize(img_array, (img_size, img_size))
gray = cv2.cvtColor(new_array, cv2.COLOR_BGR2GRAY)
return gray
## Load Test and Background Image
base_img = loadImages(base_img_path)
base_img = np.array(base_img)
back_img = loadImages(back_img_path)
back_img = np.array(back_img)
## Perform SHAP Explanations
background = back_img.reshape(1, 40, 40, 1)
e = shap.DeepExplainer(model, background)
shap_values = e.shap_values(base_img.reshape(1, 40, 40, 1))
shap.image_plot(shap_values, back_img.reshape(1, 40, 40, 1), show=False)
## Save SHAP Explanations
with open('shap_explanations.data', 'wb') as filehandle:
pickle.dump(shap_values, filehandle)