def make_gradcam(image, prediction, model):
prediction = prediction.squeeze()
if prediction >= 0.5:
# highlight the regions contributing towards '1-damaged' classification
grad_modifier = None
else:
# highlight the regions contributing towards '0-non-damaged' classification
grad_modifier = 'negate'
# squeeze out the 'batch' dimension for keras-vis
image = image.squeeze()
gradient = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=image,
backprop_modifier=None,
grad_modifier=grad_modifier)
image = normalize_tf_image(image)
gradient_tmp = np.expand_dims(gradient, axis=2)
# print(image.shape, gradient_tmp.shape)
image_gradient = np.concatenate((image, gradient_tmp), axis=2)
# print(image_gradient.shape)
figure, ax = plt.subplots(1, 1)
# figure.suptitle('Prediction: {:.2f}'.format(prediction), y=0.8, horizontalalignment='center')
ax.imshow(image_gradient)
ax.contour(gradient, 3, colors='k')
plt.axis('off')
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
return figure
def visualize_activations(self, stacked_shreds):
# TODO: hard-coded left and right indexes. change if you like. don't want to spend time.
if self._comparator.t > 2:
left, right = self._comparator.t + 1, self._comparator.t + 2
else:
left, right = 0, 1
tensor = ComparatorCNN._prepare_left_right_check(
stacked_shreds[0][left], stacked_shreds[0][right])
root = os.path.join(os.path.dirname(__file__), 'class_activation_maps')
os.makedirs('class_activation_maps', exist_ok=True)
for i, layer in enumerate(self._comparator._model.layers[1:]):
try:
layer_name = layer.name
if 'conv' not in layer_name:
continue
cam = visualize_cam(self._comparator._model, i, None, tensor)
plt.imshow(stacked_shreds[0][left], cmap='gray')
plt.imshow(cam,
cmap='hot',
interpolation='nearest',
alpha=0.15)
plt.title('#{}:{}'.format(i, layer_name))
current_milli_time = lambda: int(round(time.time() * 1000))
plt.savefig(
os.path.join(
root, '{}_{}_{}.png'.format(i, layer_name,
current_milli_time())))
plt.clf()
print(cam)
except:
print("Exception!")
def run(input_img, model):
# preprocess
img = Image.fromarray(input_img)
img_resized = resize(img)
img_scaled = transform_image(img_resized)
# labels
labels = [
'Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass',
'Nodule', 'Pneumonia', 'Pneumothorax', 'Consolidation', 'Edema',
'Emphysema', 'Fibrosis', 'Pleural_Thickening', 'Hernia'
]
# predict for image
prediction = model.predict(np.array([img_scaled]))[0]
pred_idx = prediction.argmax()
prob = prediction.max()
label = labels[pred_idx]
# create region of interest map
layer_idx = len(model.layers) - 1
cam = visualization.visualize_cam(model, layer_idx, pred_idx,
np.array([img_scaled]))
img_overlay = visualization.overlay(cam, img_resized, .3)
return [label, prob, img_overlay]
def save_img(path, savepath, origimg, typeimg, layeridx):
img = load_img(path, target_size=(224,224))
x = img_to_array(img) #numpy array
x = x.reshape(x.shape) #adds on dimension for keras
model.layers[layeridx].activation = activations.linear
if typeimg == 'activation':
img = visualize_activation(model, layeridx, 20, x)
if typeimg == 'saliency':
img = visualize_saliency(model, layeridx, 1, x)
if typeimg == 'cam':
img = visualize_cam(model, layeridx, 1, x)
if not os.path.exists('layer-' + savepath):
os.makedirs('layer-' + savepath)
if not os.path.exists('image-' + savepath):
os.makedirs('image-' + savepath)
combined = str(savepath) + '/' + str(origimg)
plt.imshow(img)
plt.savefig('layer-' + combined, dpi=600)
def get_guided_cam(model,
img1,
layer_idx=None,
predict_class=None,
penultimate_layer_idx=None,
backprop_modifier='guided'):
if predict_class is None:
prob = model.predict(np.expand_dims(img1, axis=0))
predict_class = np.argmax(prob)
if layer_idx is None:
layer_idx = len(model.layers) - 1
if penultimate_layer_idx is None:
for i in range(len(model.layers) - 1, -1, -1):
if isinstance(model.layers[i], Conv2D) or \
isinstance(model.layers[i], MaxPool2D) or\
isinstance(model.layers[i], SeparableConv2D):
penultimate_layer_idx = i
break
# penultimate_layer_idx = LIBS.find_layer_idx(model, 'mixed10')
grads = visualize_cam(model,
layer_idx,
filter_indices=[predict_class],
seed_input=img1,
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier)
str_uuid = str(uuid.uuid1())
filename = '/tmp/' + str_uuid + '.png'
jet_heatmap = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET)
cv2.imwrite('1111.png', jet_heatmap)
img1 /= 2.0
img1 += 0.5
img1 *= 255.
img1 = img1.astype(np.uint8)
# x_train /= 255.
# x_train -= 0.5
# x_train *= 2.
img_cam = overlay(jet_heatmap, img1)
cv2.imwrite(filename, img_cam)
# plt.title('Guided-CAM')
# plt.imshow(overlay(jet_heatmap, img1))
# fig = plt.gcf()
# fig.set_size_inches(3, 3)
# fig.savefig(filename, dpi=100)
# plt.savefig('Guided_CAM1.png')
# plt.close()
return filename
def generate_cam_from_image(image, model=None, returnAsImage=True, layer=None):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
if model is None:
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
if layer is None:
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
else:
layer_idx = layer
#seed_img = Image.open(image)
seed_img = Image.fromarray(image)
seed_img = seed_img.resize((configs.img_width, configs.img_height))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(np.asarray(seed_img))
img_input = np.expand_dims(bgr_img, axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class],
np.asarray(seed_img))
if returnAsImage is True:
return Image.fromarray(heatmap)
else:
return heatmap
def generate_gradcam(fm: FineModel,
image: np.ndarray,
penultimate_layer_idx: int = None,
color=True,
class_index: int = None):
"""
Generate gradcam image for given image, classification and class. If `class_index`
is unspecified, we run the image through the classifier and use the predicted class.
Penultimate layer does not need to be manually specified for our current models.
"""
if not class_index:
class_index = fm.predict(image)
preprocess = fm._get_preprocess_func()
processed = preprocess(image)
gradcam = visualize_cam(fm.get_model(),
fm.get_depths()[0],
filter_indices=class_index,
seed_input=processed,
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=None)
if color:
gradcam = np.uint8(cm.jet(gradcam)[..., :3] * 255)
return gradcam
def visualize(img_dir):
#TODO: test this method
print('Loading Model...')
model = keras.models.load_model(MODEL_PATH)
model.layers.pop()
model.add(Activation('linear'))
utils.apply_modifications(model)
for img_path in glob.glob(img_dir + '*.png'):
print(f'Working on {img_path}')
plt.figure()
f, ax = plt.subplots(1, 4)
img = load_img(img_path,
color_mode='grayscale',
target_size=(IMAGE_DIM, IMAGE_DIM),
interpolation='lanczos')
background = img.convert('RGB')
img = img_to_array(img)
saliency_grads = visualize_saliency(model,
-1,
filter_indices=0,
seed_input=img,
backprop_modifier='guided')
ax[0].imshow(background)
ax[1].imshow(saliency_grads, cmap='jet')
cam_grads = visualize_cam(model,
-1,
filter_indices=0,
seed_input=img,
backprop_modifier='guided')
cam_heatmap = np.uint8(cm.jet(cam_grads)[..., :3] * 255)
saliency_heatmap = np.uint8(cm.jet(saliency_grads)[..., :3] * 255)
ax[2].imshow(overlay(saliency_heatmap, img_to_array(background)))
ax[3].imshow(overlay(cam_heatmap, img_to_array(background)))
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
cv2.imshow(
'Attention - {}'.format(utils.get_imagenet_label(pred_class)),
heatmap)
cv2.waitKey(0)
def plot_convs_heatmap(self):
layer_idx = -1
self.model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(self.model)
names = []
for layer in self.model.layers:
if isinstance(layer, _Conv):
names.append(layer.name)
pred_class = np.argmax(self.model.predict(self.x))
fig = plt.figure()
for i in range(len(names)):
name = names[i]
print('Calculating heatmap for ', name)
penult_layer_idx = utils.find_layer_idx(model, name)
heatmap = visualize_cam(model, layer_idx, filter_indices=[pred_class], seed_input=self.seed_img,penultimate_layer_idx=penult_layer_idx, backprop_modifier=None)
sp = fig.add_subplot(6, 9, i + 1)
sp.set_title(name,fontsize=7)
sp.imshow(overlay(self.seed_img, heatmap))
sp.get_xaxis().set_visible(False)
sp.get_yaxis().set_visible(False)
plt.show()
def explain(self, image, target_class):
# generate images with removed parts
try:
grad_cam_res = visualize_cam(
model=self.model,
layer_idx=self.layer,
filter_indices=target_class, seed_input=image)
except TypeError:
print("GradCam need to have input dimension of the model defined."
"Please define it or use other approach.")
raise
guided_bprop = GuidedBackprop(self.model, output_index=target_class)
guided_backprop_res = guided_bprop.get_mask(image)
# ignoring negative activations since they come only from the
# last layer, since guided backprop does not propagate negative values
# through the ReLU
guided_backprop_res[guided_backprop_res < 0] = 0
guided_backprop_res = np.sum(guided_backprop_res, axis=2)
# mask = np.abs(mask)
# values should be between 0 and 1
guided_backprop_res /= guided_backprop_res.max()
# pintvise multiplication
res = guided_backprop_res * grad_cam_res
# values should be between 0 and 1
res /= res.max()
return res, None
def showSaliencyMap(showModel, layerName, showImage, imageSize):
layer_idx = utils.find_layer_idx(showModel, layerName)
showModel.layers[layer_idx].activation = activations.linear
showModel = utils.apply_modifications(showModel)
plt.rcParams['figure.figsize'] = (18, 6)
for modifier in [None, 'guided', 'relu']:
plt.figure()
showImage = np.array(showImage)
f, ax = plt.subplots(1, len(showImage))
plt.suptitle("vanilla" if modifier is None else modifier)
for i, img in enumerate(showImage):
img = np.array(img)
grads = visualize_cam(showModel,
layer_idx,
filter_indices=3,
seed_input=img,
backprop_modifier=modifier)
# overlay the heatmap onto original image.
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = np.array(np.reshape(jet_heatmap,
(imageSize, imageSize, 3)),
dtype=np.uint8)
if len(showImage) == 1:
# for single image
ax.imshow(overlay(jet_heatmap, img))
else:
# for multi images
ax[i].imshow(overlay(jet_heatmap, img))
plt.show()
def visualize_attention_maps():
model = load_regression_model()
print(model.summary())
plt.figure(figsize=(20, 4))
for ind, img_name in enumerate(SAMPLES):
img, bgr_img = load_image(img_name)
img_input = np.expand_dims(img_to_array(img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=bgr_img,
grad_modifier='small_values')
jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255)
ax = plt.subplot(2, len(SAMPLES), ind + 1)
plt.imshow(img)
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
ax = plt.subplot(2, len(SAMPLES), ind + len(SAMPLES) + 1)
plt.imshow(overlay(img, jet_heatmap, alpha=0.3))
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# plt.show()
plt.savefig('heatmaps_c3.png', bbox_inches='tight')
def plot_map(self):
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
if self.arch == 'vgg16':
layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3')
elif self.arch == 'resnet50':
layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3_3x3')
elif self.arch == 'senet50':
layer_idx = vis_utils.find_layer_idx(self.model, layer_name='conv5_3_3x3') # o también conv5_3_1x1_up ó conv5_3_1x1_down
class_idxs_sorted = np.argsort(self.scores.flatten())[::-1]
class_idx = class_idxs_sorted[0]
seed_input = self.img
grad_top = visualize_cam(self.model, layer_idx, class_idx, seed_input,
penultimate_layer_idx = layer_idx,
backprop_modifier = None,
grad_modifier = None)
fig, axes = plt.subplots(1, 2, figsize=(14,5))
axes[0].imshow(self._img)
axes[1].imshow(self._img)
i = axes[1].imshow(grad_top/255., cmap="jet", alpha=0.8)
fig.colorbar(i)
plt.suptitle("Pr[img|label={}] = {:5.2f}".format(
self.inv_classes[class_idx],
self.scores[0,class_idx]))
plt.show()
def plt_cam(model, img, ax, idx, layer_idx=None):
"""
Plot Class Activation Map(CAM), which represents the activation
at the end of all convolutional layer;
Reference: https://arxiv.org/pdf/1610.02391v1.pdf
Args:
model: Model to plot.
img: Seed image.
ax: Matplotlib axis.
idx: Index of the plot to be shown on the axis.
layer_idx: Index of the layer to plot Grad-CAM. Optional.
Returns: None
"""
pred_layer_idx = vutils.find_layer_idx(model, "predictions")
hmap = vvis.visualize_cam(model,
pred_layer_idx,
filter_indices=None,
seed_input=img)
if img.shape[2] == 1:
input_img = np.stack((img.reshape(img.shape[0:2]), ) * 3, -1)
else:
input_img = img
ax[idx].imshow(vvis.overlay(hmap, input_img))
ax[idx].set_title("Heatmap")
def model_vis(model):
images = np.asarray(load_anomaly_example(anomaly=False))
input_img = images[0]
plt_img = cv2.cvtColor(input_img, cv2.COLOR_YUV2BGR)
titles = ["Input", "Attention"]
subplot_args = {
'nrows': 1,
'ncols': 2,
'figsize': (9, 3),
'subplot_kw': {
'xticks': [],
'yticks': []
}
}
f, ax = plt.subplots(**subplot_args)
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=input_img,
grad_modifier=None)
jet_heatmap = np.uint8(cm.jet(heatmap)[..., :3] * 255)
for i, modifier in enumerate(titles):
ax[i].set_title(titles[i], fontsize=14)
ax[0].imshow(plt_img)
ax[1].imshow(overlay(plt_img, jet_heatmap, alpha=0.75))
plt.tight_layout()
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def anomaly_detection_attention(model, logdir="../tests/ducks.log"):
model = cbcNetv2.get_anomaly_inference(model)
observation, prediction, anomaly = read_dataset(logdir)
writer = imageio.get_writer("../TrainingDataVisualization.mp4",
format='mp4',
mode='I',
fps=60.0)
for a_observation, a_gt in zip(observation, anomaly):
observation = np.expand_dims(a_observation, axis=0)
output = model.predict(observation)
heatmap = visualize_cam(model,
layer_idx=-1,
filter_indices=0,
seed_input=a_observation,
grad_modifier=None)
jet_heatmap = cv2.cvtColor(np.uint8(cm.jet(heatmap)[..., :3] * 255),
cv2.COLOR_BGR2RGB)
plt_img = cv2.cvtColor(a_observation, cv2.COLOR_YUV2RGB)
fin = cv2.addWeighted(plt_img, 0.6, jet_heatmap, 0.4, 0)
string_to_show = "Guess: {}, GT: {}".format(
round(float(output[0][0]), 2), a_gt)
x, y, w, h = 0, 0, 30, 30
# Add text
cv2.putText(fin, string_to_show, (x + int(w / 10), y + int(h / 2)),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
cv2.imshow("Output", fin)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
writer.append_data(cv2.cvtColor(fin, cv2.COLOR_RGB2BGR))
writer.close()
def apply(self, model, image, layer, filter):
# Changer l'activation softmax par linear
model.layers[layer].activation = activations.linear
model = utils.apply_modifications(model)
penultimate_layer_idx = None
if layer == 2 and isinstance(
model.layers[1], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)):
penultimate_layer_idx = 1
elif layer <= 2 and isinstance(
model.layers[0], (_Conv, _Pooling1D, _Pooling2D, _Pooling3D)):
penultimate_layer_idx = 0
grads = visualize_cam(model,
layer,
filter_indices=filter,
backprop_modifier="guided",
grad_modifier=None,
seed_input=image,
penultimate_layer_idx=penultimate_layer_idx)
shape = model.layers[0].input_shape
if shape[len(shape) - 1] == 1:
img = None
if K.image_data_format() == 'channels_first':
img = np.zeros(shape=(int(model.input.shape[2]),
int(model.input.shape[3]), 3))
else:
img = np.zeros(shape=(int(model.input.shape[1]),
int(model.input.shape[2]), 3))
for i in range(len(image)):
for j in range(len(image[i])):
img[i][j][0] = image[i][j][0]
img[i][j][1] = image[i][j][0]
img[i][j][2] = image[i][j][0]
image = img
grads = overlay(grads, image)
return grads
def show_attention_evolution(self, num_epoch, class_name, image_path):
if class_name not in self.labels:
print("ラベル が間違っているよ" + str(class_name))
class_index = self.labels.index(class_name)
two_ep = int(num_epoch / 4)
thr_ep = int(num_epoch / 2)
four_ep = int(num_epoch * 3 / 4)
model_epochs = [1, two_ep, thr_ep, four_ep, num_epoch]
for idx, epoch in enumerate(model_epochs):
print("epoch" + str(epoch))
self.load_model(epoch, silent=True)
layer_idx = vis_utils.find_layer_idx(self.model, 'predictions')
# Swap softmax with linear
self.model.layers[layer_idx].activation = activations.linear
vis_model = vis_utils.apply_modifications(self.model)
x = vis_utils.load_img(image_path,
target_size=(self.img_width,
self.img_height))
grads = visualize_cam(vis_model,
layer_idx,
filter_indices=class_index,
seed_input=x,
backprop_modifier="guided")
show_img_array(grads)
show_img_array(np.squeeze(x))
def plotAttention(model, layer_idx, im, im_idx, label, fold,
output_folder_path):
grads = visualize_saliency(
model,
layer_idx,
filter_indices=None,
seed_input=im,
backprop_modifier="guided",
)
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB)
cv2.imwrite(
os.path.join(output_folder_path,
label + "-" + fold + "-sm-" + str(im_idx) + ".png"),
jet_heatmap,
)
grads = visualize_cam(
model,
layer_idx,
filter_indices=None,
seed_input=im,
backprop_modifier="guided",
penultimate_layer_idx=utils.find_layer_idx(model, "block5_pool"),
)
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
jet_heatmap = cv2.cvtColor(jet_heatmap, cv2.COLOR_BGR2RGB)
cv2.imwrite(
os.path.join(output_folder_path,
label + "-" + fold + "-cam-" + str(im_idx) + ".png"),
overlay(jet_heatmap, cv2.cvtColor(im, cv2.COLOR_GRAY2RGB), 0.2),
)
def view_grad_cam(model,
X,
prediction_layer_name,
cnn_layer_name,
filters=None):
pred_layer_idx = utils.find_layer_idx(cnn_model, prediction_layer_name)
last_cnn_layer_idx = utils.find_layer_idx(cnn_model, cnn_layer_name)
grads = visualize_cam(
cnn_model,
layer_idx=pred_layer_idx,
filter_indices=
filters, # filter/class indices within the layer to be maximized (None = all filters/classes are visualized).
penultimate_layer_idx=last_cnn_layer_idx,
seed_input=X,
backprop_modifier=None)
# the channel-wise mean of the resulting feature map is the heatmap of the class activation
heatmap = np.mean(grads, axis=-1)
# heatmap post processing for visualisation purposes
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.matshow(heatmap)
plt.show()
img2 = cv2.imread(args.test_image)
heatmap = cv2.resize(heatmap, (img2.shape[1], img2.shape[0]))
heatmap = np.uint8(255 * heatmap)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
superimposed_img = np.uint8(heatmap * 0.6 + img2 * 0.4)
cv2.imshow('heatmap superimposed on image', superimposed_img)
cv2.waitKey(100)
def predict_and_visualize(model, indices, results_dir):
f = h5py.File(workdir + 'ibis.hdf5', 'r')
images = f['ibis_t1']
labels = f['qc_label']
filenames = f['filename']
predictions = []
with open(results_dir + 'test_images.csv', 'w') as output_file:
output_writer = csv.writer(output_file)
output_writer.writerow(['Filename', 'Probability'])
for index in indices:
img = images[index, target_size[0]//2, ...][np.newaxis, ..., np.newaxis]
label = labels[index, ...]
prediction = model.predict(img, batch_size=1)
print('probs:', prediction[0])
output_writer.writerow([filenames[index, ...], prediction[0][0], np.argmax(label)])
predictions.append(np.argmax(prediction[0]))
for i, (index, prediction) in enumerate(zip(indices, predictions)):
layer_idx = utils.find_layer_idx(model, 'predictions')
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
grads = visualize_cam(model, layer_idx, filter_indices=prediction, seed_input=img[0, ...], backprop_modifier='guided')
heatmap = np.uint8(cm.jet(grads)[:,:,0,:3]*255)
gray = np.uint8(img[0, :, :, :]*255)
gray3 = np.dstack((gray,)*3)
print('image shape, heatmap shape', gray3.shape, heatmap.shape)
plt.imshow(overlay(heatmap, gray3, alpha=0.25))
actual = np.argmax(labels[index, ...])
if prediction == actual:
decision = '_right_'
else:
decision = '_wrong_'
if actual == 1:
qc_status = 'PASS'
else:
qc_status = 'FAIL'
# filename = qc_status + decision + filenames[index, ...][:-4] + '.png'
filename = str(i) + decision + qc_status + '.png'
plt.axis('off')
plt.savefig(results_dir + filename, bbox_inches='tight')
plt.clf()
f.close()
def plot_multiple_grad_cam(
images,
model,
layer,
penultimate_layer=None,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None,
experts=None,
expert_spacing=0.1,
):
rows = 2
if experts is not None:
rows = 3
fig, ax = plt.subplots(
rows, len(images), figsize=(4 * len(images), 4 * rows))
ax = ax.flatten()
penultimate_layer_idx = None
if penultimate_layer:
penultimate_layer_idx = find_layer_idx(model, penultimate_layer)
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(
model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
if experts:
for i, filename in enumerate(images):
for j, expert in enumerate(experts):
if i == 0:
message = "expert {}: {}".format(j + 1, expert[i])
ax[i + 2 * len(images)].text(
0.3,
1 - (expert_spacing * j),
message,
horizontalalignment='left',
verticalalignment='center')
else:
message = "{}".format(expert[i])
ax[i + 2 * len(images)].text(
0.5,
1 - (expert_spacing * j),
message,
horizontalalignment='center',
verticalalignment='center')
ax[i + 2 * len(images)].axis('off')
return fig, ax
def cam(model, img, gt=None,
nclasses = 2,
save_path = None,
layer_idx = -1,
threshold = 0.5,
dice = True,
modifier = 'guided'):
"""
"""
model.layers[layer_idx].activation = activations.linear
# model = utils.apply_modifications(model)
# print(model.summary())
num_layers = sum([model.layers[layer].name.__contains__('conv') for layer in range(1, len(model.layers))])
layer_dice = np.zeros((num_layers, nclasses))
layer_cams = []
counter = 0
for layer in range(1, len(model.layers)):
if 'conv' not in model.layers[layer].name: continue
nclass_cam = []
if save_path:
plt.figure(figsize=(30, 10))
gs = gridspec.GridSpec(1, nclasses)
gs.update(wspace=0.025, hspace=0.05)
for class_ in range(nclasses):
grads_ = visualize_cam(model, layer_idx, filter_indices=class_, penultimate_layer_idx = layer,
seed_input = img[None, ...], backprop_modifier = modifier)
nclass_cam.append(grads_)
if save_path:
ax = plt.subplot(gs[class_])
im = ax.imshow(grads_, cmap='jet')
ax.set_xticklabels([])
ax.set_yticklabels([])
ax.set_aspect('equal')
ax.tick_params(bottom='off', top='off', labelbottom='off' )
if class_ == nclasses:
divider = make_axes_locatable(ax)
cax = divider.append_axes("right", size="5%", pad=0.2)
cb = plt.colorbar(im, ax=ax, cax=cax )
if dice:
thresh_image = grads_ > threshold
gt_mask = gt == class_
score = (np.sum(thresh_image*gt_mask))*2.0/(np.sum(gt_mask*1. + thresh_image*1.) + 1.e-3)
layer_dice[counter][class_ -1] += score
counter += 1
if save_path:
os.makedirs(save_path, exist_ok = True)
plt.savefig(os.path.join(save_path, model.layers[layer].name.replace('/', '_')+'.png'), bbox_inches='tight')
layer_cams.append(nclass_cam)
if dice:
return np.array(layer_cams), layer_dice
return np.array(layer_cams)
def visualize_class_activation_map(model, image):
image = cv2.resize(image, (320, 160))
heatmap = visualize_cam(model, layer_idx=-1, filter_indices=0, seed_input=image, grad_modifier=None)
heatmap = cv2.cvtColor(heatmap, cv2.COLOR_RGB2BGR)
heatmap = cv2.addWeighted(image, 1.0, heatmap, 0.5, 0)
heatmap = cv2.resize(heatmap, (640, 480))
return heatmap
def reason(model, image, layer, filters):
# Changer l'activation softmax par linear
model.layers[layer].activation = activations.linear
model = apply_modifications(model)
grads = visualize_cam(model, layer, filter_indices=filters, backprop_modifier="guided", grad_modifier=None, seed_input=image)
grads = overlay(grads, image)
return grads
def server_grad_cam(model_no,
img_source,
pred,
preprocess=True,
blend_original_image=True):
model = dicts_models[model_no]['model_original']
image_size = dicts_models[model_no]['image_size']
if preprocess:
img_preprocess = my_preprocess.do_preprocess(img_source, crop_size=384)
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_preprocess, image_shape=(image_size, image_size, 3))
else:
img_input = LIBS.ImgPreprocess.my_image_helper.my_gen_img_tensor(
img_source, image_shape=(image_size, image_size, 3))
penultimate_layer = get_last_conv_layer_number(model)
layer_idx = len(model.layers) - 1
modifier = 'guided' # [None, 'guided', 'relu']
# too slow
grads = visualize_cam(model,
layer_idx,
filter_indices=[pred],
seed_input=img_input,
penultimate_layer_idx=penultimate_layer,
backprop_modifier=modifier)
cam = cv2.applyColorMap(np.uint8(255 * grads), cv2.COLORMAP_JET)
if blend_original_image:
# Return to BGR [0..255] from the preprocessed image
image_original = img_input[0, :]
from LIBS.ImgPreprocess.my_image_norm import input_norm_reverse
image_original = input_norm_reverse(image_original)
image_original = image_original.astype(np.uint8)
image_original -= np.min(image_original)
image_original = np.minimum(image_original, 255)
cam = np.float32(cam) + np.float32(image_original)
cam = 255 * cam / np.max(cam)
# 传过来的是web目录
str_uuid = str(uuid.uuid1())
filename_CAM = os.path.join(BASE_DIR_SAVE, str_uuid,
'Grad_CAM{}.jpg'.format(pred))
if not os.path.exists(os.path.dirname(filename_CAM)):
os.makedirs(os.path.dirname(filename_CAM))
cv2.imwrite(filename_CAM, cam)
return filename_CAM
def detect(self,
filename,
answer_text,
verbose=False,
layer_index=-1,
mode="local"):
if mode == "local":
input_tensor = image_to_tensor(filename, self.img_height,
self.img_width)
else:
# if True:
try:
input_tensor = image_to_tensor(filename, self.img_height,
self.img_width)
except:
raise ValueError("This URL is not supported!! Use other one.")
detection = self.model.predict(input_tensor)[0]
a = np.array(detection)
detect_label = self.labels[a.argmax(0)]
if verbose is True:
print("結果 .... " + str(answer_text[detect_label]))
img1 = vis_utils.load_img(filename,
target_size=(self.img_height,
self.img_width))
# Swap softmax with linear
layer_idx = vis_utils.find_layer_idx(self.model, 'predictions')
self.model.layers[layer_idx].activation = activations.linear
vis_model = vis_utils.apply_modifications(self.model)
filter_index = a.argmax(0)
grads = visualize_cam(
vis_model,
layer_idx,
filter_index, #クラス番号
img1[:, :, :],
backprop_modifier='guided')
a = np.array(detection)
fig = plt.figure(figsize=(10, 5))
ax1 = fig.add_subplot(1, 2, 1)
ax1.tick_params(labelbottom="off", bottom="off")
ax1.grid(False)
ax1.tick_params(labelleft="off", left=False)
plt.yticks(color="None")
ax1.set_xticklabels([])
ax1.imshow(img1)
ax1.imshow(grads, cmap='jet', alpha=0.6)
ax1.set_title("Heat Map")
sns.set(style="white", context="talk")
f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True)
sns.barplot(self.labels, detection, palette="PiYG", ax=ax1)
ax1.set_ylabel("Value")
plt.tick_params(length=0)
plt.grid(False)
plt.show()
else:
print(detect_label)
print(detection)
print("detectメソッドが完了しました．")
def grad_cam(self,model_input,kcc_id):
final_layer=-1
grad_top = visualize_cam(self.model, final_layer,kcc_id, model_input,
penultimate_layer_idx = self.conv_layer,#None,
backprop_modifier = None,
grad_modifier = None)
return grad_top
