def test():
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'predictions')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
plt.rcParams['figure.figsize'] = (18, 6)
img1 = utils.load_img('images/ouzel1.jpg', target_size=(224, 224))
img2 = utils.load_img('images/ouzel2.jpg', target_size=(224, 224))
# f, ax = plt.subplots(1, 2)
# ax[0].imshow(img1)
# ax[1].imshow(img2)
f, ax = plt.subplots(1, 2)
for i, img in enumerate([img1, img2]):
# 20 is the imagenet index corresponding to `ouzel`
# heatmap = saliency.visualize_cam(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier='guided')
heatmap = saliency.visualize_saliency(model, layer_idx, filter_indices=20, seed_input=img,backprop_modifier=None)
print (np.shape(heatmap))
# Lets overlay the heatmap onto original image.
ax[i].imshow(overlay(heatmap,img))
plt.show()
def plot_multiple_saliency(images,
model,
layer,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None):
fig, ax = plt.subplots(2, len(images), figsize=(4 * len(images), 4))
ax = ax.flatten()
for i, filename in enumerate(images):
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
grad = visualize_saliency(model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
image = load_img(filename,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
return fig
def plot_multiple_grad_cam(
images,
model,
layer,
penultimate_layer=None,
filter_idx=None,
backprop_modifier=None,
grad_modifier=None,
experts=None,
expert_spacing=0.1,
):
rows = 2
if experts is not None:
rows = 3
fig, ax = plt.subplots(
rows, len(images), figsize=(4 * len(images), 4 * rows))
ax = ax.flatten()
penultimate_layer_idx = None
if penultimate_layer:
penultimate_layer_idx = find_layer_idx(model, penultimate_layer)
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
ax[i].imshow(image)
ax[i].axis('off')
for i, filename in enumerate(images):
image = load_img(
filename, target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(
model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
penultimate_layer_idx=penultimate_layer_idx,
backprop_modifier=backprop_modifier,
grad_modifier=grad_modifier)
ax[i + len(images)].imshow(overlay(grad, image))
ax[i + len(images)].axis('off')
if experts:
for i, filename in enumerate(images):
for j, expert in enumerate(experts):
if i == 0:
message = "expert {}: {}".format(j + 1, expert[i])
ax[i + 2 * len(images)].text(
0.3,
1 - (expert_spacing * j),
message,
horizontalalignment='left',
verticalalignment='center')
else:
message = "{}".format(expert[i])
ax[i + 2 * len(images)].text(
0.5,
1 - (expert_spacing * j),
message,
horizontalalignment='center',
verticalalignment='center')
ax[i + 2 * len(images)].axis('off')
return fig, ax
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
cv2.imshow(
'Attention - {}'.format(utils.get_imagenet_label(pred_class)),
heatmap)
cv2.waitKey(0)
def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
if show:
cv2.imshow(
'Saliency - {}'.format(utils.get_imagenet_label(pred_class)),
heatmap)
cv2.waitKey(0)
def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Saliency - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def show_attention_evolution(self, num_epoch, class_name, image_path):
if class_name not in self.labels:
print("ラベル が間違っているよ" + str(class_name))
class_index = self.labels.index(class_name)
two_ep = int(num_epoch / 4)
thr_ep = int(num_epoch / 2)
four_ep = int(num_epoch * 3 / 4)
model_epochs = [1, two_ep, thr_ep, four_ep, num_epoch]
for idx, epoch in enumerate(model_epochs):
print("epoch" + str(epoch))
self.load_model(epoch, silent=True)
layer_idx = vis_utils.find_layer_idx(self.model, 'predictions')
# Swap softmax with linear
self.model.layers[layer_idx].activation = activations.linear
vis_model = vis_utils.apply_modifications(self.model)
x = vis_utils.load_img(image_path,
target_size=(self.img_width,
self.img_height))
grads = visualize_cam(vis_model,
layer_idx,
filter_indices=class_index,
seed_input=x,
backprop_modifier="guided")
show_img_array(grads)
show_img_array(np.squeeze(x))
def recognition(img, conn, x):
Stime = time.time()
img = 'upload/' + x + ".jpg"
print(img)
img1 = utils.load_img(img, target_size=(299, 299))
im_normal2 = img1.reshape(1, img1.shape[0], img1.shape[1],
img1.shape[2]).astype('float32') / 255
probabilities = model.predict(im_normal2)
predict = np.argmax(probabilities, axis=1)
i = predict
'''print("class: %s, acc: %.2f" % (list(load_dict.keys())[list(load_dict.values()).index(i)],
(probabilities[0][i])))'''
data_to_client = {
'class': list(load_dict.keys())[list(load_dict.values()).index(i)],
'acc': (probabilities[0][i])
}
conn.send(bytes(data_to_client['class'], encoding="utf8"))
print(data_to_client['class'], data_to_client['acc'])
#寫入資料庫
inserst(i, img)
Etime = time.time()
print("spend: %f" % (Etime - Stime) + 's')
return
def main():
if (len(sys.argv) != 2):
print('Give the model path.')
return
drive = DriveRun(sys.argv[1])
config = Config()
csv_fname = '/home/mir-alb/Ninad_Thesis/Test/Test.csv'
csv_header = ['image_fname', 'steering_angle']
df = pd.read_csv(csv_fname, names=csv_header, index_col=False)
num_data = len(df)
text = open('/home/mir-lab/Ninad_Thesis/Test/Salient/Salient.txt', 'w+')
bar = ProgressBar()
image_process = ImageProcess()
for i in bar(range(num_data)):
image_name = df.loc[i]['image_fname']
steering = df.loc[i]['steering_angle']
image_path = '/home/mir-lab/Ninad_Thesis/Test/' + image_name + '.jpg'
image = utils.load_img(image_path, target_size=(config.image_size[1],
config.image_size[0]))
image = image_process.process(image)
prediction = drive.run(image)
text.write(str(image_name) + '\t' + str(steering) + '\t' + str(prediction))
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
heatmap = visualize_saliency(drive.net_model.model, layer_idx=-1,
filter_indices=0, seed_input=image,
grad_modifier=modifier, keepdims=True)
final = overlay(image, heatmap, alpha=0.5)
cv2.imwrite('/home/mir-lab/Ninad_Thesis/Test/Salient/' + image_name + '_' + str(i) + '.jpg', final)
def activation_vis(layer_name, overlay_image):
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
# Images corresponding to tiger, penguin, dumbbell, speedboat, spider
# image_paths = [
# "http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg",
# "http://www.slate.com/content/dam/slate/articles/health_and_science/wild_things/2013/10/131025_WILD_AdeliePenguin.jpg.CROP.promo-mediumlarge.jpg",
# "https://www.kshs.org/cool2/graphics/dumbbell1lg.jpg",
# "http://tampaspeedboatadventures.com/wp-content/uploads/2010/10/DSC07011.jpg",
# "http://ichef-1.bbci.co.uk/news/660/cpsprodpb/1C24/production/_85540270_85540265.jpg"
# ]
# Predict the corresponding class for use in `visualize_saliency`.
seed_img = utils.load_img(overlay_image, target_size=target_size)
pred_class = np.argmax(
model.predict(np.array([img_to_array(seed_img)])))
# Here we are asking it to show attention such that prob of `pred_class` is maximized.
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
filename = general_dict["model_definition_id"] + "_" + general_dict["model_training_id"] + \
"_" + layer_name + "_" + overlay_image
print "Saving activation map for layer %s overlaid onto image %s" % (
layer_name, overlay_image)
imsave(os.path.join(activation_maps_dir, filename), heatmap, 'png')
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def load_image(img_name):
img = utils.load_img(f'../../scraped_database_tags_new/{img_name}',
target_size=IMAGE_SIZE)
# plt.imshow(img)
bgr_img = utils.bgr2rgb(img)
return img, bgr_img
def detect(self,
filename,
answer_text,
verbose=False,
layer_index=-1,
mode="local"):
if mode == "local":
input_tensor = image_to_tensor(filename, self.img_height,
self.img_width)
else:
# if True:
try:
input_tensor = image_to_tensor(filename, self.img_height,
self.img_width)
except:
raise ValueError("This URL is not supported!! Use other one.")
detection = self.model.predict(input_tensor)[0]
a = np.array(detection)
detect_label = self.labels[a.argmax(0)]
if verbose is True:
print("結果 .... " + str(answer_text[detect_label]))
img1 = vis_utils.load_img(filename,
target_size=(self.img_height,
self.img_width))
# Swap softmax with linear
layer_idx = vis_utils.find_layer_idx(self.model, 'predictions')
self.model.layers[layer_idx].activation = activations.linear
vis_model = vis_utils.apply_modifications(self.model)
filter_index = a.argmax(0)
grads = visualize_cam(
vis_model,
layer_idx,
filter_index, #クラス番号
img1[:, :, :],
backprop_modifier='guided')
a = np.array(detection)
fig = plt.figure(figsize=(10, 5))
ax1 = fig.add_subplot(1, 2, 1)
ax1.tick_params(labelbottom="off", bottom="off")
ax1.grid(False)
ax1.tick_params(labelleft="off", left=False)
plt.yticks(color="None")
ax1.set_xticklabels([])
ax1.imshow(img1)
ax1.imshow(grads, cmap='jet', alpha=0.6)
ax1.set_title("Heat Map")
sns.set(style="white", context="talk")
f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True)
sns.barplot(self.labels, detection, palette="PiYG", ax=ax1)
ax1.set_ylabel("Value")
plt.tick_params(length=0)
plt.grid(False)
plt.show()
else:
print(detect_label)
print(detection)
print("detectメソッドが完了しました．")
def check_load_image(model, path):
if os.path.exists(path):
if K.image_data_format() == 'channels_first':
image = utils.load_img(path,
target_size=(int(model.input.shape[2]),
int(model.input.shape[3])))
image = image.reshape(model.input.shape[1], model.input.shape[2],
model.input.shape[3])
elif K.image_data_format() == 'channels_last':
image = utils.load_img(path,
target_size=(int(model.input.shape[1]),
int(model.input.shape[2])))
image = image.reshape(model.input.shape[1], model.input.shape[2],
model.input.shape[3])
return image
else:
print("The path for the image doesn't exist.")
sys.exit(1)
def __init__(self,model_path,img_path):
K.set_image_data_format('channels_last')
self.model_path = model_path
self.img_path = img_path
self.model = load_model(model_path)
self.inv_dico = {v: k for k, v in get_dico().items()}
self.seed_img = utils.load_img(img_path, target_size=(224, 224))
x = image.img_to_array(self.seed_img)
x = np.expand_dims(x, axis=0)
x = preprocess_input(x)
self.x = x
def on_epoch_end(self, epoch, logs={}):
clear_output(wait = True)
index = 1
print('------------------------------------------------------------------------------------------------')
print('------------------------------------------------------------------------------------------------')
if self.verbose is True:
for image in self.test_img_list:
if self.color_mode == "rgb":
input_tensor = image_to_tensor(image, self.img_height, self.img_width)
elif self.color_mode == "grayscale":
input_tensor = image_to_tensor(image, self.img_height, self.img_width, color_mode="grayscale")
detection = self.model.predict(input_tensor)[0]
layer_idx = utils.find_layer_idx(self.model, 'predictions')
test_label = image.split("/")[-2]
filter_index = self.labels.index(test_label)
print(filter_index)
img1 = utils.load_img(image, target_size=(self.img_height, self.img_width))
grads = visualize_cam(
self.model,
layer_idx,
filter_indices=None,
seed_input=img1,
backprop_modifier='guided'
)
print('\nIndex' + str(index))
print(detection)
a = np.array(detection)
print('Estimation：' + self.labels[a.argmax(0)])
fig = plt.figure(figsize=(10, 5))
ax1 = fig.add_subplot(1, 2, 1)
ax1.tick_params(labelbottom="off",bottom="off")
ax1.tick_params(labelleft="off",left="off")
ax1.set_xticklabels([])
ax1.imshow(overlay(grads, img1))
ax2 = fig.add_subplot(1, 2, 2)
ax2.tick_params(labelbottom="off",bottom="off")
ax2.tick_params(labelleft="off",left="off")
ax2.set_xticklabels([])
ax2.imshow(Image.open(image))
plt.show()
sns.set(style="white", context="talk")
f, ax1 = plt.subplots(1, 1, figsize=(8, 6), sharex=True)
sns.barplot(self.labels, detection, palette="PiYG", ax=ax1)
ax1.set_ylabel("Value")
plt.show()
index = index + 1
if self.now_epoch % 5 == 0 or self.now_epoch == 1:
_index = str(self.now_epoch)
if self.now_epoch < 10:
_index = "0" + _index
self.model.save("./models/"+ self.task_name+"/"+"epoch_"+ _index +".hdf5")
self.now_epoch = self.now_epoch + 1
def visualize_attention(model, img_path, target):
"""Visualize attention for self driving `model`.
Args:
model: The keras model.
img_path: The image path to use as input.
target: One of 'right', 'left', 'same' to indicate whether regression target should 'increase', 'decrease'
or remain 'same'.
"""
img = utils.load_img(img_path, target_size=(FRAME_H, FRAME_W))
# Convert to BGR, create input with batch_size: 1.
bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
if target == 'right':
t = pred + 1
elif target == 'left':
t = pred - 1
else:
t = pred
# Generate saliency visualization.
saliency = visualize_regression_saliency(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img)
saliency = overlay(img, saliency, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('Saliency to steer: {}'.format(target))
plt.imshow(saliency)
# Generate grad-CAM visualization.
cam = visualize_regression_cam(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img,
penultimate_layer_idx=5)
cam = overlay(img, cam, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('grad-CAM to steer: {}'.format(target))
plt.imshow(cam)
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# # Build the VGG16 network with ImageNet weights
# model = VGG16(weights='imagenet', include_top=True)
# print('Model loaded.')
# Build the InceptionV3 network with ImageNet weights
# model = InceptionV3(weights='imagenet', include_top=True)
# print('Model loaded.')
json_file = open('inception_v3_tf_cat_dog_top_layer.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
# loaded_model.load_weights("vgg16_tf_cat_dog_final_dense2.h5")
loaded_model.load_weights("inception_v3_tf_cat_dog_top_layer.h5")
print("Loaded model from disk")
model = loaded_model
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'../resources/tiger.jpg', '../resources/ouzel.jpg',
'../resources/ouzel_1.jpg'
]:
# seed_img = utils.load_img(path, target_size=(224, 224))
seed_img = utils.load_img(path, target_size=(299, 299))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def plot_grad_cam(image_file,
model,
layer,
filter_idx=None,
backprop_modifier="relu"):
image = load_img(image_file,
target_size=(config.IMAGE_SIZE, config.IMAGE_SIZE))
grad = visualize_cam(model,
find_layer_idx(model, layer),
filter_idx,
normalize(image),
backprop_modifier=backprop_modifier)
fig, ax = plt.subplots(1, 2)
ax[0].imshow(overlay(grad, image))
ax[0].axis('off')
ax[1].imshow(image)
ax[1].axis('off')
return fig
def plot(self, vis_func, img_path, label_list, figsize):
img = utils.load_img(img_path, target_size=self.img_shape_)
img = img[:, :, :3]
predictions = self.model_.predict(img2tensor(img, self.img_shape_))
predictions = softmax(predictions)
if not label_list:
prediction_text = decode_predictions(predictions)[0]
def _plot(label_id):
label_id = int(label_id)
text_label = get_pred_text_label(label_id)
label_proba = np.round(predictions[0, label_id], 4)
heatmap = vis_func(img, label_id)
for p in prediction_text:
print(p[1:])
plt.figure(figsize=figsize)
plt.subplot(1, 2, 1)
plt.title('label:%s\nscore:%s' % (text_label, label_proba))
plt.imshow(overlay(heatmap, img))
plt.subplot(1, 2, 2)
plt.imshow(img)
plt.show()
else:
def _plot(label_id):
print(pd.DataFrame(predictions, columns=label_list))
label_id = int(label_id)
text_label = label_list[label_id]
label_proba = np.round(predictions[0, label_id], 4)
heatmap = vis_func(img, label_id)
plt.figure(figsize=figsize)
plt.subplot(1, 2, 1)
plt.title('label:%s\nscore:%s' % (text_label, label_proba))
plt.imshow(overlay(heatmap, img))
plt.subplot(1, 2, 2)
plt.imshow(img)
plt.show()
return interact(_plot, label_id='352')
def main():
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# 20 is the imagenet category for 'ouzel'
heatmap = visualize_saliency(model.input, layer_dict[layer_name], [20],
seed_img)
cv2.imshow('Importance map', heatmap)
cv2.waitKey(0)
def segmentation(self, path):
self.path = 'model/unet.h5'
self.model1 = load_model(self.path,
custom_objects={
'generalized_dice_loss':
generalized_dice_loss,
'mean_iou': mean_iou,
'dice_coef': dice_coef
})
self.img = utils.load_img(path, target_size=(224, 224))
self.img = image.img_to_array(self.img)
self.img = np.expand_dims(self.img, axis=0)
self.img = self.img.astype('float32')
self.img /= 255.
self.img_mask = self.model1.predict(self.img, verbose=1)
self.imgs = self.img_mask
for i in range(self.imgs.shape[0]):
self.img = self.imgs[i]
self.img = array_to_img(self.img)
self.img.save("image/mask.jpg")
def visualize():
model = load_model('fisheries.h5')
path = './input/train/ALB/img_00003.jpg'
img = utils.load_img(path, target_size=(dataset.SIZE, dataset.SIZE))
X = dataset.load_image(path)
print(X.shape)
# print(X[0])
preds = model.predict(X)
pred_class = preds.argmax()
print('Predicted:' + str(preds))
print('Predicted:' + dataset.TYPES[pred_class])
plt.imshow(img)
plt.show()
idx = [2, 6, 8, 13]
for i in idx:
print(model.layers[i])
heatmap = visualize_cam(model, i, [pred_class], img)
plt.imshow(heatmap)
plt.show()
def heatmap(self, path):
# Grad-CAM requires the category of the image as input. So we need predict image to get the label first.
self.img = utils.load_img(path, target_size=(224, 224))
self.x = image.img_to_array(self.img)
self.x = np.expand_dims(self.x, axis=0)
self.y = self.model.predict(self.x, verbose=1)
self.y = self.y.argmax(axis=-1)
self.label = [
'dyed-lifted-polyps', 'dyed-resection-margins', 'esophagitis',
'normal-cecum', 'normal-pylorus', 'normal-z-line', 'polyps',
'ulcerative-colitis'
]
self.label = np.array(self.label)
self.layer_idx = utils.find_layer_idx(self.model, 'dense_1')
# Swap softmax with linear
self.model.layers[self.layer_idx].activation = activations.linear
self.model = utils.apply_modifications(self.model)
for modifier in ['guided']:
f, ax = plt.subplots(1, 1)
plt.suptitle(self.label[self.y])
# Model, layer id, class ,image as input.
self.grads = visualize_cam(self.model,
self.layer_idx,
filter_indices=self.y,
seed_input=self.img,
backprop_modifier=modifier)
# Lets overlay the heatmap onto original image.
self.jet_heatmap = np.uint8(cm.jet(self.grads)[..., :3] * 255)
self.a = ax.imshow(overlay(self.jet_heatmap, self.img))
plt.savefig("image/attention.jpg")
return self.a
print('Predicted:', decoded)
topK_synsets = [triple[0] for triple in decoded]
topK_names = [triple[1] for triple in decoded]
topK_scores = [triple[2] for triple in decoded]
class_names = ['boxer', 'tiger_cat']
topK_ndx = []
imagenet_ndx = [] # indexes into the softmax entries of final layer
for i, name in enumerate(class_names):
ndx = topK_names.index(name)
topK_ndx.append(ndx)
imagenet_ndx.append(np.argwhere(preds[0] == topK_scores[ndx])[0][0])
# 282 = Tiger cat, 242 = Boxer (0 indexed)
img = utils.load_img(img_path, target_size=(224, 224))
N = len(class_names)
fig, ax = plt.subplots(1, N + 1)
ax[0].imshow(img)
ax[0].axis('off')
# Lets overlay the heatmap for each desired class onto original image.
for i in range(N):
ndx = imagenet_ndx[i]
layer_idx = utils.find_layer_idx(model, 'predictions') # final layer
grads = visualize_cam(model,
layer_idx,
filter_indices=ndx,
seed_input=img,
backprop_modifier='guided')
jet_heatmap = np.uint8(cm.jet(grads)[..., :3] * 255)
#CAM on images for InceptionV3 network.
im_file = "husky.jpg"
img1 = image.load_img(im_file, target_size=(299, 299))
img1 = image.img_to_array(img1)
img1 = np.expand_dims(img1, axis=0)
img1 = preprocess_input(img1)
layer_idx = utils.find_layer_idx(model_origin, 'predictions')
heatmap = visualize_cam(model_origin,
layer_idx,
filter_indices=248,
seed_input=img1[0, :, :, :])
# In[11]:
img_init = utils.load_img(im_file, target_size=(299, 299))
plt.imshow(overlay(img_init, heatmap))
plt.show()
# The red region represents the area of the image on which the network focuses. This result is very nice as main attention is given at the head of the dog.
#
# ### Custom Model :
#
# We run the same process over the custom model trained for car crash detection.
#
#
# The activation of the last FC layer is removed :
# In[5]:
#With Custom Model
from vis.visualization import visualize_activation
from matplotlib import pyplot as plt
#%matplotlib inline
plt.rcParams['figure.figsize'] = (18, 6)
# 20 is the imagenet category for 'ouzel'
img = visualize_activation(model, layer_idx, filter_indices=20, verbose=True)
plt.imshow(img)
"""
https://github.com/raghakot/keras-vis/blob/master/examples/vggnet/attention.ipynb
"""
img1 = utils.load_img(
'/home/innereye/keras-vis/examples/vggnet/images/ouzel1.jpg',
target_size=(224, 224))
img2 = utils.load_img(
'/home/innereye/keras-vis/examples/vggnet/images/ouzel2.jpg',
target_size=(224, 224))
f, ax = plt.subplots(1, 2)
ax[0].imshow(img1)
ax[1].imshow(img2)
from vis.visualization import visualize_saliency
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'predictions')
model = ResNet50(weights='imagenet', include_top=True)
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'fc1000')
# Swap softmax with linear
model.layers[layer_idx].activation = activations.linear
model = utils.apply_modifications(model)
from vis.utils import utils
from matplotlib import pyplot as plt
plt.rcParams['figure.figsize'] = (18, 6)
img1 = utils.load_img('../vggnet/images/ouzel1.jpg', target_size=(224, 224))
img2 = utils.load_img('../vggnet/images/ouzel2.jpg', target_size=(224, 224))
f, ax = plt.subplots(1, 2)
ax[0].imshow(img1)
ax[1].imshow(img2)
from vis.visualization import visualize_saliency, overlay
from vis.utils import utils
from keras import activations
# Utility to search for layer index by name.
# Alternatively we can specify this as -1 since it corresponds to the last layer.
layer_idx = utils.find_layer_idx(model, 'fc1000')
f, ax = plt.subplots(1, 2)
# Images corresponding to tiger, penguin, dumbbell, speedboat, spider
image_paths = [
"http://www.tigerfdn.com/wp-content/uploads/2016/05/How-Much-Does-A-Tiger-Weigh.jpg",
"http://www.slate.com/content/dam/slate/articles/health_and_science/wild_things/2013/10/131025_WILD_AdeliePenguin.jpg.CROP.promo-mediumlarge.jpg",
"http://www.kshs.org/cool2/graphics/dumbbell1lg.jpg",
"http://media1.britannica.com/eb-media/80/150980-004-EE46999B.jpg",
]
heatmaps = []
# vis_images = []
for path in image_paths:
# For InceptionV3
seed_img = utils.load_img(path, target_size=(299, 299))
pred = model.predict(preprocess_input(np.expand_dims(img_to_array(seed_img), axis=0)))
# For VGG16
# seed_img = utils.load_img(path, target_size=(224, 224))
# pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)])))
# print(utils.get_imagenet_label(pred_class))
# seed_img = utils.load_img(path, target_size=(224, 224))
# pred_class = np.argmax(model.predict(np.array([img_to_array(seed_img)])))
print('Predicted:', decode_predictions(pred))
print('Predicted:', decode_predictions(pred)[0][0][1])
# pred_class = np.argmax(model.predict(preprocess_input(np.array([img_to_array(seed_img)]))))
model = cnn.define_CNN(x_train, num_classes, dropouts, 1)
print(model.summary())
model.load_weights(filepath, by_name=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'fc2_out'
layer_idx = [
idx for idx, layer in enumerate(model.layers) if layer.name == layer_name
][0]
heatmaps = []
#for path in image_paths:
for i in range(5):
seed_img = utils.load_img(path, target_size=(32, 32))
print(seed_img.shape)
seed_img = x_train[np.random.randint(i * 13, 50000)]
print(seed_img.shape)
x = np.expand_dims(img_to_array(seed_img), axis=0)
x = preprocess_input(x)
pred_class = np.argmax(model.predict(x))
# Here we are asking it to show attention such that prob of `pred_class` is maximized.
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
heatmaps.append(heatmap)
plt.axis('off')
plt.imshow(utils.stitch_images(heatmaps))
plt.title('Activation map')
plt.savefig('activation_map_cnn')