def generate_saliceny_map(show=True):
"""Generates a heatmap indicating the pixels that contributed the most towards
maximizing the filter output. First, the class prediction is determined, then we generate heatmap
to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in ['../resources/ouzel.jpg', '../resources/ouzel_1.jpg']:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_saliency(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Saliency - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'https://upload.wikimedia.org/wikipedia/commons/thumb/1/1c/Tigerwater_edit2.jpg/170px-Tigerwater_edit2.jpg'
]:
seed_img = utils.load_img(path, target_size=(224, 224))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def generate_cam_from_image(image, model=None, returnAsImage=True, layer=None):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# Build the VGG16 network with ImageNet weights
if model is None:
model = VGG16(weights='imagenet', include_top=True)
print('Model loaded.')
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
if layer is None:
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
else:
layer_idx = layer
#seed_img = Image.open(image)
seed_img = Image.fromarray(image)
seed_img = seed_img.resize((configs.img_width, configs.img_height))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(np.asarray(seed_img))
img_input = np.expand_dims(bgr_img, axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class],
np.asarray(seed_img))
if returnAsImage is True:
return Image.fromarray(heatmap)
else:
return heatmap
def load_image(img_name):
img = utils.load_img(f'../../scraped_database_tags_new/{img_name}',
target_size=IMAGE_SIZE)
# plt.imshow(img)
bgr_img = utils.bgr2rgb(img)
return img, bgr_img
def generate_saliency(self, img, label_id):
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(self.model_.layers)
if layer.name == layer_name
][0]
bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
heatmap = visualize_saliency(self.model_, layer_idx, [label_id], img)
return heatmap
def generate_cam(show=True):
"""Generates a heatmap via grad-CAM method.
First, the class prediction is determined, then we generate heatmap to visualize that class.
"""
# # Build the VGG16 network with ImageNet weights
# model = VGG16(weights='imagenet', include_top=True)
# print('Model loaded.')
# Build the InceptionV3 network with ImageNet weights
# model = InceptionV3(weights='imagenet', include_top=True)
# print('Model loaded.')
json_file = open('inception_v3_tf_cat_dog_top_layer.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
# loaded_model.load_weights("vgg16_tf_cat_dog_final_dense2.h5")
loaded_model.load_weights("inception_v3_tf_cat_dog_top_layer.h5")
print("Loaded model from disk")
model = loaded_model
# The name of the layer we want to visualize
# (see model definition in vggnet.py)
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers)
if layer.name == layer_name
][0]
for path in [
'../resources/tiger.jpg', '../resources/ouzel.jpg',
'../resources/ouzel_1.jpg'
]:
# seed_img = utils.load_img(path, target_size=(224, 224))
seed_img = utils.load_img(path, target_size=(299, 299))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
if show:
plt.axis('off')
plt.imshow(heatmap)
plt.title('Attention - {}'.format(
utils.get_imagenet_label(pred_class)))
plt.show()
def visualize_attention(model, img_path, target):
"""Visualize attention for self driving `model`.
Args:
model: The keras model.
img_path: The image path to use as input.
target: One of 'right', 'left', 'same' to indicate whether regression target should 'increase', 'decrease'
or remain 'same'.
"""
img = utils.load_img(img_path, target_size=(FRAME_H, FRAME_W))
# Convert to BGR, create input with batch_size: 1.
bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
if target == 'right':
t = pred + 1
elif target == 'left':
t = pred - 1
else:
t = pred
# Generate saliency visualization.
saliency = visualize_regression_saliency(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img)
saliency = overlay(img, saliency, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('Saliency to steer: {}'.format(target))
plt.imshow(saliency)
# Generate grad-CAM visualization.
cam = visualize_regression_cam(model,
layer_idx=-1,
output_indices=0,
targets=t,
seed_input=bgr_img,
penultimate_layer_idx=5)
cam = overlay(img, cam, alpha=0.7)
plt.figure()
plt.axis('off')
plt.title('grad-CAM to steer: {}'.format(target))
plt.imshow(cam)
plt.show()
from matplotlib import pyplot as plt
#%matplotlib inline
from model import build_model, FRAME_W, FRAME_H
from keras.preprocessing.image import img_to_array
from vis.utils import utils
model = keras.models.load_model("model.hdf5")
print(model.summary())
#img = utils.load_img('images/test.jpg', target_size=(FRAME_H, FRAME_W))
img = utils.load_img('images/test.jpg')
plt.imshow(img)
# Convert to BGR, create input with batch_size: 1.
bgr_img = utils.bgr2rgb(img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred = model.predict(img_input)[0][0]
print('Predicted {}'.format(pred))
from vis.visualization import visualize_saliency, overlay
titles = ['right steering', 'left steering', 'maintain steering']
modifiers = [None, 'negate', 'small_values']
for i, modifier in enumerate(modifiers):
heatmap = visualize_saliency(model,
layer_idx=-1,
filter_indices=0,
seed_input=bgr_img,
grad_modifier=modifier)
plt.figure()
layer_name = 'predictions'
layer_idx = [
idx for idx, layer in enumerate(model.layers) if layer.name == layer_name
][0]
image_paths = [
"data/test/1.jpg",
"data/test/2.jpg",
"data/test/3.jpg",
"data/test/4.jpg",
"data/test/5.jpg",
"data/test/6.jpg",
"data/test/7.jpg",
"data/test/8.jpg",
]
for path in image_paths:
seed_img = utils.load_img(path, target_size=(299, 299))
# Convert to BGR, create input with batch_size: 1, and predict.
bgr_img = utils.bgr2rgb(seed_img)
img_input = np.expand_dims(img_to_array(bgr_img), axis=0)
pred_class = np.argmax(model.predict(img_input))
heatmap = visualize_cam(model, layer_idx, [pred_class], seed_img)
plt.axis('off')
# plt.imshow(seed_img)
plt.imshow(heatmap)
plt.title('Attention - {}'.format(get_cat_dog_label(pred_class)))
plt.show()
print camera['X'].shape
pygame.init()
size = (320, 240)
pygame.display.set_caption("comma.ai data viewer")
screen = pygame.display.set_mode(size)
camera_surface = pygame.surface.Surface((320, 240), 0, 24).convert()
for i in range(0, camera['X'].shape[0]):
angle_steers = camera['angle'][i]
speed_ms = camera['speed'][i]
imagem = camera['X'][i]
#imshow(imagem)
print imagem.shape
pygame.surfarray.blit_array(camera_surface, imagem.swapaxes(0, 2))
#camera_surface_2x = pygame.transform.scale2x(camera_surface)
screen.blit(camera_surface, (0, 0))
pygame.display.flip()
data = numpy.array(imagem.swapaxes(0, 2).swapaxes(0, 1))
file = 'test.jpeg'
data = utils.bgr2rgb(data)
cv2.imwrite(file, data)
#plt.imshow(data)
cv2.imshow('cv_img', data)
#cv2.imwrite('teste', image_np)
cv2.waitKey(0)