def draw_features(module, input, output, work_dir='./'):
x = output.cpu().numpy()
out_channels = list(output.shape)[1]
height = int(math.sqrt(out_channels))
width = height
if list(output.shape)[2] < 128:
return
fig = plt.figure(figsize=(32, 32))
fig.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95, wspace=0.05, hspace=0.05)
for i in range(height * width):
plt.subplot(height, width, i + 1)
plt.axis('off')
img = x[0, i, :, :]
pmin = np.min(img)
pmax = np.max(img)
img = ((img - pmin) / (pmax - pmin + 0.000001))*255 # float在[0，1]之间，转换成0-255
img = img.astype(np.uint8) # 转成unit8
img = cv.applyColorMap(img, cv.COLORMAP_JET) # 生成heat map
img = img[:, :, ::-1] # 注意cv2（BGR）和matplotlib(RGB)通道是相反的
plt.imshow(img)
# print("{}/{}".format(i,width*height))
savename = get_image_name_for_hook(module, work_dir)
fig.savefig(savename, dpi=100)
fig.clf()
plt.close()
def arr2heatmap(arr):
heatmap = cv2.normalize(arr,
dst=None,
alpha=0,
beta=255,
norm_type=cv2.NORM_MINMAX,
dtype=cv2.CV_8U)
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
return heatmap
def overlay_heatmap(self, heatmap, image, alpha=0.5, colormap=cv2.COLORMAP_VIRIDIS):
# apply the supplied color map to the heatmap and then
# overlay the heatmap on the input image
heatmap = cv2.applyColorMap(heatmap, colormap)
# if the input image is grey-scale, convert it to 3-dim
if len(image.shape) == 2 or image.shape[-1] != 3:
image = cv2.cvtColor(src=image, code=cv2.COLOR_GRAY2RGB)
# if image px values are in [0, 1], upscale to [0, 255]
if np.max(image) <= 1.0:
image = image * 255.0
output = cv2.addWeighted(image.astype('uint8'), alpha, heatmap, 1 - alpha, 0)
return output
def getScreenDepthVis(client, depth_coordinates, depth_list):
#get image from airsim
responses = client.simGetImages(
[airsim.ImageRequest(0, airsim.ImageType.DepthPlanner, True, False)])
# condition the data
img1d = np.array(responses[0].image_data_float, dtype=np.float)
img1d = 255 / np.maximum(np.ones(img1d.size), img1d)
img2d = np.reshape(img1d, (responses[0].height, responses[0].width))
image = np.invert(
np.array(Image.fromarray(img2d.astype(np.uint8), mode='L')))
factor = 10
maxIntensity = 255.0 # depends on dtype of image data
# Decrease intensity such that dark pixels become much darker, bright pixels become slightly dark
newImage1 = (maxIntensity) * (image / maxIntensity)**factor
newImage1 = newImage1.astype(np.uint8)
color_img = cv2.applyColorMap(newImage1, cv2.COLORMAP_JET)
# divide view into 3x3 matrix
pxstep = int(newImage1.shape[1] / 3)
pystep = int(newImage1.shape[0] / 3)
gx = pxstep
gy = pystep
while gx < newImage1.shape[1]:
cv2.line(color_img, (gx, 0), (gx, newImage1.shape[0]),
color=(0, 0, 0),
thickness=1)
gx += pxstep
while gy < newImage1.shape[0]:
cv2.line(color_img, (0, gy), (newImage1.shape[1], gy),
color=(0, 0, 0),
thickness=1)
gy += pystep
# print circle, and depth values on the screen
for i in range(len(depth_list)):
if depth_list[i] <= DEPTH_RANGE_M[1]:
color_img = cv2.circle(
color_img,
(int(depth_coordinates[i][0]), int(depth_coordinates[i][1])),
5, (0, 0, 0), 5)
color_img = cv2.putText(color_img, str(
round(depth_list[i],
2)), (int(pxstep * (1 / 4 + i % 3)),
int(pystep * (1 / 3 + math.floor(i / 3)))),
cv2.FONT_HERSHEY_SIMPLEX, 0.75,
(0, 0, 255), 2)
cv2.imshow("Depth Vis", color_img)
cv2.waitKey(1)
def apply_gradcam(setup_dict, idx, hm_intensity=0.5, save_hm=True):
'''
Make a prediction and overlay a heatmap depicting the gradient of the predicted class with respect to the output of
a layer of the model.
:param setup_dict: dict containing important information and objects for Grad-CAM
:param idx: index of image in test set to explain
:param save_hm: Boolean indicating whether to save the heatmap visualization
'''
# Get i'th preprocessed image in test set
setup_dict['TEST_GENERATOR'].reset()
for i in range(idx + 1):
x, y = setup_dict['TEST_GENERATOR'].next()
# Get the corresponding original image (no preprocessing)
orig_img = cv2.imread(setup_dict['RAW_DATA_PATH'] + setup_dict['TEST_SET']['filename'][idx])
new_dim = tuple(setup_dict['IMG_DIM'])
orig_img = cv2.resize(orig_img, new_dim, interpolation=cv2.INTER_NEAREST) # Resize image
# Predict this example
probs = predict_instance(x, setup_dict['MODEL'])
# Rearrange prediction probability vector to reflect original ordering of classes in project config
probs = [probs[0][setup_dict['CLASSES'].index(c)] for c in setup_dict['TEST_GENERATOR'].class_indices]
with tf.GradientTape() as tape:
last_conv_layer = setup_dict['MODEL'].get_layer(setup_dict['LAYER_NAME'])
iterate = Model([setup_dict['MODEL'].inputs], [setup_dict['MODEL'].output, last_conv_layer.output])
model_out, last_conv_layer = iterate(x)
class_out = model_out[:, np.argmax(model_out[0])]
grads = tape.gradient(class_out, last_conv_layer)
pooled_grads = tf.keras.backend.mean(grads, axis=(0, 1, 2))
heatmap = tf.reduce_mean(tf.multiply(pooled_grads, last_conv_layer), axis=-1)
heatmap = np.maximum(heatmap, 0.0) # Equivalent of passing through ReLU
heatmap /= np.max(heatmap)
heatmap = heatmap.squeeze(axis=0)
heatmap = cv2.resize(heatmap, tuple(setup_dict['IMG_DIM']))
heatmap = cv2.applyColorMap(np.uint8(255 * heatmap), cv2.COLORMAP_JET)
heatmap_img = cv2.addWeighted(heatmap, hm_intensity, orig_img, 1.0 - hm_intensity, 0)
# Visualize the Grad-CAM heatmap and optionally save it to disk
if save_hm:
file_path = setup_dict['IMG_PATH']
else:
file_path = None
img_filename = setup_dict['TEST_SET']['filename'][idx]
label = setup_dict['TEST_SET']['label'][idx]
_ = visualize_heatmap(orig_img, heatmap_img, img_filename, label, probs, setup_dict['CLASSES'],
dir_path=file_path)
return heatmap
def __print_debug_info(self, sample_shape: np.array) -> None:
algo = self.algo()
if config.DEBUG and algo in [
RecognitionAlgo.EIGEN, RecognitionAlgo.FISHER
]:
color_map = cv2.COLORMAP_JET if algo == RecognitionAlgo.EIGEN else cv2.COLORMAP_BONE
eigenvectors = self.__rec.getEigenVectors()
for i in range(np.shape(eigenvectors)[1]):
image = img.normalized(
np.reshape(eigenvectors[:, i], sample_shape))
image = cv2.applyColorMap(image, color_map)
img.save(image, config.Paths.DEBUG_DIR + '/ev{}.png'.format(i))
def __show_image_grid_heatmap(self,
heatmap,
tensor=None,
ratio=0.3,
colormap=2,
normalize=True,
name=None,
caption=None,
env_appendix="",
opts=None,
image_args=None,
**kwargs):
"""
Internal show_image_grid_heatmap method, called by the internal process.
This function does all the magic.
"""
from cv2 import cv2
if opts is None: opts = {}
if image_args is None: image_args = {}
map_grid = np_make_grid(heatmap, normalize=normalize)
map_ = np.clip(map_grid * 255, a_min=0, a_max=255)
map_ = map_.astype(np.uint8)
map_ = cv2.applyColorMap(map_.transpose(1, 2, 0), colormap=colormap)
map_ = cv2.cvtColor(map_, cv2.COLOR_BGR2RGB)
map_ = map_.transpose(2, 0, 1)
fuse_img = map_
if tensor is not None:
img_grid = np_make_grid(tensor, **image_args)
image = np.clip(img_grid * 255, a_min=0, a_max=255)
image = image.astype(np.uint8)
fuse_img = (1.0 - ratio) * image + ratio * map_
opts = opts.copy()
opts.update(dict(title=name, caption=caption))
win = self.vis.image(img=fuse_img,
win=name,
env=self.name + env_appendix,
opts=opts)
return win
def show_image_grid_heatmap(self, heatmap, background=None, ratio=0.3, normalize=True,
colormap=cv2.COLORMAP_JET, name="heatmap", n_iter=None,
prefix=False, iter_format="{:05d}", image_args=None, **kwargs):
"""
Creates heat map from the given map and if given combines it with the background and then
displays results with as image grid.
Args:
heatmap: 4d- tensor (N, C, H, W) to be converted to a heatmap
background: 4d- tensor (N, C, H, W) background/ context of the heatmap (to be underlayed)
name: The name of the window
ratio: The ratio to mix the map with the background (0 = only background, 1 = only map)
n_iter: The iteration number, formatted with the iter_format and added to the model name (if not None)
iter_format: The format string, which indicates how n_iter will be formated as a string
prefix: If True, the formated n_iter will be appended as a prefix, otherwise as a suffix
image_args: Arguments for the tensorvision save image method
"""
if image_args is None: image_args = {}
if n_iter is not None:
name = name_and_iter_to_filename(name=name, n_iter=n_iter, ending=".png", iter_format=iter_format,
prefix=prefix)
elif not name.endswith(".png"):
name += ".png"
file_name = os.path.join(self.img_dir, name)
map_grid = np_make_grid(heatmap, normalize=normalize)
map_ = np.clip(map_grid * 255, a_min=0, a_max=255)
map_ = map_.astype(np.uint8)
map_ = cv2.applyColorMap(map_.transpose(1, 2, 0), colormap=colormap)
map_ = cv2.cvtColor(map_, cv2.COLOR_BGR2RGB)
map_ = map_.transpose(2, 0, 1)
fuse_img = map_
if background is not None:
img_grid = np_make_grid(background, **image_args)
image = np.clip(img_grid * 255, a_min=0, a_max=255)
image = image.astype(np.uint8)
fuse_img = (1.0 - ratio) * image + ratio * map_
imsave(file_name, fuse_img.transpose(1, 2, 0))
def create_mask_montage(self, image, predictions):
"""
Create a montage showing the probability heatmaps for each one one of the
detected objects
Arguments:
image (np.ndarray): an image as returned by OpenCV
predictions (BoxList): the result of the computation by the model.
It should contain the field `mask`.
"""
masks = predictions.get_field("mask")
masks_per_dim = self.masks_per_dim
masks = torch.nn.functional.interpolate(masks.float(),
scale_factor=1 /
masks_per_dim).byte()
height, width = masks.shape[-2:]
max_masks = masks_per_dim**2
masks = masks[:max_masks]
# handle case where we have less detections than max_masks
if len(masks) < max_masks:
masks_padded = torch.zeros(max_masks,
1,
height,
width,
dtype=torch.uint8)
masks_padded[:len(masks)] = masks
masks = masks_padded
masks = masks.reshape(masks_per_dim, masks_per_dim, height, width)
result = torch.zeros((masks_per_dim * height, masks_per_dim * width),
dtype=torch.uint8)
for y in range(masks_per_dim):
start_y = y * height
end_y = (y + 1) * height
for x in range(masks_per_dim):
start_x = x * width
end_x = (x + 1) * width
result[start_y:end_y, start_x:end_x] = masks[y, x]
return cv2.applyColorMap(result.numpy(), cv2.COLORMAP_JET)
def arr2heatmap(arr):
# ax = cv2.applyColorMap( (arr * cv2.getTrackbarPos("Scale", "Trackbar")/122).astype('uint8'), cv2.COLORMAP_JET)
ax = cv2.applyColorMap((arr * 2.245).astype("uint8"), cv2.COLORMAP_JET)
return ax
import numpy as np
from keras.preprocessing import image
test_image = image.load_img('corona.jpg', target_size=(64, 64))
test_image = image.img_to_array(test_image)
test_image = np.expand_dims(test_image, axis=0)
result = classifier.predict(test_image)
training_set.class_indices
if result[0][0] == 1:
prediction = 'Patient is affected with Corona'
else:
prediction = 'Patient is Healthy'
print("\nOutcome : ", prediction)
import matplotlib.pyplot as plt
# reads the image
img = cv2.imread('corona.jpg')
im_gray = cv2.imread('corona.jpg', cv2.IMREAD_GRAYSCALE)
heatmap_img = cv2.applyColorMap(im_gray, cv2.COLORMAP_JET)
fin = cv2.addWeighted(heatmap_img, 0.7, img, 0.3, 0)
# plot heat map image
fig = plt.figure(figsize=(18, 18))
fig.add_subplot(1, 3, 1)
plt.imshow(img)
fig.add_subplot(1, 3, 2)
plt.imshow(heatmap_img)
fig.add_subplot(1, 3, 3)
plt.imshow(fin)
plt.savefig('output.png')
obstacles = seq_obstacles[i]
lzFinder = LzFinder("aeroscapes")
lzs_ranked, risk_map = lzFinder.get_ranked_lz(obstacles, img, segImg)
img = lzFinder.draw_lzs_obs(lzs_ranked[-5:], obstacles, img)
if EXTRACT_METRICS:
gtSeg = glob.glob(PATH_GT + "*.png")
if SIMULATE:
print("hello")
if SAVE_TO_FILE:
cv.imwrite(
"/content/Safe-UAV-Landing/data/results/riskMaps/" + fileName +
"_risk.jpg",
cv.applyColorMap(risk_map, cv.COLORMAP_JET),
)
cv.imwrite(
"/content/Safe-UAV-Landing/data/results/landingZones/" +
fileName + "_lzs.jpg",
img,
)
if not HEADLESS:
cv.imshow("best landing zones",
cv.applyColorMap(risk_map, cv.COLORMAP_JET))
cv.waitKey(0)
cv.imshow("best landing zones", img)
cv.waitKey(0)
cv.destroyAllWindows()
# 将特征图组数的每个通道乘以该通道对于大象类别的重要程度
for i in range(512):
conv_layer_output_value[:, :, i] *= pooled_grads_value[i]
# 得到的特征图的逐通道平均值即为类激活的热力图
heatmap = np.mean(conv_layer_output_value, axis=-1)
# 将热力图标准化到0~1范围内
heatmap = np.maximum(heatmap, 0)
heatmap /= np.max(heatmap)
plt.matshow(heatmap)
plt.savefig("03_heatmap.png")
plt.show()
"""
Part 4. 叠加热力图
"""
from cv2 import cv2 as cv2
img = cv2.imread(img_path)
heatmap = cv2.resize(heatmap, (img.shape[1], img.shape[0]))
heatmap = np.uint8(255 * heatmap)
# 将热力图添加到原始图像上
heatmap = cv2.applyColorMap(heatmap, cv2.COLORMAP_JET)
# 0.4 是热力图的强度因子
superimposed_img = heatmap * 0.4 + img
cv2.imwrite('03_result.png', superimposed_img)