def create_images(X_val,
y_val,
model: TargetNetwork,
class_index: int,
layer_name: str,
segment: Segment,
amount: int = 10,
save=True):
validation_data = (X_val, y_val)
explainer = GradCAM()
heatmaps = explainer.explain(validation_data,
model.model,
class_index=class_index,
layer_name=layer_name,
image_weight=0.8)
images = random.choices(heatmaps, k=amount)
images = [cv2.resize(image, (400, 400)) for image in images]
#files = [File(image) for image in images]
files = [
base64.b64encode(cv2.imencode('.jpg', img)[1]).decode()
for img in images
]
if save:
[
Image(image=file, segment=segment,
ground_truth=class_index).save() for file in files
]
def on_epoch_end(self, epoch, logs):
# Initialize GRADCam Class
cam = GradCAM(self.model, self.layer_name)
count = 0
foldername = self.save_dir + '/epoch{}'.format(epoch)
if not os.path.exists(foldername):
os.makedirs(foldername)
for data in self.activation_data:
image = data[0]
image = np.expand_dims(image, 0)
pred = model.predict(image)
classIDx = np.argmax(pred[0])
# Compute Heatmap
heatmap = cam.compute_heatmap(image, classIDx)
image = image.reshape(image.shape[1:])
image = image * 255
image = image.astype(np.uint8)
# Overlay heatmap on original image
heatmap = cv2.resize(heatmap, (image.shape[1], image.shape[0]))
plt.imshow(np.squeeze(image))
plt.imshow(heatmap, alpha=.6, cmap='inferno')
plt.axis('off')
plt.savefig(self.save_dir +
'/epoch{}/out{}.png'.format(epoch, count),
bbox_inches='tight',
transparent=True,
pad_inches=0)
plt.clf()
count += 1
make_grid(foldername)
def main():
device = get_device(args.cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[args.arch]()
model = model.cuda()
check_point = torch.load(args.checkpoint,
map_location=lambda storage, loc: storage.cuda(0))
model.load_state_dict(check_point['state_dict'])
model.to(device)
model.eval()
gcam = GradCAM(model=model)
_ = gcam.forward(images)
def main():
# 注意输入的图片必须是32的整数倍
# 否则由于padding的原因会出现注意力飘逸的问题
img_size = 224
assert img_size % 32 == 0
model = swin_base_patch4_window7_224()
# https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224.pth
weights_path = "./swin_base_patch4_window7_224.pth"
model.load_state_dict(torch.load(weights_path,
map_location="cpu")["model"],
strict=False)
target_layers = [model.norm]
data_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# load image
img_path = "both.png"
assert os.path.exists(img_path), "file: '{}' dose not exist.".format(
img_path)
img = Image.open(img_path).convert('RGB')
img = np.array(img, dtype=np.uint8)
img = center_crop_img(img, img_size)
# [C, H, W]
img_tensor = data_transform(img)
# expand batch dimension
# [C, H, W] -> [N, C, H, W]
input_tensor = torch.unsqueeze(img_tensor, dim=0)
cam = GradCAM(model=model,
target_layers=target_layers,
use_cuda=False,
reshape_transform=ResizeTransform(im_h=img_size,
im_w=img_size))
target_category = 281 # tabby, tabby cat
# target_category = 254 # pug, pug-dog
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
grayscale_cam = grayscale_cam[0, :]
visualization = show_cam_on_image(img / 255., grayscale_cam, use_rgb=True)
plt.imshow(visualization)
plt.show()
def main():
model = models.mobilenet_v3_large(pretrained=True)
target_layers = [model.features[-1]]
# model = models.vgg16(pretrained=True)
# target_layers = [model.features]
# model = models.resnet34(pretrained=True)
# target_layers = [model.layer4]
# model = models.regnet_y_800mf(pretrained=True)
# target_layers = [model.trunk_output]
# model = models.efficientnet_b0(pretrained=True)
# target_layers = [model.features]
data_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# load image
img_path = "both.png"
assert os.path.exists(img_path), "file: '{}' dose not exist.".format(
img_path)
img = Image.open(img_path).convert('RGB')
img = np.array(img, dtype=np.uint8)
# img = center_crop_img(img, 224)
# [C, H, W]
img_tensor = data_transform(img)
# expand batch dimension
# [C, H, W] -> [N, C, H, W]
input_tensor = torch.unsqueeze(img_tensor, dim=0)
cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False)
target_category = 281 # tabby, tabby cat
# target_category = 254 # pug, pug-dog
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
grayscale_cam = grayscale_cam[0, :]
visualization = show_cam_on_image(img.astype(dtype=np.float32) / 255.,
grayscale_cam,
use_rgb=True)
plt.imshow(visualization)
plt.show()
def visualize(data_loader: DataLoader, model: torch.nn.Module,
args: argparse.Namespace):
"""
获取最后一层卷积层激活函数后的激活输出，根据选取的可视化方法，可视化所有类别的激活图
efficientnet的最后一层卷积层激活函数输出模块名为 _swish_fc，如果要可视化其他模型，这里需要做对应修改
:param data_loader: 待可视化的数据集
:param model: 待可视化模型
:param args: 可视化超参
"""
model.eval()
cam_inferences = []
if args.visual_method in ('cam', 'all'):
cam_inferences.append(CAM(model, '_swish_fc'))
if args.visual_method in ('grad-cam', 'all'):
cam_inferences.append(GradCAM(model, '_swish_fc'))
if args.visual_method in ('grad-cam++', 'all'):
cam_inferences.append(GradCamPlusPlus(model, '_swish_fc'))
if len(cam_inferences) == 0:
raise NotImplementedError(
'--visual_method must be in (cam, grad-cam, grad-cam++)')
for i, (images, labels, paths, _) in enumerate(data_loader):
batch_images, batch_cams = [], []
uint8_images = np.uint8(images.numpy().transpose(
(0, 2, 3, 1))[..., ::-1] * 255)
for cam_inference in cam_inferences:
outputs, cams = cam_inference(images, args)
batch_cams.append(cams)
batch_images.append(uint8_images)
batch_cams = np.concatenate(batch_cams, axis=2)
batch_images = np.concatenate(batch_images, axis=1)
if args.cuda:
labels = labels.cuda(args.gpu, non_blocking=True)
match_results, labels, predictions = Visualize._assess(
outputs, labels)
for path, image, cams, is_match, label, pred in \
zip(paths, batch_images, batch_cams, match_results, labels, predictions):
# 预测错误，则需要画出错误的热图和正确的热图
logging.info(
f'{is_match} path: {path},{data_loader.dataset.classes[label]},'
f'{data_loader.dataset.classes[pred]}')
cv2.imwrite(f'visual_images/{os.path.basename(path)}',
HeatMapTool.add_heat(image, cams))
def main():
model = vit_base_patch16_224()
# 链接: https://pan.baidu.com/s/1zqb08naP0RPqqfSXfkB2EA 密码: eu9f
weights_path = "./vit_base_patch16_224.pth"
model.load_state_dict(torch.load(weights_path, map_location="cpu"))
# Since the final classification is done on the class token computed in the last attention block,
# the output will not be affected by the 14x14 channels in the last layer.
# The gradient of the output with respect to them, will be 0!
# We should chose any layer before the final attention block.
target_layers = [model.blocks[-1].norm1]
data_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
# load image
img_path = "both.png"
assert os.path.exists(img_path), "file: '{}' dose not exist.".format(
img_path)
img = Image.open(img_path).convert('RGB')
img = np.array(img, dtype=np.uint8)
img = center_crop_img(img, 224)
# [N, C, H, W]
img_tensor = data_transform(img)
# expand batch dimension
input_tensor = torch.unsqueeze(img_tensor, dim=0)
cam = GradCAM(model=model,
target_layers=target_layers,
use_cuda=False,
reshape_transform=ReshapeTransform(model))
target_category = 281 # tabby, tabby cat
# target_category = 254 # pug, pug-dog
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
grayscale_cam = grayscale_cam[0, :]
visualization = show_cam_on_image(img / 255., grayscale_cam, use_rgb=True)
plt.imshow(visualization)
plt.show()