def create_images(X_val, y_val, model: TargetNetwork, class_index: int, layer_name: str, segment: Segment, amount: int = 10, save=True): validation_data = (X_val, y_val) explainer = GradCAM() heatmaps = explainer.explain(validation_data, model.model, class_index=class_index, layer_name=layer_name, image_weight=0.8) images = random.choices(heatmaps, k=amount) images = [cv2.resize(image, (400, 400)) for image in images] #files = [File(image) for image in images] files = [ base64.b64encode(cv2.imencode('.jpg', img)[1]).decode() for img in images ] if save: [ Image(image=file, segment=segment, ground_truth=class_index).save() for file in files ]
def on_epoch_end(self, epoch, logs): # Initialize GRADCam Class cam = GradCAM(self.model, self.layer_name) count = 0 foldername = self.save_dir + '/epoch{}'.format(epoch) if not os.path.exists(foldername): os.makedirs(foldername) for data in self.activation_data: image = data[0] image = np.expand_dims(image, 0) pred = model.predict(image) classIDx = np.argmax(pred[0]) # Compute Heatmap heatmap = cam.compute_heatmap(image, classIDx) image = image.reshape(image.shape[1:]) image = image * 255 image = image.astype(np.uint8) # Overlay heatmap on original image heatmap = cv2.resize(heatmap, (image.shape[1], image.shape[0])) plt.imshow(np.squeeze(image)) plt.imshow(heatmap, alpha=.6, cmap='inferno') plt.axis('off') plt.savefig(self.save_dir + '/epoch{}/out{}.png'.format(epoch, count), bbox_inches='tight', transparent=True, pad_inches=0) plt.clf() count += 1 make_grid(foldername)
def main(): device = get_device(args.cuda) # Synset words classes = get_classtable() # Model from torchvision model = models.__dict__[args.arch]() model = model.cuda() check_point = torch.load(args.checkpoint, map_location=lambda storage, loc: storage.cuda(0)) model.load_state_dict(check_point['state_dict']) model.to(device) model.eval() gcam = GradCAM(model=model) _ = gcam.forward(images)
def main(): # 注意输入的图片必须是32的整数倍 # 否则由于padding的原因会出现注意力飘逸的问题 img_size = 224 assert img_size % 32 == 0 model = swin_base_patch4_window7_224() # https://github.com/SwinTransformer/storage/releases/download/v1.0.0/swin_base_patch4_window7_224.pth weights_path = "./swin_base_patch4_window7_224.pth" model.load_state_dict(torch.load(weights_path, map_location="cpu")["model"], strict=False) target_layers = [model.norm] data_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) # load image img_path = "both.png" assert os.path.exists(img_path), "file: '{}' dose not exist.".format( img_path) img = Image.open(img_path).convert('RGB') img = np.array(img, dtype=np.uint8) img = center_crop_img(img, img_size) # [C, H, W] img_tensor = data_transform(img) # expand batch dimension # [C, H, W] -> [N, C, H, W] input_tensor = torch.unsqueeze(img_tensor, dim=0) cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False, reshape_transform=ResizeTransform(im_h=img_size, im_w=img_size)) target_category = 281 # tabby, tabby cat # target_category = 254 # pug, pug-dog grayscale_cam = cam(input_tensor=input_tensor, target_category=target_category) grayscale_cam = grayscale_cam[0, :] visualization = show_cam_on_image(img / 255., grayscale_cam, use_rgb=True) plt.imshow(visualization) plt.show()
def main(): model = models.mobilenet_v3_large(pretrained=True) target_layers = [model.features[-1]] # model = models.vgg16(pretrained=True) # target_layers = [model.features] # model = models.resnet34(pretrained=True) # target_layers = [model.layer4] # model = models.regnet_y_800mf(pretrained=True) # target_layers = [model.trunk_output] # model = models.efficientnet_b0(pretrained=True) # target_layers = [model.features] data_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) # load image img_path = "both.png" assert os.path.exists(img_path), "file: '{}' dose not exist.".format( img_path) img = Image.open(img_path).convert('RGB') img = np.array(img, dtype=np.uint8) # img = center_crop_img(img, 224) # [C, H, W] img_tensor = data_transform(img) # expand batch dimension # [C, H, W] -> [N, C, H, W] input_tensor = torch.unsqueeze(img_tensor, dim=0) cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False) target_category = 281 # tabby, tabby cat # target_category = 254 # pug, pug-dog grayscale_cam = cam(input_tensor=input_tensor, target_category=target_category) grayscale_cam = grayscale_cam[0, :] visualization = show_cam_on_image(img.astype(dtype=np.float32) / 255., grayscale_cam, use_rgb=True) plt.imshow(visualization) plt.show()
def visualize(data_loader: DataLoader, model: torch.nn.Module, args: argparse.Namespace): """ 获取最后一层卷积层激活函数后的激活输出,根据选取的可视化方法,可视化所有类别的激活图 efficientnet的最后一层卷积层激活函数输出模块名为 _swish_fc,如果要可视化其他模型,这里需要做对应修改 :param data_loader: 待可视化的数据集 :param model: 待可视化模型 :param args: 可视化超参 """ model.eval() cam_inferences = [] if args.visual_method in ('cam', 'all'): cam_inferences.append(CAM(model, '_swish_fc')) if args.visual_method in ('grad-cam', 'all'): cam_inferences.append(GradCAM(model, '_swish_fc')) if args.visual_method in ('grad-cam++', 'all'): cam_inferences.append(GradCamPlusPlus(model, '_swish_fc')) if len(cam_inferences) == 0: raise NotImplementedError( '--visual_method must be in (cam, grad-cam, grad-cam++)') for i, (images, labels, paths, _) in enumerate(data_loader): batch_images, batch_cams = [], [] uint8_images = np.uint8(images.numpy().transpose( (0, 2, 3, 1))[..., ::-1] * 255) for cam_inference in cam_inferences: outputs, cams = cam_inference(images, args) batch_cams.append(cams) batch_images.append(uint8_images) batch_cams = np.concatenate(batch_cams, axis=2) batch_images = np.concatenate(batch_images, axis=1) if args.cuda: labels = labels.cuda(args.gpu, non_blocking=True) match_results, labels, predictions = Visualize._assess( outputs, labels) for path, image, cams, is_match, label, pred in \ zip(paths, batch_images, batch_cams, match_results, labels, predictions): # 预测错误,则需要画出错误的热图和正确的热图 logging.info( f'{is_match} path: {path},{data_loader.dataset.classes[label]},' f'{data_loader.dataset.classes[pred]}') cv2.imwrite(f'visual_images/{os.path.basename(path)}', HeatMapTool.add_heat(image, cams))
def main(): model = vit_base_patch16_224() # 链接: https://pan.baidu.com/s/1zqb08naP0RPqqfSXfkB2EA 密码: eu9f weights_path = "./vit_base_patch16_224.pth" model.load_state_dict(torch.load(weights_path, map_location="cpu")) # Since the final classification is done on the class token computed in the last attention block, # the output will not be affected by the 14x14 channels in the last layer. # The gradient of the output with respect to them, will be 0! # We should chose any layer before the final attention block. target_layers = [model.blocks[-1].norm1] data_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]) ]) # load image img_path = "both.png" assert os.path.exists(img_path), "file: '{}' dose not exist.".format( img_path) img = Image.open(img_path).convert('RGB') img = np.array(img, dtype=np.uint8) img = center_crop_img(img, 224) # [N, C, H, W] img_tensor = data_transform(img) # expand batch dimension input_tensor = torch.unsqueeze(img_tensor, dim=0) cam = GradCAM(model=model, target_layers=target_layers, use_cuda=False, reshape_transform=ReshapeTransform(model)) target_category = 281 # tabby, tabby cat # target_category = 254 # pug, pug-dog grayscale_cam = cam(input_tensor=input_tensor, target_category=target_category) grayscale_cam = grayscale_cam[0, :] visualization = show_cam_on_image(img / 255., grayscale_cam, use_rgb=True) plt.imshow(visualization) plt.show()