def visualize_grad_cam(model): cuda = torch.cuda.is_available() # returns True/False device = torch.device("cuda" if cuda else "cpu") url = 'https://www.cs.toronto.edu/~kriz/cifar-10-sample/dog4.png' # sample image for GradCam testing on Custom Model response = requests.get(url) pil_img = PIL.Image.open(BytesIO(response.content)) pil_img # print('pre-processing the input') torch_img = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor() ])(pil_img).to(device) normed_torch_img = transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])(torch_img)[None] # print('loading the models:') # resnet = models.resnet18(pretrained=True) # vgg = models.vgg16(pretrained=True) # print('loading the models Finished!') configs = [ dict(model_type='resnet', arch=model, layer_name='layer3'), # My Model # dict(model_type='resnet', arch=resnet, layer_name='layer3'), # Reference # dict(model_type='vgg', arch=vgg, layer_name='features_29') # Reference ] for config in configs: config['arch'].to(device).eval() cams = [ [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs ] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) mask_pp, _ = gradcam_pp(normed_torch_img) heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) images = make_grid(images, nrow=5, scale_each=False, padding=10, pad_value=140, range=5) output_dir = '/home/prakash/Prakash/EVA4/Session-9/Notebooks' os.makedirs(output_dir, exist_ok=True) output_name = 'output.jpeg' output_path = os.path.join(output_dir, output_name) # print(output_path) save_image(images, output_path) grad_cam_output = PIL.Image.open(output_path) print(grad_cam_output.size) resized_output = grad_cam_output.resize((500, 120)) plt.imshow(np.asarray(resized_output))
def grad_cam(img, model, layer): configs = [dict(model_type='resnet', arch=model, layer_name=layer)] for config in configs: config['arch'].to(device).eval() torch_img = transforms.Compose([transforms.ToTensor()])(img).to(device) normed_torch_img = transforms.Normalize( [0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(torch_img)[None] cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) mask_pp, _ = gradcam_pp(normed_torch_img) heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) images.extend( [torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) return images
def show_gradcam(tensor, model): # Feedforward image, calculate GradCAM and gather heatmap if model.__class__.__name__ == 'ResNet': target_layer = model.layer4[-1].conv2 gradcam = GradCAM(model, target_layer) elif model.__class__.__name__ == 'DenseNet': target_layer = model.features.norm5 gradcam = GradCAM(model, target_layer) elif model.__class__.__name__ == 'DataParallel': target_layer = model.module.densenet121.features.norm5 gradcam = GradCAM(model.module.densenet121, target_layer) else: raise ValueError('improper model') mask, _ = gradcam(tensor) heatmap, _ = visualize_cam(mask, tensor) # heatmap from torch.tensor to numpy.array mask = mask[0].permute(1, 2, 0).detach().cpu().numpy() heatmap = heatmap.permute(1, 2, 0).numpy() return heatmap, mask
def generate_saliency_map(img, img_name, directory, model_dict): # Normalize PIL Image, Must be in PIL format normalizer = Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) torch_img = torch.from_numpy(np.asarray(img)).permute( 2, 0, 1).unsqueeze(0).float().div(255).cuda() torch_img = F.upsample(torch_img, size=(512, 512), mode='bilinear', align_corners=False) normed_torch_img = normalizer(torch_img) # gradcam = GradCAM(model_dict, True) gradcam_pp = GradCAMpp(model_dict, True) mask_pp, _ = gradcam_pp(normed_torch_img) mask_pp_np = mask_pp.detach().cpu().numpy() heatmap_pp, result_pp = visualize_cam(mask_pp_np, torch_img) # Only going to use result_pp (Saliency Map from GradCAM++) os.makedirs(directory, exist_ok=True) output_name = img_name output_path = os.path.join(directory, output_name) save_image(result_pp, output_path) return output_path
def plot_cam(img, mean, sd, device): """ Plots the cam """ # convert to PIL pil_img = F.to_pil_image(img) # normed_torch_img = transform_test_albu(pil_img).to(device) # call the transformation. To keep it simple, we are calling PyTorch way of transform torch_img = transforms.Compose([transforms.ToTensor()])(pil_img).to(device) normed_torch_img = transforms.Normalize(mean=mean, std=sd)(torch_img)[None] # Call the GridCAM config = dict(model_type='resnet', arch=net, layer_name='layer4') gradcam = GradCAM.from_config(**config) images = [] mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) images.extend([torch_img.cpu(), heatmap, result]) grid_image = make_grid(images, nrow=5) return transforms.ToPILImage()(grid_image)
def grad_cam_gen(model, img, path, apex = False, device = 'cuda'): if apex: model, optim = amp.initialize(model, optim, opt_level='O1') configs = [dict(model_type='seresnext', arch=model, layer_name='layer4_2_se_module_fc2')] # with amp.disable_casts(): for config in configs: config['arch'].eval() cams = [ [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs] indices = {0: 'Epidural', 1: 'Intraparenchymal', 2: 'Intraventricular', 3: 'Subarachnoid', 4:'Subdural', 5:'any'} for _, gradcam_pp in cams: for cls_idx in range(6): mask_pp, _ = gradcam_pp(img, cls_idx) heatmap_pp, result_pp = visualize_cam(mask_pp, img) result_pp = result_pp.cpu().numpy() #convert image back to Height,Width,Channels result_pp = np.transpose(result_pp, (1,2,0)) path = path.split('/')[-1].split('.')[0] plt.imsave('uploads/{}_grad_cam_{}.png'.format(path, indices[cls_idx]), np.transpose(result_pp, (1, 0, 2))) plt.show()
def grad_cam(model, x_batch): gradcam = GradCAM.from_config(arch=model._modules['resnet'], model_type='resnet', layer_name='7') mask, _ = gradcam(x_batch) heatmap, result = visualize_cam(mask, x_batch) result = result.numpy().transpose(1, 2, 0) return heatmap, result
def predict(): print(request.method) f = request.files['image'] file_path = os.path.join(root_dir, 'uploads', f.filename) print(file_path) f.save(file_path) #file_path='/home/ramkik/covid19/static/uploads/E63574A7-4188-4C8D-8D17-9D67A18A1AFA.jpeg' image = Image.open(file_path).convert('RGB') image = transform(image) normalize_image = normalize(image) normalize_image = normalize_image.unsqueeze(0) if torch.cuda.is_available(): normalize_image = normalize_image.cuda() #print(normalize_image.shape) output = model(normalize_image) print(output) # output = F.softmax(output) # print(output) prediction_score, pred_label_idx = torch.topk(output, 1) pred_label_idx.squeeze_() print(normalize_image.shape) mask, _ = gradcam(normalize_image) heatmap, result = visualize_cam(mask, image) mask_pp, _ = gradcam_pp(normalize_image) heatmap_pp, result_pp = visualize_cam(mask_pp, image, alpha=1.0) save_image(result, 'result.png', nrow=1) # grad = transforms.ToPILImage()(result_pp) # plt.imshow(grad) response = {} print('output : ', output) print(pred_label_idx) print(prediction_score) response['class'] = classes[pred_label_idx.item()] response['score'] = str(prediction_score.item()) return response
def grad_cam(model, model_type, layer_name, normed_torch_img, torch_img): config = dict(model_type=model_type, arch=model, layer_name=layer_name) config['arch'].eval() #.to(device) cam = GradCAM.from_config(**config) mask, _ = cam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) return (transforms.ToPILImage()(result))
def TileMaps(tensor, mask_1, mask_2, mask_3, image_list=[]): assert torch.is_tensor(tensor) assert torch.is_tensor(mask_1) assert torch.is_tensor(mask_2) assert torch.is_tensor(mask_3) heatmap_1, _ = visualize_cam(mask_1, tensor) heatmap_2, _ = visualize_cam(mask_2, tensor) heatmap_3, _ = visualize_cam(mask_3, tensor) image_list.append( torch.stack([ tensor.squeeze().cpu(), heatmap_1.cpu(), heatmap_2.cpu(), heatmap_3.cpu() ], 0)) return image_list
def Grad_Cam(model, train_datasets): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.eval() target_layer = model.features gradcam = GradCAM(model, target_layer) gradcam_pp = GradCAMpp(model, target_layer) images = [] for i in range(10): index = random.randint(0, 212) first_inputs, _ = train_datasets.__getitem__(index) inputs = first_inputs.to(device).unsqueeze(0) mask, _ = gradcam(inputs) heatmap, result = visualize_cam(mask, first_inputs) mask_pp, _ = gradcam_pp(inputs) heatmap_pp, result_pp = visualize_cam(mask_pp, first_inputs) images.extend([first_inputs.cpu(), heatmap, heatmap_pp, result, result_pp]) grid_image = make_grid(images, nrow=5) return transforms.ToPILImage()(grid_image)
def grad_cam(): net = ResNet18() configs = [dict(model_type='resnet', arch=net, layer_name='layer4')] for config in configs: config['arch'].to(device).eval() cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) mask_pp, _ = gradcam_pp(normed_torch_img) heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) images.extend( [torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) return make_grid(images, nrow=5)
def gradcam_exp(gradcam, gradcam_pp, inp, image, layer_name, f_size): """ Compute the explanations obtained with GradCAM and GradCAM++ before showing them. Parameters ---------- gradcam : GradCAM object the GradCAM explanation model gradcam_pp : GradCAMpp object the GradCAM++ explanation model inp : torch tensor the input to the model (should have requires_grad = False) image : numpy array the image corresponding to the input layer_name : string name of the layer analyzed f_size : tuple of ints size of each figure in the matplotlib.pyplot.figure object """ mask, _ = gradcam(inp) heatmap, result = visualize_cam(mask, inp) mask_pp, _ = gradcam_pp(inp) heatmap_pp, result_pp = visualize_cam(mask_pp, inp) display_gradients(heatmap.detach().numpy(), f_size).suptitle("Grad-CAM for an image with label 0", size="xx-large") display_gradients(heatmap_pp.detach().numpy(), f_size).suptitle("Grad-CAM++ for an image with label 0", size="xx-large") # heatmap_show = np.swapaxes(np.swapaxes(heatmap, 0, 1), 1, 2) # heatmap_pp_show = np.swapaxes(np.swapaxes(heatmap_pp, 0, 1), 1, 2) result_show = np.swapaxes(np.swapaxes(result.detach(), 0, 1), 1, 2) result_pp_show = np.swapaxes(np.swapaxes(result_pp.detach(), 0, 1), 1, 2) fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(2*f_size[0], f_size[1])) ax1.imshow(result_show) ax1.set_title("With GradCAM") ax2.imshow(result_pp_show) ax2.set_title("With GradCAM++") fig.suptitle("With respect to %s" % layer_name, size="xx-large")
def plot_imgs(imgs, figsize, label): plt.figure(figsize = (figsize[0],figsize[1])) for i in range(len(imgs)): img = transform_img(imgs[i]).cuda() configs = [ dict(model_type='densenet', arch=densenet121, layer_name='features'), ] for config in configs: config['arch'].cuda().eval() cams = [ [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs ] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(img) heatmap, result = visualize_cam(mask, img) mask_pp, _ = gradcam_pp(img) heatmap_pp, result_pp_densenet = visualize_cam(mask_pp, img) images.extend([img.cpu(), heatmap, heatmap_pp, result, result_pp_densenet]) plt.subplot(1,len(imgs),i+1) plt.title('Grad-CAM', fontsize = 30) plt.imshow(transforms.ToPILImage()(result_pp_densenet)) plt.tight_layout() plt.figure(figsize=(figsize[0],figsize[1])) for i in range(len(imgs)): img = transform_img(imgs[i]) plt.subplot(2,len(imgs),i+1) plt.title(label[i], fontsize = 30) plt.imshow(img[0,0,:,:]) plt.tight_layout()
def TileOutput(tensor, mask, mask_func, image_list = []): assert torch.is_tensor(tensor) assert torch.is_tensor(mask) assert isinstance(image_list,list) assert callable(mask_func) heatmap, result = visualize_cam(mask, tensor) hard_masked,_ = mask_func(tensor, mask) hard_masked = hard_masked.squeeze(0) masked = AlphaMask(tensor, mask).squeeze(0) masked = RangeNormalize(masked) image_list.append(torch.stack([tensor.squeeze().cpu(), heatmap.cpu(), result.cpu(), masked.cpu(), hard_masked.cpu()], 0)) return image_list
def main(config): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_transform = transforms.Compose([ transforms.Scale(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor() ]) val_transform = transforms.Compose( [transforms.Resize((224, 224)), transforms.ToTensor()]) test_transform = transforms.Compose([transforms.ToTensor()]) trainset = AVADataset(csv_file=config.train_csv_file, root_dir=config.train_img_path, transform=train_transform) valset = AVADataset(csv_file=config.val_csv_file, root_dir=config.val_img_path, transform=val_transform) train_loader = torch.utils.data.DataLoader( trainset, batch_size=config.train_batch_size, shuffle=True, num_workers=config.num_workers) val_loader = torch.utils.data.DataLoader(valset, batch_size=config.val_batch_size, shuffle=False, num_workers=config.num_workers) # base_model = models.vgg16(pretrained=True) # base_model = models.resnet18(pretrained=True) base_model = models.resnet101(pretrained=True, progress=False) # base_model = models.inception_v3(pretrained=True) model = NIMA(base_model) # model = NIMA() if config.warm_start: model.load_state_dict( torch.load( os.path.join(config.ckpt_path, 'epoch-%d.pkl' % config.warm_start_epoch))) print('Successfully loaded model epoch-%d.pkl' % config.warm_start_epoch) if config.multi_gpu: model.features = torch.nn.DataParallel(model.features, device_ids=config.gpu_ids) model = model.to(device) else: model = model.to(device) conv_base_lr = config.conv_base_lr dense_lr = config.dense_lr optimizer = optim.SGD([{ 'params': model.features.parameters(), 'lr': conv_base_lr }, { 'params': model.classifier.parameters(), 'lr': dense_lr }], momentum=0.9) # optimizer = optim.Adam( model.parameters(), lr = conv_base_lr, betas=(0.9,0.999)) # Loss functions # criterion = torch.nn.L1Loss() criterion = torch.nn.CrossEntropyLoss() # send hyperparams lrs.send({ 'title': 'EMD Loss', 'train_batch_size': config.train_batch_size, 'val_batch_size': config.val_batch_size, 'optimizer': 'SGD', 'conv_base_lr': config.conv_base_lr, 'dense_lr': config.dense_lr, 'momentum': 0.9 }) param_num = 0 for param in model.parameters(): param_num += int(np.prod(param.shape)) print('Trainable params: %.2f million' % (param_num / 1e6)) if config.test: # start.record() print('Testing') model.load_state_dict( torch.load( os.path.join(config.ckpt_path, 'epoch-%d.pkl' % config.warm_start_epoch))) target_layer = model.features # compute mean score test_transform = test_transform #val_transform testset = AVADataset(csv_file=config.test_csv_file, root_dir=config.test_img_path, transform=val_transform) test_loader = torch.utils.data.DataLoader( testset, batch_size=config.test_batch_size, shuffle=False, num_workers=config.num_workers) ypreds = [] ylabels = [] im_ids = [] # std_preds = [] count = 0 gradcam = GradCAM(model, target_layer) for data in test_loader: im_id = data['img_id'] im_name = os.path.split(im_id[0]) myname = os.path.splitext(im_name[1]) image = data['image'].to(device) mask, _ = gradcam(image) heatmap, result = visualize_cam(mask, image) im = transforms.ToPILImage()(result) im.save(myname[0] + ".jpg") labels = data['annotations'].to(device).long() output = model(image) output = output.view(-1, 2) bpred = output.to(torch.device("cpu")) cpred = bpred.data.numpy() blabel = labels.to(torch.device("cpu")) clabel = blabel.data.numpy() # predicted_mean, predicted_std = 0.0, 0.0 # for i, elem in enumerate(output, 1): # predicted_mean += i * elem # for j, elem in enumerate(output, 1): # predicted_std += elem * (i - predicted_mean) ** 2 ypreds.append(cpred) ylabels.append(clabel) im_name = os.path.split(im_id[0]) im_ids.append(im_name[1]) count = count + 1 np.savez('Test_results_16.npz', Label=ylabels, Predict=ypreds) df = pd.DataFrame(data={'Label': ylabels, "Predict": ypreds}) print(df.dtypes) df.to_pickle("./Test_results_19_resnet.pkl")
# In[29]: raw_tensor = misc.LoadImageToTensor(load_image_name, device, norm=False) raw_tensor = F.interpolate(raw_tensor, size=(224, 224), mode='bilinear', align_corners=False) # Now we will create illustrations of the combined saliency map. # In[30]: masked_tensor_raw, drop_map = getMask(raw_tensor, csmap) cs_heatmap, cs_result = visualize_cam(csmap, raw_tensor) cs_masked = misc.AlphaMask(raw_tensor, csmap).squeeze(0) cs_masked = misc.RangeNormalize(cs_masked) images = [] images.append( torch.stack([ raw_tensor.squeeze().cpu(), cs_heatmap.cpu(), cs_result.cpu(), cs_masked.cpu(), masked_tensor_raw[0, ].cpu() ], 0)) # Now, lets get the Grad-CAM++ saliency map only. # In[31]:
torch_img, _, _ = dataset.__getitem__(0) normed_torch_img = torch_img gradient = [] model = Combine(1) model = torch.load( "C:\\Users\\EthanZhu\\Box Sync\\Project\\CNN_LSTM\\Github\\dlfnirs\\train6\\model_5\\model\\Kfold_5_epoch_21.pth" ) for layer in model.children(): if type(layer) == nn.Conv2d: layer.register_hook(gradient.append(grad)) configs = [dict(model_type='vgg', arch=model, layer_name='features')] for config in configs: config['arch'].to('cuda').eval() cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) mask_pp, _ = gradcam_pp(normed_torch_img) heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) grid_image = make_grid(images, nrow=5)
def visualize_grad_cam(model, image_list, classes): cuda = torch.cuda.is_available() # returns True/False device = torch.device("cuda" if cuda else "cpu") # url = 'https://www.cs.toronto.edu/~kriz/cifar-10-sample/dog4.png' # sample image for GradCam testing on Custom Model # response = requests.get(url) # pil_img = PIL.Image.open(BytesIO(response.content)) # pil_img master_img_arr = [] for idx in range(len(image_list)): pil_img = image_list[idx]['data'] pil_img = (pil_img - pil_img.min()) / (pil_img.max() - pil_img.min()) pil_img = (pil_img * 255).astype(np.uint8) pil_img = transforms.ToPILImage(mode='RGB')(pil_img) target_label = image_list[idx]['target'] prediction_label = image_list[idx]['prediction'] # print('pre-processing the input') torch_img = transforms.Compose([ transforms.Resize((32, 32)), transforms.ToTensor() ])(pil_img).to(device) normed_torch_img = transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])(torch_img)[None] # print('loading the models:') # resnet = models.resnet18(pretrained=True) # vgg = models.vgg16(pretrained=True) # print('loading the models Finished!') configs = [ dict(model_type='resnet', arch=model, layer_name='layer3'), # My Model # dict(model_type='resnet', arch=resnet, layer_name='layer3'), # Reference # dict(model_type='vgg', arch=vgg, layer_name='features_29') # Reference ] for config in configs: config['arch'].to(device).eval() cams = [ [cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs ] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) mask_pp, _ = gradcam_pp(normed_torch_img) heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) for i in range(len(images)): master_img_arr.append({'image': images[i], 'target_label': target_label, 'prediction_label': prediction_label}) # === Using Matplotlib figure = plt.figure(figsize=(10, 50)) num_of_images = 125 for index in range(1, num_of_images + 1): ax = plt.subplot(25, 5, index) plt.tight_layout(pad=1.0) plt.axis('off') image = master_img_arr[index - 1]['image'] target_label = master_img_arr[index - 1]['target_label'] prediction_label = master_img_arr[index - 1]['prediction_label'] image = image.view(3, 32, 32).cpu().numpy().swapaxes(0, 2).swapaxes(0, 1) image = (image - image.min()) / (image.max() - image.min()) image = (image * 255).astype(np.uint8) plt.imshow(image) ax.set_title(f"target: {classes[target_label]} \n prediction: {classes[prediction_label]}") # Storing Image as output output_dir = '/home/prakash/Prakash/EVA4/Session-10/Notebooks' os.makedirs(output_dir, exist_ok=True) output_name = 'output_misclassified_gradcam.jpeg' output_path = os.path.join(output_dir, output_name) figure.savefig(output_path, bbox_inches="tight")
def generate_visual_result(gradcam, original_image, transformed_image, prediction, file_name): output_filename_visual_response = COVID19_CT_PATH_SAVE_VISUAL_RESPONSE + file_name mask, _ = gradcam( transformed_image ) # Create a GradCAM(Gradient-weighted Class Activation Mapping) based on http://gradcam.cloudcv.org/ heatmap, result = visualize_cam( mask, transformed_image ) # Based on the mask is created a heatmap visual response mask2 = np.zeros((224, 224)) mask2[...] = mask[0, 0, :, :] # Changing the tensor shape to a numpy array in a standar image shape heatmap = heatmap.numpy() np_heatmap = np.zeros( (heatmap.shape[1], heatmap.shape[2], heatmap.shape[0])) np_heatmap[..., 0] = heatmap[0, ...] np_heatmap[..., 1] = heatmap[1, ...] np_heatmap[..., 2] = heatmap[2, ...] # Image.fromarray(np.uint8(np_heatmap*255)).convert("RGBA").show() # Image.fromarray(np.uint8(original_image)).convert("RGBA").show() np_original_image = np.array(original_image) np_mask_resized = cv2.resize(mask2, np_original_image.shape[:2]) np_heatmap_resized = cv2.resize(np_heatmap, np_original_image.shape[:2]) np_heatmap_resized = np.uint8(255 * np_heatmap_resized) original_copy = np_original_image.copy() np_mask_resized[np.where(np_mask_resized < 0.3)] = 0 np_heatmap_resized[np.where(np_mask_resized == 0)] = 0 original_copy[np.where(np_mask_resized == 0)] = 0 np_original_image[np.where(np_mask_resized != 0)] = 0 # visual_response = np_heatmap_resized * 0.4 + np_original_image + original_copy * 0.5 visual_response = np.array(original_image) h, w, _ = visual_response.shape font = cv2.FONT_HERSHEY_SIMPLEX scale = 1 * (h / 512) # text = 'Stella AI Report' #thickness = 2 #color_text = (92, 6, 18) #textsize = cv2.getTextSize(text, font, scale, thickness)[0] # Get coords based on boundary #textX = int((w - textsize[0]) / 2) # Coord to put the image centered #textY = int(h - h * 0.03) # Over 3% of the image height #cv2.putText(img=visual_response, # text=text, # org=(textX, textY), # fontFace=font, # fontScale=scale, # This one scale the image proportionaly to its size (512 is the reference) # color=color_text, # Red color # thickness=thickness) text = 'COVID-19' scale = scale * 0.5 thickness = 2 color_text = (92, 6, 18) textsize_covid = cv2.getTextSize(text, font, scale, thickness)[0] x_initial_covid_text = round(w * 0.05) y_initial_covid_text = round(h - h * 0.1) cv2.putText( img=visual_response, text=text, org=(x_initial_covid_text, y_initial_covid_text), fontFace=font, fontScale= scale, # This one scale the image proportionaly to its size (512 is the reference) color=color_text, # Red color thickness=thickness) rectangle_height = h * 0.02 x_initial_pink_rectangle = round( x_initial_covid_text + w * 0.02 + textsize_covid[0] ) # x_initial_covid_text + 2% of image width + text width x_final_pink_rectangle = round(w - w * 0.05) # Image width - 5% of Image width y_initial_pink_rectangle = y_initial_covid_text y_final_pink_rectangle = round(y_initial_covid_text - rectangle_height) cv2.rectangle(img=visual_response, pt1=(x_initial_pink_rectangle, y_initial_pink_rectangle), pt2=(x_final_pink_rectangle, y_final_pink_rectangle), color=(237, 192, 198), thickness=cv2.FILLED) x_initial_red_rectangle = x_initial_pink_rectangle x_final_red_rectangle = round( x_initial_pink_rectangle + (x_final_pink_rectangle - x_initial_pink_rectangle) * prediction) cv2.rectangle(img=visual_response, pt1=(x_initial_red_rectangle, y_initial_pink_rectangle), pt2=(x_final_red_rectangle, y_final_pink_rectangle), color=(92, 6, 18), thickness=cv2.FILLED) font = cv2.FONT_HERSHEY_SIMPLEX text = str(round(prediction * 100)) + '%' scale = 0.3 * (h / 512) thickness = 1 color_text = (92, 6, 18) textsize_porcentage = cv2.getTextSize(text, font, scale, thickness)[0] cv2.putText( img=visual_response, text=text, org=(round(x_final_red_rectangle + w * 0.01), round(h - h * 0.1 - (rectangle_height - textsize_porcentage[1]) / 2)), fontFace=font, fontScale=scale, # This one is the scale of the image. It is proportionaly to its size (512 is the reference) color=color_text, # Red color thickness=thickness) # Image.fromarray(np.uint8(visual_response)).convert("RGBA").show() cv2.imwrite(output_filename_visual_response, cv2.cvtColor(visual_response, cv2.COLOR_RGB2BGR)) cv2.destroyAllWindows() return output_filename_visual_response
def make_plot_and_save(input_img, img_name, no_norm_image, segm, model, train_or_val, epoch=None, vis_prefix=None): global is_server # get Grad-CAM results and prepare them to show on the plot target_layer = model.layer4 gradcam = GradCAM(model, target_layer=target_layer) gradcam_pp = GradCAMpp(model, target_layer=target_layer) # sam_output shapes: # [1, 1, 56, 56]x3 , [1, 1, 28, 28]x4 [1, 1, 14, 14]x6 , [1, 1, 7, 7]x3 mask, no_norm_mask, logit, sam_output = gradcam(input_img) sam1_show = torch.squeeze(sam_output[0].cpu()).detach().numpy() sam4_show = torch.squeeze(sam_output[3].cpu()).detach().numpy() sam8_show = torch.squeeze(sam_output[7].cpu()).detach().numpy() sam14_show = torch.squeeze(sam_output[13].cpu()).detach().numpy() heatmap, result = visualize_cam(mask, no_norm_image) result_show = np.moveaxis(torch.squeeze(result).detach().numpy(), 0, -1) mask_pp, no_norm_mask_pp, logit_pp, sam_output_pp = gradcam_pp(input_img) heatmap_pp, result_pp = visualize_cam(mask_pp, no_norm_image) result_pp_show = np.moveaxis( torch.squeeze(result_pp).detach().numpy(), 0, -1) # prepare mask and original image to show on the plot segm_show = torch.squeeze(segm.cpu()).detach().numpy() segm_show = np.moveaxis(segm_show, 0, 2) input_show = np.moveaxis( torch.squeeze(no_norm_image).detach().numpy(), 0, -1) # draw and save the plot plt.close('all') fig, axs = plt.subplots(nrows=2, ncols=6, figsize=(24, 9)) plt.suptitle(f'{train_or_val}-Image: {img_name}') axs[1][0].imshow(segm_show) axs[1][0].set_title('Mask') axs[0][0].imshow(input_show) axs[0][0].set_title('Original Image') axs[0][1].imshow(result_show) axs[0][1].set_title('Grad-CAM') axs[1][1].imshow(result_pp_show) axs[1][1].set_title('Grad-CAM++') axs[1][2].imshow(sam1_show, cmap='gray') axs[1][2].set_title('SAM-1 relative') axs[0][2].imshow(sam1_show, vmin=0., vmax=1., cmap='gray') axs[0][2].set_title('SAM-1 absolute') axs[1][3].imshow(sam4_show, cmap='gray') axs[1][3].set_title('SAM-4 relative') axs[0][3].imshow(sam4_show, vmin=0., vmax=1., cmap='gray') axs[0][3].set_title('SAM-4 absolute') axs[1][4].imshow(sam8_show, cmap='gray') axs[1][4].set_title('SAM-8 relative') axs[0][4].imshow(sam8_show, vmin=0., vmax=1., cmap='gray') axs[0][4].set_title('SAM-8 absolute') axs[1][5].imshow(sam14_show, cmap='gray') axs[1][5].set_title('SAM-14 relative') axs[0][5].imshow(sam14_show, vmin=0., vmax=1., cmap='gray') axs[0][5].set_title('SAM-14 absolute') plt.show() if vis_prefix is not None: plt.savefig(f'vis/{vis_prefix}/{train_or_val}/{img_name}.png', bbox_inches='tight') if is_server: if epoch is not None: wandb.log({f'{train_or_val}/{img_name}': fig}, step=epoch) else: wandb.log({f'{train_or_val}/{img_name}': fig})
arr = torch.tensor(arr) arr = arr.view(-1, 1, 160, 160) arr = arr.cuda() arr = arr.type(torch.float32) img = torch.tensor(img) img = img.cuda() img = img.type(torch.float32) model_path = r'C:\Users\Fabian\stanford\fed_learning\rsync\fl\experiments\incl_subjects_site_one_slices_dataset_full\resnet_model_08-30_17-48_lr_0_001_pretrained_30epochs_rod_0_1_da_10.pth' resnet = torch.load(model_path) # resnet = nn.Sequential(*list(resnet.children())[:-2]) configs = [dict(model_type='resnet', arch=resnet, layer_name='layer4')] for config in configs: config['arch'].to(device).eval() cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)] for config in configs] images = [] for gradcam, gradcam_pp in cams: mask, _ = gradcam(arr) test = mask.cpu().numpy()[0, 0, :, :] heatmap, result = visualize_cam(mask, img) result = detach(result, transpose=True) hm = detach(heatmap, transpose=True) # mask_pp, _ = gradcam_pp(normed_torch_img) # heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img) # # images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp]) # # grid_image = make_grid(images, nrow=5) print('nice')
images = [] ct = 0 random.seed(0) for batch in valid_data: # or anything else you want to do if ct == 6: break target, distractors, idx = batch target = target[0].unsqueeze(0) distractors = [distractors[0][0].unsqueeze(0)] sm = simpleModel(model, distractors, word_counts, 's_t') sm.eval() gradcam = GradCAM(sm, sm.model.cnn.conv_net[8]) mask, _ = gradcam(target) heatmap_s, result = visualize_cam(mask, target) model.zero_grad() if use_distractors_in_sender: sm = simpleModel(model, target, word_counts, 's_d') sm.eval() gradcam = GradCAM(sm, sm.model.cnn.conv_net[6]) mask, _ = gradcam(distractors[0]) heatmap_s_d, result = visualize_cam(mask, distractors[0]) model.zero_grad() sm = simpleModel(model, distractors, word_counts, 'r_t') sm.train() gradcam = GradCAM(sm, sm.model.cnn.conv_net[6]) mask, _ = gradcam(target) heatmap_r, result = visualize_cam(mask, target)
def main(): # Initialize the model for this run model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True) model_ft.to(device) # Temporary header # directory - normal, bacteria, TB, COVID-19, virus dir_test = '/home/ubuntu/segmentation/output/COVID-19/' label = 3 # set 3 for COVID-19 for virus class # Data loader test_masked_images = sorted(glob.glob(dir_test + '*.npz')) #test_masks = sorted(glob.glob(dir_test + '*.mask.npy')) for masked_img in test_masked_images: test_masked_img = np.load(masked_img) #test_mask = np.load(mask) test_masked_img = Image.fromarray(test_masked_img).resize((1024, 1024)) #test_mask = Image.fromarray(test_mask).resize((1024,1024)) #test_img = np.asarray(test_img) #test_mask = np.round(np.asarray(test_mask)) #test_masked = np.multiply(test_img, test_mask) test_normalized = test_masked_img h_whole = test_normalized.shape[0] # original w w_whole = test_normalized.shape[1] # original h background = np.zeros((h_whole, w_whole)) background_indicer = np.zeros((h_whole, w_whole)) sum_prob_wt = 0.0 for i in range(header.repeat): non_zero_list = np.nonzero(test_normalized) random_index = random.randint(0, len(non_zero_list[0]) - 1) non_zero_row = non_zero_list[0][ random_index] # random non-zero row index non_zero_col = non_zero_list[1][ random_index] # random non-zero col index X_patch = test_normalized[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] X_patch_img = data_transforms( augmentation(Image.fromarray(X_patch), rand_p=0.0, mode='test')) X_patch_img_ = np.squeeze(np.asarray(X_patch_img)) X_patch_1 = np.expand_dims(X_patch_img_, axis=0) X_patch_2 = np.expand_dims(X_patch_img_, axis=0) X_patch_3 = np.expand_dims(X_patch_img_, axis=0) X_ = np.concatenate((X_patch_1, X_patch_2, X_patch_3), axis=0) X_ = np.expand_dims(X_, axis=0) X = torch.from_numpy(X_) X = X.to(device) checkpoint = torch.load( os.path.join(header.save_dir, str(header.inference_epoch) + '.pth')) model_ft.load_state_dict(checkpoint['model_state_dict']) model_ft.eval() outputs = model_ft(X) outputs_prob = F.softmax(outputs) prob = outputs_prob[0][label] prob_wt = prob.detach().cpu().numpy() gradcam = GradCAM.from_config(model_type='resnet', arch=model_ft, layer_name='layer4') mask, logit = gradcam(X, class_idx=label) mask_np = np.squeeze(mask.detach().cpu().numpy()) indicer = np.ones((224, 224)) mask_np = np.asarray( cv2.resize( mask_np, dsize=( int(min(w_whole, non_zero_col + (header.img_size / 2))) - int(max(0, non_zero_col - (header.img_size / 2))), int(min(h_whole, non_zero_row + (header.img_size / 2))) - int(max(0, non_zero_row - (header.img_size / 2)))))) indicer = np.asarray( cv2.resize( indicer, dsize=( int(min(w_whole, non_zero_col + (header.img_size / 2))) - int(max(0, non_zero_col - (header.img_size / 2))), int(min(h_whole, non_zero_row + (header.img_size / 2))) - int(max(0, non_zero_row - (header.img_size / 2)))))) mask_add = np.zeros((1024, 1024)) mask_add[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] = mask_np mask_add = mask_add * prob_wt indicer_add = np.zeros((1024, 1024)) indicer_add[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] = indicer indicer_add = indicer_add background = background + mask_add background_indicer = background_indicer + indicer_add # number in this indicer means how many time the area included. sum_prob_wt = sum_prob_wt + prob_wt final_mask = np.divide(background, background_indicer + 1e-7) final_mask = np.expand_dims(np.expand_dims(final_mask, axis=0), axis=0) torch_final_mask = torch.from_numpy(final_mask) test_img = np.asarray(Image.fromarray(test_img).resize((1024, 1024))) test_img = (test_img - test_img.min()) / test_img.max() test_img = np.expand_dims(test_img, axis=0) test_img = np.concatenate((test_img, test_img, test_img), axis=0) torch_final_img = torch.from_numpy(np.expand_dims(test_img, axis=0)) final_cam, cam_result = visualize_cam(torch_final_mask, torch_final_img) final_cam = (final_cam - final_cam.min()) / final_cam.max() final_cam_np = np.swapaxes(np.swapaxes(np.asarray(final_cam), 0, 2), 0, 1) test_img_np = np.swapaxes(np.swapaxes(test_img, 0, 2), 0, 1) final_combined = test_img_np + final_cam_np final_combined = (final_combined - final_combined.min()) / final_combined.max() plt.imshow(final_combined) plt.savefig( test_masked_img.split('.image.npy')[0] + '.patch.heatmap_' + '.png')