def DisplayGradCamImages(model, model_type, layer, dataloader, classes, device, count=10): gradcam = GradCAM.from_config( **dict(model_type=model_type, arch=model, layer_name=layer)) dataiter = iter(dataloader) images, labels = dataiter.next() outputs = model(images.to(device)) _, predicted = torch.max(outputs.data, 1) for i in range(count): imagestodisplay = [] mask, _ = gradcam(images[i][np.newaxis, :].to(device)) heatmap, result = visualize_cam(mask, images[i][np.newaxis, :]) imagestodisplay.extend([images[i].cpu(), heatmap, result]) grid_image = make_grid(imagestodisplay, nrow=3) plt.figure(figsize=(20, 20)) plt.imshow(np.transpose(grid_image, (1, 2, 0))) plt.show() print( f"Prediction : {classes[predicted[i]]}, Actual : {classes[labels[i]]}" )
def DisplayMisclassifiedGradCamImages(model, model_type, layer, misclassified_indexes, device, classes): gradcam = GradCAM.from_config( **dict(model_type=model_type, arch=model, layer_name=layer)) x, y = 0, 0 fig, axs = plt.subplots(5, 5, figsize=(20, 20)) plt.setp(axs, xticks=[], yticks=[]) fig.subplots_adjust(wspace=0.7) images = list(misclassified_indexes.items())[:25] for index, results in images: img = results['data'] img = torch.from_numpy(img) actual_class = classes[results['actual']] predicted_class = classes[results['predicted']] mask, _ = gradcam(img[np.newaxis, :].to(device)) heatmap, result = visualize_cam(mask, img[np.newaxis, :]) result = np.transpose(result.cpu().numpy(), (1, 2, 0)) axs[x, y].imshow(result) axs[x, y].set_title('Actual Class:' + str(actual_class) + "\nPredicted class: " + str(predicted_class)) if y == 4: x += 1 y = 0 else: y += 1
def grad_cam(model, x_batch): gradcam = GradCAM.from_config(arch=model._modules['resnet'], model_type='resnet', layer_name='7') mask, _ = gradcam(x_batch) heatmap, result = visualize_cam(mask, x_batch) result = result.numpy().transpose(1, 2, 0) return heatmap, result
def grad_cam(model, model_type, layer_name, normed_torch_img, torch_img): config = dict(model_type=model_type, arch=model, layer_name=layer_name) config['arch'].eval() #.to(device) cam = GradCAM.from_config(**config) mask, _ = cam(normed_torch_img) heatmap, result = visualize_cam(mask, torch_img) return (transforms.ToPILImage()(result))
def __init__(self, weights=None, model_metadata=None): super().__init__(weights, model_metadata) # Se carga la arquitectura de una DenseNet169 desde torchvision. Adicionalmente, se carga los preentrenados # disponibles para esta arquitectura. Finalmente, se dispone de este modelo en la CPU. Nota: Esto debe ajustarse # para disponer de la posibilidad de alojar el modelo y sus pesos en la GPU si esta se encuentra disponible. self.model = models.densenet169(pretrained=True).cpu() # Se cargan los pre entrenados pretrained_net = torch.load(weights, map_location='cpu') self.model.load_state_dict(pretrained_net) self.model.eval() self.gradcam = GradCAM.from_config(model_type='densenet', arch=self.model, layer_name='features_norm5')
print("Image {}".format(f)) input_image_name = f # Our input image to process save_prefix = os.path.split(os.path.splitext(input_image_name)[0])[-1] # Chop the file extension and path load_image_name = os.path.join(input_dir, input_image_name) os.makedirs(output_dir, exist_ok=True) # Lets load in our image. We will do a simple resize on it. in_tensor = misc.LoadImageToTensor(load_image_name, device) in_tensor = F.interpolate(in_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False) # Now, lets get the Grad-CAM++ saliency map only. resnet_gradcam = GradCAM.from_config(model_type='resnet', arch=model, layer_name='layer4') cam_map, logit = resnet_gradcam(in_tensor) # Create our saliency map object. We hand it our Torch model and names for the layers we want to tap. get_salmap = maps.SaliencyModel(model, layers, output_size=[in_height,in_width], weights=weights, norm_method=norm_method) # Get Forward sal map csmap,smaps,_ = get_salmap(in_tensor) # Let's get our original input image back. We will just use this one for visualization. raw_tensor = misc.LoadImageToTensor(load_image_name, device, norm=False) raw_tensor = F.interpolate(raw_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False)
def main(): # Initialize the model for this run model_ft, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True) model_ft.to(device) # Temporary header # directory - normal, bacteria, TB, COVID-19, virus dir_test = '/home/ubuntu/segmentation/output/COVID-19/' label = 3 # set 3 for COVID-19 for virus class # Data loader test_masked_images = sorted(glob.glob(dir_test + '*.npz')) #test_masks = sorted(glob.glob(dir_test + '*.mask.npy')) for masked_img in test_masked_images: test_masked_img = np.load(masked_img) #test_mask = np.load(mask) test_masked_img = Image.fromarray(test_masked_img).resize((1024, 1024)) #test_mask = Image.fromarray(test_mask).resize((1024,1024)) #test_img = np.asarray(test_img) #test_mask = np.round(np.asarray(test_mask)) #test_masked = np.multiply(test_img, test_mask) test_normalized = test_masked_img h_whole = test_normalized.shape[0] # original w w_whole = test_normalized.shape[1] # original h background = np.zeros((h_whole, w_whole)) background_indicer = np.zeros((h_whole, w_whole)) sum_prob_wt = 0.0 for i in range(header.repeat): non_zero_list = np.nonzero(test_normalized) random_index = random.randint(0, len(non_zero_list[0]) - 1) non_zero_row = non_zero_list[0][ random_index] # random non-zero row index non_zero_col = non_zero_list[1][ random_index] # random non-zero col index X_patch = test_normalized[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] X_patch_img = data_transforms( augmentation(Image.fromarray(X_patch), rand_p=0.0, mode='test')) X_patch_img_ = np.squeeze(np.asarray(X_patch_img)) X_patch_1 = np.expand_dims(X_patch_img_, axis=0) X_patch_2 = np.expand_dims(X_patch_img_, axis=0) X_patch_3 = np.expand_dims(X_patch_img_, axis=0) X_ = np.concatenate((X_patch_1, X_patch_2, X_patch_3), axis=0) X_ = np.expand_dims(X_, axis=0) X = torch.from_numpy(X_) X = X.to(device) checkpoint = torch.load( os.path.join(header.save_dir, str(header.inference_epoch) + '.pth')) model_ft.load_state_dict(checkpoint['model_state_dict']) model_ft.eval() outputs = model_ft(X) outputs_prob = F.softmax(outputs) prob = outputs_prob[0][label] prob_wt = prob.detach().cpu().numpy() gradcam = GradCAM.from_config(model_type='resnet', arch=model_ft, layer_name='layer4') mask, logit = gradcam(X, class_idx=label) mask_np = np.squeeze(mask.detach().cpu().numpy()) indicer = np.ones((224, 224)) mask_np = np.asarray( cv2.resize( mask_np, dsize=( int(min(w_whole, non_zero_col + (header.img_size / 2))) - int(max(0, non_zero_col - (header.img_size / 2))), int(min(h_whole, non_zero_row + (header.img_size / 2))) - int(max(0, non_zero_row - (header.img_size / 2)))))) indicer = np.asarray( cv2.resize( indicer, dsize=( int(min(w_whole, non_zero_col + (header.img_size / 2))) - int(max(0, non_zero_col - (header.img_size / 2))), int(min(h_whole, non_zero_row + (header.img_size / 2))) - int(max(0, non_zero_row - (header.img_size / 2)))))) mask_add = np.zeros((1024, 1024)) mask_add[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] = mask_np mask_add = mask_add * prob_wt indicer_add = np.zeros((1024, 1024)) indicer_add[ int(max(0, non_zero_row - (header.img_size / 2)) ):int(min(h_whole, non_zero_row + (header.img_size / 2))), int(max(0, non_zero_col - (header.img_size / 2)) ):int(min(w_whole, non_zero_col + (header.img_size / 2)))] = indicer indicer_add = indicer_add background = background + mask_add background_indicer = background_indicer + indicer_add # number in this indicer means how many time the area included. sum_prob_wt = sum_prob_wt + prob_wt final_mask = np.divide(background, background_indicer + 1e-7) final_mask = np.expand_dims(np.expand_dims(final_mask, axis=0), axis=0) torch_final_mask = torch.from_numpy(final_mask) test_img = np.asarray(Image.fromarray(test_img).resize((1024, 1024))) test_img = (test_img - test_img.min()) / test_img.max() test_img = np.expand_dims(test_img, axis=0) test_img = np.concatenate((test_img, test_img, test_img), axis=0) torch_final_img = torch.from_numpy(np.expand_dims(test_img, axis=0)) final_cam, cam_result = visualize_cam(torch_final_mask, torch_final_img) final_cam = (final_cam - final_cam.min()) / final_cam.max() final_cam_np = np.swapaxes(np.swapaxes(np.asarray(final_cam), 0, 2), 0, 1) test_img_np = np.swapaxes(np.swapaxes(test_img, 0, 2), 0, 1) final_combined = test_img_np + final_cam_np final_combined = (final_combined - final_combined.min()) / final_combined.max() plt.imshow(final_combined) plt.savefig( test_masked_img.split('.image.npy')[0] + '.patch.heatmap_' + '.png')