def test_register_hooks_to_relu_layers(mocker, available_models): print() for name, model_module in available_models: print(f'Testing model: {name}', end='\r') stdout.write('\x1b[2K') model = model_module() relu_layers = find_relu_layers(model, nn.ReLU) for layer in relu_layers: mocker.spy(layer, 'register_forward_hook') mocker.spy(layer, 'register_backward_hook') backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) if 'inception' in name: input_ = torch.zeros([1, 3, 299, 299]) make_mock_output(mocker, model, target_class) backprop.calculate_gradients(input_, target_class, guided=True) for layer in relu_layers: layer.register_forward_hook.assert_called_once() layer.register_backward_hook.assert_called_once()
def test_handle_greyscale_input(mocker, model_grayscale): backprop = Backprop(model_grayscale) input_ = torch.zeros([1, 1, 224, 224], requires_grad=True) gradients = backprop.calculate_gradients(input_) assert gradients.shape == (1, 224, 224)
def test_register_hook_to_first_conv_layer(mocker, name, model_module): model = model_module() conv_layer = find_first_conv_layer(model, nn.modules.conv.Conv2d, 3) mocker.spy(conv_layer, 'register_backward_hook') Backprop(model) conv_layer.register_backward_hook.assert_called_once()
def multiple_saliency_maps(model, tensors, start, num, k=0, guide = True): backprop = Backprop(model) for i in range(start,start+num): cur_image = tensors[i,:,:,:].unsqueeze(0).requires_grad_(requires_grad=True) backprop.visualize(cur_image, k, guided=guide) plt.show() print("Range: ", start, " - ", start+num) start += num return start
def test_checks_input_size_for_inception_model(mocker): with pytest.raises(ValueError) as error: model = models.inception_v3() backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) backprop.calculate_gradients(input_, target_class) assert 'Image must be 299x299 for Inception models.' in str(error.value)
def test_handle_binary_classifier(mocker, model): backprop = Backprop(model) target_class = 0 input_ = torch.zeros([1, 3, 224, 224]) mock_output = torch.tensor([0.8]) mock_output.requires_grad = True mocker.patch.object(model, 'forward', return_value=mock_output) backprop.calculate_gradients(input_, target_class)
def test_zero_out_gradients(mocker, model): backprop = Backprop(model) mocker.spy(model, 'zero_grad') target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) make_mock_output(mocker, model, target_class) backprop.calculate_gradients(input_, target_class) model.zero_grad.assert_called_once()
def test_warn_when_prediction_is_wrong(mocker, model): backprop = Backprop(model) top_class = torch.tensor(1) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) make_mock_output(mocker, model, top_class) with pytest.warns(UserWarning): backprop.calculate_gradients(input_, target_class)
def test_return_max_across_color_channels_if_specified(mocker, model): backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) make_mock_output(mocker, model, target_class) gradients = backprop.calculate_gradients(input_, target_class, take_max=True) assert gradients.shape == (1, 224, 224)
def test_visualize_calls_calculate_gradients_twice(mocker, model): backprop = Backprop(model) mocker.spy(backprop, 'calculate_gradients') top_class = 5 target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) make_expected_gradient_target(top_class) make_mock_output(mocker, model, target_class) backprop.visualize(input_, target_class, use_gpu=True) assert backprop.calculate_gradients.call_count == 2
def test_register_backward_hook_to_first_conv_layer(mocker, available_models): print() for name, model_module in available_models: print(f'Testing model: {name}', end='\r') stdout.write('\x1b[2K') model = model_module() conv_layer = find_first_conv_layer(model, nn.modules.conv.Conv2d, 3) mocker.spy(conv_layer, 'register_backward_hook') Backprop(model) conv_layer.register_backward_hook.assert_called_once()
def saliency_map(model, img_p): """ Return saliency map over the image : shape (255, 255) """ # Load and preprocess image. X = load_and_preprocess_img(img_p) # (3, 255, 255 torch tensor) X.requires_grad_() # This is critical to actually get gradients. """ # Predict y. ypred = 4 # (for now: set ypred manually) # Can I get any gradients at all. with torch.set_grad_enabled(True): out = model(X) out = out.reshape((5,)) out.backward(torch.FloatTensor([1., 1., 1., 1., 1.])) saliency = X.grad print(saliency) """ # Predict grade and get gradient. Use flashtorch library. # See https://mc.ai/feature-visualisation-in-pytorch%E2%80%8A-%E2%80%8Asaliency-maps/ with torch.set_grad_enabled(True): backprop = Backprop(model) # flashtorch.saliency Backprop object. gradients = backprop.calculate_gradients(X, take_max=True, guided=False) # (1, 255, 255) # Cast image, saliency maps to numpy arrays. X = X.detach() # must 'detach' from gradients before slicing. img_np = X.numpy()[0] # (3, 255, 255) img_np = img_np.swapaxes(0, 1) # (255, 3, 255) img_np = img_np.swapaxes(1, 2) # (255, 255, 3) saliency_map_np = gradients.numpy()[0] # (255, 255) print(max(np.max(saliency_map_np, axis=0))) print(saliency_map_np) print(img_np.shape) print(saliency_map_np.shape) # Smooth heatmap. saliency_map_np = gaussian_filter(saliency_map_np, sigma=10) # Plot image and overlay saliency map. heatmap = sns.heatmap(saliency_map_np, alpha=0.5) heatmap.imshow(img_np, cmap="YlGnBu") plt.show() return saliency_map_np
def test_calc_gradients_of_top_class_if_target_not_provided(mocker, model): backprop = Backprop(model) top_class = 5 input_ = torch.zeros([1, 3, 224, 224]) target = make_expected_gradient_target(top_class) mock_output = make_mock_output(mocker, model, top_class) backprop.calculate_gradients(input_) args, kwargs = mock_output.backward.call_args assert torch.all(kwargs['gradient'].eq(target))
def di_saliency(): hockey_path_violence = '/media/david/datos/Violence DATA/HockeyFights/frames/violence' hockey_path_noviolence = '/media/david/datos/Violence DATA/HockeyFights/frames/nonviolence' datasetAll, labelsAll, numFramesAll = createDataset(hockey_path_violence, hockey_path_noviolence) #ordered print(len(datasetAll), len(labelsAll), len(numFramesAll)) input_size = 224 data_transforms = createTransforms(input_size) dataset_source = "frames" debugg_mode = False avgmaxDuration = 1.66 interval_duration = 0.3 numDiPerVideos = 1 batch_size = 1 num_workers = 1 # model = torch.load('models/alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar') path = os.path.join('models','alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar') # model, input_size = initialize_model( model_name='alexnet', num_classes=2, feature_extract=False, numDiPerVideos=1, joinType='OnPlateau', use_pretrained=True) # model = load_checkpoint(model,'models/alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar') # model.load_state_dict(torch.load(path)) model = torch.load(path) model.cuda() backprop = Backprop(model) image_datasets = { "train": ViolenceDatasetVideos( dataset=datasetAll, labels=labelsAll, spatial_transform=data_transforms["train"], source=dataset_source, interval_duration=interval_duration,difference=3, maxDuration=avgmaxDuration, nDynamicImages=numDiPerVideos, debugg_mode=debugg_mode, ), "test": ViolenceDatasetVideos( dataset=datasetAll, labels=labelsAll, spatial_transform=data_transforms["test"], source=dataset_source, interval_duration=interval_duration, difference=3, maxDuration=avgmaxDuration, nDynamicImages=numDiPerVideos, debugg_mode=debugg_mode, ) } dataloaders_dict = { "train": torch.utils.data.DataLoader( image_datasets["train"], batch_size=batch_size, shuffle=False, num_workers=num_workers, ), "test": torch.utils.data.DataLoader( image_datasets["test"], batch_size=batch_size, shuffle=False, num_workers=num_workers, ), } count = 0 max_plots = 5 for inputs, labels in dataloaders_dict["test"]: count += 1 if count > max_plots: break print('*' * 12) print('inputs size: ',inputs.size()) # inputs = inputs.permute(1, 0, 2, 3, 4) inputs = inputs.cuda() # labels = labels.to(self.device) backprop.visualize(inputs, target_class=None, guided=False, use_gpu=True, di=True)
def test_visualize_passes_gpu_flag(mocker, model): backprop = Backprop(model) mocker.spy(backprop, 'calculate_gradients') top_class = 5 target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) make_expected_gradient_target(top_class) make_mock_output(mocker, model, target_class) backprop.visualize(input_, target_class, use_gpu=True) _, _, kwargs = backprop.calculate_gradients.mock_calls[0] assert kwargs['use_gpu']
def visualise_cnn(model_name, image, class_label, model_path=None, title=None): if model_path is None: model = get_model(model_name) else: model = get_model(model_name) model = load_model(model, model_path) model.eval() backprop = Backprop(model) # Transform the input image to a tensor img = apply_transforms(image) # Set a target class from ImageNet task: 24 in case of great gray owl imagenet = ImageNetIndex() target_class = imagenet[class_label] # Ready to roll! backprop.visualize(img, target_class, guided=True, title=title)
def get_saliency_map(model, img_p, ypred): """ Return saliency map over the image : shape (255, 255) Parameters ---------- model (PyTorch model object) img_p (str) path to image ypred (int) 0-4. Used to control saliency map Returns ----------- img_np : img as (255, 255, 3) numpy array saliency_map_np : saliency map as (255, 255) numpy array TODO: Examine this code more closely. At the moment, saliency maps don't change much across classes. I think a different saliency mapping technique is needed. The Guided=True flag may be possiblw but at the moment causes NaN errors. """ # Load and preprocess image: (1, 3, 255, 255) torch tensor. X = load_and_preprocess_img(img_p) # Require gradient. X.requires_grad_() # This is critical to actually get gradients. # Get gradient using flashtorch. with torch.set_grad_enabled(True): backprop = Backprop(model) gradients = backprop.calculate_gradients(input_=X, target_class=ypred, take_max=True, guided=False) # (1, 255, 255) # Cast image and saliency maps to numpy arrays. X = X.detach() img_np = X.numpy()[0] # (3, 255, 255) img_np = img_np.transpose(1, 2, 0) # (255, 255, 3) saliency_map_np = gradients[0].numpy() #saliency_map_np = np.absolute(saliency_map_np) # absolute value # Smooth heatmap. saliency_map_np = gaussian_filter(saliency_map_np, sigma=10) return img_np, saliency_map_np
def test_calculate_gradients_for_all_models(mocker, name, model_module): model = model_module() backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) if 'inception' in name: input_ = torch.zeros([1, 3, 299, 299]) make_mock_output(mocker, model, target_class) gradients = backprop.calculate_gradients(input_, target_class, use_gpu=True) assert gradients.shape == input_.size()[1:]
def test_calc_gradients_of_top_class_if_prediction_is_wrong(mocker, model): backprop = Backprop(model) top_class = torch.tensor(5) target_class = 7 input_ = torch.zeros([1, 3, 224, 224]) target = make_expected_gradient_target(top_class) mock_output = make_mock_output(mocker, model, top_class) with pytest.warns(UserWarning): backprop.calculate_gradients(input_, target_class) args, kwargs = mock_output.backward.call_args assert torch.all(kwargs['gradient'].eq(target))
def saliency(): """### 1. Load an image""" buho = 'images/great_grey_owl.jpg' di1 = 'images/1.png' image = load_image(buho) # image = load_image(buho) # plt.imshow(image) # plt.title('Original image'+str(type(image))) # plt.axis('off'); # plt.show() """### 2. Load a pre-trained Model""" model = models.alexnet(pretrained=True) # model = torch.load('/content/alexnet-frames-Finetuned:False-1di-tempMaxPool-OnPlateau.tar') """### 3. Create an instance of Backprop with the model""" backprop = Backprop(model) """### 4. Visualize saliency maps""" # Transform the input image to a tensor owl = apply_transforms(image) # print(owl.size()) #torch.Size([/1, 3, 224, 224]) # input_size = 224 # data_transforms = createTransforms(input_size) # owl = data_transforms['test'](image) # owl = owl.unsqueeze(dim=0) # owl = owl.unsqueeze(dim=0) # owl = owl.permute(1, 0, 2, 3, 4) # print(owl.size()) # Set a target class from ImageNet task: 24 in case of great gray owl target_class = 24 # Ready to roll! backprop.visualize(owl, target_class=target_class, guided=True, use_gpu=True)
def test_calculate_gradients_for_all_models(mocker, available_models): print() for name, model in available_models: print(f'Testing model: {name}', end='\r') stdout.write('\x1b[2K') model = model() backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) if 'inception' in name: input_ = torch.zeros([1, 3, 299, 299]) make_mock_output(mocker, model, target_class) gradients = backprop.calculate_gradients(input_, target_class) assert gradients.shape == input_.size()[1:]
def test_register_hooks_to_relu_layers(mocker, name, model_module): model = model_module() relu_layers = find_relu_layers(model, nn.ReLU) for layer in relu_layers: mocker.spy(layer, 'register_forward_hook') mocker.spy(layer, 'register_backward_hook') backprop = Backprop(model) target_class = 5 input_ = torch.zeros([1, 3, 224, 224]) if 'inception' in name: input_ = torch.zeros([1, 3, 299, 299]) make_mock_output(mocker, model, target_class) backprop.calculate_gradients(input_, target_class, guided=True) for layer in relu_layers: layer.register_forward_hook.assert_called_once() layer.register_backward_hook.assert_called_once()
NIH_CXR_BASE = CXR_BASE.joinpath("nih/v1").resolve() test_df = pd.read_csv("~/cxr-jingyi/Age/NIH_test_2500.csv") path1 = test_df.iloc[0]['path'] path1 = NIH_CXR_BASE.joinpath(path1).resolve() import matplotlib.pyplot as plt from flashtorch.utils import apply_transforms, load_image from flashtorch.saliency import Backprop image = load_image(str(path1)) plt.imshow(image) model = MobileNet(16) checkpoint = torch.load( '/home/jingyi/cxr-jingyi/Age/result/supervised/model_best.pth.tar') model.load_state_dict(checkpoint['state_dict']) backprop = Backprop(model) # Transform the input image to a tensor owl = apply_transforms(image) # Set a target class from ImageNet task: 24 in case of great gray owl target_class = 16 # Ready to roll! backprop.visualize(owl, target_class, guided=True)
def test_set_model_to_eval_mode(mocker, model): mocker.spy(model, 'eval') Backprop(model) model.eval.assert_called_once()
import matplotlib.pyplot as plt import torchvision.models as models from flashtorch.utils import load_image from flashtorch.saliency import Backprop from flashtorch.utils import apply_transforms image = load_image("G:\EEGNet/test/test.jpg") plt.imshow(image) net = models.vgg16(pretrained=1) backprop = Backprop(net) input_ = apply_transforms(image) target_class = 24 backprop.visualize(input_, target_class, guided=True)
def visualize_helper(model_module, tensor=img, k=854): model = model_module(pretrained=True).float() backprop = Backprop(model) backprop.visualize(tensor, k, guided=True)
def visualize_saliency(model, tensor, k=1, guide=True): backprop = Backprop(model) backprop.visualize(tensor, k, guided=guide) plt.show()