def test_register_hooks_to_relu_layers(mocker, available_models):
print()
for name, model_module in available_models:
print(f'Testing model: {name}', end='\r')
stdout.write('\x1b[2K')
model = model_module()
relu_layers = find_relu_layers(model, nn.ReLU)
for layer in relu_layers:
mocker.spy(layer, 'register_forward_hook')
mocker.spy(layer, 'register_backward_hook')
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
if 'inception' in name:
input_ = torch.zeros([1, 3, 299, 299])
make_mock_output(mocker, model, target_class)
backprop.calculate_gradients(input_, target_class, guided=True)
for layer in relu_layers:
layer.register_forward_hook.assert_called_once()
layer.register_backward_hook.assert_called_once()
def test_handle_greyscale_input(mocker, model_grayscale):
backprop = Backprop(model_grayscale)
input_ = torch.zeros([1, 1, 224, 224], requires_grad=True)
gradients = backprop.calculate_gradients(input_)
assert gradients.shape == (1, 224, 224)
def test_register_hook_to_first_conv_layer(mocker, name, model_module):
model = model_module()
conv_layer = find_first_conv_layer(model, nn.modules.conv.Conv2d, 3)
mocker.spy(conv_layer, 'register_backward_hook')
Backprop(model)
conv_layer.register_backward_hook.assert_called_once()
def multiple_saliency_maps(model, tensors, start, num, k=0, guide = True):
backprop = Backprop(model)
for i in range(start,start+num):
cur_image = tensors[i,:,:,:].unsqueeze(0).requires_grad_(requires_grad=True)
backprop.visualize(cur_image, k, guided=guide)
plt.show()
print("Range: ", start, " - ", start+num)
start += num
return start
def test_checks_input_size_for_inception_model(mocker):
with pytest.raises(ValueError) as error:
model = models.inception_v3()
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
backprop.calculate_gradients(input_, target_class)
assert 'Image must be 299x299 for Inception models.' in str(error.value)
def test_handle_binary_classifier(mocker, model):
backprop = Backprop(model)
target_class = 0
input_ = torch.zeros([1, 3, 224, 224])
mock_output = torch.tensor([0.8])
mock_output.requires_grad = True
mocker.patch.object(model, 'forward', return_value=mock_output)
backprop.calculate_gradients(input_, target_class)
def test_zero_out_gradients(mocker, model):
backprop = Backprop(model)
mocker.spy(model, 'zero_grad')
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
make_mock_output(mocker, model, target_class)
backprop.calculate_gradients(input_, target_class)
model.zero_grad.assert_called_once()
def test_warn_when_prediction_is_wrong(mocker, model):
backprop = Backprop(model)
top_class = torch.tensor(1)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
make_mock_output(mocker, model, top_class)
with pytest.warns(UserWarning):
backprop.calculate_gradients(input_, target_class)
def test_return_max_across_color_channels_if_specified(mocker, model):
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
make_mock_output(mocker, model, target_class)
gradients = backprop.calculate_gradients(input_,
target_class,
take_max=True)
assert gradients.shape == (1, 224, 224)
def test_visualize_calls_calculate_gradients_twice(mocker, model):
backprop = Backprop(model)
mocker.spy(backprop, 'calculate_gradients')
top_class = 5
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
make_expected_gradient_target(top_class)
make_mock_output(mocker, model, target_class)
backprop.visualize(input_, target_class, use_gpu=True)
assert backprop.calculate_gradients.call_count == 2
def test_register_backward_hook_to_first_conv_layer(mocker, available_models):
print()
for name, model_module in available_models:
print(f'Testing model: {name}', end='\r')
stdout.write('\x1b[2K')
model = model_module()
conv_layer = find_first_conv_layer(model, nn.modules.conv.Conv2d, 3)
mocker.spy(conv_layer, 'register_backward_hook')
Backprop(model)
conv_layer.register_backward_hook.assert_called_once()
def saliency_map(model, img_p):
"""
Return saliency map over the image : shape (255, 255)
"""
# Load and preprocess image.
X = load_and_preprocess_img(img_p) # (3, 255, 255 torch tensor)
X.requires_grad_() # This is critical to actually get gradients.
"""
# Predict y.
ypred = 4 # (for now: set ypred manually)
# Can I get any gradients at all.
with torch.set_grad_enabled(True):
out = model(X)
out = out.reshape((5,))
out.backward(torch.FloatTensor([1., 1., 1., 1., 1.]))
saliency = X.grad
print(saliency)
"""
# Predict grade and get gradient. Use flashtorch library.
# See https://mc.ai/feature-visualisation-in-pytorch%E2%80%8A-%E2%80%8Asaliency-maps/
with torch.set_grad_enabled(True):
backprop = Backprop(model) # flashtorch.saliency Backprop object.
gradients = backprop.calculate_gradients(X,
take_max=True,
guided=False) # (1, 255, 255)
# Cast image, saliency maps to numpy arrays.
X = X.detach() # must 'detach' from gradients before slicing.
img_np = X.numpy()[0] # (3, 255, 255)
img_np = img_np.swapaxes(0, 1) # (255, 3, 255)
img_np = img_np.swapaxes(1, 2) # (255, 255, 3)
saliency_map_np = gradients.numpy()[0] # (255, 255)
print(max(np.max(saliency_map_np, axis=0)))
print(saliency_map_np)
print(img_np.shape)
print(saliency_map_np.shape)
# Smooth heatmap.
saliency_map_np = gaussian_filter(saliency_map_np, sigma=10)
# Plot image and overlay saliency map.
heatmap = sns.heatmap(saliency_map_np, alpha=0.5)
heatmap.imshow(img_np, cmap="YlGnBu")
plt.show()
return saliency_map_np
def test_calc_gradients_of_top_class_if_target_not_provided(mocker, model):
backprop = Backprop(model)
top_class = 5
input_ = torch.zeros([1, 3, 224, 224])
target = make_expected_gradient_target(top_class)
mock_output = make_mock_output(mocker, model, top_class)
backprop.calculate_gradients(input_)
args, kwargs = mock_output.backward.call_args
assert torch.all(kwargs['gradient'].eq(target))
def di_saliency():
hockey_path_violence = '/media/david/datos/Violence DATA/HockeyFights/frames/violence'
hockey_path_noviolence = '/media/david/datos/Violence DATA/HockeyFights/frames/nonviolence'
datasetAll, labelsAll, numFramesAll = createDataset(hockey_path_violence, hockey_path_noviolence) #ordered
print(len(datasetAll), len(labelsAll), len(numFramesAll))
input_size = 224
data_transforms = createTransforms(input_size)
dataset_source = "frames"
debugg_mode = False
avgmaxDuration = 1.66
interval_duration = 0.3
numDiPerVideos = 1
batch_size = 1
num_workers = 1
# model = torch.load('models/alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar')
path = os.path.join('models','alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar')
# model, input_size = initialize_model( model_name='alexnet', num_classes=2, feature_extract=False, numDiPerVideos=1, joinType='OnPlateau', use_pretrained=True)
# model = load_checkpoint(model,'models/alexnet-frames-Finetuned:True-1di-tempMaxPool-OnPlateau.tar')
# model.load_state_dict(torch.load(path))
model = torch.load(path)
model.cuda()
backprop = Backprop(model)
image_datasets = {
"train": ViolenceDatasetVideos( dataset=datasetAll, labels=labelsAll, spatial_transform=data_transforms["train"], source=dataset_source,
interval_duration=interval_duration,difference=3, maxDuration=avgmaxDuration, nDynamicImages=numDiPerVideos, debugg_mode=debugg_mode, ),
"test": ViolenceDatasetVideos( dataset=datasetAll, labels=labelsAll, spatial_transform=data_transforms["test"], source=dataset_source,
interval_duration=interval_duration, difference=3, maxDuration=avgmaxDuration, nDynamicImages=numDiPerVideos, debugg_mode=debugg_mode, )
}
dataloaders_dict = {
"train": torch.utils.data.DataLoader( image_datasets["train"], batch_size=batch_size, shuffle=False, num_workers=num_workers, ),
"test": torch.utils.data.DataLoader( image_datasets["test"], batch_size=batch_size, shuffle=False, num_workers=num_workers, ),
}
count = 0
max_plots = 5
for inputs, labels in dataloaders_dict["test"]:
count += 1
if count > max_plots:
break
print('*' * 12)
print('inputs size: ',inputs.size())
# inputs = inputs.permute(1, 0, 2, 3, 4)
inputs = inputs.cuda()
# labels = labels.to(self.device)
backprop.visualize(inputs, target_class=None, guided=False, use_gpu=True, di=True)
def test_visualize_passes_gpu_flag(mocker, model):
backprop = Backprop(model)
mocker.spy(backprop, 'calculate_gradients')
top_class = 5
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
make_expected_gradient_target(top_class)
make_mock_output(mocker, model, target_class)
backprop.visualize(input_, target_class, use_gpu=True)
_, _, kwargs = backprop.calculate_gradients.mock_calls[0]
assert kwargs['use_gpu']
def visualise_cnn(model_name, image, class_label, model_path=None, title=None):
if model_path is None:
model = get_model(model_name)
else:
model = get_model(model_name)
model = load_model(model, model_path)
model.eval()
backprop = Backprop(model)
# Transform the input image to a tensor
img = apply_transforms(image)
# Set a target class from ImageNet task: 24 in case of great gray owl
imagenet = ImageNetIndex()
target_class = imagenet[class_label]
# Ready to roll!
backprop.visualize(img, target_class, guided=True, title=title)
def get_saliency_map(model, img_p, ypred):
"""
Return saliency map over the image : shape (255, 255)
Parameters
----------
model (PyTorch model object)
img_p (str) path to image
ypred (int) 0-4. Used to control saliency map
Returns
-----------
img_np : img as (255, 255, 3) numpy array
saliency_map_np : saliency map as (255, 255) numpy array
TODO:
Examine this code more closely. At the moment, saliency maps
don't change much across classes. I think a different saliency
mapping technique is needed. The Guided=True flag may be possiblw
but at the moment causes NaN errors.
"""
# Load and preprocess image: (1, 3, 255, 255) torch tensor.
X = load_and_preprocess_img(img_p)
# Require gradient.
X.requires_grad_() # This is critical to actually get gradients.
# Get gradient using flashtorch.
with torch.set_grad_enabled(True):
backprop = Backprop(model)
gradients = backprop.calculate_gradients(input_=X,
target_class=ypred,
take_max=True,
guided=False) # (1, 255, 255)
# Cast image and saliency maps to numpy arrays.
X = X.detach()
img_np = X.numpy()[0] # (3, 255, 255)
img_np = img_np.transpose(1, 2, 0) # (255, 255, 3)
saliency_map_np = gradients[0].numpy()
#saliency_map_np = np.absolute(saliency_map_np) # absolute value
# Smooth heatmap.
saliency_map_np = gaussian_filter(saliency_map_np, sigma=10)
return img_np, saliency_map_np
def test_calculate_gradients_for_all_models(mocker, name, model_module):
model = model_module()
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
if 'inception' in name:
input_ = torch.zeros([1, 3, 299, 299])
make_mock_output(mocker, model, target_class)
gradients = backprop.calculate_gradients(input_,
target_class,
use_gpu=True)
assert gradients.shape == input_.size()[1:]
def test_calc_gradients_of_top_class_if_prediction_is_wrong(mocker, model):
backprop = Backprop(model)
top_class = torch.tensor(5)
target_class = 7
input_ = torch.zeros([1, 3, 224, 224])
target = make_expected_gradient_target(top_class)
mock_output = make_mock_output(mocker, model, top_class)
with pytest.warns(UserWarning):
backprop.calculate_gradients(input_, target_class)
args, kwargs = mock_output.backward.call_args
assert torch.all(kwargs['gradient'].eq(target))
def saliency():
"""### 1. Load an image"""
buho = 'images/great_grey_owl.jpg'
di1 = 'images/1.png'
image = load_image(buho)
# image = load_image(buho)
# plt.imshow(image)
# plt.title('Original image'+str(type(image)))
# plt.axis('off');
# plt.show()
"""### 2. Load a pre-trained Model"""
model = models.alexnet(pretrained=True)
# model = torch.load('/content/alexnet-frames-Finetuned:False-1di-tempMaxPool-OnPlateau.tar')
"""### 3. Create an instance of Backprop with the model"""
backprop = Backprop(model)
"""### 4. Visualize saliency maps"""
# Transform the input image to a tensor
owl = apply_transforms(image)
# print(owl.size()) #torch.Size([/1, 3, 224, 224])
# input_size = 224
# data_transforms = createTransforms(input_size)
# owl = data_transforms['test'](image)
# owl = owl.unsqueeze(dim=0)
# owl = owl.unsqueeze(dim=0)
# owl = owl.permute(1, 0, 2, 3, 4)
# print(owl.size())
# Set a target class from ImageNet task: 24 in case of great gray owl
target_class = 24
# Ready to roll!
backprop.visualize(owl, target_class=target_class, guided=True, use_gpu=True)
def test_calculate_gradients_for_all_models(mocker, available_models):
print()
for name, model in available_models:
print(f'Testing model: {name}', end='\r')
stdout.write('\x1b[2K')
model = model()
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
if 'inception' in name:
input_ = torch.zeros([1, 3, 299, 299])
make_mock_output(mocker, model, target_class)
gradients = backprop.calculate_gradients(input_, target_class)
assert gradients.shape == input_.size()[1:]
def test_register_hooks_to_relu_layers(mocker, name, model_module):
model = model_module()
relu_layers = find_relu_layers(model, nn.ReLU)
for layer in relu_layers:
mocker.spy(layer, 'register_forward_hook')
mocker.spy(layer, 'register_backward_hook')
backprop = Backprop(model)
target_class = 5
input_ = torch.zeros([1, 3, 224, 224])
if 'inception' in name:
input_ = torch.zeros([1, 3, 299, 299])
make_mock_output(mocker, model, target_class)
backprop.calculate_gradients(input_, target_class, guided=True)
for layer in relu_layers:
layer.register_forward_hook.assert_called_once()
layer.register_backward_hook.assert_called_once()
NIH_CXR_BASE = CXR_BASE.joinpath("nih/v1").resolve()
test_df = pd.read_csv("~/cxr-jingyi/Age/NIH_test_2500.csv")
path1 = test_df.iloc[0]['path']
path1 = NIH_CXR_BASE.joinpath(path1).resolve()
import matplotlib.pyplot as plt
from flashtorch.utils import apply_transforms, load_image
from flashtorch.saliency import Backprop
image = load_image(str(path1))
plt.imshow(image)
model = MobileNet(16)
checkpoint = torch.load(
'/home/jingyi/cxr-jingyi/Age/result/supervised/model_best.pth.tar')
model.load_state_dict(checkpoint['state_dict'])
backprop = Backprop(model)
# Transform the input image to a tensor
owl = apply_transforms(image)
# Set a target class from ImageNet task: 24 in case of great gray owl
target_class = 16
# Ready to roll!
backprop.visualize(owl, target_class, guided=True)
def test_set_model_to_eval_mode(mocker, model):
mocker.spy(model, 'eval')
Backprop(model)
model.eval.assert_called_once()
import matplotlib.pyplot as plt
import torchvision.models as models
from flashtorch.utils import load_image
from flashtorch.saliency import Backprop
from flashtorch.utils import apply_transforms
image = load_image("G:\EEGNet/test/test.jpg")
plt.imshow(image)
net = models.vgg16(pretrained=1)
backprop = Backprop(net)
input_ = apply_transforms(image)
target_class = 24
backprop.visualize(input_, target_class, guided=True)
def visualize_helper(model_module, tensor=img, k=854):
model = model_module(pretrained=True).float()
backprop = Backprop(model)
backprop.visualize(tensor, k, guided=True)
def visualize_saliency(model, tensor, k=1, guide=True):
backprop = Backprop(model)
backprop.visualize(tensor, k, guided=guide)
plt.show()