def __init__(self, model, target_layers=None, postprocessor=None, retain_graph=False):
"""
"Grad-CAM: Visual Explanations from Deep Networks via Gradient-based Localization"
https://arxiv.org/pdf/1610.02391.pdf
Look at Figure 2 on page 4
"""
super(GradCAM, self).__init__(model, postprocessor=postprocessor, retain_graph=retain_graph)
self.fmap_pool = OrderedDict()
self.grad_pool = OrderedDict()
self._target_layers = target_layers
if target_layers == 'full' or target_layers == 'auto':
target_layers = medcam_utils.get_layers(self.model)
elif isinstance(target_layers, str):
target_layers = [target_layers]
self.target_layers = target_layers
def inject(model,
output_dir=None,
backend='gcam',
layer='auto',
label=None,
data_shape='default',
save_maps=False,
save_pickle=False,
save_scores=False,
evaluate=False,
metric='wioa',
threshold='otsu',
retain_graph=False,
return_score=False,
replace=False,
return_attention=False,
cudnn=True,
enabled=True):
"""
Injects a model with medcam functionality to extract attention maps from it. The model can be used as usual.
Whenever model(input) or model.forward(input) is called medcam will extract the corresponding attention maps.
Args:
model: A CNN-based model that inherits from torch.nn.Module
output_dir: The directory to save any results to
backend: One of the implemented visualization backends.
'gbp': Guided-Backpropagation
'gcam': Grad-Cam
'ggcam': Guided-Grad-Cam
'gcampp': Grad-Cam++
layer: One or multiple layer names of the model from which attention maps will be extracted.
'auto': Selects the last layer from which attention maps can be extracted.
'full': Selects every layer from which attention maps can be extracted.
(layer name): A layer name of the model as string.
[(layer name 1), (layer name 2), ...]: A list of layer names of the model as string.
Note: Guided-Backpropagation ignores this parameter.
label: A class label of interest. Alternatively this can be a class discriminator that creates a mask with only the non masked logits being backwarded through the model.
Example for class label: label=1
Example for discriminator: label=lambda x: 0.5 < x
data_shape: The shape of the resulting attention maps. The given shape should exclude batch and channel dimension.
'default': The shape of the current input data, excluding batch and channel dimension.
save_maps: If the attention maps should be saved sorted by layer in the output_dir.
save_pickle: If the attention maps should be saved as a pickle file in the output_dir.
save_scores: If the evaluation scores should be saved as an excel file in the output_dir.
evaluate: If the attention maps should be evaluated. This requires a corresponding mask when calling model.forward().
metric: An evaluation metric for comparing the attention map with the mask.
'wioa': Weighted intersection over attention. Most suited for classification.
'ioa': Intersection over attention.
'iou': Intersection over union. Not suited for classification.
(A function): An evaluation function.
threshold: A threshold used during evaluation for ignoring low attention. Most models have low amounts of attention everywhere in an attention map due to the nature of CNN-based models. The threshold can be used to ignore these low amounts if wanted.
'otsu': Uses the otsu algorithm to determine a threshold.
(float): A value between 0 and 1 that is used as threshold.
retain_graph: If the computation graph should be retained or not.
return_score: If the evaluation evaluation of the current input should be returned in addition to the model output.
replace: If the model output should be replaced with the extracted attention map.
return_attention: If the attention map should be returned along with the unmodified model output.
cudnn: If cudnn should be disabled. Some models (e.g. LSTMs) crash when using medcam with enabled cudnn.
enabled: If medcam should be enabled.
Returns: A shallow copy of the model injected with medcam functionality.
"""
if _already_injected(model):
return model
if not cudnn:
torch.backends.cudnn.enabled = False
if output_dir is not None:
Path(output_dir).mkdir(parents=True, exist_ok=True)
model_clone = copy.copy(model)
model_clone.eval()
# Save the original forward of the model
# This forward will be called by the backend, so if someone writes a new backend they only need to call model.model_forward and not model.medcam_dict['model_forward']
setattr(model_clone, 'model_forward', model_clone.forward)
# Save every other attribute in a dict which is added to the model attributes
# It is ugly but it avoids name conflicts
medcam_dict = {}
medcam_dict['output_dir'] = output_dir
medcam_dict['layer'] = layer
medcam_dict['counter'] = 0
medcam_dict['save_scores'] = save_scores
medcam_dict['save_maps'] = save_maps
medcam_dict['save_pickle'] = save_pickle
medcam_dict['evaluate'] = evaluate
medcam_dict['metric'] = metric
medcam_dict['return_score'] = return_score
medcam_dict['_replace_output'] = replace
medcam_dict['return_attention'] = return_attention
medcam_dict['threshold'] = threshold
medcam_dict['label'] = label
medcam_dict['channels'] = 1 # TODO: Remove in a later version
medcam_dict['data_shape'] = data_shape
medcam_dict['pickle_maps'] = []
if evaluate:
medcam_dict['Evaluator'] = Evaluator(
output_dir + "/",
metric=metric,
threshold=threshold,
layer_ordering=medcam_utils.get_layers(model_clone))
medcam_dict['current_attention_map'] = None
medcam_dict['current_layer'] = None
medcam_dict['device'] = next(model_clone.parameters()).device
medcam_dict['tested'] = False
medcam_dict['enabled'] = enabled
setattr(model_clone, 'medcam_dict', medcam_dict)
if output_dir is None and (save_scores or save_maps or save_pickle
or evaluate):
raise ValueError(
"output_dir needs to be set if save_scores, save_maps, save_pickle or evaluate is set to true"
)
# Append methods methods to the model
model_clone.get_layers = types.MethodType(get_layers, model_clone)
model_clone.get_attention_map = types.MethodType(get_attention_map,
model_clone)
model_clone.save_attention_map = types.MethodType(save_attention_map,
model_clone)
model_clone.replace_output = types.MethodType(replace_output, model_clone)
model_clone.dump = types.MethodType(dump, model_clone)
model_clone.forward = types.MethodType(forward, model_clone)
model_clone.enable_medcam = types.MethodType(enable_medcam, model_clone)
model_clone.disable_medcam = types.MethodType(disable_medcam, model_clone)
model_clone.test_run = types.MethodType(test_run, model_clone)
model_clone._assign_backend = types.MethodType(_assign_backend,
model_clone)
model_clone._process_attention_maps = types.MethodType(
_process_attention_maps, model_clone)
model_clone._save_attention_map = types.MethodType(_save_attention_map,
model_clone)
model_clone._replace_output = types.MethodType(_replace_output,
model_clone)
model_backend, heatmap = _assign_backend(
backend, model_clone, layer, None,
retain_graph) # TODO: Remove postprocessor in a later version
medcam_dict['model_backend'] = model_backend
medcam_dict['heatmap'] = heatmap
return model_clone
def get_layers(model, reverse=False):
return medcam_utils.get_layers(model, reverse)
def layers(self, reverse=False):
"""Returns the layers of the model. Optionally reverses the order of the layers."""
return medcam_utils.get_layers(self.model, reverse)