def DisplayGradCamImages(model,
model_type,
layer,
dataloader,
classes,
device,
count=10):
gradcam = GradCAM.from_config(
**dict(model_type=model_type, arch=model, layer_name=layer))
dataiter = iter(dataloader)
images, labels = dataiter.next()
outputs = model(images.to(device))
_, predicted = torch.max(outputs.data, 1)
for i in range(count):
imagestodisplay = []
mask, _ = gradcam(images[i][np.newaxis, :].to(device))
heatmap, result = visualize_cam(mask, images[i][np.newaxis, :])
imagestodisplay.extend([images[i].cpu(), heatmap, result])
grid_image = make_grid(imagestodisplay, nrow=3)
plt.figure(figsize=(20, 20))
plt.imshow(np.transpose(grid_image, (1, 2, 0)))
plt.show()
print(
f"Prediction : {classes[predicted[i]]}, Actual : {classes[labels[i]]}"
)
def DisplayMisclassifiedGradCamImages(model, model_type, layer,
misclassified_indexes, device, classes):
gradcam = GradCAM.from_config(
**dict(model_type=model_type, arch=model, layer_name=layer))
x, y = 0, 0
fig, axs = plt.subplots(5, 5, figsize=(20, 20))
plt.setp(axs, xticks=[], yticks=[])
fig.subplots_adjust(wspace=0.7)
images = list(misclassified_indexes.items())[:25]
for index, results in images:
img = results['data']
img = torch.from_numpy(img)
actual_class = classes[results['actual']]
predicted_class = classes[results['predicted']]
mask, _ = gradcam(img[np.newaxis, :].to(device))
heatmap, result = visualize_cam(mask, img[np.newaxis, :])
result = np.transpose(result.cpu().numpy(), (1, 2, 0))
axs[x, y].imshow(result)
axs[x, y].set_title('Actual Class:' + str(actual_class) +
"\nPredicted class: " + str(predicted_class))
if y == 4:
x += 1
y = 0
else:
y += 1
def grad_cam(model, x_batch):
gradcam = GradCAM.from_config(arch=model._modules['resnet'],
model_type='resnet',
layer_name='7')
mask, _ = gradcam(x_batch)
heatmap, result = visualize_cam(mask, x_batch)
result = result.numpy().transpose(1, 2, 0)
return heatmap, result
def grad_cam(model, model_type, layer_name, normed_torch_img, torch_img):
config = dict(model_type=model_type, arch=model, layer_name=layer_name)
config['arch'].eval() #.to(device)
cam = GradCAM.from_config(**config)
mask, _ = cam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
return (transforms.ToPILImage()(result))
def __init__(self, weights=None, model_metadata=None):
super().__init__(weights, model_metadata)
# Se carga la arquitectura de una DenseNet169 desde torchvision. Adicionalmente, se carga los preentrenados
# disponibles para esta arquitectura. Finalmente, se dispone de este modelo en la CPU. Nota: Esto debe ajustarse
# para disponer de la posibilidad de alojar el modelo y sus pesos en la GPU si esta se encuentra disponible.
self.model = models.densenet169(pretrained=True).cpu()
# Se cargan los pre entrenados
pretrained_net = torch.load(weights, map_location='cpu')
self.model.load_state_dict(pretrained_net)
self.model.eval()
self.gradcam = GradCAM.from_config(model_type='densenet',
arch=self.model,
layer_name='features_norm5')
print("Image {}".format(f))
input_image_name = f # Our input image to process
save_prefix = os.path.split(os.path.splitext(input_image_name)[0])[-1] # Chop the file extension and path
load_image_name = os.path.join(input_dir, input_image_name)
os.makedirs(output_dir, exist_ok=True)
# Lets load in our image. We will do a simple resize on it.
in_tensor = misc.LoadImageToTensor(load_image_name, device)
in_tensor = F.interpolate(in_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False)
# Now, lets get the Grad-CAM++ saliency map only.
resnet_gradcam = GradCAM.from_config(model_type='resnet', arch=model, layer_name='layer4')
cam_map, logit = resnet_gradcam(in_tensor)
# Create our saliency map object. We hand it our Torch model and names for the layers we want to tap.
get_salmap = maps.SaliencyModel(model, layers, output_size=[in_height,in_width], weights=weights,
norm_method=norm_method)
# Get Forward sal map
csmap,smaps,_ = get_salmap(in_tensor)
# Let's get our original input image back. We will just use this one for visualization.
raw_tensor = misc.LoadImageToTensor(load_image_name, device, norm=False)
raw_tensor = F.interpolate(raw_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False)
def main():
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
model_ft.to(device)
# Temporary header
# directory - normal, bacteria, TB, COVID-19, virus
dir_test = '/home/ubuntu/segmentation/output/COVID-19/'
label = 3 # set 3 for COVID-19 for virus class
# Data loader
test_masked_images = sorted(glob.glob(dir_test + '*.npz'))
#test_masks = sorted(glob.glob(dir_test + '*.mask.npy'))
for masked_img in test_masked_images:
test_masked_img = np.load(masked_img)
#test_mask = np.load(mask)
test_masked_img = Image.fromarray(test_masked_img).resize((1024, 1024))
#test_mask = Image.fromarray(test_mask).resize((1024,1024))
#test_img = np.asarray(test_img)
#test_mask = np.round(np.asarray(test_mask))
#test_masked = np.multiply(test_img, test_mask)
test_normalized = test_masked_img
h_whole = test_normalized.shape[0] # original w
w_whole = test_normalized.shape[1] # original h
background = np.zeros((h_whole, w_whole))
background_indicer = np.zeros((h_whole, w_whole))
sum_prob_wt = 0.0
for i in range(header.repeat):
non_zero_list = np.nonzero(test_normalized)
random_index = random.randint(0, len(non_zero_list[0]) - 1)
non_zero_row = non_zero_list[0][
random_index] # random non-zero row index
non_zero_col = non_zero_list[1][
random_index] # random non-zero col index
X_patch = test_normalized[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col + (header.img_size / 2)))]
X_patch_img = data_transforms(
augmentation(Image.fromarray(X_patch), rand_p=0.0,
mode='test'))
X_patch_img_ = np.squeeze(np.asarray(X_patch_img))
X_patch_1 = np.expand_dims(X_patch_img_, axis=0)
X_patch_2 = np.expand_dims(X_patch_img_, axis=0)
X_patch_3 = np.expand_dims(X_patch_img_, axis=0)
X_ = np.concatenate((X_patch_1, X_patch_2, X_patch_3), axis=0)
X_ = np.expand_dims(X_, axis=0)
X = torch.from_numpy(X_)
X = X.to(device)
checkpoint = torch.load(
os.path.join(header.save_dir,
str(header.inference_epoch) + '.pth'))
model_ft.load_state_dict(checkpoint['model_state_dict'])
model_ft.eval()
outputs = model_ft(X)
outputs_prob = F.softmax(outputs)
prob = outputs_prob[0][label]
prob_wt = prob.detach().cpu().numpy()
gradcam = GradCAM.from_config(model_type='resnet',
arch=model_ft,
layer_name='layer4')
mask, logit = gradcam(X, class_idx=label)
mask_np = np.squeeze(mask.detach().cpu().numpy())
indicer = np.ones((224, 224))
mask_np = np.asarray(
cv2.resize(
mask_np,
dsize=(
int(min(w_whole, non_zero_col +
(header.img_size / 2))) -
int(max(0, non_zero_col - (header.img_size / 2))),
int(min(h_whole, non_zero_row +
(header.img_size / 2))) -
int(max(0, non_zero_row - (header.img_size / 2))))))
indicer = np.asarray(
cv2.resize(
indicer,
dsize=(
int(min(w_whole, non_zero_col +
(header.img_size / 2))) -
int(max(0, non_zero_col - (header.img_size / 2))),
int(min(h_whole, non_zero_row +
(header.img_size / 2))) -
int(max(0, non_zero_row - (header.img_size / 2))))))
mask_add = np.zeros((1024, 1024))
mask_add[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col +
(header.img_size / 2)))] = mask_np
mask_add = mask_add * prob_wt
indicer_add = np.zeros((1024, 1024))
indicer_add[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col +
(header.img_size / 2)))] = indicer
indicer_add = indicer_add
background = background + mask_add
background_indicer = background_indicer + indicer_add # number in this indicer means how many time the area included.
sum_prob_wt = sum_prob_wt + prob_wt
final_mask = np.divide(background, background_indicer + 1e-7)
final_mask = np.expand_dims(np.expand_dims(final_mask, axis=0), axis=0)
torch_final_mask = torch.from_numpy(final_mask)
test_img = np.asarray(Image.fromarray(test_img).resize((1024, 1024)))
test_img = (test_img - test_img.min()) / test_img.max()
test_img = np.expand_dims(test_img, axis=0)
test_img = np.concatenate((test_img, test_img, test_img), axis=0)
torch_final_img = torch.from_numpy(np.expand_dims(test_img, axis=0))
final_cam, cam_result = visualize_cam(torch_final_mask,
torch_final_img)
final_cam = (final_cam - final_cam.min()) / final_cam.max()
final_cam_np = np.swapaxes(np.swapaxes(np.asarray(final_cam), 0, 2), 0,
1)
test_img_np = np.swapaxes(np.swapaxes(test_img, 0, 2), 0, 1)
final_combined = test_img_np + final_cam_np
final_combined = (final_combined -
final_combined.min()) / final_combined.max()
plt.imshow(final_combined)
plt.savefig(
test_masked_img.split('.image.npy')[0] + '.patch.heatmap_' +
'.png')