Ejemplo n.º 1
0
def DisplayGradCamImages(model,
                         model_type,
                         layer,
                         dataloader,
                         classes,
                         device,
                         count=10):

    gradcam = GradCAM.from_config(
        **dict(model_type=model_type, arch=model, layer_name=layer))
    dataiter = iter(dataloader)
    images, labels = dataiter.next()
    outputs = model(images.to(device))
    _, predicted = torch.max(outputs.data, 1)

    for i in range(count):
        imagestodisplay = []
        mask, _ = gradcam(images[i][np.newaxis, :].to(device))
        heatmap, result = visualize_cam(mask, images[i][np.newaxis, :])
        imagestodisplay.extend([images[i].cpu(), heatmap, result])
        grid_image = make_grid(imagestodisplay, nrow=3)
        plt.figure(figsize=(20, 20))
        plt.imshow(np.transpose(grid_image, (1, 2, 0)))
        plt.show()
        print(
            f"Prediction : {classes[predicted[i]]}, Actual : {classes[labels[i]]}"
        )
Ejemplo n.º 2
0
def DisplayMisclassifiedGradCamImages(model, model_type, layer,
                                      misclassified_indexes, device, classes):

    gradcam = GradCAM.from_config(
        **dict(model_type=model_type, arch=model, layer_name=layer))

    x, y = 0, 0
    fig, axs = plt.subplots(5, 5, figsize=(20, 20))
    plt.setp(axs, xticks=[], yticks=[])
    fig.subplots_adjust(wspace=0.7)
    images = list(misclassified_indexes.items())[:25]
    for index, results in images:
        img = results['data']
        img = torch.from_numpy(img)

        actual_class = classes[results['actual']]
        predicted_class = classes[results['predicted']]

        mask, _ = gradcam(img[np.newaxis, :].to(device))
        heatmap, result = visualize_cam(mask, img[np.newaxis, :])
        result = np.transpose(result.cpu().numpy(), (1, 2, 0))

        axs[x, y].imshow(result)
        axs[x, y].set_title('Actual Class:' + str(actual_class) +
                            "\nPredicted class: " + str(predicted_class))

        if y == 4:
            x += 1
            y = 0
        else:
            y += 1
Ejemplo n.º 3
0
def grad_cam(model, x_batch):
    gradcam = GradCAM.from_config(arch=model._modules['resnet'],
                                  model_type='resnet',
                                  layer_name='7')
    mask, _ = gradcam(x_batch)
    heatmap, result = visualize_cam(mask, x_batch)
    result = result.numpy().transpose(1, 2, 0)
    return heatmap, result
def grad_cam(model, model_type, layer_name, normed_torch_img, torch_img):
    config = dict(model_type=model_type, arch=model, layer_name=layer_name)
    config['arch'].eval()  #.to(device)

    cam = GradCAM.from_config(**config)
    mask, _ = cam(normed_torch_img)
    heatmap, result = visualize_cam(mask, torch_img)

    return (transforms.ToPILImage()(result))
Ejemplo n.º 5
0
    def __init__(self, weights=None, model_metadata=None):
        super().__init__(weights, model_metadata)

        # Se carga la arquitectura de una DenseNet169 desde torchvision. Adicionalmente, se carga los preentrenados
        # disponibles para esta arquitectura. Finalmente, se dispone de este modelo en la CPU. Nota: Esto debe ajustarse
        # para disponer de la posibilidad de alojar el modelo y sus pesos en la GPU si esta se encuentra disponible.
        self.model = models.densenet169(pretrained=True).cpu()

        # Se cargan los pre entrenados
        pretrained_net = torch.load(weights, map_location='cpu')
        self.model.load_state_dict(pretrained_net)
        self.model.eval()
        self.gradcam = GradCAM.from_config(model_type='densenet',
                                           arch=self.model,
                                           layer_name='features_norm5')
Ejemplo n.º 6
0
    
    print("Image {}".format(f))
    
    input_image_name    = f    # Our input image to process 
    
    save_prefix         = os.path.split(os.path.splitext(input_image_name)[0])[-1]  # Chop the file extension and path
    load_image_name     = os.path.join(input_dir, input_image_name)
    
    os.makedirs(output_dir, exist_ok=True)

    # Lets load in our image. We will do a simple resize on it.
    in_tensor           = misc.LoadImageToTensor(load_image_name, device)
    in_tensor           = F.interpolate(in_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False)
    
    # Now, lets get the Grad-CAM++ saliency map only.    
    resnet_gradcam      = GradCAM.from_config(model_type='resnet', arch=model, layer_name='layer4')
    cam_map, logit      = resnet_gradcam(in_tensor)
    
    # Create our saliency map object. We hand it our Torch model and names for the layers we want to tap. 
    get_salmap          = maps.SaliencyModel(model, layers, output_size=[in_height,in_width], weights=weights, 
                                             norm_method=norm_method)

    
    # Get Forward sal map
    csmap,smaps,_       = get_salmap(in_tensor)


    # Let's get our original input image back. We will just use this one for visualization. 
    raw_tensor          = misc.LoadImageToTensor(load_image_name, device, norm=False)
    raw_tensor          = F.interpolate(raw_tensor, size=(in_height, in_width), mode='bilinear', align_corners=False)
    
def main():
    # Initialize the model for this run
    model_ft, input_size = initialize_model(model_name,
                                            num_classes,
                                            feature_extract,
                                            use_pretrained=True)
    model_ft.to(device)

    # Temporary header
    # directory - normal, bacteria, TB, COVID-19, virus
    dir_test = '/home/ubuntu/segmentation/output/COVID-19/'
    label = 3  # set 3 for COVID-19 for virus class

    # Data loader
    test_masked_images = sorted(glob.glob(dir_test + '*.npz'))
    #test_masks = sorted(glob.glob(dir_test + '*.mask.npy'))

    for masked_img in test_masked_images:

        test_masked_img = np.load(masked_img)
        #test_mask = np.load(mask)

        test_masked_img = Image.fromarray(test_masked_img).resize((1024, 1024))
        #test_mask = Image.fromarray(test_mask).resize((1024,1024))

        #test_img = np.asarray(test_img)
        #test_mask = np.round(np.asarray(test_mask))

        #test_masked = np.multiply(test_img, test_mask)

        test_normalized = test_masked_img

        h_whole = test_normalized.shape[0]  # original w
        w_whole = test_normalized.shape[1]  # original h

        background = np.zeros((h_whole, w_whole))
        background_indicer = np.zeros((h_whole, w_whole))

        sum_prob_wt = 0.0

        for i in range(header.repeat):

            non_zero_list = np.nonzero(test_normalized)

            random_index = random.randint(0, len(non_zero_list[0]) - 1)

            non_zero_row = non_zero_list[0][
                random_index]  # random non-zero row index
            non_zero_col = non_zero_list[1][
                random_index]  # random non-zero col index

            X_patch = test_normalized[
                int(max(0, non_zero_row - (header.img_size / 2))
                    ):int(min(h_whole, non_zero_row + (header.img_size / 2))),
                int(max(0, non_zero_col - (header.img_size / 2))
                    ):int(min(w_whole, non_zero_col + (header.img_size / 2)))]

            X_patch_img = data_transforms(
                augmentation(Image.fromarray(X_patch), rand_p=0.0,
                             mode='test'))
            X_patch_img_ = np.squeeze(np.asarray(X_patch_img))

            X_patch_1 = np.expand_dims(X_patch_img_, axis=0)
            X_patch_2 = np.expand_dims(X_patch_img_, axis=0)
            X_patch_3 = np.expand_dims(X_patch_img_, axis=0)

            X_ = np.concatenate((X_patch_1, X_patch_2, X_patch_3), axis=0)
            X_ = np.expand_dims(X_, axis=0)

            X = torch.from_numpy(X_)
            X = X.to(device)

            checkpoint = torch.load(
                os.path.join(header.save_dir,
                             str(header.inference_epoch) + '.pth'))
            model_ft.load_state_dict(checkpoint['model_state_dict'])
            model_ft.eval()
            outputs = model_ft(X)
            outputs_prob = F.softmax(outputs)

            prob = outputs_prob[0][label]
            prob_wt = prob.detach().cpu().numpy()

            gradcam = GradCAM.from_config(model_type='resnet',
                                          arch=model_ft,
                                          layer_name='layer4')

            mask, logit = gradcam(X, class_idx=label)
            mask_np = np.squeeze(mask.detach().cpu().numpy())
            indicer = np.ones((224, 224))

            mask_np = np.asarray(
                cv2.resize(
                    mask_np,
                    dsize=(
                        int(min(w_whole, non_zero_col +
                                (header.img_size / 2))) -
                        int(max(0, non_zero_col - (header.img_size / 2))),
                        int(min(h_whole, non_zero_row +
                                (header.img_size / 2))) -
                        int(max(0, non_zero_row - (header.img_size / 2))))))

            indicer = np.asarray(
                cv2.resize(
                    indicer,
                    dsize=(
                        int(min(w_whole, non_zero_col +
                                (header.img_size / 2))) -
                        int(max(0, non_zero_col - (header.img_size / 2))),
                        int(min(h_whole, non_zero_row +
                                (header.img_size / 2))) -
                        int(max(0, non_zero_row - (header.img_size / 2))))))

            mask_add = np.zeros((1024, 1024))
            mask_add[
                int(max(0, non_zero_row - (header.img_size / 2))
                    ):int(min(h_whole, non_zero_row + (header.img_size / 2))),
                int(max(0, non_zero_col - (header.img_size / 2))
                    ):int(min(w_whole, non_zero_col +
                              (header.img_size / 2)))] = mask_np
            mask_add = mask_add * prob_wt

            indicer_add = np.zeros((1024, 1024))
            indicer_add[
                int(max(0, non_zero_row - (header.img_size / 2))
                    ):int(min(h_whole, non_zero_row + (header.img_size / 2))),
                int(max(0, non_zero_col - (header.img_size / 2))
                    ):int(min(w_whole, non_zero_col +
                              (header.img_size / 2)))] = indicer
            indicer_add = indicer_add

            background = background + mask_add
            background_indicer = background_indicer + indicer_add  # number in this indicer means how many time the area included.

            sum_prob_wt = sum_prob_wt + prob_wt

        final_mask = np.divide(background, background_indicer + 1e-7)

        final_mask = np.expand_dims(np.expand_dims(final_mask, axis=0), axis=0)
        torch_final_mask = torch.from_numpy(final_mask)

        test_img = np.asarray(Image.fromarray(test_img).resize((1024, 1024)))
        test_img = (test_img - test_img.min()) / test_img.max()
        test_img = np.expand_dims(test_img, axis=0)
        test_img = np.concatenate((test_img, test_img, test_img), axis=0)
        torch_final_img = torch.from_numpy(np.expand_dims(test_img, axis=0))

        final_cam, cam_result = visualize_cam(torch_final_mask,
                                              torch_final_img)

        final_cam = (final_cam - final_cam.min()) / final_cam.max()
        final_cam_np = np.swapaxes(np.swapaxes(np.asarray(final_cam), 0, 2), 0,
                                   1)
        test_img_np = np.swapaxes(np.swapaxes(test_img, 0, 2), 0, 1)

        final_combined = test_img_np + final_cam_np
        final_combined = (final_combined -
                          final_combined.min()) / final_combined.max()

        plt.imshow(final_combined)
        plt.savefig(
            test_masked_img.split('.image.npy')[0] + '.patch.heatmap_' +
            '.png')