def visualize_grad_cam(model):
cuda = torch.cuda.is_available() # returns True/False
device = torch.device("cuda" if cuda else "cpu")
url = 'https://www.cs.toronto.edu/~kriz/cifar-10-sample/dog4.png' # sample image for GradCam testing on Custom Model
response = requests.get(url)
pil_img = PIL.Image.open(BytesIO(response.content))
pil_img
# print('pre-processing the input')
torch_img = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor()
])(pil_img).to(device)
normed_torch_img = transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])(torch_img)[None]
# print('loading the models:')
# resnet = models.resnet18(pretrained=True)
# vgg = models.vgg16(pretrained=True)
# print('loading the models Finished!')
configs = [
dict(model_type='resnet', arch=model, layer_name='layer3'), # My Model
# dict(model_type='resnet', arch=resnet, layer_name='layer3'), # Reference
# dict(model_type='vgg', arch=vgg, layer_name='features_29') # Reference
]
for config in configs:
config['arch'].to(device).eval()
cams = [
[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs
]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
mask_pp, _ = gradcam_pp(normed_torch_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
images = make_grid(images, nrow=5, scale_each=False, padding=10, pad_value=140, range=5)
output_dir = '/home/prakash/Prakash/EVA4/Session-9/Notebooks'
os.makedirs(output_dir, exist_ok=True)
output_name = 'output.jpeg'
output_path = os.path.join(output_dir, output_name)
# print(output_path)
save_image(images, output_path)
grad_cam_output = PIL.Image.open(output_path)
print(grad_cam_output.size)
resized_output = grad_cam_output.resize((500, 120))
plt.imshow(np.asarray(resized_output))
def grad_cam(img, model, layer):
configs = [dict(model_type='resnet', arch=model, layer_name=layer)]
for config in configs:
config['arch'].to(device).eval()
torch_img = transforms.Compose([transforms.ToTensor()])(img).to(device)
normed_torch_img = transforms.Normalize(
[0.485, 0.456, 0.406], [0.229, 0.224, 0.225])(torch_img)[None]
cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
mask_pp, _ = gradcam_pp(normed_torch_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
images.extend(
[torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
return images
def show_gradcam(tensor, model):
# Feedforward image, calculate GradCAM and gather heatmap
if model.__class__.__name__ == 'ResNet':
target_layer = model.layer4[-1].conv2
gradcam = GradCAM(model, target_layer)
elif model.__class__.__name__ == 'DenseNet':
target_layer = model.features.norm5
gradcam = GradCAM(model, target_layer)
elif model.__class__.__name__ == 'DataParallel':
target_layer = model.module.densenet121.features.norm5
gradcam = GradCAM(model.module.densenet121, target_layer)
else:
raise ValueError('improper model')
mask, _ = gradcam(tensor)
heatmap, _ = visualize_cam(mask, tensor)
# heatmap from torch.tensor to numpy.array
mask = mask[0].permute(1, 2, 0).detach().cpu().numpy()
heatmap = heatmap.permute(1, 2, 0).numpy()
return heatmap, mask
def generate_saliency_map(img, img_name, directory, model_dict):
# Normalize PIL Image, Must be in PIL format
normalizer = Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
torch_img = torch.from_numpy(np.asarray(img)).permute(
2, 0, 1).unsqueeze(0).float().div(255).cuda()
torch_img = F.upsample(torch_img,
size=(512, 512),
mode='bilinear',
align_corners=False)
normed_torch_img = normalizer(torch_img)
# gradcam = GradCAM(model_dict, True)
gradcam_pp = GradCAMpp(model_dict, True)
mask_pp, _ = gradcam_pp(normed_torch_img)
mask_pp_np = mask_pp.detach().cpu().numpy()
heatmap_pp, result_pp = visualize_cam(mask_pp_np, torch_img)
# Only going to use result_pp (Saliency Map from GradCAM++)
os.makedirs(directory, exist_ok=True)
output_name = img_name
output_path = os.path.join(directory, output_name)
save_image(result_pp, output_path)
return output_path
def plot_cam(img, mean, sd, device):
"""
Plots the cam
"""
# convert to PIL
pil_img = F.to_pil_image(img)
# normed_torch_img = transform_test_albu(pil_img).to(device)
# call the transformation. To keep it simple, we are calling PyTorch way of transform
torch_img = transforms.Compose([transforms.ToTensor()])(pil_img).to(device)
normed_torch_img = transforms.Normalize(mean=mean, std=sd)(torch_img)[None]
# Call the GridCAM
config = dict(model_type='resnet', arch=net, layer_name='layer4')
gradcam = GradCAM.from_config(**config)
images = []
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
images.extend([torch_img.cpu(), heatmap, result])
grid_image = make_grid(images, nrow=5)
return transforms.ToPILImage()(grid_image)
def grad_cam_gen(model, img, path, apex = False, device = 'cuda'):
if apex:
model, optim = amp.initialize(model, optim, opt_level='O1')
configs = [dict(model_type='seresnext', arch=model, layer_name='layer4_2_se_module_fc2')]
# with amp.disable_casts():
for config in configs:
config['arch'].eval()
cams = [
[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs]
indices = {0: 'Epidural', 1: 'Intraparenchymal', 2: 'Intraventricular', 3: 'Subarachnoid', 4:'Subdural', 5:'any'}
for _, gradcam_pp in cams:
for cls_idx in range(6):
mask_pp, _ = gradcam_pp(img, cls_idx)
heatmap_pp, result_pp = visualize_cam(mask_pp, img)
result_pp = result_pp.cpu().numpy()
#convert image back to Height,Width,Channels
result_pp = np.transpose(result_pp, (1,2,0))
path = path.split('/')[-1].split('.')[0]
plt.imsave('uploads/{}_grad_cam_{}.png'.format(path, indices[cls_idx]), np.transpose(result_pp, (1, 0, 2)))
plt.show()
def grad_cam(model, x_batch):
gradcam = GradCAM.from_config(arch=model._modules['resnet'],
model_type='resnet',
layer_name='7')
mask, _ = gradcam(x_batch)
heatmap, result = visualize_cam(mask, x_batch)
result = result.numpy().transpose(1, 2, 0)
return heatmap, result
def predict():
print(request.method)
f = request.files['image']
file_path = os.path.join(root_dir, 'uploads', f.filename)
print(file_path)
f.save(file_path)
#file_path='/home/ramkik/covid19/static/uploads/E63574A7-4188-4C8D-8D17-9D67A18A1AFA.jpeg'
image = Image.open(file_path).convert('RGB')
image = transform(image)
normalize_image = normalize(image)
normalize_image = normalize_image.unsqueeze(0)
if torch.cuda.is_available():
normalize_image = normalize_image.cuda()
#print(normalize_image.shape)
output = model(normalize_image)
print(output)
# output = F.softmax(output)
# print(output)
prediction_score, pred_label_idx = torch.topk(output, 1)
pred_label_idx.squeeze_()
print(normalize_image.shape)
mask, _ = gradcam(normalize_image)
heatmap, result = visualize_cam(mask, image)
mask_pp, _ = gradcam_pp(normalize_image)
heatmap_pp, result_pp = visualize_cam(mask_pp, image, alpha=1.0)
save_image(result, 'result.png', nrow=1)
# grad = transforms.ToPILImage()(result_pp)
# plt.imshow(grad)
response = {}
print('output : ', output)
print(pred_label_idx)
print(prediction_score)
response['class'] = classes[pred_label_idx.item()]
response['score'] = str(prediction_score.item())
return response
def grad_cam(model, model_type, layer_name, normed_torch_img, torch_img):
config = dict(model_type=model_type, arch=model, layer_name=layer_name)
config['arch'].eval() #.to(device)
cam = GradCAM.from_config(**config)
mask, _ = cam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
return (transforms.ToPILImage()(result))
def TileMaps(tensor, mask_1, mask_2, mask_3, image_list=[]):
assert torch.is_tensor(tensor)
assert torch.is_tensor(mask_1)
assert torch.is_tensor(mask_2)
assert torch.is_tensor(mask_3)
heatmap_1, _ = visualize_cam(mask_1, tensor)
heatmap_2, _ = visualize_cam(mask_2, tensor)
heatmap_3, _ = visualize_cam(mask_3, tensor)
image_list.append(
torch.stack([
tensor.squeeze().cpu(),
heatmap_1.cpu(),
heatmap_2.cpu(),
heatmap_3.cpu()
], 0))
return image_list
def Grad_Cam(model, train_datasets):
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model.eval()
target_layer = model.features
gradcam = GradCAM(model, target_layer)
gradcam_pp = GradCAMpp(model, target_layer)
images = []
for i in range(10):
index = random.randint(0, 212)
first_inputs, _ = train_datasets.__getitem__(index)
inputs = first_inputs.to(device).unsqueeze(0)
mask, _ = gradcam(inputs)
heatmap, result = visualize_cam(mask, first_inputs)
mask_pp, _ = gradcam_pp(inputs)
heatmap_pp, result_pp = visualize_cam(mask_pp, first_inputs)
images.extend([first_inputs.cpu(), heatmap, heatmap_pp, result, result_pp])
grid_image = make_grid(images, nrow=5)
return transforms.ToPILImage()(grid_image)
def grad_cam():
net = ResNet18()
configs = [dict(model_type='resnet', arch=net, layer_name='layer4')]
for config in configs:
config['arch'].to(device).eval()
cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
mask_pp, _ = gradcam_pp(normed_torch_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
images.extend(
[torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
return make_grid(images, nrow=5)
def gradcam_exp(gradcam, gradcam_pp, inp, image, layer_name, f_size):
""" Compute the explanations obtained with GradCAM and GradCAM++ before showing them.
Parameters
----------
gradcam : GradCAM object
the GradCAM explanation model
gradcam_pp : GradCAMpp object
the GradCAM++ explanation model
inp : torch tensor
the input to the model (should have requires_grad = False)
image : numpy array
the image corresponding to the input
layer_name : string
name of the layer analyzed
f_size : tuple of ints
size of each figure in the matplotlib.pyplot.figure object
"""
mask, _ = gradcam(inp)
heatmap, result = visualize_cam(mask, inp)
mask_pp, _ = gradcam_pp(inp)
heatmap_pp, result_pp = visualize_cam(mask_pp, inp)
display_gradients(heatmap.detach().numpy(), f_size).suptitle("Grad-CAM for an image with label 0", size="xx-large")
display_gradients(heatmap_pp.detach().numpy(), f_size).suptitle("Grad-CAM++ for an image with label 0", size="xx-large")
# heatmap_show = np.swapaxes(np.swapaxes(heatmap, 0, 1), 1, 2)
# heatmap_pp_show = np.swapaxes(np.swapaxes(heatmap_pp, 0, 1), 1, 2)
result_show = np.swapaxes(np.swapaxes(result.detach(), 0, 1), 1, 2)
result_pp_show = np.swapaxes(np.swapaxes(result_pp.detach(), 0, 1), 1, 2)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(2*f_size[0], f_size[1]))
ax1.imshow(result_show)
ax1.set_title("With GradCAM")
ax2.imshow(result_pp_show)
ax2.set_title("With GradCAM++")
fig.suptitle("With respect to %s" % layer_name, size="xx-large")
def plot_imgs(imgs, figsize, label):
plt.figure(figsize = (figsize[0],figsize[1]))
for i in range(len(imgs)):
img = transform_img(imgs[i]).cuda()
configs = [
dict(model_type='densenet', arch=densenet121, layer_name='features'),
]
for config in configs:
config['arch'].cuda().eval()
cams = [
[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs
]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(img)
heatmap, result = visualize_cam(mask, img)
mask_pp, _ = gradcam_pp(img)
heatmap_pp, result_pp_densenet = visualize_cam(mask_pp, img)
images.extend([img.cpu(), heatmap, heatmap_pp, result, result_pp_densenet])
plt.subplot(1,len(imgs),i+1)
plt.title('Grad-CAM', fontsize = 30)
plt.imshow(transforms.ToPILImage()(result_pp_densenet))
plt.tight_layout()
plt.figure(figsize=(figsize[0],figsize[1]))
for i in range(len(imgs)):
img = transform_img(imgs[i])
plt.subplot(2,len(imgs),i+1)
plt.title(label[i], fontsize = 30)
plt.imshow(img[0,0,:,:])
plt.tight_layout()
def TileOutput(tensor, mask, mask_func, image_list = []):
assert torch.is_tensor(tensor)
assert torch.is_tensor(mask)
assert isinstance(image_list,list)
assert callable(mask_func)
heatmap, result = visualize_cam(mask, tensor)
hard_masked,_ = mask_func(tensor, mask)
hard_masked = hard_masked.squeeze(0)
masked = AlphaMask(tensor, mask).squeeze(0)
masked = RangeNormalize(masked)
image_list.append(torch.stack([tensor.squeeze().cpu(), heatmap.cpu(),
result.cpu(), masked.cpu(), hard_masked.cpu()], 0))
return image_list
def main(config):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
train_transform = transforms.Compose([
transforms.Scale(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
val_transform = transforms.Compose(
[transforms.Resize((224, 224)),
transforms.ToTensor()])
test_transform = transforms.Compose([transforms.ToTensor()])
trainset = AVADataset(csv_file=config.train_csv_file,
root_dir=config.train_img_path,
transform=train_transform)
valset = AVADataset(csv_file=config.val_csv_file,
root_dir=config.val_img_path,
transform=val_transform)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=config.train_batch_size,
shuffle=True,
num_workers=config.num_workers)
val_loader = torch.utils.data.DataLoader(valset,
batch_size=config.val_batch_size,
shuffle=False,
num_workers=config.num_workers)
# base_model = models.vgg16(pretrained=True)
# base_model = models.resnet18(pretrained=True)
base_model = models.resnet101(pretrained=True, progress=False)
# base_model = models.inception_v3(pretrained=True)
model = NIMA(base_model)
# model = NIMA()
if config.warm_start:
model.load_state_dict(
torch.load(
os.path.join(config.ckpt_path,
'epoch-%d.pkl' % config.warm_start_epoch)))
print('Successfully loaded model epoch-%d.pkl' %
config.warm_start_epoch)
if config.multi_gpu:
model.features = torch.nn.DataParallel(model.features,
device_ids=config.gpu_ids)
model = model.to(device)
else:
model = model.to(device)
conv_base_lr = config.conv_base_lr
dense_lr = config.dense_lr
optimizer = optim.SGD([{
'params': model.features.parameters(),
'lr': conv_base_lr
}, {
'params': model.classifier.parameters(),
'lr': dense_lr
}],
momentum=0.9)
# optimizer = optim.Adam( model.parameters(), lr = conv_base_lr, betas=(0.9,0.999))
# Loss functions
# criterion = torch.nn.L1Loss()
criterion = torch.nn.CrossEntropyLoss()
# send hyperparams
lrs.send({
'title': 'EMD Loss',
'train_batch_size': config.train_batch_size,
'val_batch_size': config.val_batch_size,
'optimizer': 'SGD',
'conv_base_lr': config.conv_base_lr,
'dense_lr': config.dense_lr,
'momentum': 0.9
})
param_num = 0
for param in model.parameters():
param_num += int(np.prod(param.shape))
print('Trainable params: %.2f million' % (param_num / 1e6))
if config.test:
# start.record()
print('Testing')
model.load_state_dict(
torch.load(
os.path.join(config.ckpt_path,
'epoch-%d.pkl' % config.warm_start_epoch)))
target_layer = model.features
# compute mean score
test_transform = test_transform #val_transform
testset = AVADataset(csv_file=config.test_csv_file,
root_dir=config.test_img_path,
transform=val_transform)
test_loader = torch.utils.data.DataLoader(
testset,
batch_size=config.test_batch_size,
shuffle=False,
num_workers=config.num_workers)
ypreds = []
ylabels = []
im_ids = []
# std_preds = []
count = 0
gradcam = GradCAM(model, target_layer)
for data in test_loader:
im_id = data['img_id']
im_name = os.path.split(im_id[0])
myname = os.path.splitext(im_name[1])
image = data['image'].to(device)
mask, _ = gradcam(image)
heatmap, result = visualize_cam(mask, image)
im = transforms.ToPILImage()(result)
im.save(myname[0] + ".jpg")
labels = data['annotations'].to(device).long()
output = model(image)
output = output.view(-1, 2)
bpred = output.to(torch.device("cpu"))
cpred = bpred.data.numpy()
blabel = labels.to(torch.device("cpu"))
clabel = blabel.data.numpy()
# predicted_mean, predicted_std = 0.0, 0.0
# for i, elem in enumerate(output, 1):
# predicted_mean += i * elem
# for j, elem in enumerate(output, 1):
# predicted_std += elem * (i - predicted_mean) ** 2
ypreds.append(cpred)
ylabels.append(clabel)
im_name = os.path.split(im_id[0])
im_ids.append(im_name[1])
count = count + 1
np.savez('Test_results_16.npz', Label=ylabels, Predict=ypreds)
df = pd.DataFrame(data={'Label': ylabels, "Predict": ypreds})
print(df.dtypes)
df.to_pickle("./Test_results_19_resnet.pkl")
# In[29]:
raw_tensor = misc.LoadImageToTensor(load_image_name, device, norm=False)
raw_tensor = F.interpolate(raw_tensor,
size=(224, 224),
mode='bilinear',
align_corners=False)
# Now we will create illustrations of the combined saliency map.
# In[30]:
masked_tensor_raw, drop_map = getMask(raw_tensor, csmap)
cs_heatmap, cs_result = visualize_cam(csmap, raw_tensor)
cs_masked = misc.AlphaMask(raw_tensor, csmap).squeeze(0)
cs_masked = misc.RangeNormalize(cs_masked)
images = []
images.append(
torch.stack([
raw_tensor.squeeze().cpu(),
cs_heatmap.cpu(),
cs_result.cpu(),
cs_masked.cpu(), masked_tensor_raw[0, ].cpu()
], 0))
# Now, lets get the Grad-CAM++ saliency map only.
# In[31]:
torch_img, _, _ = dataset.__getitem__(0)
normed_torch_img = torch_img
gradient = []
model = Combine(1)
model = torch.load(
"C:\\Users\\EthanZhu\\Box Sync\\Project\\CNN_LSTM\\Github\\dlfnirs\\train6\\model_5\\model\\Kfold_5_epoch_21.pth"
)
for layer in model.children():
if type(layer) == nn.Conv2d:
layer.register_hook(gradient.append(grad))
configs = [dict(model_type='vgg', arch=model, layer_name='features')]
for config in configs:
config['arch'].to('cuda').eval()
cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
mask_pp, _ = gradcam_pp(normed_torch_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
grid_image = make_grid(images, nrow=5)
def visualize_grad_cam(model, image_list, classes):
cuda = torch.cuda.is_available() # returns True/False
device = torch.device("cuda" if cuda else "cpu")
# url = 'https://www.cs.toronto.edu/~kriz/cifar-10-sample/dog4.png' # sample image for GradCam testing on Custom Model
# response = requests.get(url)
# pil_img = PIL.Image.open(BytesIO(response.content))
# pil_img
master_img_arr = []
for idx in range(len(image_list)):
pil_img = image_list[idx]['data']
pil_img = (pil_img - pil_img.min()) / (pil_img.max() - pil_img.min())
pil_img = (pil_img * 255).astype(np.uint8)
pil_img = transforms.ToPILImage(mode='RGB')(pil_img)
target_label = image_list[idx]['target']
prediction_label = image_list[idx]['prediction']
# print('pre-processing the input')
torch_img = transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor()
])(pil_img).to(device)
normed_torch_img = transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010])(torch_img)[None]
# print('loading the models:')
# resnet = models.resnet18(pretrained=True)
# vgg = models.vgg16(pretrained=True)
# print('loading the models Finished!')
configs = [
dict(model_type='resnet', arch=model, layer_name='layer3'), # My Model
# dict(model_type='resnet', arch=resnet, layer_name='layer3'), # Reference
# dict(model_type='vgg', arch=vgg, layer_name='features_29') # Reference
]
for config in configs:
config['arch'].to(device).eval()
cams = [
[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs
]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(normed_torch_img)
heatmap, result = visualize_cam(mask, torch_img)
mask_pp, _ = gradcam_pp(normed_torch_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
for i in range(len(images)):
master_img_arr.append({'image': images[i], 'target_label': target_label, 'prediction_label': prediction_label})
# === Using Matplotlib
figure = plt.figure(figsize=(10, 50))
num_of_images = 125
for index in range(1, num_of_images + 1):
ax = plt.subplot(25, 5, index)
plt.tight_layout(pad=1.0)
plt.axis('off')
image = master_img_arr[index - 1]['image']
target_label = master_img_arr[index - 1]['target_label']
prediction_label = master_img_arr[index - 1]['prediction_label']
image = image.view(3, 32, 32).cpu().numpy().swapaxes(0, 2).swapaxes(0, 1)
image = (image - image.min()) / (image.max() - image.min())
image = (image * 255).astype(np.uint8)
plt.imshow(image)
ax.set_title(f"target: {classes[target_label]} \n prediction: {classes[prediction_label]}")
# Storing Image as output
output_dir = '/home/prakash/Prakash/EVA4/Session-10/Notebooks'
os.makedirs(output_dir, exist_ok=True)
output_name = 'output_misclassified_gradcam.jpeg'
output_path = os.path.join(output_dir, output_name)
figure.savefig(output_path, bbox_inches="tight")
def generate_visual_result(gradcam, original_image, transformed_image,
prediction, file_name):
output_filename_visual_response = COVID19_CT_PATH_SAVE_VISUAL_RESPONSE + file_name
mask, _ = gradcam(
transformed_image
) # Create a GradCAM(Gradient-weighted Class Activation Mapping) based on http://gradcam.cloudcv.org/
heatmap, result = visualize_cam(
mask, transformed_image
) # Based on the mask is created a heatmap visual response
mask2 = np.zeros((224, 224))
mask2[...] = mask[0, 0, :, :]
# Changing the tensor shape to a numpy array in a standar image shape
heatmap = heatmap.numpy()
np_heatmap = np.zeros(
(heatmap.shape[1], heatmap.shape[2], heatmap.shape[0]))
np_heatmap[..., 0] = heatmap[0, ...]
np_heatmap[..., 1] = heatmap[1, ...]
np_heatmap[..., 2] = heatmap[2, ...]
# Image.fromarray(np.uint8(np_heatmap*255)).convert("RGBA").show()
# Image.fromarray(np.uint8(original_image)).convert("RGBA").show()
np_original_image = np.array(original_image)
np_mask_resized = cv2.resize(mask2, np_original_image.shape[:2])
np_heatmap_resized = cv2.resize(np_heatmap, np_original_image.shape[:2])
np_heatmap_resized = np.uint8(255 * np_heatmap_resized)
original_copy = np_original_image.copy()
np_mask_resized[np.where(np_mask_resized < 0.3)] = 0
np_heatmap_resized[np.where(np_mask_resized == 0)] = 0
original_copy[np.where(np_mask_resized == 0)] = 0
np_original_image[np.where(np_mask_resized != 0)] = 0
# visual_response = np_heatmap_resized * 0.4 + np_original_image + original_copy * 0.5
visual_response = np.array(original_image)
h, w, _ = visual_response.shape
font = cv2.FONT_HERSHEY_SIMPLEX
scale = 1 * (h / 512)
# text = 'Stella AI Report'
#thickness = 2
#color_text = (92, 6, 18)
#textsize = cv2.getTextSize(text, font, scale, thickness)[0]
# Get coords based on boundary
#textX = int((w - textsize[0]) / 2) # Coord to put the image centered
#textY = int(h - h * 0.03) # Over 3% of the image height
#cv2.putText(img=visual_response,
# text=text,
# org=(textX, textY),
# fontFace=font,
# fontScale=scale, # This one scale the image proportionaly to its size (512 is the reference)
# color=color_text, # Red color
# thickness=thickness)
text = 'COVID-19'
scale = scale * 0.5
thickness = 2
color_text = (92, 6, 18)
textsize_covid = cv2.getTextSize(text, font, scale, thickness)[0]
x_initial_covid_text = round(w * 0.05)
y_initial_covid_text = round(h - h * 0.1)
cv2.putText(
img=visual_response,
text=text,
org=(x_initial_covid_text, y_initial_covid_text),
fontFace=font,
fontScale=
scale, # This one scale the image proportionaly to its size (512 is the reference)
color=color_text, # Red color
thickness=thickness)
rectangle_height = h * 0.02
x_initial_pink_rectangle = round(
x_initial_covid_text + w * 0.02 + textsize_covid[0]
) # x_initial_covid_text + 2% of image width + text width
x_final_pink_rectangle = round(w -
w * 0.05) # Image width - 5% of Image width
y_initial_pink_rectangle = y_initial_covid_text
y_final_pink_rectangle = round(y_initial_covid_text - rectangle_height)
cv2.rectangle(img=visual_response,
pt1=(x_initial_pink_rectangle, y_initial_pink_rectangle),
pt2=(x_final_pink_rectangle, y_final_pink_rectangle),
color=(237, 192, 198),
thickness=cv2.FILLED)
x_initial_red_rectangle = x_initial_pink_rectangle
x_final_red_rectangle = round(
x_initial_pink_rectangle +
(x_final_pink_rectangle - x_initial_pink_rectangle) * prediction)
cv2.rectangle(img=visual_response,
pt1=(x_initial_red_rectangle, y_initial_pink_rectangle),
pt2=(x_final_red_rectangle, y_final_pink_rectangle),
color=(92, 6, 18),
thickness=cv2.FILLED)
font = cv2.FONT_HERSHEY_SIMPLEX
text = str(round(prediction * 100)) + '%'
scale = 0.3 * (h / 512)
thickness = 1
color_text = (92, 6, 18)
textsize_porcentage = cv2.getTextSize(text, font, scale, thickness)[0]
cv2.putText(
img=visual_response,
text=text,
org=(round(x_final_red_rectangle + w * 0.01),
round(h - h * 0.1 -
(rectangle_height - textsize_porcentage[1]) / 2)),
fontFace=font,
fontScale=scale,
# This one is the scale of the image. It is proportionaly to its size (512 is the reference)
color=color_text, # Red color
thickness=thickness)
# Image.fromarray(np.uint8(visual_response)).convert("RGBA").show()
cv2.imwrite(output_filename_visual_response,
cv2.cvtColor(visual_response, cv2.COLOR_RGB2BGR))
cv2.destroyAllWindows()
return output_filename_visual_response
def make_plot_and_save(input_img,
img_name,
no_norm_image,
segm,
model,
train_or_val,
epoch=None,
vis_prefix=None):
global is_server
# get Grad-CAM results and prepare them to show on the plot
target_layer = model.layer4
gradcam = GradCAM(model, target_layer=target_layer)
gradcam_pp = GradCAMpp(model, target_layer=target_layer)
# sam_output shapes:
# [1, 1, 56, 56]x3 , [1, 1, 28, 28]x4 [1, 1, 14, 14]x6 , [1, 1, 7, 7]x3
mask, no_norm_mask, logit, sam_output = gradcam(input_img)
sam1_show = torch.squeeze(sam_output[0].cpu()).detach().numpy()
sam4_show = torch.squeeze(sam_output[3].cpu()).detach().numpy()
sam8_show = torch.squeeze(sam_output[7].cpu()).detach().numpy()
sam14_show = torch.squeeze(sam_output[13].cpu()).detach().numpy()
heatmap, result = visualize_cam(mask, no_norm_image)
result_show = np.moveaxis(torch.squeeze(result).detach().numpy(), 0, -1)
mask_pp, no_norm_mask_pp, logit_pp, sam_output_pp = gradcam_pp(input_img)
heatmap_pp, result_pp = visualize_cam(mask_pp, no_norm_image)
result_pp_show = np.moveaxis(
torch.squeeze(result_pp).detach().numpy(), 0, -1)
# prepare mask and original image to show on the plot
segm_show = torch.squeeze(segm.cpu()).detach().numpy()
segm_show = np.moveaxis(segm_show, 0, 2)
input_show = np.moveaxis(
torch.squeeze(no_norm_image).detach().numpy(), 0, -1)
# draw and save the plot
plt.close('all')
fig, axs = plt.subplots(nrows=2, ncols=6, figsize=(24, 9))
plt.suptitle(f'{train_or_val}-Image: {img_name}')
axs[1][0].imshow(segm_show)
axs[1][0].set_title('Mask')
axs[0][0].imshow(input_show)
axs[0][0].set_title('Original Image')
axs[0][1].imshow(result_show)
axs[0][1].set_title('Grad-CAM')
axs[1][1].imshow(result_pp_show)
axs[1][1].set_title('Grad-CAM++')
axs[1][2].imshow(sam1_show, cmap='gray')
axs[1][2].set_title('SAM-1 relative')
axs[0][2].imshow(sam1_show, vmin=0., vmax=1., cmap='gray')
axs[0][2].set_title('SAM-1 absolute')
axs[1][3].imshow(sam4_show, cmap='gray')
axs[1][3].set_title('SAM-4 relative')
axs[0][3].imshow(sam4_show, vmin=0., vmax=1., cmap='gray')
axs[0][3].set_title('SAM-4 absolute')
axs[1][4].imshow(sam8_show, cmap='gray')
axs[1][4].set_title('SAM-8 relative')
axs[0][4].imshow(sam8_show, vmin=0., vmax=1., cmap='gray')
axs[0][4].set_title('SAM-8 absolute')
axs[1][5].imshow(sam14_show, cmap='gray')
axs[1][5].set_title('SAM-14 relative')
axs[0][5].imshow(sam14_show, vmin=0., vmax=1., cmap='gray')
axs[0][5].set_title('SAM-14 absolute')
plt.show()
if vis_prefix is not None:
plt.savefig(f'vis/{vis_prefix}/{train_or_val}/{img_name}.png',
bbox_inches='tight')
if is_server:
if epoch is not None:
wandb.log({f'{train_or_val}/{img_name}': fig}, step=epoch)
else:
wandb.log({f'{train_or_val}/{img_name}': fig})
arr = torch.tensor(arr)
arr = arr.view(-1, 1, 160, 160)
arr = arr.cuda()
arr = arr.type(torch.float32)
img = torch.tensor(img)
img = img.cuda()
img = img.type(torch.float32)
model_path = r'C:\Users\Fabian\stanford\fed_learning\rsync\fl\experiments\incl_subjects_site_one_slices_dataset_full\resnet_model_08-30_17-48_lr_0_001_pretrained_30epochs_rod_0_1_da_10.pth'
resnet = torch.load(model_path)
# resnet = nn.Sequential(*list(resnet.children())[:-2])
configs = [dict(model_type='resnet', arch=resnet, layer_name='layer4')]
for config in configs:
config['arch'].to(device).eval()
cams = [[cls.from_config(**config) for cls in (GradCAM, GradCAMpp)]
for config in configs]
images = []
for gradcam, gradcam_pp in cams:
mask, _ = gradcam(arr)
test = mask.cpu().numpy()[0, 0, :, :]
heatmap, result = visualize_cam(mask, img)
result = detach(result, transpose=True)
hm = detach(heatmap, transpose=True)
# mask_pp, _ = gradcam_pp(normed_torch_img)
# heatmap_pp, result_pp = visualize_cam(mask_pp, torch_img)
#
# images.extend([torch_img.cpu(), heatmap, heatmap_pp, result, result_pp])
#
# grid_image = make_grid(images, nrow=5)
print('nice')
images = []
ct = 0
random.seed(0)
for batch in valid_data: # or anything else you want to do
if ct == 6:
break
target, distractors, idx = batch
target = target[0].unsqueeze(0)
distractors = [distractors[0][0].unsqueeze(0)]
sm = simpleModel(model, distractors, word_counts, 's_t')
sm.eval()
gradcam = GradCAM(sm, sm.model.cnn.conv_net[8])
mask, _ = gradcam(target)
heatmap_s, result = visualize_cam(mask, target)
model.zero_grad()
if use_distractors_in_sender:
sm = simpleModel(model, target, word_counts, 's_d')
sm.eval()
gradcam = GradCAM(sm, sm.model.cnn.conv_net[6])
mask, _ = gradcam(distractors[0])
heatmap_s_d, result = visualize_cam(mask, distractors[0])
model.zero_grad()
sm = simpleModel(model, distractors, word_counts, 'r_t')
sm.train()
gradcam = GradCAM(sm, sm.model.cnn.conv_net[6])
mask, _ = gradcam(target)
heatmap_r, result = visualize_cam(mask, target)
def main():
# Initialize the model for this run
model_ft, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
model_ft.to(device)
# Temporary header
# directory - normal, bacteria, TB, COVID-19, virus
dir_test = '/home/ubuntu/segmentation/output/COVID-19/'
label = 3 # set 3 for COVID-19 for virus class
# Data loader
test_masked_images = sorted(glob.glob(dir_test + '*.npz'))
#test_masks = sorted(glob.glob(dir_test + '*.mask.npy'))
for masked_img in test_masked_images:
test_masked_img = np.load(masked_img)
#test_mask = np.load(mask)
test_masked_img = Image.fromarray(test_masked_img).resize((1024, 1024))
#test_mask = Image.fromarray(test_mask).resize((1024,1024))
#test_img = np.asarray(test_img)
#test_mask = np.round(np.asarray(test_mask))
#test_masked = np.multiply(test_img, test_mask)
test_normalized = test_masked_img
h_whole = test_normalized.shape[0] # original w
w_whole = test_normalized.shape[1] # original h
background = np.zeros((h_whole, w_whole))
background_indicer = np.zeros((h_whole, w_whole))
sum_prob_wt = 0.0
for i in range(header.repeat):
non_zero_list = np.nonzero(test_normalized)
random_index = random.randint(0, len(non_zero_list[0]) - 1)
non_zero_row = non_zero_list[0][
random_index] # random non-zero row index
non_zero_col = non_zero_list[1][
random_index] # random non-zero col index
X_patch = test_normalized[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col + (header.img_size / 2)))]
X_patch_img = data_transforms(
augmentation(Image.fromarray(X_patch), rand_p=0.0,
mode='test'))
X_patch_img_ = np.squeeze(np.asarray(X_patch_img))
X_patch_1 = np.expand_dims(X_patch_img_, axis=0)
X_patch_2 = np.expand_dims(X_patch_img_, axis=0)
X_patch_3 = np.expand_dims(X_patch_img_, axis=0)
X_ = np.concatenate((X_patch_1, X_patch_2, X_patch_3), axis=0)
X_ = np.expand_dims(X_, axis=0)
X = torch.from_numpy(X_)
X = X.to(device)
checkpoint = torch.load(
os.path.join(header.save_dir,
str(header.inference_epoch) + '.pth'))
model_ft.load_state_dict(checkpoint['model_state_dict'])
model_ft.eval()
outputs = model_ft(X)
outputs_prob = F.softmax(outputs)
prob = outputs_prob[0][label]
prob_wt = prob.detach().cpu().numpy()
gradcam = GradCAM.from_config(model_type='resnet',
arch=model_ft,
layer_name='layer4')
mask, logit = gradcam(X, class_idx=label)
mask_np = np.squeeze(mask.detach().cpu().numpy())
indicer = np.ones((224, 224))
mask_np = np.asarray(
cv2.resize(
mask_np,
dsize=(
int(min(w_whole, non_zero_col +
(header.img_size / 2))) -
int(max(0, non_zero_col - (header.img_size / 2))),
int(min(h_whole, non_zero_row +
(header.img_size / 2))) -
int(max(0, non_zero_row - (header.img_size / 2))))))
indicer = np.asarray(
cv2.resize(
indicer,
dsize=(
int(min(w_whole, non_zero_col +
(header.img_size / 2))) -
int(max(0, non_zero_col - (header.img_size / 2))),
int(min(h_whole, non_zero_row +
(header.img_size / 2))) -
int(max(0, non_zero_row - (header.img_size / 2))))))
mask_add = np.zeros((1024, 1024))
mask_add[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col +
(header.img_size / 2)))] = mask_np
mask_add = mask_add * prob_wt
indicer_add = np.zeros((1024, 1024))
indicer_add[
int(max(0, non_zero_row - (header.img_size / 2))
):int(min(h_whole, non_zero_row + (header.img_size / 2))),
int(max(0, non_zero_col - (header.img_size / 2))
):int(min(w_whole, non_zero_col +
(header.img_size / 2)))] = indicer
indicer_add = indicer_add
background = background + mask_add
background_indicer = background_indicer + indicer_add # number in this indicer means how many time the area included.
sum_prob_wt = sum_prob_wt + prob_wt
final_mask = np.divide(background, background_indicer + 1e-7)
final_mask = np.expand_dims(np.expand_dims(final_mask, axis=0), axis=0)
torch_final_mask = torch.from_numpy(final_mask)
test_img = np.asarray(Image.fromarray(test_img).resize((1024, 1024)))
test_img = (test_img - test_img.min()) / test_img.max()
test_img = np.expand_dims(test_img, axis=0)
test_img = np.concatenate((test_img, test_img, test_img), axis=0)
torch_final_img = torch.from_numpy(np.expand_dims(test_img, axis=0))
final_cam, cam_result = visualize_cam(torch_final_mask,
torch_final_img)
final_cam = (final_cam - final_cam.min()) / final_cam.max()
final_cam_np = np.swapaxes(np.swapaxes(np.asarray(final_cam), 0, 2), 0,
1)
test_img_np = np.swapaxes(np.swapaxes(test_img, 0, 2), 0, 1)
final_combined = test_img_np + final_cam_np
final_combined = (final_combined -
final_combined.min()) / final_combined.max()
plt.imshow(final_combined)
plt.savefig(
test_masked_img.split('.image.npy')[0] + '.patch.heatmap_' +
'.png')
