def grad_cam_image(model, image, input_tensor, img_class, layer, class_labels,
device):
if not next(model.parameters()).is_cuda:
model.to(device)
model.eval()
pred = model(input_tensor.to(device))
preds = pred.cpu().detach().numpy()
print(preds)
sorted_ids = preds.argsort()
print(sorted_ids)
for i in range(1, 10):
print(sorted_ids[0][-i])
print(class_labels[sorted_ids[0][-i]][1])
class_pred = np.argmax(pred.cpu().detach().numpy())
print(class_pred)
print(class_labels[class_pred][1], '\n')
target_layer = model.features[layer]
method = 'gradcam++' # Can be gradcam/gradcam++/scorecam
cam = CAM(model=model, target_layer=target_layer, use_cuda=True)
grayscale_cam = cam(input_tensor=input_tensor, method=method)
visualization = show_cam_on_image(
np.transpose(image[0].numpy(), (1, 2, 0)), grayscale_cam)
#visualization = show_cam_on_image(np.asarray(image), grayscale_cam)
return visualization, class_pred
def demo():
from pytorch_grad_cam import GradCAM, ScoreCAM, GradCAMPlusPlus, AblationCAM, XGradCAM, EigenCAM
from pytorch_grad_cam.utils.image import show_cam_on_image
from torchvision.models import resnet50
model = resnet50(pretrained=True)
target_layers = [model.layer4[-1]]
input_tensor = None # Create an input tensor image for your model..
# Note: input_tensor can be a batch tensor with several images!
# Construct the CAM object once, and then re-use it on many images:
cam = GradCAM(model=model,
target_layers=target_layers,
use_cuda=args.use_cuda)
# You can also use it within a with statement, to make sure it is freed,
# In case you need to re-create it inside an outer loop:
# with GradCAM(model=model, target_layers=target_layers, use_cuda=args.use_cuda) as cam:
# ...
# If target_category is None, the highest scoring category
# will be used for every image in the batch.
# target_category can also be an integer, or a list of different integers
# for every image in the batch.
target_category = 281
# You can also pass aug_smooth=True and eigen_smooth=True, to apply smoothing.
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
# In this example grayscale_cam has only one image in the batch:
grayscale_cam = grayscale_cam[0, :]
visualization = show_cam_on_image(rgb_img, grayscale_cam)
def show_cam_grad(grayscale_cam, src_img, title, out_path=None):
"""fuse src_img and grayscale_cam and show or save."""
grayscale_cam = grayscale_cam[0, :]
src_img = np.float32(src_img) / 255
visualization_img = show_cam_on_image(src_img,
grayscale_cam,
use_rgb=False)
if out_path:
mmcv.imwrite(visualization_img, str(out_path))
else:
mmcv.imshow(visualization_img, win_name=title)
def predict_all_classifier_data(self):
if self.classifier_data == []:
self.get_classifier_data()
if torch.cuda.is_available():
model = torch.load("./model/{}".format(classifier_data_model))
else:
model = torch.load("./model/{}".format(classifier_data_model),
map_location=torch.device('cpu'))
result = np.load("./predict/bounding_box_wider_data_predict.npz",
allow_pickle=True)
predict = []
grad_cam_image = []
for index, image in enumerate(self.classifier_data):
x1, y1, x2, y2 = result["predict"][index]
image = image[y1:y2, x1:x2]
image = cv2.resize(
image,
(classifier_data_training_size, classifier_data_training_size),
interpolation=cv2.INTER_LINEAR).astype(np.float32) / 255
output = model(
torch.tensor(np.array([image.transpose(
(2, 0, 1))])).float().to(self.device))
_, pre = torch.max(torch.nn.functional.softmax(output, dim=1), 1)
predict.append(pre.data.cpu()[0])
target_layers = [model[0].layer4[-1]]
input_tensor = torch.tensor(np.array([image.transpose(
(2, 0, 1))])).float().to(self.device)
cam = GradCAM(model=model,
target_layers=target_layers,
use_cuda=torch.cuda.is_available())
grayscale_cam = cam(input_tensor=input_tensor)
grayscale_cam = grayscale_cam[0, :]
visualization = show_cam_on_image(image,
grayscale_cam,
use_rgb=True)
grad_cam_image.append(visualization)
if not os.path.exists("./predict/"):
os.makedirs("./predict/")
np.savez("./predict/classifier.npz",
label=np.array(["Fracture", "Normal"]),
predict=predict,
goundTruth=self.classifier_target,
grad_cam_image=grad_cam_image)
def visualize():
app = Application()
cam = AblationCAM(
app.model,
app.model.layer4[-1]
)
source = [(i[0].cpu(), i[1].cpu()) for i in app.source if i[1].item() >= 0.2610552]
for i, datum in enumerate(source[0:10]):
img, _ = datum
print(_ * 25 + 15)
grayscale = cam(
input_tensor=img.unsqueeze(0),
# aug_smooth=True
)
grayscale = grayscale[0, :]
rgb_img = numpy.array(img.cpu()).squeeze(0)
rgb_img = cvtColor(rgb_img, COLOR_GRAY2RGB)
visualization = show_cam_on_image(rgb_img, grayscale)
imwrite(str(i) + '_cam.jpg', visualization)
def image2cam(image_path, cam, outdir):
rgb_img = cv2.imread(image_path, 1)[:, :, ::-1]
rgb_img = cv2.resize(rgb_img, (image_size, image_size))
rgb_img = np.float32(rgb_img) / 255
input_tensor = preprocess_image(rgb_img,
mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
# If None, returns the map for the highest scoring category.
# Otherwise, targets the requested category.
target_category = None
# AblationCAM and ScoreCAM have batched implementations.
# You can override the internal batch size for faster computation.
cam.batch_size = 32
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category,
eigen_smooth=args.eigen_smooth,
aug_smooth=args.aug_smooth)
# Here grayscale_cam has only one image in the batch
grayscale_cam = grayscale_cam[0, :]
cam_image = show_cam_on_image(rgb_img,
grayscale_cam,
use_rgb=False,
colormap=cv2.COLORMAP_JET)
# cv2.imshow('image', cam_image)
# cv2.waitKey(0)
img_name = image_path.split('/')[-1].split('.')[0]
if '\\' in img_name:
img_name = img_name.split('\\')[-1]
cv2.imwrite(os.path.join(out_dir, f'{img_name}_{args.method}.jpg'),
cam_image)
one_img[2] = one_img[2] * 0.5 + B
reshape_img[:, :, 0] = one_img[2]
reshape_img[:, :, 1] = one_img[1]
reshape_img[:, :, 2] = one_img[0]
model.eval()
class_output = model(input_data=input_tensor.clone())
pred = torch.max(class_output.data, 1)[1].cpu()
if True: # pred != label[i]:
cam = CAM(model=model,
target_layer=target_layer,
use_cuda=True)
grayscale_cam = cam(input_tensor=input_tensor,
target_category=pred,
method=method)
visualization = show_cam_on_image(reshape_img,
grayscale_cam)
# print(visualization.shape)
heatmap = cv2.applyColorMap(np.uint8(255 * grayscale_cam),
cv2.COLORMAP_JET)
heatmap = np.float32(heatmap) / 255
heatmap = np.uint8(255 * heatmap)
cv2.imwrite('./GradCAM/original' + str(i) + '.png',
np.uint8(255 * reshape_img),
[int(cv2.IMWRITE_PNG_COMPRESSION), 0])
cv2.imwrite('./GradCAM/mix' + str(i) + '.png',
visualization,
[int(cv2.IMWRITE_PNG_COMPRESSION), 0])
cv2.imwrite('./GradCAM/heatmap' + str(i) + '.png', heatmap,
[int(cv2.IMWRITE_PNG_COMPRESSION), 0])
break
model = model.cuda()
target_layer = model.blocks[-1].norm1
if args.method not in methods:
raise Exception(f"Method {args.method} not implemented")
cam = methods[args.method](model=model,
target_layer=target_layer,
use_cuda=args.use_cuda,
reshape_transform=reshape_transform)
rgb_img = cv2.imread(args.image_path, 1)[:, :, ::-1]
rgb_img = cv2.resize(rgb_img, (224, 224))
rgb_img = np.float32(rgb_img) / 255
input_tensor = preprocess_image(rgb_img, mean=[0.5, 0.5, 0.5],
std=[0.5, 0.5, 0.5])
# If None, returns the map for the highest scoring category.
# Otherwise, targets the requested category.
target_category = None
# AblationCAM and ScoreCAM have batched implementations.
# You can override the internal batch size for faster computation.
cam.batch_size = 32
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
cam_image = show_cam_on_image(rgb_img, grayscale_cam)
cv2.imwrite(f'{args.method}_cam.jpg', cam_image)
transform = transforms.Compose([
# transforms.Resize((224,224)),
transforms.ToTensor()
])
input_tensor = transform(input)
# Create an input tensor image for your model..
# Note: input_tensor can be a batch tensor with several images!
# Construct the CAM object once, and then re-use it on many images:
cam = GradCAM(model=model, target_layer=target_layer, use_cuda=False)
# If target_category is None, the highest scoring category
# will be used for every image in the batch.
# target_category can also be an integer, or a list of different integers
# for every image in the batch.
target_category = 1
# You can also pass aug_smooth=True and eigen_smooth=True, to apply smoothing.
grayscale_cam = cam(input_tensor=input_tensor.unsqueeze(0), target_category=target_category)
# In this example grayscale_cam has only one image in the batch:
grayscale_cam = grayscale_cam[0, :]
x = input_tensor.numpy()
x = np.transpose(x, (1,2,0))
visualization = show_cam_on_image(x, grayscale_cam)
print(grayscale_cam.shape)
plt.imsave('awef3.jpg',visualization)
plt.imshow(visualization)
def test(opt):
""" model configuration """
if 'CTC' in opt.Prediction:
converter = CTCLabelConverter(opt.character)
else:
converter = AttnLabelConverter(opt.character)
opt.num_class = len(converter.character)
if opt.rgb:
opt.input_channel = 3
model = Model(opt)
print('model input parameters', opt.imgH, opt.imgW, opt.num_fiducial,
opt.input_channel, opt.output_channel, opt.hidden_size,
opt.num_class, opt.batch_max_length, opt.Transformation,
opt.FeatureExtraction, opt.SequenceModeling, opt.Prediction)
model = torch.nn.DataParallel(model).to(device)
# load model
print('loading pretrained model from %s' % opt.saved_model)
model.load_state_dict(torch.load(opt.saved_model, map_location=device))
opt.exp_name = '_'.join(opt.saved_model.split('/')[1:])
# print(model)
# return model
AlignCollate_evaluation = AlignCollate(imgH=opt.imgH,
imgW=opt.imgW,
keep_ratio_with_pad=opt.PAD)
eval_data, eval_data_log = hierarchical_dataset(root=opt.eval_data,
opt=opt)
evaluation_loader = torch.utils.data.DataLoader(
eval_data,
batch_size=opt.batch_size,
shuffle=False,
num_workers=int(opt.workers),
collate_fn=AlignCollate_evaluation,
pin_memory=True)
# _, accuracy_by_best_model, _, _, _, _, _, _ = validation(
# model, criterion, evaluation_loader, converter, opt)
for i, (image_tensors, labels) in enumerate(evaluation_loader):
# batch_size = image_tensors.size(0)
# text_for_pred = torch.LongTensor(batch_size, opt.batch_max_length + 1).fill_(0).to(device)
target_layer = model.module.FeatureExtraction.ConvNet.layer4[-1]
# model.eval()
# handle =model.module.Transformation.register_forward_hook(hook)
# model(image_tensors,text_for_pred)
input_tensor = image_tensors
# print(labels)
# print(input_tensor.shape,'input_tensor.shape')
# handle.remove()
# print(input_tensor)
# Create an input tensor image for your model..
# input_tensor=image_tensors
# Note: input_tensor can be a batch tensor with several images!
# print(labels)
# Construct the CAM object once, and then re-use it on many images:
cam = EigenCAM(model=model,
target_layer=target_layer,
use_cuda=opt.use_cuda)
# If target_category is None, the highest scoring category
# will be used for every image in the batch.
# target_category can also be an integer, or a list of different integers
# for every image in the batch.
text_for_loss, length_for_loss = converter.encode(
labels, batch_max_length=opt.batch_max_length)
target_category = text_for_loss
# You can also pass aug_smooth=True and eigen_smooth=True, to apply smoothing.
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category)
# In this example grayscale_cam has only one image in the batch:
grayscale_cam = grayscale_cam[0, :]
loader = transforms.ToPILImage()
sourc_image = loader(image_tensors[0])
sourc_image = cv2.cvtColor(np.asarray(sourc_image), cv2.COLOR_RGB2BGR)
# rgb_img=loader(input_tensor[0].cpu())
# rgb_img.save('rgb_visual.bmp')
# rgb_img2=cv2.imread('rgb_visual.bmp')
rgb_img2 = np.float32(sourc_image) / 255
visualization = show_cam_on_image(rgb_img2, grayscale_cam)
sourc_image = cv2.resize(sourc_image, (0, 0),
fx=5,
fy=5,
interpolation=cv2.INTER_CUBIC)
visualization = cv2.resize(visualization, (0, 0),
fx=5,
fy=5,
interpolation=cv2.INTER_CUBIC)
cat_image = np.vstack((sourc_image, visualization))
cv2.imwrite('visual_image2/' + str(i) + '_2.bmp', cat_image)
# Otherwise, targets the requested category.
target_category = None
# AblationCAM and ScoreCAM have batched implementations.
# You can override the internal batch size for faster computation.
cam.batch_size = 32
grayscale_cam = cam(input_tensor=input_tensor,
target_category=target_category,
aug_smooth=args.aug_smooth,
eigen_smooth=args.eigen_smooth)
# Here grayscale_cam has only one image in the batch
grayscale_cam = grayscale_cam[0, :]
cam_image = show_cam_on_image(rgb_img, grayscale_cam, use_rgb=True)
# cam_image is RGB encoded whereas "cv2.imwrite" requires BGR encoding.
cam_image = cv2.cvtColor(cam_image, cv2.COLOR_RGB2BGR)
gb_model = GuidedBackpropReLUModel(model=model, use_cuda=args.use_cuda)
gb = gb_model(input_tensor, target_category=target_category)
cam_mask = cv2.merge([grayscale_cam, grayscale_cam, grayscale_cam])
cam_gb = deprocess_image(cam_mask * gb)
gb = deprocess_image(gb)
cv2.imwrite(f'{args.method}_cam.jpg', cam_image)
cv2.imwrite(f'{args.method}_gb.jpg', gb)
cv2.imwrite(f'{args.method}_cam_gb.jpg', cam_gb)