def grad_cam(image):
model = models.vgg19_bn(pretrained=True)
model.to(device)
model.eval()
image = cv2.resize(image, (224, 224))
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(image).unsqueeze(0)
image = image.to(device)
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image)
output = None
for i in range(0, 5):
gcam.backward(idx=idx[i])
temp = gcam.generate(target_layer='features.52')
if not output is None:
output += temp
else:
output = temp
output /= 5
return output
class GradCamExplainer(ImageExplainer):
def __init__(self, model_wrapper, target_layer, label):
self.model = model_wrapper.model
self.explainer = GradCAM(model=self.model)
self.target_layer = target_layer
self.label = label
def explain(self, instance, budget):
instance = instance.unsqueeze(0).cuda()
probs, ids = self.explainer.forward(instance)
# print(probs, ids)
target_ids = torch.LongTensor([[self.label]]).cuda()
self.explainer.backward(ids=target_ids)
regions = self.explainer.generate(target_layer=self.target_layer)
explanation_3d = regions[0].double().detach().cpu().numpy()
explanation_2d = np.sum(np.abs(explanation_3d), axis=0)
top_percentile = np.percentile(explanation_2d, 100 - budget)
# only return above percentile
explanation_2d[explanation_2d < top_percentile] = 0.0
explanation_2d[explanation_2d >= top_percentile] = 1.0
return explanation_2d.astype(np.float32)
def main():
# Dataset
print('Creating dataset...')
transform_val = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize(MEAN, STD)])
valset = torchvision.datasets.CIFAR100(root='./data',
train=False,
download=True,
transform=transform_val)
val_loader = DataLoader(valset,
batch_size=100,
shuffle=False,
num_workers=4,
pin_memory=True)
# Model
model_path = os.path.join(args.checkpoint, args.model, 'best_model.pt')
print('Loading model...')
model = get_model(args.model)
if os.path.exists(model_path):
model.load_state_dict(torch.load(model_path))
else:
raise Exception('Cannot find model', model_path)
if torch.cuda.device_count() > 1:
print("Using", torch.cuda.device_count(), "GPUs!")
model = nn.DataParallel(model)
model.cuda()
cudnn.benchmark = True
# result
result_path = os.path.join('results', args.model)
if not os.path.exists(result_path):
os.makedirs(result_path)
gcam = GradCAM(model=model)
model.eval()
acc = 0
for i, (inputs, labels) in enumerate(val_loader):
inputs, labels = (Variable(inputs.cuda()), Variable(labels.cuda()))
outputs = model(inputs)
outputs, labels = outputs.data, labels.data
_, preds = outputs.topk(1, 1, True, True)
corrects = preds.eq(labels.view(-1, 1).expand_as(preds))
acc += torch.sum(corrects)
for sample in SAMPLES:
image = get_image(inputs[sample])
probs, idx = gcam.forward(inputs[sample].unsqueeze(0))
gcam.backward(idx=idx[1])
output = gcam.generate(target_layer='layer4.2')
save_gradcam(result_path, '{}_gcam.png'.format(sample), output,
image)
save_image(result_path, image, sample, preds[sample],
labels[sample])
break
acc = acc.item() / len(valset) * 100
print('Finish!!!')
def demo2(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# The four residual layers
target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"]
target_class = 243 # "bull mastif"
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print(
"\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])
)
)
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "resnet152", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def main():
device = torch.device(
'cuda' if args.cuda and torch.cuda.is_available() else 'cpu')
if args.cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Load the LFLSeg module (ResNet-101 backbone)
LFLSeg_model = models.resnet101()
num_ftrs = LFLSeg_model.fc.in_features
LFLSeg_model.fc = nn.Linear(
num_ftrs, 3
) # Replace final layer with 3 outputs (full leaf, partial leaf, non-leaf)
# Dowload the pretrained model: https://drive.google.com/drive/folders/1HqBYjUGXxl1eAkzhURoV5JAqWHvBvvTp?usp=sharing
load_path = '/path/to/LFLSeg_resnet101.pth'
LFLSeg_model.load_state_dict(torch.load(load_path), strict=True)
LFLSeg_model.to(device)
LFLSeg_model.eval()
# Load the GradCAM function
gcam = GradCAM(model=LFLSeg_model)
if (os.path.exists(args.input) == False):
print("The image path doesn't exist!")
return
else:
# If output folder is not exists, create a new one
if not os.path.exists(args.output):
os.makedirs(args.output)
filename = os.path.basename(args.input)
raw_image = Image.open(args.input).convert('RGB')
image = transforms.Compose([
transforms.Resize(size=(224, 224), interpolation=3),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
])(raw_image).unsqueeze(0)
probs, idx = gcam.forward(image.to(device))
# Only get the heatmap for the "full leaf" class (i.e., idx=0)
gcam.backward(idx=0)
output = gcam.generate(target_layer=args.target_layer)
save_gradcam('{}/{}_gcam.png'.format(args.output, filename[:-4]),
output,
raw_image,
threshold=args.threshold,
is_segment=args.segment)
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, model):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
gradcam_list = []
# Synset words
classes = get_classtable()
# Model from torchvision
if model is None:
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
#print(image_paths)
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
#print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Grad-CAM
filename = osp.join(
output_dir,
"{}-{}-gradcam-{}.png".format(j, arch, target_layer))
gradcam_list.append(filename)
save_gradcam(filename, gcam=regions[j, 0], raw_image=raw_images[j])
return gradcam_list
def VideoSpatialPrediction(vid_name,
target,
net,
num_categories,
num_samples=25,
new_size=299,
batch_size=2):
gc = GradCAM(model=net)
clip_mean = [0.5] * num_samples
clip_std = [0.226] * num_samples
normalize = video_transforms.Normalize(mean=clip_mean, std=clip_std)
val_transform = video_transforms.Compose([
video_transforms.ToTensor(),
normalize,
])
deep = 1
# inception = 299,299, resnet = 224,224
dims = (new_size, new_size, deep, num_samples)
rgb = np.zeros(shape=dims, dtype=np.float64)
rgb_flip = np.zeros(shape=dims, dtype=np.float64)
for i in range(num_samples):
img_file = os.path.join(vid_name, 'vr_{0:02d}.png'.format(i))
img = cv2.imread(img_file, cv2.IMREAD_GRAYSCALE)
rgb[:, :, 0, i] = img
rgb_flip[:, :, 0, i] = img[:, ::-1]
_, _, _, c = rgb.shape
rgb_list = []
for c_index in range(c):
cur_img = rgb[:, :, :, c_index]
cur_img_tensor = val_transform(cur_img)
rgb_list.append(np.expand_dims(cur_img_tensor.numpy(), 0))
rgb_np = np.concatenate(rgb_list, axis=0)
prediction = np.zeros((num_categories, rgb.shape[3]))
index = 50
input_data = rgb_np[index:index + 1, :, :, :]
imgDataTensor = torch.from_numpy(input_data).type(torch.FloatTensor).cuda()
imgDataVar = torch.autograd.Variable(imgDataTensor)
probs, ids = gc.forward(imgDataVar)
ids_ = torch.LongTensor([[target]] * len(imgDataVar)).to(
torch.device("cuda"))
gc.backward(ids=ids_)
regions = gc.generate(target_layer="Mixed_7c")
save_gradcam(vid_name.split("/")[-1] + ".png",
gcam=regions[0, 0],
raw_image=rgb[:, :, :, index])
return prediction
def demo2(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# The four residual layers
target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"]
target_class = 243 # "bull mastif"
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def gradcam_classify(model):
image_paths = []
for f in os.listdir("samples/"):
fname = os.path.join("samples/", f)
image_paths.append(fname)
#print("my image paths")
#print(image_paths)
#for i in range(1,26):
# image_paths.append('samples/'+str(i)+'.PNG')
target_layer = 'layer4'
topk = 1
output_dir = 'results'
classes = getclasses()
#model.to(device)
model.eval()
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
# =========================================================================
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
# =========================================================================
print("Grad-CAM in action:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
print('len images = ', len(images))
for j in range(len(images)):
# Grad-CAM
global filename
filename = osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, 'resnet', target_layer,
classes[ids[j, i]]),
)
#print(f'gradcam generated filenames {i}')
save_gradcam(
filename=filename,
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def process_a_batch(image_paths, target_layer, arch, topk, output_dir, cuda):
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
images = load_images_only(image_paths)
images = torch.stack(images).to(device)
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
# =========================================================================
print("Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
regions = regions.cpu().numpy()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(image_paths[j],
classes[ids[j, i]], probs[j, i]))
# Grad-CAM
save_cam(filename=os.path.join(
output_dir,
"{}-{}-{}-{:.3}.png".format(os.path.basename(image_paths[j]),
classes[ids[j, i]], ids[j, i],
probs[j, i]),
),
gcam=regions[j, 0])
bp.remove_hook()
gcam.remove_hook()
del bp
del images
del gcam
del model
del regions
del probs
del ids
del _
torch.cuda.empty_cache()
def main(image_path, cuda):
device = torch.device(
"cuda" if cuda and torch.cuda.is_available() else "cpu")
if cuda:
current_device = torch.cuda.current_device()
print("Running on the GPU:",
torch.cuda.get_device_name(current_device))
else:
print("Running on the CPU")
# Synset words
classes = list()
with open("samples/synset_words.txt") as lines:
for line in lines:
line = line.strip().split(" ", 1)[1]
line = line.split(", ", 1)[0].replace(" ", "_")
classes.append(line)
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (224, ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])(raw_image).unsqueeze(0)
image = image.to(device)
gcam = GradCAM(model=model)
predictions = gcam.forward(image)
top_idx = predictions[0][1]
for target_layer in ["layer1", "layer2", "layer3", "layer4"]:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
gcam.backward(idx=top_idx)
region = gcam.generate(target_layer=target_layer)
save_gradcam(
"results/{}-gradcam-{}-{}.png".format("resnet152", target_layer,
classes[top_idx]),
region,
raw_image,
)
def demo2(image_paths, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# The four residual layers
target_layers = ["layer1", "layer2", "layer3", "layer4"]
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
top_ids = ids[:, [0]]
gcam.backward(ids=top_ids)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, 0]], probs[j,
0]))
save_gradcam(
filename="results/{}-{}-gradcam-{}-{}.png".format(
j, "resnet152", target_layer, classes[top_ids[j]]),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo4(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = torch.load('best_all.pth') # load model
model.to(device)
model.eval()
# The layers you want to see
target_layers = ["base_network.layer4.2.conv3"]
target_class = 4 # "lake"
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def extract_object(original_image, cuda = 1):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
output_dir = "./output"
# device = get_device(cuda)
device = torch.device("cuda" if cuda else "cpu")
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
# device = "cuda"
# model.to(device)
model.eval()
target_layer = "layer4"
target_class = 243 # "bull mastif"
# Images
# images = original_image.to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(original_image)
# ids_ = torch.LongTensor([[target_class]] * len(original_image)).to(device)
# pdb.set_trace()
gcam.backward(ids=ids)
# Grad-CAM
dogs = []
red_bboxs = []
regions = gcam.generate(target_layer=target_layer)
for j in range(len(original_image)):
dog, red_bbox = save_gradcam(gcam=regions[j, 0], raw_image=original_image[j])
dogs.append(dog)
red_bboxs.append(red_bbox)
print(red_bbox)
# tensor_dogs = torch.stack(dogs)
# tensor_red_bbox = torch.stack(red_bboxs)
#del gcam,model,dogs,red_bboxs
return dogs, red_bboxs
def guided_backprop_eye(image, name, net):
img = torch.stack([image[name]])
bp = BackPropagation(model=net)
probs, ids = bp.forward(img)
gcam = GradCAM(model=net)
_ = gcam.forward(img)
gbp = GuidedBackPropagation(model=net)
_ = gbp.forward(img)
# Guided Backpropagation
actual_status = ids[:, 0]
gbp.backward(ids=actual_status.reshape(1, 1))
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=actual_status.reshape(1, 1))
regions = gcam.generate(target_layer='last_conv')
# Get Images
prob = probs.data[:, 0]
if actual_status == 0:
prob = probs.data[:, 1]
prob_image = np.zeros((shape[0], 60, 3), np.uint8)
cv2.putText(prob_image, '%.1f%%' % (prob * 100), (5, 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv2.LINE_AA)
guided_bpg_image = get_gradient_image(gradients[0])
guided_bpg_image = cv2.merge(
(guided_bpg_image, guided_bpg_image, guided_bpg_image))
grad_cam_image = get_gradcam_image(gcam=regions[0, 0],
raw_image=image[name + '_raw'])
guided_gradcam_image = get_gradient_image(torch.mul(regions, gradients)[0])
guided_gradcam_image = cv2.merge(
(guided_gradcam_image, guided_gradcam_image, guided_gradcam_image))
#print(image['path'],classes[actual_status.data], probs.data[:,0] * 100)
print(classes[actual_status.data], probs.data[:, 0] * 100)
return cv2.hconcat([
image[name + '_raw'], prob_image, guided_bpg_image, grad_cam_image,
guided_gradcam_image
])
def guided_gradcam(model, device, raw_image, image, CONFIG, topk):
# =========================================================================
print('Guided Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
results.append(find_gradient(output))
print('[{:.5f}] {}'.format(probs[i], idx[i].cpu().numpy()))
return (results, probs, idx)
def get_grad_cam(device, model, classes, images, labels, target_layers):
# move the model to device
model.to(device)
# set the model in evaluation mode
model.eval()
# get the grad cam
gcam = GradCAM(model=model, candidate_layers=target_layers)
# images = torch.stack(images).to(device)
# predicted probabilities and class ids
pred_probs, pred_ids = gcam.forward(images)
# actual class ids
# target_ids = torch.LongTensor(labels).view(len(images), -1).to(device)
target_ids = labels.view(len(images), -1).to(device)
# backward pass wrt to the actual ids
gcam.backward(ids=target_ids)
# we will store the layers and correspondings images activations here
layers_region = {}
# fetch the grad cam layers of all the images
for target_layer in target_layers:
# logger.info(f'generating Grad-CAM for {target_layer}')
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
layers_region[target_layer] = regions
# we are done here, remove the hooks
gcam.remove_hook()
return layers_region, pred_probs, pred_ids
def demo3(image_paths, topk, output_dir, cuda):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
model = torch.hub.load("kazuto1011/pytorch-ported-models",
"xception_v1",
pretrained=True)
model.to(device)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
target_layer = "exit_flow.conv4"
# Preprocessing
def _preprocess(image_path):
raw_image = cv2.imread(image_path)
raw_image = cv2.resize(raw_image, model.image_shape)
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image -= model.mean
image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "xception_v1",
target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
model = build_model(cfg, join(opt.model, 'best.ckpt'))
dataset = ImageDataset(opt.dataset, cfg, mode='heatmap')
dataloader = DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=False,
drop_last=False)
gcam = GradCAM(model=model)
layer = 'backbone.features.denseblock4.denselayer15'
for images, paths, labels in dataloader:
probs, ids = gcam.forward(images)
processed_images = [[] for _ in range(len(images))]
for i in range(ids.shape[1]):
gcam.backward(ids[:, [i]])
regions = gcam.generate(target_layer=layer)
for j in range(len(images)):
print(f"#{j}: {classes[ids[j, i]]} ({probs[j, i]:.5f})")
# Grad-CAM
raw_image = imread(paths[j])
combined = combine_image_and_gcam(regions[j, 0], raw_image)
processed_images[j].append(combined.astype(np.uint8))
for j, (image_list, path) in enumerate(zip(processed_images, paths)):
plt.figure(figsize=(16, 4))
for i, image in enumerate(image_list):
plt.subplot(1,
len(image_list),
i + 1,
xticks=[],
yticks=[],
def main(image_path, arch, topk, cuda, target_layer):
CONFIG = {
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
'se_resnet': {
'target_layer': target_layer,
'input_size': 32
},
}.get(arch)
cuda = cuda and torch.cuda.is_available()
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = list()
with open('samples/synset_words.txt') as lines:
for line in lines:
line = line.strip().split(' ', 1)[1]
line = line.split(', ', 1)[0].replace(' ', '_')
classes.append(line)
# Model
#model = models.__dict__[arch](pretrained=True)
model = models.se_resnet.se_resnet50(num_classes=100)
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])(raw_image)
if cuda:
model.cuda()
image = image.cuda()
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(to_var(image))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
#save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch), output, raw_image) # NOQA
save_gradcam('results/{}.png'.format(target_layer), output,
raw_image) # NOQA
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def fulltest(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
fold = 0
device = get_device(cuda)
# Synset words
#classes = {0:'normal',1:'covid'}
classes = get_classtable()
# Model
#model = models.resnet34(pretrained=False,num_classes=config.num_classes)
best_model = torch.load(config.weights + 'ct-cn/' + 'model_best.pth.tar')
print(best_model["state_dict"].keys())
model = make_model(arch, num_classes=config.num_classes, pretrained=True)
#best_model = torch.load(config.weights +'model_best.pth.tar')
model.load_state_dict(best_model["state_dict"])
print(best_model["state_dict"].keys())
model.to(device)
model.eval()
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
print(probs)
print(ids)
for i in range(topk):
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
for i in range(topk):
# In this example, we specify the high confidence classes
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]
),
),
gradient=torch.mul(regions, gradients)[j],
)
def demo3(image_paths, topk, output_dir, cuda):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
model = torch.hub.load(
"kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True
)
model.to(device)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
target_layer = "exit_flow.conv4"
# Preprocessing
def _preprocess(image_path):
raw_image = cv2.imread(image_path)
raw_image = cv2.resize(raw_image, model.image_shape)
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image -= model.mean
image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "xception_v1", target_layer, classes[ids[j, i]]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
if use_gpu:
inputs = inputs.cuda()
inputs = inputs.view(1, inputs.size(0), inputs.size(1), inputs.size(2))
probs, idx = gcam.forward(inputs)
if (args.subtype == None):
comp_idx = idx[0]
item_id = 0
else:
comp_idx = WBC_id
item_id = (np.where(idx.cpu().numpy() == (WBC_id)))[0][0]
gcam.backward(idx=comp_idx)
output = gcam.generate(target_layer = 'layer4.2') # for resnet
heatmap = output
original = inputs.data.cpu().numpy()
original = np.transpose(original, (0,2,3,1))[0]
original = original * cf.std + cf.mean
original = np.uint8(original * 255.0)
mask = np.uint8(heatmap * 255.0)
blank_heatmap[:, (mode*160):480+(mode*160)] = cv2.resize(heatmap, (480, 480))
mode += 1
blank_heatmap[blank_heatmap > 1] = 1
blank_heatmap = cv2.GaussianBlur(blank_heatmap, (15, 15), 0)
def grad_cam(model, image_paths, topk, output_dir, cuda, model_name,
checkpoint):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = [
'apple', # id 0
'aquarium_fish',
'baby',
'bear',
'beaver',
'bed',
'bee',
'beetle',
'bicycle',
'bottle',
'bowl',
'boy',
'bridge',
'bus',
'butterfly',
'camel',
'can',
'castle',
'caterpillar',
'cattle',
'chair',
'chimpanzee',
'clock',
'cloud',
'cockroach',
'couch',
'crab',
'crocodile',
'cup',
'dinosaur',
'dolphin',
'elephant',
'flatfish',
'forest',
'fox',
'girl',
'hamster',
'house',
'kangaroo',
'computer_keyboard',
'lamp',
'lawn_mower',
'leopard',
'lion',
'lizard',
'lobster',
'man',
'maple_tree',
'motorcycle',
'mountain',
'mouse',
'mushroom',
'oak_tree',
'orange',
'orchid',
'otter',
'palm_tree',
'pear',
'pickup_truck',
'pine_tree',
'plain',
'plate',
'poppy',
'porcupine',
'possum',
'rabbit',
'raccoon',
'ray',
'road',
'rocket',
'rose',
'sea',
'seal',
'shark',
'shrew',
'skunk',
'skyscraper',
'snail',
'snake',
'spider',
'squirrel',
'streetcar',
'sunflower',
'sweet_pepper',
'table',
'tank',
'telephone',
'television',
'tiger',
'tractor',
'train',
'trout',
'tulip',
'turtle',
'wardrobe',
'whale',
'willow_tree',
'wolf',
'woman',
'worm',
]
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
checkpoint = checkpoint
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
model.eval()
# Check available layer names
'''print("Layers:")
for m in model.named_modules():
print("\t", m[0])'''
# Here we choose the last convolution layer
target_layer = "module.layer4"
#print(target_layer)
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"top{}-{}-{}-gradcam-{}-{}-prob-({:.5f}).png".format(
i + 1, j, model_name, target_layer, classes[ids[j, i]],
probs[j, i]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo3(image_paths, topk, output_dir, cuda, num_class, da_model):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
#classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
#model = torch.hub.load(
# "kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True
#)
#target_images = ['mediumresidential01','mediumresidential03', 'mediumresidential14',\
# 'mediumresidential21', 'mediumresidential36','mediumresidential37'] + \
#['mediumresidential38','mediumresidential39', 'mediumresidential40',\
# 'mediumresidential41', 'mediumresidential43','mediumresidential44']
num_class, = num_class
#target_images = ['baseballdiamond96', 'airplane18', 'runway42', 'sparseresidential77', 'mediumresidential21']
target_images = [
'runway00', 'runway02', 'runway03', 'runway04', 'runway05', 'runway06',
'runway07', 'runway08'
]
classes = [str(i) for i in range(num_class)]
if da_model == 'STA':
feature_extractor = ResNetFc(
model_name='resnet50',
model_path=
'/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/resnet50.pth'
)
cls = CLS(feature_extractor.output_num(),
num_class,
bottle_neck_dim=256)
model = nn.Sequential(feature_extractor, cls).cuda()
model.load_state_dict(
torch.load(
'/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/Only_source_classifier.pth'
))
elif da_model == 'OPDA':
G, C = get_model('vgg', num_class=num_class, unit_size=1000)
load_model(
G, C,
'/home/at7133/Research/Domain_adaptation/OPDA_BP/checkpoint/checkpoint_99'
)
model = nn.Sequential(G, C).cuda()
model.to(device)
#unset_training(model)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
#target_layer = "exit_flow.conv4"
if da_model == 'STA':
target_layer = "0.model_resnet.layer4.2.conv3"
elif da_model == 'OPDA':
target_layer = "0.lower.36" #TODO find proper target layer
# Preprocessing
def _Normalize(img, mean, std):
if isinstance(img, torch.FloatTensor):
mean = torch.FloatTensor(mean)
std = torch.FloatTensor(std)
else:
raise TypeError(f'Expected Torch floattensor, got {type(img)}')
return (img - mean) / std
def _preprocess(image_path, img_shape):
raw_image = cv2.imread(image_path)
#raw_image = cv2.resize(raw_image, model.image_shape)
raw_image = cv2.resize(raw_image, (img_shape, img_shape))
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image = image / 255.0
if da_model == "OPDA":
image = _Normalize(image, [0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
#pdb.set_trace()
#image -= model.mean
#image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
def Load_txt(img_paths):
assert isfile(img_paths), f"Image path {img_paths} doesn't exist"
with open(img_paths, 'rb') as fr:
image_paths = [
img_path.split()[0].decode("utf-8")
for img_path in fr.readlines()
]
return image_paths
def Filter_imgages(image_paths, req_images):
img_files = list(
map(lambda x: x.split('/')[-1].split('.')[0], image_paths))
satisfied_images = []
for item in req_images:
try:
[[idx]] = np.argwhere(np.isin(img_files, item))
except:
raise ValueError(f'{item} not found in the given paths')
satisfied_images.append(image_paths[idx])
return satisfied_images
def Load_images(image_paths, req_images):
if image_paths[0].endswith('.txt'):
assert len(image_paths
) == 1 #make sure only one text file is given as input
image_paths = Load_txt(image_paths[0])
image_paths = Filter_imgages(image_paths, req_images)
assert len(image_paths) == len(
req_images), " All target images are not found"
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path, 224)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
return images, raw_images
images, raw_images = Load_images(image_paths, target_images)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
if not isdir(output_dir):
os.makedirs(output_dir)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "xception_v1",
target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, checkpoint,
distribute):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch]()
model = model.cuda()
# print(model)
checkpoint = checkpoint + arch + '/model_best.pth.tar'
# print(checkpoint)
check_point = torch.load(checkpoint,
map_location=lambda storage, loc: storage.cuda(0))
distributed_model = (distribute > 0.5)
only_CAM = True
if distributed_model == True:
# create new OrderedDict that does not contain `module.`
from collections import OrderedDict
new_check_point = OrderedDict()
for k, v in check_point['state_dict'].items():
# name = k[7:] # remove `module.`
# name = k[9:] # remove `module.1.`
if k.startswith('module.1.'):
name = k[9:]
else:
name = k[7:]
new_check_point[name] = v
# load params
model.load_state_dict(new_check_point)
else:
model.load_state_dict(check_point['state_dict'])
model.to(device)
model.eval()
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
for i in range(topk):
# In this example, we specify the high confidence classes
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch,
classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(
j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-{}.png".format(
# j, arch, target_layer, classes[ids[j, i]] image_path
osp.splitext(image_paths[j])[0],
arch,
target_layer),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def main():
"""
Visualize model responses given multiple images
"""
device = torch.device("cuda")
test_dataset = CXRDataset(args.image_dir, ['TEST'])
test_loader = DataLoader(test_dataset,
batch_size=args.batch_size,
pin_memory=True)
model = CombineNet().to(device)
model = nn.DataParallel(model)
print("Parameters: {}".format(sum(p.numel() for p in model.parameters())))
model.eval()
if args.load_model:
checkpoint = torch.load(args.load_model + "/best.ckpt")
#model_weights = OrderedDict()
#for k, v in checkpoint['model'].items():
# name = k[7:] # remove `module.`
# model_weights[name] = v
model.load_state_dict(checkpoint['model'])
epoch = checkpoint['epoch']
print('Loading model: {}, from epoch: {}'.format(
args.load_model, epoch))
else:
print('Model: {} not found'.format(args.load_model))
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
target_layer = 'module.feature_model.layer_block.3.1.conv2'
#print(model.module.keys())
print(model.module.feature_model.layer_block[3][1].conv2)
epoch_iterator = tqdm(test_loader, desc="Iteration")
for i, batch in enumerate(epoch_iterator, 0):
raw_images, _, _ = batch
batch_device = tuple(t.to(device, non_blocking=True) for t in batch)
images, labels, img_ids = batch_device
if (labels.cpu().detach().numpy() == np.array([0, 0, 0, 1])).all():
probs, ids = gcam.forward(images)
gcam.backward(ids=ids[:, [0]])
regions = gcam.generate(target_layer=target_layer)
for j in range(args.batch_size):
gcam_im = save_gradcam(
filename=osp.join(
args.output_dir,
"{}-gradcam-{}_severe_{}.png".format(
j, target_layer, i),
),
gcam=regions[j, 0],
raw_image=images[j],
)
img_path = osp.join(
args.output_dir,
"{}-image-{}_severe_{}.png".format(j, target_layer, i),
)
raw_image = np.array(raw_images[j])
xray = save_xray(img_path, raw_image)
'''
# TODO: Blending code
#cv2.imwrite(img_path, raw_images[j])
img_path=osp.join(
args.output_dir,
"{}-blended-{}_severe_{}.png".format(j, target_layer, i),
)
beta = (1.0 - args.alpha)
xray = np.array(xray)
xray = xray.squeeze()
xray = np.stack([xray, xray, xray],axis=2)
print(xray.dtype)
print(gcam.dtype)
blended = cv2.addWeighted(xray, args.alpha, gcam, beta, 0.0)
save_xray(img_path, blended)
'''
print('finished!')
def main():
root_path = '/media/palm/Unimportant/pdr2018/typesep_validate/Tomato/'
image_name = 'c9ebc74c2177ce60a8230855333fb9e7.jpg'
folder_name = '14_Tomato_Spider_Mite_Damage_Serious'
# image_path = root_path+'/14_Tomato_Spider_Mite_Damage_Serious/1c0f1ae1374d01c2933069232735a331.jpg'
image_path = os.path.join(root_path, folder_name, image_name)
topk = 1
cuda = 'cuda'
arch = 'densenet201'
CONFIG = {
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
}.get(arch)
a, b, c = getlookup()
device = torch.device(
'cuda' if cuda and torch.cuda.is_available() else 'cpu')
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = c['Tomato']
# Model
model = getmodel(20)
checkpoint = torch.load('checkpoint/try_4_densesep-Tomatotemp.t7')
model.load_state_dict(checkpoint['net'])
model.to('cuda')
model.eval()
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(raw_image).unsqueeze(0)
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
save_gradcam(
'results/{}_{}_gcam_{}.png'.format(image_name, classes[idx[i]],
arch), output, raw_image)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Vanilla Backpropagation')
# =========================================================================
bp = BackPropagation(model=model)
probs, idx = bp.forward(image.to(device))
for i in range(0, topk):
bp.backward(idx=idx[i])
output = bp.generate()
save_gradient(
'results/{}_{}_bp_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Deconvolution')
# =========================================================================
deconv = Deconvolution(
model=copy.deepcopy(model)) # TODO: remove hook func in advance
probs, idx = deconv.forward(image.to(device))
for i in range(0, topk):
deconv.backward(idx=idx[i])
output = deconv.generate()
save_gradient(
'results/{}_{}_deconv_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Guided Backpropagation/Guided Grad-CAM')
# =========================================================================
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
save_gradient(
'results/{}_{}_gbp_{}.png'.format(image_name, classes[idx[i]],
arch), feature)
save_gradient(
'results/{}_{}_ggcam_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def main(image_path, arch, topk, cuda):
CONFIG = {
'resnet18': {
'target_layer': 'layer4.1',
'input_size': 224
},
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
}.get(arch)
device = torch.device(
'cuda' if cuda and torch.cuda.is_available() else 'cpu')
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = ["other", "rori"]
# with open('samples/synset_words.txt') as lines:
# for line in lines:
# line = line.strip().split(' ', 1)[1]
# line = line.split(', ', 1)[0].replace(' ', '_')
# classes.append(line)
# Model
model = models.__dict__[arch](pretrained=True)
num_features = model.fc.in_features
model.fc = nn.Linear(num_features, 200) #これにより512->2の層に変わった
model.add_module('relu_fc', nn.ReLU())
model.add_module('fc2', nn.Linear(200, 2))
param = torch.load('weight_resnet18_3.pth')
model.load_state_dict(param)
model.to(device)
model.eval()
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(raw_image).unsqueeze(0)
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch),
output, raw_image)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Vanilla Backpropagation')
# =========================================================================
bp = BackPropagation(model=model)
probs, idx = bp.forward(image.to(device))
for i in range(0, topk):
bp.backward(idx=idx[i])
output = bp.generate()
save_gradient('results/{}_bp_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Deconvolution')
# =========================================================================
deconv = Deconvolution(
model=copy.deepcopy(model)) # TODO: remove hook func in advance
probs, idx = deconv.forward(image.to(device))
for i in range(0, topk):
deconv.backward(idx=idx[i])
output = deconv.generate()
save_gradient('results/{}_deconv_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Guided Backpropagation/Guided Grad-CAM')
# =========================================================================
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
save_gradient('results/{}_gbp_{}.png'.format(classes[idx[i]], arch),
feature)
save_gradient('results/{}_ggcam_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def validate(self,test_epoch=False):
self.model.train()
if test_epoch:
out = osp.join(self.out, 'test')
else:
out = osp.join(self.out, 'visualization')
mkdir(out)
log_file = osp.join(out, 'test_accurarcy.txt')
fv = open(log_file, 'a')
log_file2 = osp.join(out, 'test_accurarcy_perepoch.txt')
fv2 = open(log_file2, 'a')
log_file3 = osp.join(out, 'test_recall_f1_acc_perepoch.txt')
fv3 = open(log_file3, 'a')
correct = 0
correct_binary = 0
pred_history=[]
target_history=[]
loss_history=[]
sofar = 0
for batch_idx, (data,target,sub_name) in tqdm.tqdm(
# enumerate(self.test_loader), total=len(self.test_loader),
enumerate(self.test_loader), total=len(self.test_loader),
desc='Valid epoch=%d' % self.epoch, ncols=80,
leave=False):
if self.cuda:
data, target = data.cuda(), target.cuda()
# data, target = Variable(data), Variable(target)
data, target = Variable(data,volatile=True), Variable(target,volatile=True)
if self.dual_network:
if self.add_calcium_mask:
data = data[:,0:2,: :,:]
else:
data = data[:,0,:,:,:]
data = torch.unsqueeze(data, 1)
pred_prob = self.model(data)
if test_epoch:
#get attention
gcam = GradCAM(model=self.model)
probs, idx = gcam.forward(data)
topk = 3
target_layer = 'ec6.2'
# target_layer = 'ec1.2'
test_attention_out = osp.join(out, target_layer)
mkdir(test_attention_out)
input_img = data[0, 0].data.cpu().numpy()
input_size = (input_img.shape[0], input_img.shape[1], input_img.shape[2])
input_mask = data[0, 1].data.cpu().numpy()
nii_img = nib.Nifti1Image(input_img, affine=np.eye(4))
output_img_file = os.path.join(out, ('%s_img.nii.gz' % sub_name[0]))
nib.save(nii_img, output_img_file)
nii_mask = nib.Nifti1Image(input_mask, affine=np.eye(4))
output_mask_file = os.path.join(out, ('%s_mask.nii.gz' % sub_name[0]))
nib.save(nii_mask, output_mask_file)
del input_img,input_mask
del nii_img, nii_mask
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=target_layer)
output = resize(output,input_size , mode='constant', preserve_range=True)
nii_seg = nib.Nifti1Image(output, affine=np.eye(4))
output_att_file = os.path.join(test_attention_out, ('%s_test_att%d_clss%d.nii.gz' % (sub_name[0],i,idx[i])))
nib.save(nii_seg, output_att_file)
gcam.backward_del(idx=idx[i])
del gcam, output, nii_seg, probs
#training attention
subnum = data.size(0)
for subi in range(subnum):
attentions = []
i = 1
self.input = data
fmap = self.get_feature_maps('compatibility_score%d' % i, upscale=False)
try:
attmap = fmap[1][1]
except:
aaaa = 1
attention = attmap[subi,0].cpu().numpy()
# attention = attention[:, :]
# attention = numpy.expand_dims(resize(attention, (fmap_size[0], fmap_size[1]), mode='constant', preserve_range=True), axis=2)
attention = resize(attention, input_size, mode='constant', preserve_range=True)
attention = (attention-np.min(attention))/(np.max(attention)-np.min(attention))
# this one is useless
# plotNNFilter(fmap_0, figure_id=i+3, interp='bilinear', colormap=cm.jet, title='compat. feature %d' %i)
nii_seg = nib.Nifti1Image(attention, affine=np.eye(4))
output_att_file = os.path.join(out, ('%s_train_att.nii.gz' % sub_name[subi]))
nib.save(nii_seg, output_att_file)
del nii_seg, fmap, attmap, attention
# plotNNFilterOverlay(input_img, attention, figure_id=i, interp='bilinear', colormap=cm.jet,
# title='a', alpha=0.5)
# attentions.append(attention)
pred_clss = F.log_softmax(pred_prob)
pred = pred_clss.data.max(1)[1] # get th
correct += pred.eq(target.data).cpu().sum()
pred_binary = pred>0
target_binary = target.data>0
correct_binary += pred_binary.eq(target_binary).cpu().sum()
sofar += data.size(0)
test_loss = F.nll_loss(pred_clss, target)
for batch_num in range(data.size(0)):
# test_loss /= len(self.test_loader) # loss function already averages over batch size
results_strs = '[Epoch %04d] True=[%d],Pred=[%d],Pred_prob=%s,Test set: Average loss: %.4f, Accuracy: %d/%d (%.3f) binary (%.3f), subname=[%s]\n' % (
self.epoch, target.data.cpu().numpy()[batch_num], pred.cpu().numpy()[batch_num], np.array2string(pred_clss[batch_num].data.cpu().numpy()), test_loss.data[0], correct, sofar,
100. * float(correct) / sofar, 100 * float(correct_binary) / sofar, sub_name[batch_num])
print(results_strs)
fv.write(results_strs)
loss_history.append(test_loss.data.cpu().numpy().tolist())
pred_history += pred_binary.cpu().numpy().tolist()
target_history += target_binary.data.cpu().numpy().tolist()
f1 = f1_score(target_history, pred_history)
recall = recall_score(target_history, pred_history)
precision = precision_score(target_history, pred_history)
accuracy = accuracy_score(target_history, pred_history)
print_str='test epoch='+str(self.epoch)+',accuracy='+str(accuracy)+",f1="+str(f1)+",recall="+str(recall)+',precision='+str(precision)+",loss="+str(np.mean(loss_history))+"\n"
fv2.write(results_strs)
fv3.write(print_str)
fv.close()
fv2.close()
fv3.close()
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
for i in range(topk):
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def test(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
fold = 0
device = get_device(cuda)
# Synset words
classes = {0: 'normal', 1: 'covid'}
# Model
#model = models.resnet34(pretrained=False,num_classes=config.num_classes)
best_model = torch.load(config.best_models + config.model_name + os.sep +
str(fold) + os.sep + 'model_best.pth.tar')
print(best_model["state_dict"].keys())
model = make_model('mynet',
num_classes=config.num_classes,
pretrained=False)
#best_model = torch.load(config.weights +'model_best.pth.tar')
model.load_state_dict(best_model["state_dict"])
print(best_model["state_dict"].keys())
model.to(device)
model.eval()
# The four residual layers
target_layers = [
"_features.0", "_features.1", "_features.2", "_features.3",
"_features.4", "_features.5", "_features.6", "_features.7"
]
target_class = 1 # "bull mastif"
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
print(ids_.shape)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)