def demo2(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# The four residual layers
target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"]
target_class = 243 # "bull mastif"
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print(
"\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])
)
)
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "resnet152", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo2(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
model.to(device)
model.eval()
# The four residual layers
target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"]
target_class = 243 # "bull mastif"
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def guided_backprop_eye(image, name, net):
img = torch.stack([image[name]])
bp = BackPropagation(model=net)
probs, ids = bp.forward(img)
gcam = GradCAM(model=net)
_ = gcam.forward(img)
gbp = GuidedBackPropagation(model=net)
_ = gbp.forward(img)
# Guided Backpropagation
actual_status = ids[:, 0]
gbp.backward(ids=actual_status.reshape(1, 1))
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=actual_status.reshape(1, 1))
regions = gcam.generate(target_layer='last_conv')
# Get Images
prob = probs.data[:, 0]
if actual_status == 0:
prob = probs.data[:, 1]
prob_image = np.zeros((shape[0], 60, 3), np.uint8)
cv2.putText(prob_image, '%.1f%%' % (prob * 100), (5, 15),
cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv2.LINE_AA)
guided_bpg_image = get_gradient_image(gradients[0])
guided_bpg_image = cv2.merge(
(guided_bpg_image, guided_bpg_image, guided_bpg_image))
grad_cam_image = get_gradcam_image(gcam=regions[0, 0],
raw_image=image[name + '_raw'])
guided_gradcam_image = get_gradient_image(torch.mul(regions, gradients)[0])
guided_gradcam_image = cv2.merge(
(guided_gradcam_image, guided_gradcam_image, guided_gradcam_image))
#print(image['path'],classes[actual_status.data], probs.data[:,0] * 100)
print(classes[actual_status.data], probs.data[:, 0] * 100)
return cv2.hconcat([
image[name + '_raw'], prob_image, guided_bpg_image, grad_cam_image,
guided_gradcam_image
])
def extract_object(original_image, cuda = 1):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
output_dir = "./output"
# device = get_device(cuda)
device = torch.device("cuda" if cuda else "cpu")
# Synset words
classes = get_classtable()
# Model
model = models.resnet152(pretrained=True)
# device = "cuda"
# model.to(device)
model.eval()
target_layer = "layer4"
target_class = 243 # "bull mastif"
# Images
# images = original_image.to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(original_image)
# ids_ = torch.LongTensor([[target_class]] * len(original_image)).to(device)
# pdb.set_trace()
gcam.backward(ids=ids)
# Grad-CAM
dogs = []
red_bboxs = []
regions = gcam.generate(target_layer=target_layer)
for j in range(len(original_image)):
dog, red_bbox = save_gradcam(gcam=regions[j, 0], raw_image=original_image[j])
dogs.append(dog)
red_bboxs.append(red_bbox)
print(red_bbox)
# tensor_dogs = torch.stack(dogs)
# tensor_red_bbox = torch.stack(red_bboxs)
#del gcam,model,dogs,red_bboxs
return dogs, red_bboxs
def main():
root_path = '/media/palm/Unimportant/pdr2018/typesep_validate/Tomato/'
image_name = 'c9ebc74c2177ce60a8230855333fb9e7.jpg'
folder_name = '14_Tomato_Spider_Mite_Damage_Serious'
# image_path = root_path+'/14_Tomato_Spider_Mite_Damage_Serious/1c0f1ae1374d01c2933069232735a331.jpg'
image_path = os.path.join(root_path, folder_name, image_name)
topk = 1
cuda = 'cuda'
arch = 'densenet201'
CONFIG = {
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
}.get(arch)
a, b, c = getlookup()
device = torch.device(
'cuda' if cuda and torch.cuda.is_available() else 'cpu')
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = c['Tomato']
# Model
model = getmodel(20)
checkpoint = torch.load('checkpoint/try_4_densesep-Tomatotemp.t7')
model.load_state_dict(checkpoint['net'])
model.to('cuda')
model.eval()
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(raw_image).unsqueeze(0)
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
save_gradcam(
'results/{}_{}_gcam_{}.png'.format(image_name, classes[idx[i]],
arch), output, raw_image)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Vanilla Backpropagation')
# =========================================================================
bp = BackPropagation(model=model)
probs, idx = bp.forward(image.to(device))
for i in range(0, topk):
bp.backward(idx=idx[i])
output = bp.generate()
save_gradient(
'results/{}_{}_bp_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Deconvolution')
# =========================================================================
deconv = Deconvolution(
model=copy.deepcopy(model)) # TODO: remove hook func in advance
probs, idx = deconv.forward(image.to(device))
for i in range(0, topk):
deconv.backward(idx=idx[i])
output = deconv.generate()
save_gradient(
'results/{}_{}_deconv_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Guided Backpropagation/Guided Grad-CAM')
# =========================================================================
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
save_gradient(
'results/{}_{}_gbp_{}.png'.format(image_name, classes[idx[i]],
arch), feature)
save_gradient(
'results/{}_{}_ggcam_{}.png'.format(image_name, classes[idx[i]],
arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, checkpoint,
distribute):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch]()
model = model.cuda()
# print(model)
checkpoint = checkpoint + arch + '/model_best.pth.tar'
# print(checkpoint)
check_point = torch.load(checkpoint,
map_location=lambda storage, loc: storage.cuda(0))
distributed_model = (distribute > 0.5)
only_CAM = True
if distributed_model == True:
# create new OrderedDict that does not contain `module.`
from collections import OrderedDict
new_check_point = OrderedDict()
for k, v in check_point['state_dict'].items():
# name = k[7:] # remove `module.`
# name = k[9:] # remove `module.1.`
if k.startswith('module.1.'):
name = k[9:]
else:
name = k[7:]
new_check_point[name] = v
# load params
model.load_state_dict(new_check_point)
else:
model.load_state_dict(check_point['state_dict'])
model.to(device)
model.eval()
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
for i in range(topk):
# In this example, we specify the high confidence classes
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch,
classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(
j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-{}.png".format(
# j, arch, target_layer, classes[ids[j, i]] image_path
osp.splitext(image_paths[j])[0],
arch,
target_layer),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
if not only_CAM:
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
for i in range(topk):
# In this example, we specify the high confidence classes
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]
),
),
gradient=torch.mul(regions, gradients)[j],
)
def main(image_path, arch, topk, cuda):
CONFIG = {
'resnet18': {
'target_layer': 'layer4.1',
'input_size': 224
},
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
}.get(arch)
device = torch.device(
'cuda' if cuda and torch.cuda.is_available() else 'cpu')
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = ["other", "rori"]
# with open('samples/synset_words.txt') as lines:
# for line in lines:
# line = line.strip().split(' ', 1)[1]
# line = line.split(', ', 1)[0].replace(' ', '_')
# classes.append(line)
# Model
model = models.__dict__[arch](pretrained=True)
num_features = model.fc.in_features
model.fc = nn.Linear(num_features, 200) #これにより512->2の層に変わった
model.add_module('relu_fc', nn.ReLU())
model.add_module('fc2', nn.Linear(200, 2))
param = torch.load('weight_resnet18_3.pth')
model.load_state_dict(param)
model.to(device)
model.eval()
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(raw_image).unsqueeze(0)
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch),
output, raw_image)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Vanilla Backpropagation')
# =========================================================================
bp = BackPropagation(model=model)
probs, idx = bp.forward(image.to(device))
for i in range(0, topk):
bp.backward(idx=idx[i])
output = bp.generate()
save_gradient('results/{}_bp_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Deconvolution')
# =========================================================================
deconv = Deconvolution(
model=copy.deepcopy(model)) # TODO: remove hook func in advance
probs, idx = deconv.forward(image.to(device))
for i in range(0, topk):
deconv.backward(idx=idx[i])
output = deconv.generate()
save_gradient('results/{}_deconv_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Guided Backpropagation/Guided Grad-CAM')
# =========================================================================
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
save_gradient('results/{}_gbp_{}.png'.format(classes[idx[i]], arch),
feature)
save_gradient('results/{}_ggcam_{}.png'.format(classes[idx[i]], arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
# check if CUDA is available
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('CUDA is not available. Training on CPU ...')
else:
print('CUDA is available! Training on GPU ...')
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt'
model_name = 'resnet'
num_classes = 9
feature_extract = False
model, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
if train_on_gpu:
state = torch.load(PRE_MODEL_DIR)
else:
state = torch.load(PRE_MODEL_DIR, map_location='cpu')
# Loading weights in restnet architecture
model.load_state_dict(state['state_dict'])
model.to(device)
model.eval()
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
for i in range(topk):
# In this example, we specify the high confidence classes
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def demo3(image_paths, topk, output_dir, cuda):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
model = torch.hub.load("kazuto1011/pytorch-ported-models",
"xception_v1",
pretrained=True)
model.to(device)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
target_layer = "exit_flow.conv4"
# Preprocessing
def _preprocess(image_path):
raw_image = cv2.imread(image_path)
raw_image = cv2.resize(raw_image, model.image_shape)
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image -= model.mean
image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "xception_v1",
target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images) #.to(device)
# Here we choose the last convolution layer TODO: This will likely be wrong!
target_layer = "exit_flow.conv4"
target_class = 1
# run grad cam on all images!
gcam = GradCAM(model=vgg16)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
# Make the target class male. The second half of the images are all of men. Please excuse the hack.
if j > (len(images)/2):
target_class = 0
def cam(config):
## prepare data
classes = {}
classes[0] = 'non-merger'
classes[1] = 'merger'
dsets = {}
# dset_loaders = {}
#sampling WOR, i guess we leave the 10 in the middle to validate?
pristine_indices = torch.randperm(len(pristine_x))
pristine_x_test = pristine_x[
pristine_indices[int(np.floor(.8 * len(pristine_x))):]]
pristine_y_test = pristine_y[
pristine_indices[int(np.floor(.8 * len(pristine_x))):]]
noisy_indices = torch.randperm(len(noisy_x))
noisy_x_test = noisy_x[noisy_indices[int(np.floor(.8 * len(noisy_x))):]]
noisy_y_test = noisy_y[noisy_indices[int(np.floor(.8 * len(noisy_x))):]]
dsets["source"] = pristine_x_test
dsets["target"] = noisy_x_test
class_num = config["network"]["params"]["class_num"]
# load checkpoint
print('load model from {}'.format(config['ckpt_path']))
ckpt = torch.load(config['ckpt_path'] + '/best_model.pth.tar')
## set base network
net_config = config["network"]
base_network = net_config["name"](**net_config["params"])
base_network.load_state_dict(ckpt['base_network'])
centroids = None
if 'center_criterion' in ckpt.keys():
centroids = ckpt['center_criterion']['centers'].cpu()
target_centroids = None
if 'target_center_criterion' in ckpt.keys():
target_centroids = ckpt['target_center_criterion']['centers'].cpu()
use_gpu = torch.cuda.is_available()
if use_gpu:
base_network = base_network.cuda()
#might not get to tstack this?
source_images = torch.stack(list(dsets["source"])).cuda()
target_images = torch.stack(list(dsets["source"])).cuda()
base_network.train(False)
model_name = config["network"]
target_layer = config[
"target_layer"] #no idea... maybe i can print this out? layer4 for resnet, relu for deepemerge
if config["class"] == 'non-merger':
target_class = 0 #non-merger
elif config["class"] == 'merger':
target_class = 1 #non-merger
else:
print("incorrect class choice")
sys.exit()
save_where = osp.join(
config["ckpt_path"],
'guided ' + str(config["which"]) + '-' + str(config["class"]))
output_dir = save_where
if not osp.exists(save_where):
os.makedirs(save_where)
else:
os.chdir(save_where)
gcam = GradCAM(model=base_network)
gbp = GuidedBackPropagation(model=base_network)
if config["which"] == 'source':
print("start source test: ")
probs, ids = gcam.forward(source_images) #see how they load in images
_ = gbp.forward(source_images) #see if you need to catch anything
#not sure what to do about this
if use_gpu:
ids_ = torch.LongTensor([[target_class]] *
len(source_images)).cuda()
else:
ids_ = torch.LongTensor([[target_class]] * len(source_images))
gcam.backward(ids=ids_)
gbp.backward(ids=ids_)
gradients = gbp.generate()
regions = gcam.generate(target_layer=target_layer)
# print("ids")
# print(ids)
# print("probs")
# print(probs)
# # print("argmax")
# # print(torch.argmax(ids, dim = 1))
# # print("equality")
# # print(torch.argmax(ids, dim = 1).cpu() == target_class)
# # print(probs)
# # print(probs[(torch.argmax(ids, dim = 1).cpu() == target_class).cpu()])
# # print(probs.cpu()[(torch.argmax(ids, dim = 1).cpu() == target_class).cpu()])
# #print(torch.argmax(probs, dim = 1).cpu() == target_class)
# print("try 1")
# print(probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]])
# print("try 2")
# a = probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]]
# print(a[:, target_class])
# a = probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]]
# a = a[:, target_class]
# print(len(regions))
# print(len(source_images)-1)
for j in range(0, len(source_images) - 1):
# print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[:, target_class][j])
# )
# )
# save_gradcam(
# filename=osp.join(
# output_dir,
# "{}-{}-gradcam-{}-{}.png".format(
# j, model_name, target_layer, classes[target_class]
# ),
# ),
# gcam=regions[j, 0],
# #raw_image=raw_images[j],
# raw_image=source_images[j],
# )
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, net_config,
classes[target_class]),
),
gradient=torch.mul(regions, gradients)
[j], #gradients[j], #torch.mul(regions, gradients)[j],
)
elif config["which"] == 'target':
print("start target test: ")
probs, ids = gcam.forward(target_images) #see how they load in images
_ = gbp.forward(target_images) #see if you need to catch anything
#not sure what to do about this
if use_gpu:
ids_ = torch.LongTensor([[target_class]] *
len(target_images)).cuda()
else:
ids_ = torch.LongTensor([[target_class]] * len(target_images))
gcam.backward(ids=ids_)
gbp.backward(ids=ids_)
gradients = gbp.generate()
regions = gcam.generate(target_layer=target_layer)
for j in range(0, len(target_images) - 1):
# print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[:, target_class][j])
# )
# )
# save_gradcam(
# filename=osp.join(
# output_dir,
# "{}-{}-gradcam-{}-{}.png".format(
# j, model_name, target_layer, classes[target_class]
# ),
# ),
# gcam=regions[j, 0],
# #raw_image=raw_images[j],
# raw_image= target_images[j],
# )
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, net_config,
classes[target_class]),
),
gradient=torch.mul(regions, gradients)[j], #gradients[j],
)
else:
print("incorrect domain choice")
sys.exit()
return ()
def main():
parser = make_parser() #creates the parser
args = parser.parse_args()
CONFIG = {
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
'resnet50': {
'target_layer': 'layer4',
'input_size': 224
},
'resnet18': {
'target_layer': 'layer4',
'input_size': 224
},
# Add your model
}.get(args.arch)
device = torch.device(
'cuda' if args.cuda and torch.cuda.is_available() else 'cpu')
if args.cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = list()
with open('samples/synset_words.txt') as lines:
for line in lines:
line = line.strip().split(' ', 1)[1]
line = line.split(', ', 1)[0].replace(' ', '_')
classes.append(line)
# Model
print("1")
class_names = ['no',
'yes'] #important to define the classes for prediction
model_ft = get_cnn(len(class_names),
args) #retrieves the cnn - architecture to be used
print("2")
criterion = nn.CrossEntropyLoss(
) #creates the criterion (used in training and testing)
optimizer_ft = get_optimizer(
model_ft, args
) #changes the weights based on error (using Stochastic Gradient Descent)
exp_lr_scheduler = lr_scheduler.StepLR(
optimizer_ft, step_size=7, gamma=0.1
) #helps with the learning rate, to be zigzagging to get into the right function
model = load_model(model_ft, args.weights) #load the model with weights
print("3")
model = model.to(device)
model.eval()
# Image
print("image path: " + str(args.image_path))
raw_image = cv2.imread(args.image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225],
)
])(raw_image).unsqueeze(0)
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(image.to(device))
for i in range(0, args.topk):
print("idx is: " + str(idx))
print("idx is: " + str(probs))
print("i is: " + str(i))
gcam.backward(idx=idx[i])
print("idx AFTER is: " + str(idx))
print("idx[i]: " + str(idx[i]))
output = gcam.generate(target_layer=CONFIG['target_layer'])
print("classes[idx[i]]: " + str(classes[idx[i]]))
save_gradcam(
'results/{}_gcam_{}.png'.format(classes[idx[i]], args.arch),
output, raw_image)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Vanilla Backpropagation')
# =========================================================================
bp = BackPropagation(model=model)
probs, idx = bp.forward(image.to(device))
for i in range(0, args.topk):
bp.backward(idx=idx[i])
output = bp.generate()
save_gradient(
'results/{}_bp_{}.png'.format(classes[idx[i]], args.arch), output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Deconvolution')
# =========================================================================
deconv = Deconvolution(
model=copy.deepcopy(model)) # TODO: remove hook func in advance
probs, idx = deconv.forward(image.to(device))
for i in range(0, args.topk):
deconv.backward(idx=idx[i])
output = deconv.generate()
save_gradient(
'results/{}_deconv_{}.png'.format(classes[idx[i]], args.arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
# =========================================================================
print('Guided Backpropagation/Guided Grad-CAM')
# =========================================================================
gbp = GuidedBackPropagation(model=model)
probs, idx = gbp.forward(image.to(device))
for i in range(0, args.topk):
gcam.backward(idx=idx[i])
region = gcam.generate(target_layer=CONFIG['target_layer'])
gbp.backward(idx=idx[i])
feature = gbp.generate()
h, w, _ = feature.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = feature * region
save_gradient(
'results/{}_gbp_{}.png'.format(classes[idx[i]], args.arch),
feature)
save_gradient(
'results/{}_ggcam_{}.png'.format(classes[idx[i]], args.arch),
output)
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, topk, output_dir, cuda):
# Get image paths
root = '../UTKFace/val'
image_paths = []
q = 0
for female_file in os.listdir(os.path.join(root, "female")):
image_paths.append(
os.path.join(root, os.path.join("female", str(female_file))))
q += 1
if (q == 10):
break
q = 0
for male_file in os.listdir(os.path.join(root, "male")):
image_paths.append(
os.path.join(root, os.path.join("male", str(male_file))))
q += 1
if (q == 10):
break
print(len(image_paths))
# preprocess each image
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images) #.to(device)
"""
CNN Gradcam
"""
"""
# Synset words
classes = get_classtable()
print("Using CNN")
cnn = CNN()
#if use_gpu:
# cnn.cuda()
optimizer = torch.optim.SGD(cnn.parameters(),lr=0.001,momentum=0.9)
SAVED_MODEL_PATH = '../cnn_model_best_final.pth.tar'
checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu')
cnn.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
epoch = checkpoint['epoch']
best_val_acc = checkpoint['best_val_acc']
print("best model saved from epoch: ", epoch)
print("best val_acc = ", best_val_acc)
cnn.eval()
# The four residual layers
target_layers = ["layer3"]
target_class = 1
# run grad cam on all images!
gcam = GradCAM(model=cnn)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
#print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[ids == target_class])
# )
# )
if j > (len(images)/2):
target_class = 0
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "CNN", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
"""
"""
# ============================================================================================
Adversary CNN
"""
"""
classes = get_classtable()
print("Using Adversary CNN")
cnn = CNN()
optimizer = torch.optim.SGD(cnn.parameters(),lr=0.001,momentum=0.9)
SAVED_MODEL_PATH ='../cnn_model_best_SGD_adversary.pth.tar'
checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu')
cnn.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
epoch = checkpoint['epoch']
best_val_acc = checkpoint['best_val_acc']
print("best model saved from epoch: ", epoch)
print("best val_acc = ", best_val_acc)
cnn.eval()
# The four residual layers
target_layers = ["layer3"]
target_class = 1
# run grad cam on all images!
gcam = GradCAM(model=cnn)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
#print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[ids == target_class])
# )
# )
# Make the target class male. The second half of the images are all of men. Please excuse the hack.
if j > (len(images)/2):
target_class = 0
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "CNNAdv", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
"""
"""
# =============================================================================================
VGG16 Gradcam
"""
"""
# TODO: waiting on .tar file.
#device = get_device(cuda)
# Synset words
classes = get_classtable()
# make model
NUM_CLASSES = 2
vgg16 = models.vgg16(pretrained = True)
# Freeze training for all layers NOTE: may or may not need this
for param in vgg16.features.parameters():
param.require_grad = False
# Newly created modules have require_grad=True by default
num_features = vgg16.classifier[6].in_features
features = list(vgg16.classifier.children())[:-1] # Remove last layer
features.extend([nn.Linear(num_features, NUM_CLASSES)]) # Add our layer with 4 outputs
vgg16.classifier = nn.Sequential(*features) # Replace the model classifier
optimizer = torch.optim.SGD(vgg16.parameters(),lr=0.001,momentum=0.9) # dont know why you need this when you're just recreating a model that's been already trained...
SAVED_MODEL_PATH = '../vgg_checkpoint.pth.tar'
checkpoint = torch.load(SAVED_MODEL_PATH)
vgg16.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
epoch = checkpoint['epoch']
best_val_acc = checkpoint['best_val_acc']
vgg16.eval() # the gcam has this eval in their code. idk if we need it though
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer TODO: This will likely be wrong!
target_layer = "exit_flow.conv4"
target_class = 0 # TODO: CHANGE THIS!!!
# run grad cam on all images!
gcam = GradCAM(model=cnn)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
"""
#print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[ids == target_class])
# )
#)
"""
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "VGG16", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
"""
"""
# =============================================================================================
VGG16 Adversarial Gradcam
"""
"""
# =============================================================================================
Resnet Gradcam
"""
"""
# Synset words
classes = get_classtable()
model = models.resnet18(pretrained=True)
for param in model.parameters():
param.requires_grad = True
#print(model)
#Source of code below:
# Parameters of newly constructed modules have requires_grad=True by default
num_ftrs = model.fc.out_features
model = nn.Sequential(model, nn.Linear(num_ftrs, 1000), nn.ReLU(), nn.Linear(1000, 2))
learning_rate = 0.002
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate,momentum=0.9)
SAVED_MODEL_PATH = '../resnet_model_best_v1.pth.tar'
checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu')
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
epoch = checkpoint['epoch']
best_val_acc = checkpoint['best_val_acc']
print("best model saved from epoch: ", epoch)
print("best val_acc = ", best_val_acc)
model.eval() # the gcam has this eval in their code. idk if we need it though
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer TODO: This will likely be wrong!
target_layers = ["0.layer4.1.bn2"] # ["layer4"] # why did they originally do "relu", "layer1", "layer2", "layer3",
target_class = 1
# run grad cam on all images!
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
"""
#print(
# "\t#{}: {} ({:.5f})".format(
# j, classes[target_class], float(probs[ids == target_class])
# )
#)
"""
# Make the target class male. The second half of the images are all of men. Please excuse the hack.
if j > (len(images)/2):
target_class = 0
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "RESNET", target_layer, classes[target_class]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
"""
"""
# =============================================================================================
Resnet Adversarial Gradcam
"""
learning_rate = 0.002
model = MyResnet()
criterion = nn.CrossEntropyLoss()
adversary = NN(1000)
nn_criterion = nn.CrossEntropyLoss()
# resnet = models.resnet18(pretrained=True)
# for param in resnet.parameters():
# param.requires_grad = True
# num_ftrs = resnet.fc.out_features
# fc1 = nn.Sequential(nn.Linear(num_ftrs, 1000),
# nn.ReLU(),
# nn.Linear(1000,2))
optimizer = torch.optim.SGD(model.parameters(),
lr=learning_rate,
momentum=0.9)
# Synset words
classes = get_classtable()
SAVED_MODEL_PATH = '../resnet_model_best.pth.tar'
checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu')
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
epoch = checkpoint['epoch']
best_val_acc = checkpoint['best_val_acc']
print("best model saved from epoch: ", epoch)
print("best val_acc = ", best_val_acc)
model.eval(
) # the gcam has this eval in their code. idk if we need it though
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer TODO: This will likely be wrong!
target_layers = [
"resnet.layer4.1.bn2"
] # ["layer4"] # why did they originally do "relu", "layer1", "layer2", "layer3",
target_class = 1
# run grad cam on all images!
gcam = GradCAM(model=model)
print(images[0])
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
# Make the target class male. The second half of the images are all of men. Please excuse the hack.
if j > (len(images) / 2):
target_class = 0
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "RESNET", target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
"""
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, model):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
if model is None:
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
print(image_paths)
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images)
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
#print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}.png".format(
j, arch, target_layer
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def main(image_path, target_layer, arch, topk, cuda):
device = torch.device(
"cuda" if cuda and torch.cuda.is_available() else "cpu")
if cuda:
current_device = torch.cuda.current_device()
print("Running on the GPU:",
torch.cuda.get_device_name(current_device))
else:
print("Running on the CPU")
# Synset words
classes = list()
# Model from torchvision
model = MGN()
"""
pickle.load = partial(pickle.load, encoding="latin1")
pickle.Unpickler = partial(pickle.Unpickler, encoding="latin1")
checkpoint = torch.load("hacnn_market_xent.pth.tar", pickle_module=pickle)
#checkpoint = torch.load(args.load_weights)
#checkpoint = torch.load("hacnn_market_xent.pth.tar")
pretrain_dict = checkpoint['state_dict']
model_dict = model.state_dict()
pretrain_dict = {k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size()}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
"""
pretrain_dict = torch.load("model_700.pt")
model_dict = model.state_dict()
pretrain_dict = {
k: v
for k, v in pretrain_dict.items()
if k in model_dict and model_dict[k].size() == v.size()
}
model_dict.update(pretrain_dict)
model.load_state_dict(model_dict)
#model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
place = []
with open("samples/new.txt") as lines:
for line in lines:
line = line.strip().split(" ", 1)[1]
line = line.split(", ", 1)[0].replace(" ", "_")
classes.append(line)
# Image preprocessing
with open("new.txt") as lines:
for line in lines:
line = line.strip()
line = "/mnt/SSD/jzwang/market1501/query/" + line
place.append(line)
for line in place:
image_path = line
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (128, 384))
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])(raw_image).unsqueeze(0)
image = image.to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with an image
3. Run backward() with a specific class
4. Run generate() to export result
"""
# =========================================================================
print("Vanilla Backpropagation")
bp = BackPropagation(model=model)
predictions = bp.forward(image)
for i in range(topk):
print("[{:.5f}] {}".format(predictions[i][0],
classes[predictions[i][1]]))
bp.backward(idx=predictions[i][1])
gradient = bp.generate()
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution")
deconv = Deconvnet(model=model)
_ = deconv.forward(image)
for i in range(topk):
print("[{:.5f}] {}".format(predictions[i][0],
classes[predictions[i][1]]))
deconv.backward(idx=predictions[i][1])
gradient = deconv.generate()
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM")
gcam = GradCAM(model=model)
_ = gcam.forward(image)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(image)
t = ['p1', 'p2', 'p3']
for i in range(topk):
print("[{:.5f}] {}".format(predictions[i][0],
classes[predictions[i][1]]))
# Grad-CAM
for target_layer in t:
gcam.backward(idx=predictions[i][1])
#print("1")
region = gcam.generate(target_layer=target_layer)
#print(2)
line = line.strip().split("/")[-1]
save_gradcam(
"results/{}-gradcam-{}.png".format(line, target_layer),
region,
raw_image,
)
#print(3)
# Guided Backpropagation
gbp.backward(idx=predictions[i][1])
gradient = gbp.generate()
# Guided Grad-CAM
h, w, _ = gradient.shape
region = cv2.resize(region, (w, h))[..., np.newaxis]
output = gradient * region
def demo3(image_paths, topk, output_dir, cuda):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
# check if CUDA is available
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('CUDA is not available. Training on CPU ...')
else:
print('CUDA is available! Training on GPU ...')
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt'
model_name = 'resnet'
num_classes = 9
feature_extract = False
model, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
if train_on_gpu:
state = torch.load(PRE_MODEL_DIR)
else:
state = torch.load(PRE_MODEL_DIR, map_location='cpu')
# Loading weights in restnet architecture
model.load_state_dict(state['state_dict'])
model.to(device)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
target_layer = "layer4.2.conv3"
# Preprocessing
def _preprocess(image_path):
raw_image = cv2.imread(image_path)
raw_image = cv2.resize(raw_image, (224, 224))
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image -= torch.FloatTensor([0.485, 0.456, 0.406])
image /= torch.FloatTensor([0.229, 0.224, 0.225])
image = image.permute(2, 0, 1)
return image, raw_image
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "xception_v1",
target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo40(image_paths, output_dir, cuda, classes_target):
"""
Generate Grad-CAM at last convolucional for each block layers of ResNet-152
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model
# check if CUDA is available
train_on_gpu = torch.cuda.is_available()
if not train_on_gpu:
print('CUDA is not available. Training on CPU ...')
else:
print('CUDA is available! Training on GPU ...')
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt'
model_name = 'resnet'
num_classes = 9
feature_extract = False
model, input_size = initialize_model(model_name,
num_classes,
feature_extract,
use_pretrained=True)
if train_on_gpu:
state = torch.load(PRE_MODEL_DIR)
else:
state = torch.load(PRE_MODEL_DIR, map_location='cpu')
# Loading weights in restnet architecture
model.load_state_dict(state['state_dict'])
model.to(device)
model.eval()
# The the last convolucinal layer for each block
target_layers = [
"layer1.0.conv3", "layer1.1.conv3", "layer1.2.conv3", "layer2.0.conv3",
"layer2.1.conv3", "layer2.2.conv3", "layer2.3.conv3", "layer2.4.conv3",
"layer2.5.conv3", "layer2.6.conv3", "layer2.7.conv3", "layer3.0.conv3",
"layer3.1.conv3", "layer3.2.conv3", "layer3.3.conv3", "layer3.4.conv3",
"layer3.5.conv3", "layer3.6.conv3", "layer3.7.conv3", "layer3.8.conv3",
"layer3.9.conv3", "layer3.10.conv3", "layer3.11.conv3",
"layer3.12.conv3", "layer3.13.conv3", "layer3.14.conv3",
"layer3.15.conv3", "layer3.16.conv3", "layer3.17.conv3",
"layer3.18.conv3", "layer3.19.conv3", "layer3.20.conv3",
"layer3.21.conv3", "layer3.22.conv3", "layer3.23.conv3",
"layer3.24.conv3", "layer3.25.conv3", "layer3.26.conv3",
"layer3.27.conv3", "layer3.28.conv3", "layer3.29.conv3",
"layer3.30.conv3", "layer3.31.conv3", "layer3.32.conv3",
"layer3.33.conv3", "layer3.34.conv3", "layer3.35.conv3",
"layer4.0.conv3", "layer4.1.conv3", "layer4.2.conv3"
]
target_class = int(classes_target)
# {'actinic-keratosis': 0,
# 'basal-cell-carcinoma': 1,
# 'dermatofibroma': 2,
# 'hemangioma': 3,
# 'intraepithelial-carcinoma': 4,
# 'malignant-melanoma': 5,
# 'melanocytic-nevus': 6,
# 'pyogenic-granuloma': 7,
# 'squamous-cell-carcinoma': 8}
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def main(args):
setup_logger(name="fvcore")
logger = setup_logger()
logger.info("Arguments: " + str(args))
cfg = setup_cfg(args)
print(cfg)
# 构建模型
model = build_model(cfg)
# 加载权重
checkpointer = DetectionCheckpointer(model)
checkpointer.load(cfg.MODEL.WEIGHTS)
# 加载图像
path = os.path.expanduser(args.input)
original_image = read_image(path, format="BGR")
height, width = original_image.shape[:2]
transform_gen = T.ResizeShortestEdge(
[cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST],
cfg.INPUT.MAX_SIZE_TEST)
image = transform_gen.get_transform(original_image).apply_image(
original_image)
image = torch.as_tensor(image.astype("float32").transpose(
2, 0, 1)).requires_grad_(True)
inputs = {"image": image, "height": height, "width": width}
# Grad-CAM
layer_name = get_last_conv_name(model)
grad_cam = GradCAM(model, layer_name)
mask, box, class_id = grad_cam(inputs) # cam mask
grad_cam.remove_handlers()
#
image_dict = {}
img = original_image[..., ::-1]
x1, y1, x2, y2 = box
image_dict['predict_box'] = img[y1:y2, x1:x2]
image_cam, image_dict['heatmap'] = gen_cam(img[y1:y2, x1:x2], mask)
# Grad-CAM++
grad_cam_plus_plus = GradCamPlusPlus(model, layer_name)
mask_plus_plus = grad_cam_plus_plus(inputs) # cam mask
_, image_dict['heatmap++'] = gen_cam(img[y1:y2, x1:x2], mask_plus_plus)
grad_cam_plus_plus.remove_handlers()
# 获取类别名称
meta = MetadataCatalog.get(
cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused")
label = meta.thing_classes[class_id]
print("label:{}".format(label))
# # GuidedBackPropagation
# gbp = GuidedBackPropagation(model)
# inputs['image'].grad.zero_() # 梯度置零
# grad = gbp(inputs)
# print("grad.shape:{}".format(grad.shape))
# gb = gen_gb(grad)
# gb = gb[y1:y2, x1:x2]
# image_dict['gb'] = gb
# # 生成Guided Grad-CAM
# cam_gb = gb * mask[..., np.newaxis]
# image_dict['cam_gb'] = norm_image(cam_gb)
save_image(image_dict, os.path.basename(path))
def train(self):
"""
Train the model on the training set.
A checkpoint of the model is saved after each epoch
and if the validation accuracy is improved upon,
a separate ckpt is created for use on the test set.
"""
# load the most recent checkpoint
model_num = 0
if self.resume:
self.load_checkpoint(model_num, best=False)
print("\n[*] Train on {} samples, validate on {} samples".format(
self.num_train, self.num_valid)
)
#Added:
self.device, self.device_name = self.get_device()
for model in self.models:
self.grad_cams.append(GradCAM(model=model))
for epoch in range(self.start_epoch, self.epochs):
for scheduler in self.schedulers:
scheduler.step(epoch)
print(
'\nEpoch: {}/{} - LR: {:.6f}'.format(
epoch+1, self.epochs, self.optimizers[0].param_groups[0]['lr'],)
)
# train for 1 epoch
train_losses, train_accs = self.train_one_epoch(epoch)
# evaluate on validation set
valid_losses, valid_accs = self.validate(epoch)
for i in range(self.model_num):
is_best = valid_accs[i].avg> self.best_valid_accs[i]
msg1 = "model_{:d}: train loss: {:.3f} - train acc: {:.3f} "
msg2 = "- val loss: {:.3f} - val acc: {:.3f}"
if is_best:
#self.counter = 0
msg2 += " [*]"
msg = msg1 + msg2
print(msg.format(i+1, train_losses[i].avg, train_accs[i].avg, valid_losses[i].avg, valid_accs[i].avg))
# check for improvement
#if not is_best:
#self.counter += 1
#if self.counter > self.train_patience:
#print("[!] No improvement in a while, stopping training.")
#return
self.best_valid_accs[i] = max(valid_accs[i].avg, self.best_valid_accs[i])
self.save_checkpoint(i,
{'epoch': epoch + 1,
'model_state': self.models[i].state_dict(),
'optim_state': self.optimizers[i].state_dict(),
'best_valid_acc': self.best_valid_accs[i],
}, is_best
)
def main(image_path, arch, topk, cuda, target_layer):
CONFIG = {
'resnet152': {
'target_layer': 'layer4.2',
'input_size': 224
},
'vgg19': {
'target_layer': 'features.36',
'input_size': 224
},
'vgg19_bn': {
'target_layer': 'features.52',
'input_size': 224
},
'inception_v3': {
'target_layer': 'Mixed_7c',
'input_size': 299
},
'densenet201': {
'target_layer': 'features.denseblock4',
'input_size': 224
},
# Add your model
'se_resnet': {
'target_layer': target_layer,
'input_size': 32
},
}.get(arch)
cuda = cuda and torch.cuda.is_available()
if cuda:
current_device = torch.cuda.current_device()
print('Running on the GPU:',
torch.cuda.get_device_name(current_device))
else:
print('Running on the CPU')
# Synset words
classes = list()
with open('samples/synset_words.txt') as lines:
for line in lines:
line = line.strip().split(' ', 1)[1]
line = line.split(', ', 1)[0].replace(' ', '_')
classes.append(line)
# Model
#model = models.__dict__[arch](pretrained=True)
model = models.se_resnet.se_resnet50(num_classes=100)
# Image
raw_image = cv2.imread(image_path)[..., ::-1]
raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2)
image = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])(raw_image)
if cuda:
model.cuda()
image = image.cuda()
# =========================================================================
print('Grad-CAM')
# =========================================================================
gcam = GradCAM(model=model)
probs, idx = gcam.forward(to_var(image))
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=CONFIG['target_layer'])
#save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch), output, raw_image) # NOQA
save_gradcam('results/{}.png'.format(target_layer), output,
raw_image) # NOQA
print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
cfg = edict(json.load(fp))
classes = [
'Cardiomegaly', 'Edema', 'Consolidation', 'Atelectasis', 'Pleural Effusion'
]
num_tasks = len(cfg.num_classes)
model = build_model(cfg, join(opt.model, 'best.ckpt'))
dataset = ImageDataset(opt.dataset, cfg, mode='heatmap')
dataloader = DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=False,
drop_last=False)
gcam = GradCAM(model=model)
layer = 'backbone.features.denseblock4.denselayer15'
for images, paths, labels in dataloader:
probs, ids = gcam.forward(images)
processed_images = [[] for _ in range(len(images))]
for i in range(ids.shape[1]):
gcam.backward(ids[:, [i]])
regions = gcam.generate(target_layer=layer)
for j in range(len(images)):
print(f"#{j}: {classes[ids[j, i]]} ({probs[j, i]:.5f})")
# Grad-CAM
raw_image = imread(paths[j])
combined = combine_image_and_gcam(regions[j, 0], raw_image)
processed_images[j].append(combined.astype(np.uint8))
sample_input = Variable(torch.randn(1,3,in_size,in_size), volatile=False)
if use_gpu:
sampe_input = sample_input.cuda()
def is_image(f):
return f.endswith(".png") or f.endswith(".jpg")
test_transform = transforms.Compose([
transforms.Scale(in_size),
transforms.CenterCrop(in_size),
transforms.ToTensor(),
transforms.Normalize(cf.mean, cf.std)
])
gcam = GradCAM(list(model._modules.items())[0][1], cuda=use_gpu)
print("\n[Phase 2] : Gradient Detection")
if args.subtype != None:
WBC_id = return_class_idx(args.subtype)
if not (args.subtype in dset_classes):
print("The given subtype does not exists!")
sys.exit(1)
if args.subtype == None:
print("| Checking All Activated Regions...")
else:
print("| Checking Activated Regions for " + dset_classes[WBC_id] + "...")
for subdir, dirs, files in os.walk(cf.test_base):
# print(label_id[pred])
print(output, pred.item())
return pred
state_dict = torch.load(image_model_path)
model.load_state_dict(state_dict)
model.eval()
label_id = {0: 'butterfly',1: 'ibis'}
grad_cam = GradCAM(model=model, feature_layer=list(model.layer4.modules())[-1])
VISUALIZE_SIZE = (224, 224)
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
image_transform = transforms.Compose([
transforms.Resize(VISUALIZE_SIZE),
transforms.ToTensor(),
normalize])
image.thumbnail(VISUALIZE_SIZE, Image.ANTIALIAS)
# save image origin size
image_orig_size = image.size # (W, H)
def demo3(image_paths, topk, output_dir, cuda):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
model = torch.hub.load(
"kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True
)
model.to(device)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
target_layer = "exit_flow.conv4"
# Preprocessing
def _preprocess(image_path):
raw_image = cv2.imread(image_path)
raw_image = cv2.resize(raw_image, model.image_shape)
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image -= model.mean
image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(
j, "xception_v1", target_layer, classes[ids[j, i]]
),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo3(image_paths, topk, output_dir, cuda, num_class, da_model):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
#classes = get_classtable()
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
#model = torch.hub.load(
# "kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True
#)
#target_images = ['mediumresidential01','mediumresidential03', 'mediumresidential14',\
# 'mediumresidential21', 'mediumresidential36','mediumresidential37'] + \
#['mediumresidential38','mediumresidential39', 'mediumresidential40',\
# 'mediumresidential41', 'mediumresidential43','mediumresidential44']
num_class, = num_class
#target_images = ['baseballdiamond96', 'airplane18', 'runway42', 'sparseresidential77', 'mediumresidential21']
target_images = [
'runway00', 'runway02', 'runway03', 'runway04', 'runway05', 'runway06',
'runway07', 'runway08'
]
classes = [str(i) for i in range(num_class)]
if da_model == 'STA':
feature_extractor = ResNetFc(
model_name='resnet50',
model_path=
'/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/resnet50.pth'
)
cls = CLS(feature_extractor.output_num(),
num_class,
bottle_neck_dim=256)
model = nn.Sequential(feature_extractor, cls).cuda()
model.load_state_dict(
torch.load(
'/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/Only_source_classifier.pth'
))
elif da_model == 'OPDA':
G, C = get_model('vgg', num_class=num_class, unit_size=1000)
load_model(
G, C,
'/home/at7133/Research/Domain_adaptation/OPDA_BP/checkpoint/checkpoint_99'
)
model = nn.Sequential(G, C).cuda()
model.to(device)
#unset_training(model)
model.eval()
# Check available layer names
print("Layers:")
for m in model.named_modules():
print("\t", m[0])
# Here we choose the last convolution layer
#target_layer = "exit_flow.conv4"
if da_model == 'STA':
target_layer = "0.model_resnet.layer4.2.conv3"
elif da_model == 'OPDA':
target_layer = "0.lower.36" #TODO find proper target layer
# Preprocessing
def _Normalize(img, mean, std):
if isinstance(img, torch.FloatTensor):
mean = torch.FloatTensor(mean)
std = torch.FloatTensor(std)
else:
raise TypeError(f'Expected Torch floattensor, got {type(img)}')
return (img - mean) / std
def _preprocess(image_path, img_shape):
raw_image = cv2.imread(image_path)
#raw_image = cv2.resize(raw_image, model.image_shape)
raw_image = cv2.resize(raw_image, (img_shape, img_shape))
image = torch.FloatTensor(raw_image[..., ::-1].copy())
image = image / 255.0
if da_model == "OPDA":
image = _Normalize(image, [0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
#pdb.set_trace()
#image -= model.mean
#image /= model.std
image = image.permute(2, 0, 1)
return image, raw_image
# Images
def Load_txt(img_paths):
assert isfile(img_paths), f"Image path {img_paths} doesn't exist"
with open(img_paths, 'rb') as fr:
image_paths = [
img_path.split()[0].decode("utf-8")
for img_path in fr.readlines()
]
return image_paths
def Filter_imgages(image_paths, req_images):
img_files = list(
map(lambda x: x.split('/')[-1].split('.')[0], image_paths))
satisfied_images = []
for item in req_images:
try:
[[idx]] = np.argwhere(np.isin(img_files, item))
except:
raise ValueError(f'{item} not found in the given paths')
satisfied_images.append(image_paths[idx])
return satisfied_images
def Load_images(image_paths, req_images):
if image_paths[0].endswith('.txt'):
assert len(image_paths
) == 1 #make sure only one text file is given as input
image_paths = Load_txt(image_paths[0])
image_paths = Filter_imgages(image_paths, req_images)
assert len(image_paths) == len(
req_images), " All target images are not found"
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = _preprocess(image_path, 224)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
return images, raw_images
images, raw_images = Load_images(image_paths, target_images)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
if not isdir(output_dir):
os.makedirs(output_dir)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "xception_v1",
target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def grad_cam(model, image_paths, topk, output_dir, cuda, model_name,
checkpoint):
"""
Generate Grad-CAM with original models
"""
device = get_device(cuda)
# Synset words
classes = [
'apple', # id 0
'aquarium_fish',
'baby',
'bear',
'beaver',
'bed',
'bee',
'beetle',
'bicycle',
'bottle',
'bowl',
'boy',
'bridge',
'bus',
'butterfly',
'camel',
'can',
'castle',
'caterpillar',
'cattle',
'chair',
'chimpanzee',
'clock',
'cloud',
'cockroach',
'couch',
'crab',
'crocodile',
'cup',
'dinosaur',
'dolphin',
'elephant',
'flatfish',
'forest',
'fox',
'girl',
'hamster',
'house',
'kangaroo',
'computer_keyboard',
'lamp',
'lawn_mower',
'leopard',
'lion',
'lizard',
'lobster',
'man',
'maple_tree',
'motorcycle',
'mountain',
'mouse',
'mushroom',
'oak_tree',
'orange',
'orchid',
'otter',
'palm_tree',
'pear',
'pickup_truck',
'pine_tree',
'plain',
'plate',
'poppy',
'porcupine',
'possum',
'rabbit',
'raccoon',
'ray',
'road',
'rocket',
'rose',
'sea',
'seal',
'shark',
'shrew',
'skunk',
'skyscraper',
'snail',
'snake',
'spider',
'squirrel',
'streetcar',
'sunflower',
'sweet_pepper',
'table',
'tank',
'telephone',
'television',
'tiger',
'tractor',
'train',
'trout',
'tulip',
'turtle',
'wardrobe',
'whale',
'willow_tree',
'wolf',
'woman',
'worm',
]
# Third-party model from my other repository, e.g. Xception v1 ported from Keras
checkpoint = checkpoint
model.load_state_dict(checkpoint['state_dict'])
model.to(device)
model.eval()
# Check available layer names
'''print("Layers:")
for m in model.named_modules():
print("\t", m[0])'''
# Here we choose the last convolution layer
target_layer = "module.layer4"
#print(target_layer)
# Images
images = []
raw_images = []
print("Images:")
for i, image_path in enumerate(image_paths):
print("\t#{}: {}".format(i, image_path))
image, raw_image = preprocess(image_path)
images.append(image)
raw_images.append(raw_image)
images = torch.stack(images).to(device)
print("Grad-CAM:")
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
for i in range(topk):
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"top{}-{}-{}-gradcam-{}-{}-prob-({:.5f}).png".format(
i + 1, j, model_name, target_layer, classes[ids[j, i]],
probs[j, i]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def main():
"""
Visualize model responses given multiple images
"""
device = torch.device("cuda")
test_dataset = CXRDataset(args.image_dir, ['TEST'])
test_loader = DataLoader(test_dataset,
batch_size=args.batch_size,
pin_memory=True)
model = CombineNet().to(device)
model = nn.DataParallel(model)
print("Parameters: {}".format(sum(p.numel() for p in model.parameters())))
model.eval()
if args.load_model:
checkpoint = torch.load(args.load_model + "/best.ckpt")
#model_weights = OrderedDict()
#for k, v in checkpoint['model'].items():
# name = k[7:] # remove `module.`
# model_weights[name] = v
model.load_state_dict(checkpoint['model'])
epoch = checkpoint['epoch']
print('Loading model: {}, from epoch: {}'.format(
args.load_model, epoch))
else:
print('Model: {} not found'.format(args.load_model))
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
target_layer = 'module.feature_model.layer_block.3.1.conv2'
#print(model.module.keys())
print(model.module.feature_model.layer_block[3][1].conv2)
epoch_iterator = tqdm(test_loader, desc="Iteration")
for i, batch in enumerate(epoch_iterator, 0):
raw_images, _, _ = batch
batch_device = tuple(t.to(device, non_blocking=True) for t in batch)
images, labels, img_ids = batch_device
if (labels.cpu().detach().numpy() == np.array([0, 0, 0, 1])).all():
probs, ids = gcam.forward(images)
gcam.backward(ids=ids[:, [0]])
regions = gcam.generate(target_layer=target_layer)
for j in range(args.batch_size):
gcam_im = save_gradcam(
filename=osp.join(
args.output_dir,
"{}-gradcam-{}_severe_{}.png".format(
j, target_layer, i),
),
gcam=regions[j, 0],
raw_image=images[j],
)
img_path = osp.join(
args.output_dir,
"{}-image-{}_severe_{}.png".format(j, target_layer, i),
)
raw_image = np.array(raw_images[j])
xray = save_xray(img_path, raw_image)
'''
# TODO: Blending code
#cv2.imwrite(img_path, raw_images[j])
img_path=osp.join(
args.output_dir,
"{}-blended-{}_severe_{}.png".format(j, target_layer, i),
)
beta = (1.0 - args.alpha)
xray = np.array(xray)
xray = xray.squeeze()
xray = np.stack([xray, xray, xray],axis=2)
print(xray.dtype)
print(gcam.dtype)
blended = cv2.addWeighted(xray, args.alpha, gcam, beta, 0.0)
save_xray(img_path, blended)
'''
print('finished!')
def validate(self,test_epoch=False):
self.model.train()
if test_epoch:
out = osp.join(self.out, 'test')
else:
out = osp.join(self.out, 'visualization')
mkdir(out)
log_file = osp.join(out, 'test_accurarcy.txt')
fv = open(log_file, 'a')
log_file2 = osp.join(out, 'test_accurarcy_perepoch.txt')
fv2 = open(log_file2, 'a')
log_file3 = osp.join(out, 'test_recall_f1_acc_perepoch.txt')
fv3 = open(log_file3, 'a')
correct = 0
correct_binary = 0
pred_history=[]
target_history=[]
loss_history=[]
sofar = 0
for batch_idx, (data,target,sub_name) in tqdm.tqdm(
# enumerate(self.test_loader), total=len(self.test_loader),
enumerate(self.test_loader), total=len(self.test_loader),
desc='Valid epoch=%d' % self.epoch, ncols=80,
leave=False):
if self.cuda:
data, target = data.cuda(), target.cuda()
# data, target = Variable(data), Variable(target)
data, target = Variable(data,volatile=True), Variable(target,volatile=True)
if self.dual_network:
if self.add_calcium_mask:
data = data[:,0:2,: :,:]
else:
data = data[:,0,:,:,:]
data = torch.unsqueeze(data, 1)
pred_prob = self.model(data)
if test_epoch:
#get attention
gcam = GradCAM(model=self.model)
probs, idx = gcam.forward(data)
topk = 3
target_layer = 'ec6.2'
# target_layer = 'ec1.2'
test_attention_out = osp.join(out, target_layer)
mkdir(test_attention_out)
input_img = data[0, 0].data.cpu().numpy()
input_size = (input_img.shape[0], input_img.shape[1], input_img.shape[2])
input_mask = data[0, 1].data.cpu().numpy()
nii_img = nib.Nifti1Image(input_img, affine=np.eye(4))
output_img_file = os.path.join(out, ('%s_img.nii.gz' % sub_name[0]))
nib.save(nii_img, output_img_file)
nii_mask = nib.Nifti1Image(input_mask, affine=np.eye(4))
output_mask_file = os.path.join(out, ('%s_mask.nii.gz' % sub_name[0]))
nib.save(nii_mask, output_mask_file)
del input_img,input_mask
del nii_img, nii_mask
for i in range(0, topk):
gcam.backward(idx=idx[i])
output = gcam.generate(target_layer=target_layer)
output = resize(output,input_size , mode='constant', preserve_range=True)
nii_seg = nib.Nifti1Image(output, affine=np.eye(4))
output_att_file = os.path.join(test_attention_out, ('%s_test_att%d_clss%d.nii.gz' % (sub_name[0],i,idx[i])))
nib.save(nii_seg, output_att_file)
gcam.backward_del(idx=idx[i])
del gcam, output, nii_seg, probs
#training attention
subnum = data.size(0)
for subi in range(subnum):
attentions = []
i = 1
self.input = data
fmap = self.get_feature_maps('compatibility_score%d' % i, upscale=False)
try:
attmap = fmap[1][1]
except:
aaaa = 1
attention = attmap[subi,0].cpu().numpy()
# attention = attention[:, :]
# attention = numpy.expand_dims(resize(attention, (fmap_size[0], fmap_size[1]), mode='constant', preserve_range=True), axis=2)
attention = resize(attention, input_size, mode='constant', preserve_range=True)
attention = (attention-np.min(attention))/(np.max(attention)-np.min(attention))
# this one is useless
# plotNNFilter(fmap_0, figure_id=i+3, interp='bilinear', colormap=cm.jet, title='compat. feature %d' %i)
nii_seg = nib.Nifti1Image(attention, affine=np.eye(4))
output_att_file = os.path.join(out, ('%s_train_att.nii.gz' % sub_name[subi]))
nib.save(nii_seg, output_att_file)
del nii_seg, fmap, attmap, attention
# plotNNFilterOverlay(input_img, attention, figure_id=i, interp='bilinear', colormap=cm.jet,
# title='a', alpha=0.5)
# attentions.append(attention)
pred_clss = F.log_softmax(pred_prob)
pred = pred_clss.data.max(1)[1] # get th
correct += pred.eq(target.data).cpu().sum()
pred_binary = pred>0
target_binary = target.data>0
correct_binary += pred_binary.eq(target_binary).cpu().sum()
sofar += data.size(0)
test_loss = F.nll_loss(pred_clss, target)
for batch_num in range(data.size(0)):
# test_loss /= len(self.test_loader) # loss function already averages over batch size
results_strs = '[Epoch %04d] True=[%d],Pred=[%d],Pred_prob=%s,Test set: Average loss: %.4f, Accuracy: %d/%d (%.3f) binary (%.3f), subname=[%s]\n' % (
self.epoch, target.data.cpu().numpy()[batch_num], pred.cpu().numpy()[batch_num], np.array2string(pred_clss[batch_num].data.cpu().numpy()), test_loss.data[0], correct, sofar,
100. * float(correct) / sofar, 100 * float(correct_binary) / sofar, sub_name[batch_num])
print(results_strs)
fv.write(results_strs)
loss_history.append(test_loss.data.cpu().numpy().tolist())
pred_history += pred_binary.cpu().numpy().tolist()
target_history += target_binary.data.cpu().numpy().tolist()
f1 = f1_score(target_history, pred_history)
recall = recall_score(target_history, pred_history)
precision = precision_score(target_history, pred_history)
accuracy = accuracy_score(target_history, pred_history)
print_str='test epoch='+str(self.epoch)+',accuracy='+str(accuracy)+",f1="+str(f1)+",recall="+str(recall)+',precision='+str(precision)+",loss="+str(np.mean(loss_history))+"\n"
fv2.write(results_strs)
fv3.write(print_str)
fv.close()
fv2.close()
fv3.close()
# set target layer for CAM
if args.model == 'vgg16' or args.model == 'densenet121':
target_layer = model.features[-1]
elif args.model == 'resnet18':
target_layer = model.layer4[-1]
# get given label's index
label = {'covid_19': 0, 'lung_opacity': 1, 'normal': 2, 'pneumonia': 3}
idx_to_label = {v: k for k, v in label.items()}
if args.label is not None:
label = label[args.label]
else:
label = None
# load and preprocess image
image = utils.load_image(args.image_path)
warnings.filterwarnings("ignore", category=UserWarning)
# pass image through model and get CAM for the given label
cam = GradCAM(model=model, target_layer=target_layer)
label, mask = cam(image, label)
print(f'GradCAM generated for label "{idx_to_label[label]}".')
# deprocess image and overlay CAM
image = utils.deprocess_image(image)
image = apply_mask(image, mask)
# save the image
utils.save_image(image, args.output_path)
def test(image_paths, output_dir, cuda):
"""
Generate Grad-CAM at different layers of ResNet-152
"""
fold = 0
device = get_device(cuda)
# Synset words
classes = {0: 'normal', 1: 'covid'}
# Model
#model = models.resnet34(pretrained=False,num_classes=config.num_classes)
best_model = torch.load(config.best_models + config.model_name + os.sep +
str(fold) + os.sep + 'model_best.pth.tar')
print(best_model["state_dict"].keys())
model = make_model('mynet',
num_classes=config.num_classes,
pretrained=False)
#best_model = torch.load(config.weights +'model_best.pth.tar')
model.load_state_dict(best_model["state_dict"])
print(best_model["state_dict"].keys())
model.to(device)
model.eval()
# The four residual layers
target_layers = [
"_features.0", "_features.1", "_features.2", "_features.3",
"_features.4", "_features.5", "_features.6", "_features.7"
]
target_class = 1 # "bull mastif"
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
gcam = GradCAM(model=model)
probs, ids = gcam.forward(images)
ids_ = torch.LongTensor([[target_class]] * len(images)).to(device)
print(ids_.shape)
gcam.backward(ids=ids_)
for target_layer in target_layers:
print("Generating Grad-CAM @{}".format(target_layer))
# Grad-CAM
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(
j, classes[target_class], float(probs[ids == target_class])))
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, "resnet152",
target_layer,
classes[target_class]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
device = get_device(cuda)
# Synset words
classes = get_classtable()
# Model from torchvision
model = models.__dict__[arch](pretrained=True)
model.to(device)
model.eval()
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
for i in range(topk):
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
def fulltest(image_paths, target_layer, arch, topk, output_dir, cuda):
"""
Visualize model responses given multiple images
"""
fold = 0
device = get_device(cuda)
# Synset words
#classes = {0:'normal',1:'covid'}
classes = get_classtable()
# Model
#model = models.resnet34(pretrained=False,num_classes=config.num_classes)
best_model = torch.load(config.weights + 'ct-cn/' + 'model_best.pth.tar')
print(best_model["state_dict"].keys())
model = make_model(arch, num_classes=config.num_classes, pretrained=True)
#best_model = torch.load(config.weights +'model_best.pth.tar')
model.load_state_dict(best_model["state_dict"])
print(best_model["state_dict"].keys())
model.to(device)
model.eval()
# Images
images, raw_images = load_images(image_paths)
images = torch.stack(images).to(device)
"""
Common usage:
1. Wrap your model with visualization classes defined in grad_cam.py
2. Run forward() with images
3. Run backward() with a list of specific classes
4. Run generate() to export results
"""
# =========================================================================
print("Vanilla Backpropagation:")
bp = BackPropagation(model=model)
probs, ids = bp.forward(images) # sorted
print(probs)
print(ids)
for i in range(topk):
bp.backward(ids=ids[:, [i]])
gradients = bp.generate()
# Save results as image files
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Remove all the hook function in the "model"
bp.remove_hook()
# =========================================================================
print("Deconvolution:")
deconv = Deconvnet(model=model)
_ = deconv.forward(images)
for i in range(topk):
deconv.backward(ids=ids[:, [i]])
gradients = deconv.generate()
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j,
i]]),
),
gradient=gradients[j],
)
deconv.remove_hook()
# =========================================================================
print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:")
gcam = GradCAM(model=model)
_ = gcam.forward(images)
gbp = GuidedBackPropagation(model=model)
_ = gbp.forward(images)
for i in range(topk):
# Guided Backpropagation
gbp.backward(ids=ids[:, [i]])
gradients = gbp.generate()
# Grad-CAM
gcam.backward(ids=ids[:, [i]])
regions = gcam.generate(target_layer=target_layer)
for j in range(len(images)):
print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j,
i]))
# Guided Backpropagation
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]),
),
gradient=gradients[j],
)
# Grad-CAM
save_gradcam(
filename=osp.join(
output_dir,
"{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer,
classes[ids[j, i]]),
),
gcam=regions[j, 0],
raw_image=raw_images[j],
)
# Guided Grad-CAM
save_gradient(
filename=osp.join(
output_dir,
"{}-{}-guided_gradcam-{}-{}.png".format(
j, arch, target_layer, classes[ids[j, i]]),
),
gradient=torch.mul(regions, gradients)[j],
)
