def demo2(image_paths, output_dir, cuda): """ Generate Grad-CAM at different layers of ResNet-152 """ device = get_device(cuda) # Synset words classes = get_classtable() # Model model = models.resnet152(pretrained=True) model.to(device) model.eval() # The four residual layers target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"] target_class = 243 # "bull mastif" # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) gcam = GradCAM(model=model) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)).to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print( "\t#{}: {} ({:.5f})".format( j, classes[target_class], float(probs[ids == target_class]) ) ) save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "resnet152", target_layer, classes[target_class] ), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def demo2(image_paths, output_dir, cuda): """ Generate Grad-CAM at different layers of ResNet-152 """ device = get_device(cuda) # Synset words classes = get_classtable() # Model model = models.resnet152(pretrained=True) model.to(device) model.eval() # The four residual layers target_layers = ["relu", "layer1", "layer2", "layer3", "layer4"] target_class = 243 # "bull mastif" # Images images, raw_images = load_images(image_paths) images = torch.stack(images).to(device) gcam = GradCAM(model=model) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)).to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format( j, classes[target_class], float(probs[ids == target_class]))) save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "resnet152", target_layer, classes[target_class]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def guided_backprop_eye(image, name, net): img = torch.stack([image[name]]) bp = BackPropagation(model=net) probs, ids = bp.forward(img) gcam = GradCAM(model=net) _ = gcam.forward(img) gbp = GuidedBackPropagation(model=net) _ = gbp.forward(img) # Guided Backpropagation actual_status = ids[:, 0] gbp.backward(ids=actual_status.reshape(1, 1)) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=actual_status.reshape(1, 1)) regions = gcam.generate(target_layer='last_conv') # Get Images prob = probs.data[:, 0] if actual_status == 0: prob = probs.data[:, 1] prob_image = np.zeros((shape[0], 60, 3), np.uint8) cv2.putText(prob_image, '%.1f%%' % (prob * 100), (5, 15), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255, 255, 255), 1, cv2.LINE_AA) guided_bpg_image = get_gradient_image(gradients[0]) guided_bpg_image = cv2.merge( (guided_bpg_image, guided_bpg_image, guided_bpg_image)) grad_cam_image = get_gradcam_image(gcam=regions[0, 0], raw_image=image[name + '_raw']) guided_gradcam_image = get_gradient_image(torch.mul(regions, gradients)[0]) guided_gradcam_image = cv2.merge( (guided_gradcam_image, guided_gradcam_image, guided_gradcam_image)) #print(image['path'],classes[actual_status.data], probs.data[:,0] * 100) print(classes[actual_status.data], probs.data[:, 0] * 100) return cv2.hconcat([ image[name + '_raw'], prob_image, guided_bpg_image, grad_cam_image, guided_gradcam_image ])
def extract_object(original_image, cuda = 1): """ Generate Grad-CAM at different layers of ResNet-152 """ output_dir = "./output" # device = get_device(cuda) device = torch.device("cuda" if cuda else "cpu") # Synset words classes = get_classtable() # Model model = models.resnet152(pretrained=True) # device = "cuda" # model.to(device) model.eval() target_layer = "layer4" target_class = 243 # "bull mastif" # Images # images = original_image.to(device) gcam = GradCAM(model=model) probs, ids = gcam.forward(original_image) # ids_ = torch.LongTensor([[target_class]] * len(original_image)).to(device) # pdb.set_trace() gcam.backward(ids=ids) # Grad-CAM dogs = [] red_bboxs = [] regions = gcam.generate(target_layer=target_layer) for j in range(len(original_image)): dog, red_bbox = save_gradcam(gcam=regions[j, 0], raw_image=original_image[j]) dogs.append(dog) red_bboxs.append(red_bbox) print(red_bbox) # tensor_dogs = torch.stack(dogs) # tensor_red_bbox = torch.stack(red_bboxs) #del gcam,model,dogs,red_bboxs return dogs, red_bboxs
def main(): root_path = '/media/palm/Unimportant/pdr2018/typesep_validate/Tomato/' image_name = 'c9ebc74c2177ce60a8230855333fb9e7.jpg' folder_name = '14_Tomato_Spider_Mite_Damage_Serious' # image_path = root_path+'/14_Tomato_Spider_Mite_Damage_Serious/1c0f1ae1374d01c2933069232735a331.jpg' image_path = os.path.join(root_path, folder_name, image_name) topk = 1 cuda = 'cuda' arch = 'densenet201' CONFIG = { 'resnet152': { 'target_layer': 'layer4.2', 'input_size': 224 }, 'vgg19': { 'target_layer': 'features.36', 'input_size': 224 }, 'vgg19_bn': { 'target_layer': 'features.52', 'input_size': 224 }, 'inception_v3': { 'target_layer': 'Mixed_7c', 'input_size': 299 }, 'densenet201': { 'target_layer': 'features.denseblock4', 'input_size': 224 }, # Add your model }.get(arch) a, b, c = getlookup() device = torch.device( 'cuda' if cuda and torch.cuda.is_available() else 'cpu') if cuda: current_device = torch.cuda.current_device() print('Running on the GPU:', torch.cuda.get_device_name(current_device)) else: print('Running on the CPU') # Synset words classes = c['Tomato'] # Model model = getmodel(20) checkpoint = torch.load('checkpoint/try_4_densesep-Tomatotemp.t7') model.load_state_dict(checkpoint['net']) model.to('cuda') model.eval() # Image raw_image = cv2.imread(image_path)[..., ::-1] raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2) image = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], ) ])(raw_image).unsqueeze(0) # ========================================================================= print('Grad-CAM') # ========================================================================= gcam = GradCAM(model=model) probs, idx = gcam.forward(image.to(device)) for i in range(0, topk): gcam.backward(idx=idx[i]) output = gcam.generate(target_layer=CONFIG['target_layer']) save_gradcam( 'results/{}_{}_gcam_{}.png'.format(image_name, classes[idx[i]], arch), output, raw_image) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Vanilla Backpropagation') # ========================================================================= bp = BackPropagation(model=model) probs, idx = bp.forward(image.to(device)) for i in range(0, topk): bp.backward(idx=idx[i]) output = bp.generate() save_gradient( 'results/{}_{}_bp_{}.png'.format(image_name, classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Deconvolution') # ========================================================================= deconv = Deconvolution( model=copy.deepcopy(model)) # TODO: remove hook func in advance probs, idx = deconv.forward(image.to(device)) for i in range(0, topk): deconv.backward(idx=idx[i]) output = deconv.generate() save_gradient( 'results/{}_{}_deconv_{}.png'.format(image_name, classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Guided Backpropagation/Guided Grad-CAM') # ========================================================================= gbp = GuidedBackPropagation(model=model) probs, idx = gbp.forward(image.to(device)) for i in range(0, topk): gcam.backward(idx=idx[i]) region = gcam.generate(target_layer=CONFIG['target_layer']) gbp.backward(idx=idx[i]) feature = gbp.generate() h, w, _ = feature.shape region = cv2.resize(region, (w, h))[..., np.newaxis] output = feature * region save_gradient( 'results/{}_{}_gbp_{}.png'.format(image_name, classes[idx[i]], arch), feature) save_gradient( 'results/{}_{}_ggcam_{}.png'.format(image_name, classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, checkpoint, distribute): """ Visualize model responses given multiple images """ device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision model = models.__dict__[arch]() model = model.cuda() # print(model) checkpoint = checkpoint + arch + '/model_best.pth.tar' # print(checkpoint) check_point = torch.load(checkpoint, map_location=lambda storage, loc: storage.cuda(0)) distributed_model = (distribute > 0.5) only_CAM = True if distributed_model == True: # create new OrderedDict that does not contain `module.` from collections import OrderedDict new_check_point = OrderedDict() for k, v in check_point['state_dict'].items(): # name = k[7:] # remove `module.` # name = k[9:] # remove `module.1.` if k.startswith('module.1.'): name = k[9:] else: name = k[7:] new_check_point[name] = v # load params model.load_state_dict(new_check_point) else: model.load_state_dict(check_point['state_dict']) model.to(device) model.eval() # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Vanilla Backpropagation:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) for i in range(topk): # In this example, we specify the high confidence classes bp.backward(ids=ids[:, [i]]) gradients = bp.generate() # Save results as image files for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) if not only_CAM: save_gradient( filename=osp.join( output_dir, "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution:") deconv = Deconvnet(model=model) _ = deconv.forward(images) for i in range(topk): deconv.backward(ids=ids[:, [i]]) gradients = deconv.generate() for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) if not only_CAM: save_gradient( filename=osp.join( output_dir, "{}-{}-deconvnet-{}.png".format( j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Guided Backpropagation if not only_CAM: save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-{}.png".format( # j, arch, target_layer, classes[ids[j, i]] image_path osp.splitext(image_paths[j])[0], arch, target_layer), ), gcam=regions[j, 0], raw_image=raw_images[j], ) # Guided Grad-CAM if not only_CAM: save_gradient( filename=osp.join( output_dir, "{}-{}-guided_gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]]), ), gradient=torch.mul(regions, gradients)[j], )
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda): """ Visualize model responses given multiple images """ device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision model = models.__dict__[arch](pretrained=True) model.to(device) model.eval() # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Vanilla Backpropagation:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) for i in range(topk): # In this example, we specify the high confidence classes bp.backward(ids=ids[:, [i]]) gradients = bp.generate() # Save results as image files for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution:") deconv = Deconvnet(model=model) _ = deconv.forward(images) for i in range(topk): deconv.backward(ids=ids[:, [i]]) gradients = deconv.generate() for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Guided Backpropagation save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]] ), ), gcam=regions[j, 0], raw_image=raw_images[j], ) # Guided Grad-CAM save_gradient( filename=osp.join( output_dir, "{}-{}-guided_gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]] ), ), gradient=torch.mul(regions, gradients)[j], )
def main(image_path, arch, topk, cuda): CONFIG = { 'resnet18': { 'target_layer': 'layer4.1', 'input_size': 224 }, 'resnet152': { 'target_layer': 'layer4.2', 'input_size': 224 }, 'vgg19': { 'target_layer': 'features.36', 'input_size': 224 }, 'vgg19_bn': { 'target_layer': 'features.52', 'input_size': 224 }, 'inception_v3': { 'target_layer': 'Mixed_7c', 'input_size': 299 }, 'densenet201': { 'target_layer': 'features.denseblock4', 'input_size': 224 }, # Add your model }.get(arch) device = torch.device( 'cuda' if cuda and torch.cuda.is_available() else 'cpu') if cuda: current_device = torch.cuda.current_device() print('Running on the GPU:', torch.cuda.get_device_name(current_device)) else: print('Running on the CPU') # Synset words classes = ["other", "rori"] # with open('samples/synset_words.txt') as lines: # for line in lines: # line = line.strip().split(' ', 1)[1] # line = line.split(', ', 1)[0].replace(' ', '_') # classes.append(line) # Model model = models.__dict__[arch](pretrained=True) num_features = model.fc.in_features model.fc = nn.Linear(num_features, 200) #これにより512->2の層に変わった model.add_module('relu_fc', nn.ReLU()) model.add_module('fc2', nn.Linear(200, 2)) param = torch.load('weight_resnet18_3.pth') model.load_state_dict(param) model.to(device) model.eval() # Image raw_image = cv2.imread(image_path)[..., ::-1] raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2) image = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], ) ])(raw_image).unsqueeze(0) # ========================================================================= print('Grad-CAM') # ========================================================================= gcam = GradCAM(model=model) probs, idx = gcam.forward(image.to(device)) for i in range(0, topk): gcam.backward(idx=idx[i]) output = gcam.generate(target_layer=CONFIG['target_layer']) save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch), output, raw_image) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Vanilla Backpropagation') # ========================================================================= bp = BackPropagation(model=model) probs, idx = bp.forward(image.to(device)) for i in range(0, topk): bp.backward(idx=idx[i]) output = bp.generate() save_gradient('results/{}_bp_{}.png'.format(classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Deconvolution') # ========================================================================= deconv = Deconvolution( model=copy.deepcopy(model)) # TODO: remove hook func in advance probs, idx = deconv.forward(image.to(device)) for i in range(0, topk): deconv.backward(idx=idx[i]) output = deconv.generate() save_gradient('results/{}_deconv_{}.png'.format(classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Guided Backpropagation/Guided Grad-CAM') # ========================================================================= gbp = GuidedBackPropagation(model=model) probs, idx = gbp.forward(image.to(device)) for i in range(0, topk): gcam.backward(idx=idx[i]) region = gcam.generate(target_layer=CONFIG['target_layer']) gbp.backward(idx=idx[i]) feature = gbp.generate() h, w, _ = feature.shape region = cv2.resize(region, (w, h))[..., np.newaxis] output = feature * region save_gradient('results/{}_gbp_{}.png'.format(classes[idx[i]], arch), feature) save_gradient('results/{}_ggcam_{}.png'.format(classes[idx[i]], arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda): """ Visualize model responses given multiple images """ # check if CUDA is available train_on_gpu = torch.cuda.is_available() if not train_on_gpu: print('CUDA is not available. Training on CPU ...') else: print('CUDA is available! Training on GPU ...') device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt' model_name = 'resnet' num_classes = 9 feature_extract = False model, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True) if train_on_gpu: state = torch.load(PRE_MODEL_DIR) else: state = torch.load(PRE_MODEL_DIR, map_location='cpu') # Loading weights in restnet architecture model.load_state_dict(state['state_dict']) model.to(device) model.eval() # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Vanilla Backpropagation:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) for i in range(topk): # In this example, we specify the high confidence classes bp.backward(ids=ids[:, [i]]) gradients = bp.generate() # Save results as image files for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution:") deconv = Deconvnet(model=model) _ = deconv.forward(images) for i in range(topk): deconv.backward(ids=ids[:, [i]]) gradients = deconv.generate() for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Guided Backpropagation save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], ) # Guided Grad-CAM save_gradient( filename=osp.join( output_dir, "{}-{}-guided_gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]]), ), gradient=torch.mul(regions, gradients)[j], )
def demo3(image_paths, topk, output_dir, cuda): """ Generate Grad-CAM with original models """ device = get_device(cuda) # Synset words classes = get_classtable() # Third-party model from my other repository, e.g. Xception v1 ported from Keras model = torch.hub.load("kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True) model.to(device) model.eval() # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer target_layer = "exit_flow.conv4" # Preprocessing def _preprocess(image_path): raw_image = cv2.imread(image_path) raw_image = cv2.resize(raw_image, model.image_shape) image = torch.FloatTensor(raw_image[..., ::-1].copy()) image -= model.mean image /= model.std image = image.permute(2, 0, 1) return image, raw_image # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = _preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) print("Grad-CAM:") gcam = GradCAM(model=model) probs, ids = gcam.forward(images) for i in range(topk): # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "xception_v1", target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images) #.to(device) # Here we choose the last convolution layer TODO: This will likely be wrong! target_layer = "exit_flow.conv4" target_class = 1 # run grad cam on all images! gcam = GradCAM(model=vgg16) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): # Make the target class male. The second half of the images are all of men. Please excuse the hack. if j > (len(images)/2): target_class = 0
def cam(config): ## prepare data classes = {} classes[0] = 'non-merger' classes[1] = 'merger' dsets = {} # dset_loaders = {} #sampling WOR, i guess we leave the 10 in the middle to validate? pristine_indices = torch.randperm(len(pristine_x)) pristine_x_test = pristine_x[ pristine_indices[int(np.floor(.8 * len(pristine_x))):]] pristine_y_test = pristine_y[ pristine_indices[int(np.floor(.8 * len(pristine_x))):]] noisy_indices = torch.randperm(len(noisy_x)) noisy_x_test = noisy_x[noisy_indices[int(np.floor(.8 * len(noisy_x))):]] noisy_y_test = noisy_y[noisy_indices[int(np.floor(.8 * len(noisy_x))):]] dsets["source"] = pristine_x_test dsets["target"] = noisy_x_test class_num = config["network"]["params"]["class_num"] # load checkpoint print('load model from {}'.format(config['ckpt_path'])) ckpt = torch.load(config['ckpt_path'] + '/best_model.pth.tar') ## set base network net_config = config["network"] base_network = net_config["name"](**net_config["params"]) base_network.load_state_dict(ckpt['base_network']) centroids = None if 'center_criterion' in ckpt.keys(): centroids = ckpt['center_criterion']['centers'].cpu() target_centroids = None if 'target_center_criterion' in ckpt.keys(): target_centroids = ckpt['target_center_criterion']['centers'].cpu() use_gpu = torch.cuda.is_available() if use_gpu: base_network = base_network.cuda() #might not get to tstack this? source_images = torch.stack(list(dsets["source"])).cuda() target_images = torch.stack(list(dsets["source"])).cuda() base_network.train(False) model_name = config["network"] target_layer = config[ "target_layer"] #no idea... maybe i can print this out? layer4 for resnet, relu for deepemerge if config["class"] == 'non-merger': target_class = 0 #non-merger elif config["class"] == 'merger': target_class = 1 #non-merger else: print("incorrect class choice") sys.exit() save_where = osp.join( config["ckpt_path"], 'guided ' + str(config["which"]) + '-' + str(config["class"])) output_dir = save_where if not osp.exists(save_where): os.makedirs(save_where) else: os.chdir(save_where) gcam = GradCAM(model=base_network) gbp = GuidedBackPropagation(model=base_network) if config["which"] == 'source': print("start source test: ") probs, ids = gcam.forward(source_images) #see how they load in images _ = gbp.forward(source_images) #see if you need to catch anything #not sure what to do about this if use_gpu: ids_ = torch.LongTensor([[target_class]] * len(source_images)).cuda() else: ids_ = torch.LongTensor([[target_class]] * len(source_images)) gcam.backward(ids=ids_) gbp.backward(ids=ids_) gradients = gbp.generate() regions = gcam.generate(target_layer=target_layer) # print("ids") # print(ids) # print("probs") # print(probs) # # print("argmax") # # print(torch.argmax(ids, dim = 1)) # # print("equality") # # print(torch.argmax(ids, dim = 1).cpu() == target_class) # # print(probs) # # print(probs[(torch.argmax(ids, dim = 1).cpu() == target_class).cpu()]) # # print(probs.cpu()[(torch.argmax(ids, dim = 1).cpu() == target_class).cpu()]) # #print(torch.argmax(probs, dim = 1).cpu() == target_class) # print("try 1") # print(probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]]) # print("try 2") # a = probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]] # print(a[:, target_class]) # a = probs[[torch.tensor(torch.argmax(ids, dim = 1).cpu() == target_class)]] # a = a[:, target_class] # print(len(regions)) # print(len(source_images)-1) for j in range(0, len(source_images) - 1): # print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[:, target_class][j]) # ) # ) # save_gradcam( # filename=osp.join( # output_dir, # "{}-{}-gradcam-{}-{}.png".format( # j, model_name, target_layer, classes[target_class] # ), # ), # gcam=regions[j, 0], # #raw_image=raw_images[j], # raw_image=source_images[j], # ) save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, net_config, classes[target_class]), ), gradient=torch.mul(regions, gradients) [j], #gradients[j], #torch.mul(regions, gradients)[j], ) elif config["which"] == 'target': print("start target test: ") probs, ids = gcam.forward(target_images) #see how they load in images _ = gbp.forward(target_images) #see if you need to catch anything #not sure what to do about this if use_gpu: ids_ = torch.LongTensor([[target_class]] * len(target_images)).cuda() else: ids_ = torch.LongTensor([[target_class]] * len(target_images)) gcam.backward(ids=ids_) gbp.backward(ids=ids_) gradients = gbp.generate() regions = gcam.generate(target_layer=target_layer) for j in range(0, len(target_images) - 1): # print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[:, target_class][j]) # ) # ) # save_gradcam( # filename=osp.join( # output_dir, # "{}-{}-gradcam-{}-{}.png".format( # j, model_name, target_layer, classes[target_class] # ), # ), # gcam=regions[j, 0], # #raw_image=raw_images[j], # raw_image= target_images[j], # ) save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, net_config, classes[target_class]), ), gradient=torch.mul(regions, gradients)[j], #gradients[j], ) else: print("incorrect domain choice") sys.exit() return ()
def main(): parser = make_parser() #creates the parser args = parser.parse_args() CONFIG = { 'resnet152': { 'target_layer': 'layer4.2', 'input_size': 224 }, 'vgg19': { 'target_layer': 'features.36', 'input_size': 224 }, 'vgg19_bn': { 'target_layer': 'features.52', 'input_size': 224 }, 'inception_v3': { 'target_layer': 'Mixed_7c', 'input_size': 299 }, 'densenet201': { 'target_layer': 'features.denseblock4', 'input_size': 224 }, 'resnet50': { 'target_layer': 'layer4', 'input_size': 224 }, 'resnet18': { 'target_layer': 'layer4', 'input_size': 224 }, # Add your model }.get(args.arch) device = torch.device( 'cuda' if args.cuda and torch.cuda.is_available() else 'cpu') if args.cuda: current_device = torch.cuda.current_device() print('Running on the GPU:', torch.cuda.get_device_name(current_device)) else: print('Running on the CPU') # Synset words classes = list() with open('samples/synset_words.txt') as lines: for line in lines: line = line.strip().split(' ', 1)[1] line = line.split(', ', 1)[0].replace(' ', '_') classes.append(line) # Model print("1") class_names = ['no', 'yes'] #important to define the classes for prediction model_ft = get_cnn(len(class_names), args) #retrieves the cnn - architecture to be used print("2") criterion = nn.CrossEntropyLoss( ) #creates the criterion (used in training and testing) optimizer_ft = get_optimizer( model_ft, args ) #changes the weights based on error (using Stochastic Gradient Descent) exp_lr_scheduler = lr_scheduler.StepLR( optimizer_ft, step_size=7, gamma=0.1 ) #helps with the learning rate, to be zigzagging to get into the right function model = load_model(model_ft, args.weights) #load the model with weights print("3") model = model.to(device) model.eval() # Image print("image path: " + str(args.image_path)) raw_image = cv2.imread(args.image_path)[..., ::-1] raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2) image = transforms.Compose([ transforms.ToTensor(), transforms.Normalize( mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], ) ])(raw_image).unsqueeze(0) # ========================================================================= print('Grad-CAM') # ========================================================================= gcam = GradCAM(model=model) probs, idx = gcam.forward(image.to(device)) for i in range(0, args.topk): print("idx is: " + str(idx)) print("idx is: " + str(probs)) print("i is: " + str(i)) gcam.backward(idx=idx[i]) print("idx AFTER is: " + str(idx)) print("idx[i]: " + str(idx[i])) output = gcam.generate(target_layer=CONFIG['target_layer']) print("classes[idx[i]]: " + str(classes[idx[i]])) save_gradcam( 'results/{}_gcam_{}.png'.format(classes[idx[i]], args.arch), output, raw_image) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Vanilla Backpropagation') # ========================================================================= bp = BackPropagation(model=model) probs, idx = bp.forward(image.to(device)) for i in range(0, args.topk): bp.backward(idx=idx[i]) output = bp.generate() save_gradient( 'results/{}_bp_{}.png'.format(classes[idx[i]], args.arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Deconvolution') # ========================================================================= deconv = Deconvolution( model=copy.deepcopy(model)) # TODO: remove hook func in advance probs, idx = deconv.forward(image.to(device)) for i in range(0, args.topk): deconv.backward(idx=idx[i]) output = deconv.generate() save_gradient( 'results/{}_deconv_{}.png'.format(classes[idx[i]], args.arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]])) # ========================================================================= print('Guided Backpropagation/Guided Grad-CAM') # ========================================================================= gbp = GuidedBackPropagation(model=model) probs, idx = gbp.forward(image.to(device)) for i in range(0, args.topk): gcam.backward(idx=idx[i]) region = gcam.generate(target_layer=CONFIG['target_layer']) gbp.backward(idx=idx[i]) feature = gbp.generate() h, w, _ = feature.shape region = cv2.resize(region, (w, h))[..., np.newaxis] output = feature * region save_gradient( 'results/{}_gbp_{}.png'.format(classes[idx[i]], args.arch), feature) save_gradient( 'results/{}_ggcam_{}.png'.format(classes[idx[i]], args.arch), output) print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
def demo1(image_paths, topk, output_dir, cuda): # Get image paths root = '../UTKFace/val' image_paths = [] q = 0 for female_file in os.listdir(os.path.join(root, "female")): image_paths.append( os.path.join(root, os.path.join("female", str(female_file)))) q += 1 if (q == 10): break q = 0 for male_file in os.listdir(os.path.join(root, "male")): image_paths.append( os.path.join(root, os.path.join("male", str(male_file)))) q += 1 if (q == 10): break print(len(image_paths)) # preprocess each image images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images) #.to(device) """ CNN Gradcam """ """ # Synset words classes = get_classtable() print("Using CNN") cnn = CNN() #if use_gpu: # cnn.cuda() optimizer = torch.optim.SGD(cnn.parameters(),lr=0.001,momentum=0.9) SAVED_MODEL_PATH = '../cnn_model_best_final.pth.tar' checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu') cnn.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) epoch = checkpoint['epoch'] best_val_acc = checkpoint['best_val_acc'] print("best model saved from epoch: ", epoch) print("best val_acc = ", best_val_acc) cnn.eval() # The four residual layers target_layers = ["layer3"] target_class = 1 # run grad cam on all images! gcam = GradCAM(model=cnn) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): #print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[ids == target_class]) # ) # ) if j > (len(images)/2): target_class = 0 save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "CNN", target_layer, classes[target_class] ), ), gcam=regions[j, 0], raw_image=raw_images[j], ) """ """ # ============================================================================================ Adversary CNN """ """ classes = get_classtable() print("Using Adversary CNN") cnn = CNN() optimizer = torch.optim.SGD(cnn.parameters(),lr=0.001,momentum=0.9) SAVED_MODEL_PATH ='../cnn_model_best_SGD_adversary.pth.tar' checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu') cnn.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) epoch = checkpoint['epoch'] best_val_acc = checkpoint['best_val_acc'] print("best model saved from epoch: ", epoch) print("best val_acc = ", best_val_acc) cnn.eval() # The four residual layers target_layers = ["layer3"] target_class = 1 # run grad cam on all images! gcam = GradCAM(model=cnn) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): #print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[ids == target_class]) # ) # ) # Make the target class male. The second half of the images are all of men. Please excuse the hack. if j > (len(images)/2): target_class = 0 save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "CNNAdv", target_layer, classes[target_class] ), ), gcam=regions[j, 0], raw_image=raw_images[j], ) """ """ # ============================================================================================= VGG16 Gradcam """ """ # TODO: waiting on .tar file. #device = get_device(cuda) # Synset words classes = get_classtable() # make model NUM_CLASSES = 2 vgg16 = models.vgg16(pretrained = True) # Freeze training for all layers NOTE: may or may not need this for param in vgg16.features.parameters(): param.require_grad = False # Newly created modules have require_grad=True by default num_features = vgg16.classifier[6].in_features features = list(vgg16.classifier.children())[:-1] # Remove last layer features.extend([nn.Linear(num_features, NUM_CLASSES)]) # Add our layer with 4 outputs vgg16.classifier = nn.Sequential(*features) # Replace the model classifier optimizer = torch.optim.SGD(vgg16.parameters(),lr=0.001,momentum=0.9) # dont know why you need this when you're just recreating a model that's been already trained... SAVED_MODEL_PATH = '../vgg_checkpoint.pth.tar' checkpoint = torch.load(SAVED_MODEL_PATH) vgg16.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) epoch = checkpoint['epoch'] best_val_acc = checkpoint['best_val_acc'] vgg16.eval() # the gcam has this eval in their code. idk if we need it though # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer TODO: This will likely be wrong! target_layer = "exit_flow.conv4" target_class = 0 # TODO: CHANGE THIS!!! # run grad cam on all images! gcam = GradCAM(model=cnn) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): """ #print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[ids == target_class]) # ) #) """ save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "VGG16", target_layer, classes[target_class] ), ), gcam=regions[j, 0], raw_image=raw_images[j], ) """ """ # ============================================================================================= VGG16 Adversarial Gradcam """ """ # ============================================================================================= Resnet Gradcam """ """ # Synset words classes = get_classtable() model = models.resnet18(pretrained=True) for param in model.parameters(): param.requires_grad = True #print(model) #Source of code below: # Parameters of newly constructed modules have requires_grad=True by default num_ftrs = model.fc.out_features model = nn.Sequential(model, nn.Linear(num_ftrs, 1000), nn.ReLU(), nn.Linear(1000, 2)) learning_rate = 0.002 optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate,momentum=0.9) SAVED_MODEL_PATH = '../resnet_model_best_v1.pth.tar' checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu') model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) epoch = checkpoint['epoch'] best_val_acc = checkpoint['best_val_acc'] print("best model saved from epoch: ", epoch) print("best val_acc = ", best_val_acc) model.eval() # the gcam has this eval in their code. idk if we need it though # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer TODO: This will likely be wrong! target_layers = ["0.layer4.1.bn2"] # ["layer4"] # why did they originally do "relu", "layer1", "layer2", "layer3", target_class = 1 # run grad cam on all images! gcam = GradCAM(model=model) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): """ #print( # "\t#{}: {} ({:.5f})".format( # j, classes[target_class], float(probs[ids == target_class]) # ) #) """ # Make the target class male. The second half of the images are all of men. Please excuse the hack. if j > (len(images)/2): target_class = 0 save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "RESNET", target_layer, classes[target_class] ), ), gcam=regions[j, 0], raw_image=raw_images[j], ) """ """ # ============================================================================================= Resnet Adversarial Gradcam """ learning_rate = 0.002 model = MyResnet() criterion = nn.CrossEntropyLoss() adversary = NN(1000) nn_criterion = nn.CrossEntropyLoss() # resnet = models.resnet18(pretrained=True) # for param in resnet.parameters(): # param.requires_grad = True # num_ftrs = resnet.fc.out_features # fc1 = nn.Sequential(nn.Linear(num_ftrs, 1000), # nn.ReLU(), # nn.Linear(1000,2)) optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9) # Synset words classes = get_classtable() SAVED_MODEL_PATH = '../resnet_model_best.pth.tar' checkpoint = torch.load(SAVED_MODEL_PATH, map_location='cpu') model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) epoch = checkpoint['epoch'] best_val_acc = checkpoint['best_val_acc'] print("best model saved from epoch: ", epoch) print("best val_acc = ", best_val_acc) model.eval( ) # the gcam has this eval in their code. idk if we need it though # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer TODO: This will likely be wrong! target_layers = [ "resnet.layer4.1.bn2" ] # ["layer4"] # why did they originally do "relu", "layer1", "layer2", "layer3", target_class = 1 # run grad cam on all images! gcam = GradCAM(model=model) print(images[0]) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)) #.to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): # Make the target class male. The second half of the images are all of men. Please excuse the hack. if j > (len(images) / 2): target_class = 0 save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "RESNET", target_layer, classes[target_class]), ), gcam=regions[j, 0], raw_image=raw_images[j], ) """
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda, model): """ Visualize model responses given multiple images """ device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision if model is None: model = models.__dict__[arch](pretrained=True) model.to(device) model.eval() # Images print(image_paths) images, raw_images = load_images(image_paths) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): #print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}.png".format( j, arch, target_layer ), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def main(image_path, target_layer, arch, topk, cuda): device = torch.device( "cuda" if cuda and torch.cuda.is_available() else "cpu") if cuda: current_device = torch.cuda.current_device() print("Running on the GPU:", torch.cuda.get_device_name(current_device)) else: print("Running on the CPU") # Synset words classes = list() # Model from torchvision model = MGN() """ pickle.load = partial(pickle.load, encoding="latin1") pickle.Unpickler = partial(pickle.Unpickler, encoding="latin1") checkpoint = torch.load("hacnn_market_xent.pth.tar", pickle_module=pickle) #checkpoint = torch.load(args.load_weights) #checkpoint = torch.load("hacnn_market_xent.pth.tar") pretrain_dict = checkpoint['state_dict'] model_dict = model.state_dict() pretrain_dict = {k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size()} model_dict.update(pretrain_dict) model.load_state_dict(model_dict) """ pretrain_dict = torch.load("model_700.pt") model_dict = model.state_dict() pretrain_dict = { k: v for k, v in pretrain_dict.items() if k in model_dict and model_dict[k].size() == v.size() } model_dict.update(pretrain_dict) model.load_state_dict(model_dict) #model = models.__dict__[arch](pretrained=True) model.to(device) model.eval() place = [] with open("samples/new.txt") as lines: for line in lines: line = line.strip().split(" ", 1)[1] line = line.split(", ", 1)[0].replace(" ", "_") classes.append(line) # Image preprocessing with open("new.txt") as lines: for line in lines: line = line.strip() line = "/mnt/SSD/jzwang/market1501/query/" + line place.append(line) for line in place: image_path = line raw_image = cv2.imread(image_path)[..., ::-1] raw_image = cv2.resize(raw_image, (128, 384)) image = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), ])(raw_image).unsqueeze(0) image = image.to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with an image 3. Run backward() with a specific class 4. Run generate() to export result """ # ========================================================================= print("Vanilla Backpropagation") bp = BackPropagation(model=model) predictions = bp.forward(image) for i in range(topk): print("[{:.5f}] {}".format(predictions[i][0], classes[predictions[i][1]])) bp.backward(idx=predictions[i][1]) gradient = bp.generate() # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution") deconv = Deconvnet(model=model) _ = deconv.forward(image) for i in range(topk): print("[{:.5f}] {}".format(predictions[i][0], classes[predictions[i][1]])) deconv.backward(idx=predictions[i][1]) gradient = deconv.generate() deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM") gcam = GradCAM(model=model) _ = gcam.forward(image) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(image) t = ['p1', 'p2', 'p3'] for i in range(topk): print("[{:.5f}] {}".format(predictions[i][0], classes[predictions[i][1]])) # Grad-CAM for target_layer in t: gcam.backward(idx=predictions[i][1]) #print("1") region = gcam.generate(target_layer=target_layer) #print(2) line = line.strip().split("/")[-1] save_gradcam( "results/{}-gradcam-{}.png".format(line, target_layer), region, raw_image, ) #print(3) # Guided Backpropagation gbp.backward(idx=predictions[i][1]) gradient = gbp.generate() # Guided Grad-CAM h, w, _ = gradient.shape region = cv2.resize(region, (w, h))[..., np.newaxis] output = gradient * region
def demo3(image_paths, topk, output_dir, cuda): """ Generate Grad-CAM with original models """ device = get_device(cuda) # Synset words classes = get_classtable() # Third-party model from my other repository, e.g. Xception v1 ported from Keras # check if CUDA is available train_on_gpu = torch.cuda.is_available() if not train_on_gpu: print('CUDA is not available. Training on CPU ...') else: print('CUDA is available! Training on GPU ...') device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt' model_name = 'resnet' num_classes = 9 feature_extract = False model, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True) if train_on_gpu: state = torch.load(PRE_MODEL_DIR) else: state = torch.load(PRE_MODEL_DIR, map_location='cpu') # Loading weights in restnet architecture model.load_state_dict(state['state_dict']) model.to(device) model.eval() # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer target_layer = "layer4.2.conv3" # Preprocessing def _preprocess(image_path): raw_image = cv2.imread(image_path) raw_image = cv2.resize(raw_image, (224, 224)) image = torch.FloatTensor(raw_image[..., ::-1].copy()) image -= torch.FloatTensor([0.485, 0.456, 0.406]) image /= torch.FloatTensor([0.229, 0.224, 0.225]) image = image.permute(2, 0, 1) return image, raw_image # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = _preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) print("Grad-CAM:") gcam = GradCAM(model=model) probs, ids = gcam.forward(images) for i in range(topk): # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "xception_v1", target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def demo40(image_paths, output_dir, cuda, classes_target): """ Generate Grad-CAM at last convolucional for each block layers of ResNet-152 """ device = get_device(cuda) # Synset words classes = get_classtable() # Model # check if CUDA is available train_on_gpu = torch.cuda.is_available() if not train_on_gpu: print('CUDA is not available. Training on CPU ...') else: print('CUDA is available! Training on GPU ...') device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision PRE_MODEL_DIR = '/content/gdrive/My Drive/UnB/TCC-1/TCC1-1-dataset-final/restnet_model152_trained_exp7.pt' model_name = 'resnet' num_classes = 9 feature_extract = False model, input_size = initialize_model(model_name, num_classes, feature_extract, use_pretrained=True) if train_on_gpu: state = torch.load(PRE_MODEL_DIR) else: state = torch.load(PRE_MODEL_DIR, map_location='cpu') # Loading weights in restnet architecture model.load_state_dict(state['state_dict']) model.to(device) model.eval() # The the last convolucinal layer for each block target_layers = [ "layer1.0.conv3", "layer1.1.conv3", "layer1.2.conv3", "layer2.0.conv3", "layer2.1.conv3", "layer2.2.conv3", "layer2.3.conv3", "layer2.4.conv3", "layer2.5.conv3", "layer2.6.conv3", "layer2.7.conv3", "layer3.0.conv3", "layer3.1.conv3", "layer3.2.conv3", "layer3.3.conv3", "layer3.4.conv3", "layer3.5.conv3", "layer3.6.conv3", "layer3.7.conv3", "layer3.8.conv3", "layer3.9.conv3", "layer3.10.conv3", "layer3.11.conv3", "layer3.12.conv3", "layer3.13.conv3", "layer3.14.conv3", "layer3.15.conv3", "layer3.16.conv3", "layer3.17.conv3", "layer3.18.conv3", "layer3.19.conv3", "layer3.20.conv3", "layer3.21.conv3", "layer3.22.conv3", "layer3.23.conv3", "layer3.24.conv3", "layer3.25.conv3", "layer3.26.conv3", "layer3.27.conv3", "layer3.28.conv3", "layer3.29.conv3", "layer3.30.conv3", "layer3.31.conv3", "layer3.32.conv3", "layer3.33.conv3", "layer3.34.conv3", "layer3.35.conv3", "layer4.0.conv3", "layer4.1.conv3", "layer4.2.conv3" ] target_class = int(classes_target) # {'actinic-keratosis': 0, # 'basal-cell-carcinoma': 1, # 'dermatofibroma': 2, # 'hemangioma': 3, # 'intraepithelial-carcinoma': 4, # 'malignant-melanoma': 5, # 'melanocytic-nevus': 6, # 'pyogenic-granuloma': 7, # 'squamous-cell-carcinoma': 8} # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) gcam = GradCAM(model=model) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)).to(device) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format( j, classes[target_class], float(probs[ids == target_class]))) save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "resnet152", target_layer, classes[target_class]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def main(args): setup_logger(name="fvcore") logger = setup_logger() logger.info("Arguments: " + str(args)) cfg = setup_cfg(args) print(cfg) # 构建模型 model = build_model(cfg) # 加载权重 checkpointer = DetectionCheckpointer(model) checkpointer.load(cfg.MODEL.WEIGHTS) # 加载图像 path = os.path.expanduser(args.input) original_image = read_image(path, format="BGR") height, width = original_image.shape[:2] transform_gen = T.ResizeShortestEdge( [cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST) image = transform_gen.get_transform(original_image).apply_image( original_image) image = torch.as_tensor(image.astype("float32").transpose( 2, 0, 1)).requires_grad_(True) inputs = {"image": image, "height": height, "width": width} # Grad-CAM layer_name = get_last_conv_name(model) grad_cam = GradCAM(model, layer_name) mask, box, class_id = grad_cam(inputs) # cam mask grad_cam.remove_handlers() # image_dict = {} img = original_image[..., ::-1] x1, y1, x2, y2 = box image_dict['predict_box'] = img[y1:y2, x1:x2] image_cam, image_dict['heatmap'] = gen_cam(img[y1:y2, x1:x2], mask) # Grad-CAM++ grad_cam_plus_plus = GradCamPlusPlus(model, layer_name) mask_plus_plus = grad_cam_plus_plus(inputs) # cam mask _, image_dict['heatmap++'] = gen_cam(img[y1:y2, x1:x2], mask_plus_plus) grad_cam_plus_plus.remove_handlers() # 获取类别名称 meta = MetadataCatalog.get( cfg.DATASETS.TEST[0] if len(cfg.DATASETS.TEST) else "__unused") label = meta.thing_classes[class_id] print("label:{}".format(label)) # # GuidedBackPropagation # gbp = GuidedBackPropagation(model) # inputs['image'].grad.zero_() # 梯度置零 # grad = gbp(inputs) # print("grad.shape:{}".format(grad.shape)) # gb = gen_gb(grad) # gb = gb[y1:y2, x1:x2] # image_dict['gb'] = gb # # 生成Guided Grad-CAM # cam_gb = gb * mask[..., np.newaxis] # image_dict['cam_gb'] = norm_image(cam_gb) save_image(image_dict, os.path.basename(path))
def train(self): """ Train the model on the training set. A checkpoint of the model is saved after each epoch and if the validation accuracy is improved upon, a separate ckpt is created for use on the test set. """ # load the most recent checkpoint model_num = 0 if self.resume: self.load_checkpoint(model_num, best=False) print("\n[*] Train on {} samples, validate on {} samples".format( self.num_train, self.num_valid) ) #Added: self.device, self.device_name = self.get_device() for model in self.models: self.grad_cams.append(GradCAM(model=model)) for epoch in range(self.start_epoch, self.epochs): for scheduler in self.schedulers: scheduler.step(epoch) print( '\nEpoch: {}/{} - LR: {:.6f}'.format( epoch+1, self.epochs, self.optimizers[0].param_groups[0]['lr'],) ) # train for 1 epoch train_losses, train_accs = self.train_one_epoch(epoch) # evaluate on validation set valid_losses, valid_accs = self.validate(epoch) for i in range(self.model_num): is_best = valid_accs[i].avg> self.best_valid_accs[i] msg1 = "model_{:d}: train loss: {:.3f} - train acc: {:.3f} " msg2 = "- val loss: {:.3f} - val acc: {:.3f}" if is_best: #self.counter = 0 msg2 += " [*]" msg = msg1 + msg2 print(msg.format(i+1, train_losses[i].avg, train_accs[i].avg, valid_losses[i].avg, valid_accs[i].avg)) # check for improvement #if not is_best: #self.counter += 1 #if self.counter > self.train_patience: #print("[!] No improvement in a while, stopping training.") #return self.best_valid_accs[i] = max(valid_accs[i].avg, self.best_valid_accs[i]) self.save_checkpoint(i, {'epoch': epoch + 1, 'model_state': self.models[i].state_dict(), 'optim_state': self.optimizers[i].state_dict(), 'best_valid_acc': self.best_valid_accs[i], }, is_best )
def main(image_path, arch, topk, cuda, target_layer): CONFIG = { 'resnet152': { 'target_layer': 'layer4.2', 'input_size': 224 }, 'vgg19': { 'target_layer': 'features.36', 'input_size': 224 }, 'vgg19_bn': { 'target_layer': 'features.52', 'input_size': 224 }, 'inception_v3': { 'target_layer': 'Mixed_7c', 'input_size': 299 }, 'densenet201': { 'target_layer': 'features.denseblock4', 'input_size': 224 }, # Add your model 'se_resnet': { 'target_layer': target_layer, 'input_size': 32 }, }.get(arch) cuda = cuda and torch.cuda.is_available() if cuda: current_device = torch.cuda.current_device() print('Running on the GPU:', torch.cuda.get_device_name(current_device)) else: print('Running on the CPU') # Synset words classes = list() with open('samples/synset_words.txt') as lines: for line in lines: line = line.strip().split(' ', 1)[1] line = line.split(', ', 1)[0].replace(' ', '_') classes.append(line) # Model #model = models.__dict__[arch](pretrained=True) model = models.se_resnet.se_resnet50(num_classes=100) # Image raw_image = cv2.imread(image_path)[..., ::-1] raw_image = cv2.resize(raw_image, (CONFIG['input_size'], ) * 2) image = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ])(raw_image) if cuda: model.cuda() image = image.cuda() # ========================================================================= print('Grad-CAM') # ========================================================================= gcam = GradCAM(model=model) probs, idx = gcam.forward(to_var(image)) for i in range(0, topk): gcam.backward(idx=idx[i]) output = gcam.generate(target_layer=CONFIG['target_layer']) #save_gradcam('results/{}_gcam_{}.png'.format(classes[idx[i]], arch), output, raw_image) # NOQA save_gradcam('results/{}.png'.format(target_layer), output, raw_image) # NOQA print('[{:.5f}] {}'.format(probs[i], classes[idx[i]]))
cfg = edict(json.load(fp)) classes = [ 'Cardiomegaly', 'Edema', 'Consolidation', 'Atelectasis', 'Pleural Effusion' ] num_tasks = len(cfg.num_classes) model = build_model(cfg, join(opt.model, 'best.ckpt')) dataset = ImageDataset(opt.dataset, cfg, mode='heatmap') dataloader = DataLoader(dataset, batch_size=opt.batch_size, shuffle=False, drop_last=False) gcam = GradCAM(model=model) layer = 'backbone.features.denseblock4.denselayer15' for images, paths, labels in dataloader: probs, ids = gcam.forward(images) processed_images = [[] for _ in range(len(images))] for i in range(ids.shape[1]): gcam.backward(ids[:, [i]]) regions = gcam.generate(target_layer=layer) for j in range(len(images)): print(f"#{j}: {classes[ids[j, i]]} ({probs[j, i]:.5f})") # Grad-CAM raw_image = imread(paths[j]) combined = combine_image_and_gcam(regions[j, 0], raw_image) processed_images[j].append(combined.astype(np.uint8))
sample_input = Variable(torch.randn(1,3,in_size,in_size), volatile=False) if use_gpu: sampe_input = sample_input.cuda() def is_image(f): return f.endswith(".png") or f.endswith(".jpg") test_transform = transforms.Compose([ transforms.Scale(in_size), transforms.CenterCrop(in_size), transforms.ToTensor(), transforms.Normalize(cf.mean, cf.std) ]) gcam = GradCAM(list(model._modules.items())[0][1], cuda=use_gpu) print("\n[Phase 2] : Gradient Detection") if args.subtype != None: WBC_id = return_class_idx(args.subtype) if not (args.subtype in dset_classes): print("The given subtype does not exists!") sys.exit(1) if args.subtype == None: print("| Checking All Activated Regions...") else: print("| Checking Activated Regions for " + dset_classes[WBC_id] + "...") for subdir, dirs, files in os.walk(cf.test_base):
# print(label_id[pred]) print(output, pred.item()) return pred state_dict = torch.load(image_model_path) model.load_state_dict(state_dict) model.eval() label_id = {0: 'butterfly',1: 'ibis'} grad_cam = GradCAM(model=model, feature_layer=list(model.layer4.modules())[-1]) VISUALIZE_SIZE = (224, 224) normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) image_transform = transforms.Compose([ transforms.Resize(VISUALIZE_SIZE), transforms.ToTensor(), normalize]) image.thumbnail(VISUALIZE_SIZE, Image.ANTIALIAS) # save image origin size image_orig_size = image.size # (W, H)
def demo3(image_paths, topk, output_dir, cuda): """ Generate Grad-CAM with original models """ device = get_device(cuda) # Synset words classes = get_classtable() # Third-party model from my other repository, e.g. Xception v1 ported from Keras model = torch.hub.load( "kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True ) model.to(device) model.eval() # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer target_layer = "exit_flow.conv4" # Preprocessing def _preprocess(image_path): raw_image = cv2.imread(image_path) raw_image = cv2.resize(raw_image, model.image_shape) image = torch.FloatTensor(raw_image[..., ::-1].copy()) image -= model.mean image /= model.std image = image.permute(2, 0, 1) return image, raw_image # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = _preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) print("Grad-CAM:") gcam = GradCAM(model=model) probs, ids = gcam.forward(images) for i in range(topk): # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format( j, "xception_v1", target_layer, classes[ids[j, i]] ), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def demo3(image_paths, topk, output_dir, cuda, num_class, da_model): """ Generate Grad-CAM with original models """ device = get_device(cuda) # Synset words #classes = get_classtable() # Third-party model from my other repository, e.g. Xception v1 ported from Keras #model = torch.hub.load( # "kazuto1011/pytorch-ported-models", "xception_v1", pretrained=True #) #target_images = ['mediumresidential01','mediumresidential03', 'mediumresidential14',\ # 'mediumresidential21', 'mediumresidential36','mediumresidential37'] + \ #['mediumresidential38','mediumresidential39', 'mediumresidential40',\ # 'mediumresidential41', 'mediumresidential43','mediumresidential44'] num_class, = num_class #target_images = ['baseballdiamond96', 'airplane18', 'runway42', 'sparseresidential77', 'mediumresidential21'] target_images = [ 'runway00', 'runway02', 'runway03', 'runway04', 'runway05', 'runway06', 'runway07', 'runway08' ] classes = [str(i) for i in range(num_class)] if da_model == 'STA': feature_extractor = ResNetFc( model_name='resnet50', model_path= '/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/resnet50.pth' ) cls = CLS(feature_extractor.output_num(), num_class, bottle_neck_dim=256) model = nn.Sequential(feature_extractor, cls).cuda() model.load_state_dict( torch.load( '/home/at7133/Research/Domain_adaptation/Separate_to_Adapt/Only_source_classifier.pth' )) elif da_model == 'OPDA': G, C = get_model('vgg', num_class=num_class, unit_size=1000) load_model( G, C, '/home/at7133/Research/Domain_adaptation/OPDA_BP/checkpoint/checkpoint_99' ) model = nn.Sequential(G, C).cuda() model.to(device) #unset_training(model) model.eval() # Check available layer names print("Layers:") for m in model.named_modules(): print("\t", m[0]) # Here we choose the last convolution layer #target_layer = "exit_flow.conv4" if da_model == 'STA': target_layer = "0.model_resnet.layer4.2.conv3" elif da_model == 'OPDA': target_layer = "0.lower.36" #TODO find proper target layer # Preprocessing def _Normalize(img, mean, std): if isinstance(img, torch.FloatTensor): mean = torch.FloatTensor(mean) std = torch.FloatTensor(std) else: raise TypeError(f'Expected Torch floattensor, got {type(img)}') return (img - mean) / std def _preprocess(image_path, img_shape): raw_image = cv2.imread(image_path) #raw_image = cv2.resize(raw_image, model.image_shape) raw_image = cv2.resize(raw_image, (img_shape, img_shape)) image = torch.FloatTensor(raw_image[..., ::-1].copy()) image = image / 255.0 if da_model == "OPDA": image = _Normalize(image, [0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) #pdb.set_trace() #image -= model.mean #image /= model.std image = image.permute(2, 0, 1) return image, raw_image # Images def Load_txt(img_paths): assert isfile(img_paths), f"Image path {img_paths} doesn't exist" with open(img_paths, 'rb') as fr: image_paths = [ img_path.split()[0].decode("utf-8") for img_path in fr.readlines() ] return image_paths def Filter_imgages(image_paths, req_images): img_files = list( map(lambda x: x.split('/')[-1].split('.')[0], image_paths)) satisfied_images = [] for item in req_images: try: [[idx]] = np.argwhere(np.isin(img_files, item)) except: raise ValueError(f'{item} not found in the given paths') satisfied_images.append(image_paths[idx]) return satisfied_images def Load_images(image_paths, req_images): if image_paths[0].endswith('.txt'): assert len(image_paths ) == 1 #make sure only one text file is given as input image_paths = Load_txt(image_paths[0]) image_paths = Filter_imgages(image_paths, req_images) assert len(image_paths) == len( req_images), " All target images are not found" images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = _preprocess(image_path, 224) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) return images, raw_images images, raw_images = Load_images(image_paths, target_images) print("Grad-CAM:") gcam = GradCAM(model=model) probs, ids = gcam.forward(images) if not isdir(output_dir): os.makedirs(output_dir) for i in range(topk): # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "xception_v1", target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def grad_cam(model, image_paths, topk, output_dir, cuda, model_name, checkpoint): """ Generate Grad-CAM with original models """ device = get_device(cuda) # Synset words classes = [ 'apple', # id 0 'aquarium_fish', 'baby', 'bear', 'beaver', 'bed', 'bee', 'beetle', 'bicycle', 'bottle', 'bowl', 'boy', 'bridge', 'bus', 'butterfly', 'camel', 'can', 'castle', 'caterpillar', 'cattle', 'chair', 'chimpanzee', 'clock', 'cloud', 'cockroach', 'couch', 'crab', 'crocodile', 'cup', 'dinosaur', 'dolphin', 'elephant', 'flatfish', 'forest', 'fox', 'girl', 'hamster', 'house', 'kangaroo', 'computer_keyboard', 'lamp', 'lawn_mower', 'leopard', 'lion', 'lizard', 'lobster', 'man', 'maple_tree', 'motorcycle', 'mountain', 'mouse', 'mushroom', 'oak_tree', 'orange', 'orchid', 'otter', 'palm_tree', 'pear', 'pickup_truck', 'pine_tree', 'plain', 'plate', 'poppy', 'porcupine', 'possum', 'rabbit', 'raccoon', 'ray', 'road', 'rocket', 'rose', 'sea', 'seal', 'shark', 'shrew', 'skunk', 'skyscraper', 'snail', 'snake', 'spider', 'squirrel', 'streetcar', 'sunflower', 'sweet_pepper', 'table', 'tank', 'telephone', 'television', 'tiger', 'tractor', 'train', 'trout', 'tulip', 'turtle', 'wardrobe', 'whale', 'willow_tree', 'wolf', 'woman', 'worm', ] # Third-party model from my other repository, e.g. Xception v1 ported from Keras checkpoint = checkpoint model.load_state_dict(checkpoint['state_dict']) model.to(device) model.eval() # Check available layer names '''print("Layers:") for m in model.named_modules(): print("\t", m[0])''' # Here we choose the last convolution layer target_layer = "module.layer4" #print(target_layer) # Images images = [] raw_images = [] print("Images:") for i, image_path in enumerate(image_paths): print("\t#{}: {}".format(i, image_path)) image, raw_image = preprocess(image_path) images.append(image) raw_images.append(raw_image) images = torch.stack(images).to(device) print("Grad-CAM:") gcam = GradCAM(model=model) probs, ids = gcam.forward(images) for i in range(topk): # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "top{}-{}-{}-gradcam-{}-{}-prob-({:.5f}).png".format( i + 1, j, model_name, target_layer, classes[ids[j, i]], probs[j, i]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def main(): """ Visualize model responses given multiple images """ device = torch.device("cuda") test_dataset = CXRDataset(args.image_dir, ['TEST']) test_loader = DataLoader(test_dataset, batch_size=args.batch_size, pin_memory=True) model = CombineNet().to(device) model = nn.DataParallel(model) print("Parameters: {}".format(sum(p.numel() for p in model.parameters()))) model.eval() if args.load_model: checkpoint = torch.load(args.load_model + "/best.ckpt") #model_weights = OrderedDict() #for k, v in checkpoint['model'].items(): # name = k[7:] # remove `module.` # model_weights[name] = v model.load_state_dict(checkpoint['model']) epoch = checkpoint['epoch'] print('Loading model: {}, from epoch: {}'.format( args.load_model, epoch)) else: print('Model: {} not found'.format(args.load_model)) # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) target_layer = 'module.feature_model.layer_block.3.1.conv2' #print(model.module.keys()) print(model.module.feature_model.layer_block[3][1].conv2) epoch_iterator = tqdm(test_loader, desc="Iteration") for i, batch in enumerate(epoch_iterator, 0): raw_images, _, _ = batch batch_device = tuple(t.to(device, non_blocking=True) for t in batch) images, labels, img_ids = batch_device if (labels.cpu().detach().numpy() == np.array([0, 0, 0, 1])).all(): probs, ids = gcam.forward(images) gcam.backward(ids=ids[:, [0]]) regions = gcam.generate(target_layer=target_layer) for j in range(args.batch_size): gcam_im = save_gradcam( filename=osp.join( args.output_dir, "{}-gradcam-{}_severe_{}.png".format( j, target_layer, i), ), gcam=regions[j, 0], raw_image=images[j], ) img_path = osp.join( args.output_dir, "{}-image-{}_severe_{}.png".format(j, target_layer, i), ) raw_image = np.array(raw_images[j]) xray = save_xray(img_path, raw_image) ''' # TODO: Blending code #cv2.imwrite(img_path, raw_images[j]) img_path=osp.join( args.output_dir, "{}-blended-{}_severe_{}.png".format(j, target_layer, i), ) beta = (1.0 - args.alpha) xray = np.array(xray) xray = xray.squeeze() xray = np.stack([xray, xray, xray],axis=2) print(xray.dtype) print(gcam.dtype) blended = cv2.addWeighted(xray, args.alpha, gcam, beta, 0.0) save_xray(img_path, blended) ''' print('finished!')
def validate(self,test_epoch=False): self.model.train() if test_epoch: out = osp.join(self.out, 'test') else: out = osp.join(self.out, 'visualization') mkdir(out) log_file = osp.join(out, 'test_accurarcy.txt') fv = open(log_file, 'a') log_file2 = osp.join(out, 'test_accurarcy_perepoch.txt') fv2 = open(log_file2, 'a') log_file3 = osp.join(out, 'test_recall_f1_acc_perepoch.txt') fv3 = open(log_file3, 'a') correct = 0 correct_binary = 0 pred_history=[] target_history=[] loss_history=[] sofar = 0 for batch_idx, (data,target,sub_name) in tqdm.tqdm( # enumerate(self.test_loader), total=len(self.test_loader), enumerate(self.test_loader), total=len(self.test_loader), desc='Valid epoch=%d' % self.epoch, ncols=80, leave=False): if self.cuda: data, target = data.cuda(), target.cuda() # data, target = Variable(data), Variable(target) data, target = Variable(data,volatile=True), Variable(target,volatile=True) if self.dual_network: if self.add_calcium_mask: data = data[:,0:2,: :,:] else: data = data[:,0,:,:,:] data = torch.unsqueeze(data, 1) pred_prob = self.model(data) if test_epoch: #get attention gcam = GradCAM(model=self.model) probs, idx = gcam.forward(data) topk = 3 target_layer = 'ec6.2' # target_layer = 'ec1.2' test_attention_out = osp.join(out, target_layer) mkdir(test_attention_out) input_img = data[0, 0].data.cpu().numpy() input_size = (input_img.shape[0], input_img.shape[1], input_img.shape[2]) input_mask = data[0, 1].data.cpu().numpy() nii_img = nib.Nifti1Image(input_img, affine=np.eye(4)) output_img_file = os.path.join(out, ('%s_img.nii.gz' % sub_name[0])) nib.save(nii_img, output_img_file) nii_mask = nib.Nifti1Image(input_mask, affine=np.eye(4)) output_mask_file = os.path.join(out, ('%s_mask.nii.gz' % sub_name[0])) nib.save(nii_mask, output_mask_file) del input_img,input_mask del nii_img, nii_mask for i in range(0, topk): gcam.backward(idx=idx[i]) output = gcam.generate(target_layer=target_layer) output = resize(output,input_size , mode='constant', preserve_range=True) nii_seg = nib.Nifti1Image(output, affine=np.eye(4)) output_att_file = os.path.join(test_attention_out, ('%s_test_att%d_clss%d.nii.gz' % (sub_name[0],i,idx[i]))) nib.save(nii_seg, output_att_file) gcam.backward_del(idx=idx[i]) del gcam, output, nii_seg, probs #training attention subnum = data.size(0) for subi in range(subnum): attentions = [] i = 1 self.input = data fmap = self.get_feature_maps('compatibility_score%d' % i, upscale=False) try: attmap = fmap[1][1] except: aaaa = 1 attention = attmap[subi,0].cpu().numpy() # attention = attention[:, :] # attention = numpy.expand_dims(resize(attention, (fmap_size[0], fmap_size[1]), mode='constant', preserve_range=True), axis=2) attention = resize(attention, input_size, mode='constant', preserve_range=True) attention = (attention-np.min(attention))/(np.max(attention)-np.min(attention)) # this one is useless # plotNNFilter(fmap_0, figure_id=i+3, interp='bilinear', colormap=cm.jet, title='compat. feature %d' %i) nii_seg = nib.Nifti1Image(attention, affine=np.eye(4)) output_att_file = os.path.join(out, ('%s_train_att.nii.gz' % sub_name[subi])) nib.save(nii_seg, output_att_file) del nii_seg, fmap, attmap, attention # plotNNFilterOverlay(input_img, attention, figure_id=i, interp='bilinear', colormap=cm.jet, # title='a', alpha=0.5) # attentions.append(attention) pred_clss = F.log_softmax(pred_prob) pred = pred_clss.data.max(1)[1] # get th correct += pred.eq(target.data).cpu().sum() pred_binary = pred>0 target_binary = target.data>0 correct_binary += pred_binary.eq(target_binary).cpu().sum() sofar += data.size(0) test_loss = F.nll_loss(pred_clss, target) for batch_num in range(data.size(0)): # test_loss /= len(self.test_loader) # loss function already averages over batch size results_strs = '[Epoch %04d] True=[%d],Pred=[%d],Pred_prob=%s,Test set: Average loss: %.4f, Accuracy: %d/%d (%.3f) binary (%.3f), subname=[%s]\n' % ( self.epoch, target.data.cpu().numpy()[batch_num], pred.cpu().numpy()[batch_num], np.array2string(pred_clss[batch_num].data.cpu().numpy()), test_loss.data[0], correct, sofar, 100. * float(correct) / sofar, 100 * float(correct_binary) / sofar, sub_name[batch_num]) print(results_strs) fv.write(results_strs) loss_history.append(test_loss.data.cpu().numpy().tolist()) pred_history += pred_binary.cpu().numpy().tolist() target_history += target_binary.data.cpu().numpy().tolist() f1 = f1_score(target_history, pred_history) recall = recall_score(target_history, pred_history) precision = precision_score(target_history, pred_history) accuracy = accuracy_score(target_history, pred_history) print_str='test epoch='+str(self.epoch)+',accuracy='+str(accuracy)+",f1="+str(f1)+",recall="+str(recall)+',precision='+str(precision)+",loss="+str(np.mean(loss_history))+"\n" fv2.write(results_strs) fv3.write(print_str) fv.close() fv2.close() fv3.close()
# set target layer for CAM if args.model == 'vgg16' or args.model == 'densenet121': target_layer = model.features[-1] elif args.model == 'resnet18': target_layer = model.layer4[-1] # get given label's index label = {'covid_19': 0, 'lung_opacity': 1, 'normal': 2, 'pneumonia': 3} idx_to_label = {v: k for k, v in label.items()} if args.label is not None: label = label[args.label] else: label = None # load and preprocess image image = utils.load_image(args.image_path) warnings.filterwarnings("ignore", category=UserWarning) # pass image through model and get CAM for the given label cam = GradCAM(model=model, target_layer=target_layer) label, mask = cam(image, label) print(f'GradCAM generated for label "{idx_to_label[label]}".') # deprocess image and overlay CAM image = utils.deprocess_image(image) image = apply_mask(image, mask) # save the image utils.save_image(image, args.output_path)
def test(image_paths, output_dir, cuda): """ Generate Grad-CAM at different layers of ResNet-152 """ fold = 0 device = get_device(cuda) # Synset words classes = {0: 'normal', 1: 'covid'} # Model #model = models.resnet34(pretrained=False,num_classes=config.num_classes) best_model = torch.load(config.best_models + config.model_name + os.sep + str(fold) + os.sep + 'model_best.pth.tar') print(best_model["state_dict"].keys()) model = make_model('mynet', num_classes=config.num_classes, pretrained=False) #best_model = torch.load(config.weights +'model_best.pth.tar') model.load_state_dict(best_model["state_dict"]) print(best_model["state_dict"].keys()) model.to(device) model.eval() # The four residual layers target_layers = [ "_features.0", "_features.1", "_features.2", "_features.3", "_features.4", "_features.5", "_features.6", "_features.7" ] target_class = 1 # "bull mastif" # Images images, raw_images = load_images(image_paths) images = torch.stack(images).to(device) gcam = GradCAM(model=model) probs, ids = gcam.forward(images) ids_ = torch.LongTensor([[target_class]] * len(images)).to(device) print(ids_.shape) gcam.backward(ids=ids_) for target_layer in target_layers: print("Generating Grad-CAM @{}".format(target_layer)) # Grad-CAM regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format( j, classes[target_class], float(probs[ids == target_class]))) save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, "resnet152", target_layer, classes[target_class]), ), gcam=regions[j, 0], raw_image=raw_images[j], )
def demo1(image_paths, target_layer, arch, topk, output_dir, cuda): """ Visualize model responses given multiple images """ device = get_device(cuda) # Synset words classes = get_classtable() # Model from torchvision model = models.__dict__[arch](pretrained=True) model.to(device) model.eval() # Images images, raw_images = load_images(image_paths) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Vanilla Backpropagation:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) # sorted for i in range(topk): bp.backward(ids=ids[:, [i]]) gradients = bp.generate() # Save results as image files for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution:") deconv = Deconvnet(model=model) _ = deconv.forward(images) for i in range(topk): deconv.backward(ids=ids[:, [i]]) gradients = deconv.generate() for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Guided Backpropagation save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], ) # Guided Grad-CAM save_gradient( filename=osp.join( output_dir, "{}-{}-guided_gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]]), ), gradient=torch.mul(regions, gradients)[j], )
def fulltest(image_paths, target_layer, arch, topk, output_dir, cuda): """ Visualize model responses given multiple images """ fold = 0 device = get_device(cuda) # Synset words #classes = {0:'normal',1:'covid'} classes = get_classtable() # Model #model = models.resnet34(pretrained=False,num_classes=config.num_classes) best_model = torch.load(config.weights + 'ct-cn/' + 'model_best.pth.tar') print(best_model["state_dict"].keys()) model = make_model(arch, num_classes=config.num_classes, pretrained=True) #best_model = torch.load(config.weights +'model_best.pth.tar') model.load_state_dict(best_model["state_dict"]) print(best_model["state_dict"].keys()) model.to(device) model.eval() # Images images, raw_images = load_images(image_paths) images = torch.stack(images).to(device) """ Common usage: 1. Wrap your model with visualization classes defined in grad_cam.py 2. Run forward() with images 3. Run backward() with a list of specific classes 4. Run generate() to export results """ # ========================================================================= print("Vanilla Backpropagation:") bp = BackPropagation(model=model) probs, ids = bp.forward(images) # sorted print(probs) print(ids) for i in range(topk): bp.backward(ids=ids[:, [i]]) gradients = bp.generate() # Save results as image files for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-vanilla-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Remove all the hook function in the "model" bp.remove_hook() # ========================================================================= print("Deconvolution:") deconv = Deconvnet(model=model) _ = deconv.forward(images) for i in range(topk): deconv.backward(ids=ids[:, [i]]) gradients = deconv.generate() for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) save_gradient( filename=osp.join( output_dir, "{}-{}-deconvnet-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) deconv.remove_hook() # ========================================================================= print("Grad-CAM/Guided Backpropagation/Guided Grad-CAM:") gcam = GradCAM(model=model) _ = gcam.forward(images) gbp = GuidedBackPropagation(model=model) _ = gbp.forward(images) for i in range(topk): # Guided Backpropagation gbp.backward(ids=ids[:, [i]]) gradients = gbp.generate() # Grad-CAM gcam.backward(ids=ids[:, [i]]) regions = gcam.generate(target_layer=target_layer) for j in range(len(images)): print("\t#{}: {} ({:.5f})".format(j, classes[ids[j, i]], probs[j, i])) # Guided Backpropagation save_gradient( filename=osp.join( output_dir, "{}-{}-guided-{}.png".format(j, arch, classes[ids[j, i]]), ), gradient=gradients[j], ) # Grad-CAM save_gradcam( filename=osp.join( output_dir, "{}-{}-gradcam-{}-{}.png".format(j, arch, target_layer, classes[ids[j, i]]), ), gcam=regions[j, 0], raw_image=raw_images[j], ) # Guided Grad-CAM save_gradient( filename=osp.join( output_dir, "{}-{}-guided_gradcam-{}-{}.png".format( j, arch, target_layer, classes[ids[j, i]]), ), gradient=torch.mul(regions, gradients)[j], )