def guided_grad_cam(self,
numpy_img,
target_pspi,
file_name_to_export='test',
save=False):
# prep image for the network
prep_img = torch.from_numpy(
cv2.resize(numpy_img,
(200, 200))[None] / 255).float().unsqueeze_(0)
prep_img = prep_img.requires_grad_().cuda()
# Grad cam
# Generate cam mask
cam = generate_cam(self.pretrained_model, prep_img, target_pspi)
# Guided backprop
GBP = GuidedBackprop(self.pretrained_model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_pspi)
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
grayscale_cam_gb = convert_to_grayscale(cam_gb)
if save:
save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
save_gradient_images(grayscale_cam_gb,
file_name_to_export + '_GGrad_Cam_gray')
return cam_gb, grayscale_cam_gb
def vis_grad(model, class_index, layer, image_path, size=[224, 224]):
original_image = cv2.imread(image_path, 1)
#plt.imshow(original_image)
#plt.show()
prep_img = preprocess_image(original_image, size)
file_name_to_export = 'model' + '_classindex_' + str(
class_index) + '-layer_' + str(layer)
# Grad cam
gcv2 = GradCam(model, target_layer=layer)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, class_index, size)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, class_index)
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
#save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
grayscale_cam_gb = convert_to_grayscale(cam_gb)
#save_gradient_images(grayscale_cam_gb, file_name_to_export + '_GGrad_Cam_gray')
print('Guided grad cam completed')
cam_gb = trans(cam_gb)
grayscale_cam_gb = trans(grayscale_cam_gb)
return cam_gb, grayscale_cam_gb
def get_Guided_backProp(image_name, model_file, resultFile):
k.clear_session()
model = load_model(model_file)
img_input = load_image(image_name)
img_orignal = load_image(image_name)
img_input = np.expand_dims(img_input, axis=0)
pred = model.predict(img_input)
class_id = np.argmax(pred, 1)
print("Class id predicted: ", class_id)
guided_bprop = GuidedBackprop(model)
Guided_backProp_mask = guided_bprop.get_mask(img_orignal)
save_image(Guided_backProp_mask, resultFile)
k.clear_session()
success = True
return success
"""
cam_gb = np.multiply(grad_cam_mask, guided_backprop_mask)
return cam_gb
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
# Grad cam
# gcv2 = GradCam(pretrained_model, target_layer=11)
gcv2 = GradCam(pretrained_model, target_layer=7)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, target_class)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(pretrained_model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_class)
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
grayscale_cam_gb = convert_to_grayscale(cam_gb)
save_gradient_images(grayscale_cam_gb,
file_name_to_export + '_GGrad_Cam_gray')
print('Guided grad cam completed')
model.add(Dense(num_classes, activation='softmax'))
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
tbCallBack = keras.callbacks.TensorBoard(log_dir='./Graph',
histogram_freq=0,
write_graph=True,
write_images=True)
model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=epochs,
verbose=1,
validation_data=(X_validation, y_validation),
callbacks=[tbCallBack])
score = model.evaluate(X_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
g_backpropagation = GuidedBackprop(model)
current_img = X_train[10]
current_img_rs = np.reshape(current_img, (50, 50))
plt.imsave("/home/stan/Desktop/pulled_code/baby_result_og.jpg", current_img_rs)
current_mask = g_backpropagation.get_mask(current_img)
# print(current_mask)
show_image(current_mask)
def eval(self, gradcam=False, rise=False, test_on_val=False):
"""The function for the meta-eval phase."""
# Load the logs
if os.path.exists(osp.join(self.args.save_path, 'trlog')):
trlog = torch.load(osp.join(self.args.save_path, 'trlog'))
else:
trlog = None
torch.manual_seed(1)
np.random.seed(1)
# Load meta-test set
test_set = Dataset('val' if test_on_val else 'test', self.args)
sampler = CategoriesSampler(test_set.label, 600, self.args.way,
self.args.shot + self.args.val_query)
loader = DataLoader(test_set,
batch_sampler=sampler,
num_workers=8,
pin_memory=True)
# Set test accuracy recorder
test_acc_record = np.zeros((600, ))
# Load model for meta-test phase
if self.args.eval_weights is not None:
weights = self.addOrRemoveModule(
self.model,
torch.load(self.args.eval_weights)['params'])
self.model.load_state_dict(weights)
else:
self.model.load_state_dict(
torch.load(osp.join(self.args.save_path,
'max_acc' + '.pth'))['params'])
# Set model to eval mode
self.model.eval()
# Set accuracy averager
ave_acc = Averager()
# Generate labels
label = torch.arange(self.args.way).repeat(self.args.val_query)
if torch.cuda.is_available():
label = label.type(torch.cuda.LongTensor)
else:
label = label.type(torch.LongTensor)
label_shot = torch.arange(self.args.way).repeat(self.args.shot)
if torch.cuda.is_available():
label_shot = label_shot.type(torch.cuda.LongTensor)
else:
label_shot = label_shot.type(torch.LongTensor)
if gradcam:
self.model.layer3 = self.model.encoder.layer3
model_dict = dict(type="resnet",
arch=self.model,
layer_name='layer3')
grad_cam = GradCAM(model_dict, True)
grad_cam_pp = GradCAMpp(model_dict, True)
self.model.features = self.model.encoder
guided = GuidedBackprop(self.model)
if rise:
self.model.layer3 = self.model.encoder.layer3
score_mod = ScoreCam(self.model)
# Start meta-test
for i, batch in enumerate(loader, 1):
if torch.cuda.is_available():
data, _ = [_.cuda() for _ in batch]
else:
data = batch[0]
k = self.args.way * self.args.shot
data_shot, data_query = data[:k], data[k:]
if i % 5 == 0:
suff = "_val" if test_on_val else ""
if self.args.rep_vec or self.args.cross_att:
print('batch {}: {:.2f}({:.2f})'.format(
i,
ave_acc.item() * 100, acc * 100))
if self.args.cross_att:
label_one_hot = self.one_hot(label).to(label.device)
_, _, logits, simMapQuer, simMapShot, normQuer, normShot = self.model(
(data_shot, label_shot, data_query),
ytest=label_one_hot,
retSimMap=True)
else:
logits, simMapQuer, simMapShot, normQuer, normShot, fast_weights = self.model(
(data_shot, label_shot, data_query),
retSimMap=True)
torch.save(
simMapQuer,
"../results/{}/{}_simMapQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
simMapShot,
"../results/{}/{}_simMapShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
data_query, "../results/{}/{}_dataQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
data_shot, "../results/{}/{}_dataShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normQuer, "../results/{}/{}_normQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normShot, "../results/{}/{}_normShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
else:
logits, normQuer, normShot, fast_weights = self.model(
(data_shot, label_shot, data_query),
retFastW=True,
retNorm=True)
torch.save(
normQuer, "../results/{}/{}_normQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
normShot, "../results/{}/{}_normShot{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
if gradcam:
print("Saving gradmaps", i)
allMasks, allMasks_pp, allMaps = [], [], []
for l in range(len(data_query)):
allMasks.append(
grad_cam(data_query[l:l + 1], fast_weights, None))
allMasks_pp.append(
grad_cam_pp(data_query[l:l + 1], fast_weights,
None))
allMaps.append(
guided.generate_gradients(data_query[l:l + 1],
fast_weights))
allMasks = torch.cat(allMasks, dim=0)
allMasks_pp = torch.cat(allMasks_pp, dim=0)
allMaps = torch.cat(allMaps, dim=0)
torch.save(
allMasks, "../results/{}/{}_gradcamQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
allMasks_pp,
"../results/{}/{}_gradcamppQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
torch.save(
allMaps, "../results/{}/{}_guidedQuer{}{}.th".format(
self.args.exp_id, self.args.model_id, i, suff))
if rise:
print("Saving risemaps", i)
allScore = []
for l in range(len(data_query)):
allScore.append(
score_mod(data_query[l:l + 1], fast_weights))
else:
if self.args.cross_att:
label_one_hot = self.one_hot(label).to(label.device)
_, _, logits = self.model(
(data_shot, label_shot, data_query),
ytest=label_one_hot)
else:
logits = self.model((data_shot, label_shot, data_query))
acc = count_acc(logits, label)
ave_acc.add(acc)
test_acc_record[i - 1] = acc
# Calculate the confidence interval, update the logs
m, pm = compute_confidence_interval(test_acc_record)
if trlog is not None:
print('Val Best Epoch {}, Acc {:.4f}, Test Acc {:.4f}'.format(
trlog['max_acc_epoch'], trlog['max_acc'], ave_acc.item()))
print('Test Acc {:.4f} + {:.4f}'.format(m, pm))
return m
vanilla_grads = Backprop.generate_gradients(noisy_img, target_class)
# Add gradients to smooth_grad
smooth_grad = smooth_grad + vanilla_grads
# Average it out
smooth_grad = smooth_grad / param_n
return smooth_grad
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_example_params(target_example)
VBP = VanillaBackprop(pretrained_model)
GBP = GuidedBackprop(
pretrained_model) # if you want to use GBP dont forget to
# change the parametre in generate_smooth_grad
param_n = 50
param_sigma_multiplier = 4
smooth_grad = generate_smooth_grad(
VBP, # ^This parameter
prep_img,
target_class,
param_n,
param_sigma_multiplier)
# Save colored gradients
save_gradient_images(smooth_grad,
file_name_to_export + '_SmoothGrad_color')
# Convert to grayscale
guided_backprop_mask (np_arr):Guided backprop mask
"""
cam_gb = np.multiply(grad_cam_mask, guided_backprop_mask)
return cam_gb
if __name__ == '__main__':
# Get params
target_example = 0 # Snake
(original_image, prep_img, target_class, file_name_to_export, pretrained_model) =\
get_params(target_example)
# Grad cam
gcv2 = GradCam(pretrained_model, target_layer=11)
# Generate cam mask
cam = gcv2.generate_cam(prep_img, target_class)
print('Grad cam completed')
# Guided backprop
GBP = GuidedBackprop(pretrained_model)
# Get gradients
guided_grads = GBP.generate_gradients(prep_img, target_class)
print('Guided backpropagation completed')
# Guided Grad cam
cam_gb = guided_grad_cam(cam, guided_grads)
save_gradient_images(cam_gb, file_name_to_export + '_GGrad_Cam')
grayscale_cam_gb = convert_to_grayscale(cam_gb)
save_gradient_images(grayscale_cam_gb, file_name_to_export + '_GGrad_Cam_gray')
print('Guided grad cam completed')