def closure(i):
#global i
if param_noise:
for n in [x for x in net.parameters() if len(x.size()) == 4]:
n.data += n.data.clone().normal_()*n.data.std()/50
out = net(net_input)
total_loss = mse(out * mask_var, img_var * mask_var)
total_loss.backward()
psrn = compare_psnr(utils.var_to_np(out), img_np)
psnr_history.append(psrn)
if report:
print ('Iteration %05d Loss %f PSNR %.3f' % (i, total_loss.data[0], psrn), '\r', end='')
if i % show_every == 0:
out_np = utils.var_to_np(out)
utils.plot_image_grid([np.clip(out_np, 0, 1)], factor=figsize, nrow=1)
#i += 1
return total_loss
def validate_models(channels):
"""
Validate trained models
:param channels: List of compressed channels used
:return: None
"""
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
test_loader = test_dataloader()
test_batch = next(iter(test_loader)).to(device)
reconstructed_images = {}
for channel in channels:
NUM_CHANNELS = channel
encoder = Encoder(NUM_CHANNELS).to(device)
generator = Generator(NUM_CHANNELS).to(device)
encoder.load_state_dict(
torch.load(f"../models/encoder_{NUM_CHANNELS}.model", map_location=torch.device("cpu"))
)
generator.load_state_dict(
torch.load(f"../models/generator_{NUM_CHANNELS}.model", map_location=torch.device("cpu"))
)
encoder.eval()
generator.eval()
reconstructed_image = generator(encoder(test_batch))
reconstructed_images[NUM_CHANNELS] = reconstructed_image
plot_image_grid(test_batch, reconstructed_images, NUM_IMAGES_GRID)
save_images(test_batch, reconstructed_images)
calculate_metric(channels)
def visualize(analysis,
text,
method_names,
visualization_transformations,
file_name=None,
scaling=1.0):
# Apply analysis postprocessing, e.g., creating a heatmap.
#postprocess = {i: METHODS[method_name].get('visualize', lambda x: x) for i, method_name in enumerate(method_names)}
# Prepare the grid as rectangular list
#grid = [[analysis[i, j] for j in range(analysis.shape[1])]
# for i in range(analysis.shape[0])]
grid = []
for i in range(analysis.shape[0]):
grid.append([])
for j in range(analysis.shape[1]):
_method_name = method_names[j]
#_p_func = METHODS[_method_name].get('f_visualize', lambda x: x)
grid[i].append(analysis[i, j])
_p_func = visualization_transformations.get(
_method_name,
visualization_transformations.get('default', None))
if _p_func is not None:
#_p_func = methods_metadata[_method_name]['f_visualize']
# batch dim has to be added before f_visualize application (otherwise bk_proj fails)
grid[i][j] = _p_func(np.expand_dims(grid[i][j], axis=0))[0]
# Prepare the labels
label, presm, prob, pred = zip(*text)
row_labels_left = [('label: {}'.format(label[i]),
'pred: {}'.format(pred[i])) for i in range(len(label))]
row_labels_right = [('logit: {}'.format(presm[i]),
'prob: {}'.format(prob[i]))
for i in range(len(label))]
col_labels = [''.join(method_name) for method_name in method_names]
# Plot the analysis.
utils.plot_image_grid(
grid,
row_labels_left,
row_labels_right,
col_labels,
#file_name=os.environ.get("plot_file_name", None),
file_name=file_name,
#figsize=figsize,
scaling=scaling,
)
def closure(i):
if input_noise:
net_input.data = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
total_loss = loss_fn(out, img_noisy_var)
total_loss.backward()
psrn = compare_psnr(utils.var_to_np(out), img_np)
psnr_history.append(psrn)
if report:
print ('Iteration %05d Loss %f PSNR %.3f' % (i, total_loss.data[0], psrn), '\r', end='')
if i % show_every == 0:
out_np = utils.var_to_np(out)
utils.plot_image_grid([np.clip(out_np, 0, 1)], factor=figsize, nrow=1)
return total_loss
def evaluate_model(net, net_input, img_np, img_noisy_np, num_iter=6000,
show_every=500, report=True, figsize=10):
loss_fn=torch.nn.MSELoss().type(dtype)
input_noise=True
LR=0.01
reg_noise_std=1./30.
net_input_saved = net_input.data.clone()
noise = net_input.data.clone()
img_noisy_var = utils.np_to_var(img_noisy_np).type(dtype)
psnr_history = []
def closure(i):
if input_noise:
net_input.data = net_input_saved + (noise.normal_() * reg_noise_std)
out = net(net_input)
total_loss = loss_fn(out, img_noisy_var)
total_loss.backward()
psrn = compare_psnr(utils.var_to_np(out), img_np)
psnr_history.append(psrn)
if report:
print ('Iteration %05d Loss %f PSNR %.3f' % (i, total_loss.data[0], psrn), '\r', end='')
if i % show_every == 0:
out_np = utils.var_to_np(out)
utils.plot_image_grid([np.clip(out_np, 0, 1)], factor=figsize, nrow=1)
return total_loss
print('Starting optimization with ADAM')
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
for j in range(num_iter):
optimizer.zero_grad()
closure(j)
optimizer.step()
if report:
out_np = utils.var_to_np(net(net_input))
q = utils.plot_image_grid([np.clip(out_np, 0, 1), img_np], factor=13);
data = {}
data['psnr_history'] = psnr_history
pickle.dump(data, open('denoising_psnr.p','wb'))
max_index, max_value = max(enumerate(psnr_history), key=operator.itemgetter(1))
return max_index, max_value
def evaluate_model(net, net_input, img_np, img_mask_np, num_iter=6000,
show_every=500, report=True, figsize=10):
pad = 'zero'
OPTIMIZER = 'adam'
INPUT = 'noise'
input_depth = 32
LR = 0.01
num_iter = 6001
param_noise = False
show_every = 500
figsize = 10
net_input_saved = net_input.data.clone()
noise = net_input.data.clone()
#img_noisy_var = utils.np_to_var(img_noisy_np).type(dtype)
# img
psnr_history = []
# Loss
mse = torch.nn.MSELoss().type(dtype)
img_var = utils.np_to_var(img_np).type(dtype)
mask_var = utils.np_to_var(img_mask_np).type(dtype)
psnr_history = []
def closure(i):
#global i
if param_noise:
for n in [x for x in net.parameters() if len(x.size()) == 4]:
n.data += n.data.clone().normal_()*n.data.std()/50
out = net(net_input)
total_loss = mse(out * mask_var, img_var * mask_var)
total_loss.backward()
psrn = compare_psnr(utils.var_to_np(out), img_np)
psnr_history.append(psrn)
if report:
print ('Iteration %05d Loss %f PSNR %.3f' % (i, total_loss.data[0], psrn), '\r', end='')
if i % show_every == 0:
out_np = utils.var_to_np(out)
utils.plot_image_grid([np.clip(out_np, 0, 1)], factor=figsize, nrow=1)
#i += 1
return total_loss
print('Starting optimization with ADAM')
optimizer = torch.optim.Adam(net.parameters(), lr=LR)
for j in range(num_iter):
optimizer.zero_grad()
closure(j)
optimizer.step()
if report:
out_np = utils.var_to_np(net(net_input))
q = utils.plot_image_grid([np.clip(out_np, 0, 1), img_np], factor=13);
data = {}
data['psnr_history'] = psnr_history
pickle.dump(data, open('inpainting_validation_psnr.p','wb'))
max_index, max_value = max(enumerate(psnr_history), key=operator.itemgetter(1))
return max_index, max_value
def compute_analysis(args):
############# DATA ##################
image_size = settings.image_size
#alpha = 3
num_classes = settings.num_classes
#n_test_examples = 2
n = image_size * image_size
k = n // 2
######################################
print('\nLoading the model')
model_path = os.path.abspath(args.model_path)[:-3]
model = load_model(model_path + '.h5')
print('Model Loaded\n')
print('Model Summary')
model.summary()
if args.img_path is None:
print('Creating a random image')
resultFolder = args.out_path + 'random_' + time.strftime(
"%Y%m%d-%H%M%S")
if not settings.random_image_flag:
print(
'Generating the random image based on the seed from settings file'
)
rng = np.random.RandomState(settings.numpy_image_seed)
original_img = rng.randint(num_classes,
size=(image_size, image_size))
else:
original_img = np.random.randint(num_classes,
size=(image_size, image_size))
original_img = original_img.astype(np.float32)
img = original_img
else:
ret = PIL.Image.open(args.img_path)
ret = ret.resize((image_size, image_size))
ret = ret.convert('L')
img = np.asarray(ret, dtype=np.uint8).astype(np.float32)
resultFolder = args.out_path + args.img_path.split('/')[-1].split(
'.')[0]
if resultFolder[-1] != '/':
resultFolder = resultFolder + '/'
if args.centre_pixel is not None:
img[image_size // 2, image_size // 2] = args.centre_pixel
#ipdb.set_trace()
y = to_categorical(img[image_size // 2, image_size // 2],
num_classes=num_classes)
y_index = np.argmax(y)
print('\nTrue Label is:', y_index)
img = img / 255
data = img.flatten()
data = np.expand_dims(data, axis=0)
preds = model.predict(data)
predict_indicies = np.argmax(preds)
print('Predicted label is:', predict_indicies)
#ipdb.set_trace()
## BTW, 2nd paramter (count starts from 0) is redundant and not used
# Heatmapping Methods
methods = [("input", {}, "Input")]
if 'grad' in args.heatmap_methods:
methods.append(("gradient", {}, "Gradient"))
if 'gb' in args.heatmap_methods:
methods.append(("guided_backprop", {}, "Guided Backprop "))
if 'deconvnet' in args.heatmap_methods:
methods.append(("deconvnet", {}, "Deconvnet"))
if 'sg' in args.heatmap_methods:
methods.append(
("smoothgrad", settings.smooth_grad_parameters, "SmoothGrad"))
if 'inpgrad' in args.heatmap_methods:
methods.append(("input_t_gradient", {}, "Input * Gradient"))
if 'ig' in args.heatmap_methods:
methods.append(
("integrated_gradients", settings.integrated_grad_parameters,
"Integrated Gradients"))
if 'lrp' in args.heatmap_methods:
methods.append(("lrp.z", {}, "LRP-Z"))
methods.append(
("lrp.epsilon", settings.lrp_epsilon_parameters, "LRP-Epsilon"))
methods.append(("lrp.alpha_beta", {
"alpha": 1,
"beta": 0
}, "LRP-alpha1_beta0"))
methods.append(("lrp.alpha_beta", settings.lrp_alpha_beta_parameters,
"LRP-alpha2_beta1"))
if 'occlusion' in args.heatmap_methods:
methods.append(("occlusion", {}, "Occlusion"))
if 'deeplift' in args.heatmap_methods:
from deepexplain.tensorflow import DeepExplain
methods.append(("deeplift", {}, "DeepLift"))
if 'shapley' in args.heatmap_methods:
methods.append(("shapley", {}, "Shapley Sampling"))
if 'pda' in args.heatmap_methods:
methods.append(("pda", {}, "Prediction Difference Analysis"))
if 'lime' in args.heatmap_methods:
methods.append(('lime', {}, "Lime"))
if 'shap' in args.heatmap_methods:
methods.append(('shap', {}, "Kernel_SHAP"))
if 'mp' in args.heatmap_methods:
methods.append(("mp", {}, "Meaningful_Perturbation"))
model_wo_softmax = iutils.keras.graph.model_wo_softmax(model)
##############################################################
# Create analyzers.
analyzers = []
for method in methods:
#ipdb.set_trace()
if method[0] == 'occlusion':
from occlusion import occlusion_analysis
kwargs = {
'image': data,
'model': model,
'num_classes': num_classes,
'img_size': image_size,
}
analyzer = occlusion_analysis(**kwargs)
elif method[0] == 'deeplift':
with DeepExplain(session=K.get_session()) as de:
input_tensor = model.layers[0].input
fModel = Model(inputs=input_tensor,
outputs=model.layers[-2].output)
target_tensor = fModel(input_tensor)
dl_bl = settings.deeplift_parameters['baseline'].flatten()
analyzer = de.get_explainer('deeplift',
target_tensor,
input_tensor,
baseline=dl_bl)
elif method[0] == 'shapley':
with DeepExplain(session=K.get_session()) as de:
input_tensor = model.layers[0].input
fModel = Model(inputs=input_tensor,
outputs=model.layers[-2].output)
target_tensor = fModel(input_tensor)
analyzer = de.get_explainer('shapley_sampling',
target_tensor,
input_tensor,
samples=2)
elif method[0] == 'pda':
from pda import prediction_difference_analysis
train_samples = settings.pda_parameters['train_samples']
# ipdb.set_trace()
kwargs = {
'image': data,
'model': model,
'num_classes': num_classes,
'img_size': image_size,
'train_samples': train_samples
}
analyzer = prediction_difference_analysis(**kwargs)
elif method[0] == 'mp':
from mp import meaningful_perturbation
#########################################
# Need to convert this in (0-255) first
mp_par = settings.mp_parameters
mp_par['num_classes'] = num_classes
mp_par['img_size'] = image_size
#ipdb.set_trace()
analyzer = meaningful_perturbation(
(data.reshape((image_size, -1)) * 255).astype('uint8'),
model_path + '.pt',
resultFolder,
**mp_par,
)
elif method[0] == 'lime':
import lime
from lime import lime_image
from lime.wrappers.scikit_image import SegmentationAlgorithm
from skimage.segmentation import mark_boundaries
# Make the explainer object
analyzer = lime_image.LimeImageExplainer()
elif method[0] == 'shap':
from shap_class import shap_analysis
from skimage.segmentation import slic as slic_super_pixel
# Make the explainer object
analyzer = shap_analysis(
np.repeat(np.expand_dims(data.reshape((image_size, -1)) * 255,
axis=-1),
3,
axis=-1), model, resultFolder,
**settings.shap_parameters)
else:
try:
analyzer = innvestigate.create_analyzer(
method[0], # analysis method identifier
model_wo_softmax, # model without softmax output
neuron_selection_mode="index",
**method[1]) # optional analysis parameters
if method[0] == "pattern.attribution":
analyzer.fit(data, batch_size=256, verbose=1)
except innvestigate.NotAnalyzeableModelException:
# Not all methods work with all models.
analyzer = None
analyzers.append(analyzer)
########## GENERATE HEATMAPS ###########
IMG_ROWS = image_size
IMG_COLS = image_size
#analysis = np.zeros([len(data), len(analyzers), IMG_ROWS, IMG_COLS]) #Use analyzer for the analysis
heatmap_grids = []
extra_info = []
for i, x in enumerate(data):
# Add batch axis.
x = np.expand_dims(x, axis=0)
y_true = (x[:, k] * 255).astype('int64')[0]
### Model predictions
pred = model.predict(x) #256 probabilities
pred_label = np.argmax(pred, axis=1)[0]
pred_prob = np.amax(pred)
neuron = pred_label
if args.clamp_label is not None:
neuron = args.clamp_label
#print('pred shape', np.shape(pred))
fiveClasses = np.argsort(-pred[0, :])[:5]
#fiveProbs = np.zeros(np.shape(fiveClasses))
fiveProbs = pred[0, fiveClasses]
##########################
analysis = np.zeros([1, len(analyzers), IMG_ROWS,
IMG_COLS]) # Use analyzer for the analysis
for aidx, analyzer in enumerate(analyzers):
print(f'Computing analysis for {methods[aidx][2]}')
if methods[aidx][0] == "input":
a = x
a = a.reshape(image_size, -1)
elif methods[aidx][0] in ['deeplift', 'shapley']:
ys = to_categorical(neuron, num_classes=num_classes)
if ys.shape[0] != x.shape[0]:
ys = np.expand_dims(ys, axis=0)
a = analyzer.run(x, ys)
a = a.reshape(image_size, -1)
a = (a - np.mean(a)) / (np.std(a) + 1e-15)
elif methods[aidx][0] == 'occlusion':
#ipdb.set_trace()
a = analyzer.explain(neuron, **settings.occlusion_parameters)
a = a.reshape(image_size, -1)
a = (a - np.mean(a)) / (np.std(a) + 1e-15)
elif methods[aidx][0] == 'pda':
print('PDA takes a lot time. Please wait...')
num = settings.pda_parameters['num']
a = analyzer.explain(neuron, num=num)
a = a.reshape(image_size, -1)
a = (a - np.mean(a)) / (np.std(a) + 1e-15)
elif methods[aidx][0] == 'mp':
print('MP takes some time. Please wait...')
a = analyzer.explain(neuron)
a = a.reshape(image_size, -1)
a = (a - np.mean(a)) / (np.std(a) + 1e-15)
elif methods[aidx][0] == 'lime':
segmenter = SegmentationAlgorithm(
'quickshift', **settings.lime_segmenter_parameters)
def lime_preprocess_input(im):
im = im[:, :, :, 0]
return np.reshape(im, (im.shape[0], -1))
def lime_predict(x):
return model.predict(lime_preprocess_input(x))
explanation = analyzer.explain_instance(
image=np.reshape(data, (image_size, -1)),
classifier_fn=lime_predict,
top_labels=settings.num_classes,
segmentation_fn=segmenter,
**settings.lime_explainer_parameters)
temp, mask = explanation.get_image_and_mask(
label=neuron, **settings.lime_mask_parameters)
bb = (mark_boundaries(temp, mask))
eutils.save_lime_mask(bb, resultFolder)
a = bb[:, :, 0]
# ipdb.set_trace()
elif methods[aidx][0] == 'shap':
print('SHAP takes some time. Please wait...')
segments_slic = slic_super_pixel(
PIL.Image.fromarray(np.uint8(analyzer.img_orig.copy())),
**settings.shap_slic_parameters)
a = analyzer.explain(segments_slic, neuron)
else:
a = analyzer.analyze(x, neuron_selection=neuron)
a = a.reshape(image_size, -1)
a = (a - np.mean(a)) / (np.std(a) + 1e-15)
analysis[i, aidx] = a
print('Done')
# Prepare the grid as rectengular list
grid = [[analysis[i, j] for j in range(analysis.shape[1])]
for i in range(analysis.shape[0])]
#ipdb.set_trace()
pred_prob = round(pred_prob, 5)
#ipdb.set_trace()
row_labels_left = [('True Label: {}'.format(y_true),
'Pred Label: {}'.format(pred_label),
'clamped Neuron: {}'.format(neuron),
'Probability: ' + str(pred_prob))]
row_labels_right = [(
'\n\n\nClass: %d' % fiveClasses[0] + ' Prob: %.5f' % fiveProbs[0],
'\nClass: %d' % fiveClasses[1] + ' Prob: %.5f' % fiveProbs[1],
'\nClass: %d' % fiveClasses[2] + ' Prob: %.5f' % fiveProbs[2],
'\nClass: %d' % fiveClasses[3] + ' Prob: %.5f' % fiveProbs[3],
'\nClass: %d' % fiveClasses[4] + ' Prob: %.5f' % fiveProbs[4],
)]
col_labels = [''.join(method[2]) for method in methods]
eutils.plot_image_grid(grid,
resultFolder,
row_labels_left,
row_labels_right,
col_labels,
file_name='heatmap_' +
time.strftime("%Y%m%d-%H%M%S") + '.png',
dpi=image_size)
heatmap_grids.append(grid)
extra_info.append([row_labels_left, row_labels_right])
return heatmap_grids, extra_info