def generate(self):
initial_learning_rate = 6
for i in range(1, 150):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, False)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
if i % 10 == 0:
# Save image
im_path = '../generated/c_specific_iteration_' + str(
i) + '.jpg'
save_image(self.created_image, im_path)
return self.processed_image
def dream(self):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, True)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas layer layers need less
optimizer = SGD([self.processed_image], lr=12, weight_decay=1e-4)
for i in range(1, 251):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.model):
# Forward
x = layer(x)
# Only need to forward until we the selected layer is reached
if index == self.selected_layer:
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image every 20 iteration
if i % 10 == 0:
print(self.created_image.shape)
im_path = '../generated/ddream_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_with_hooks(self):
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = plt.imread("../data/valid/1/0000.jpg")
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image.to(device)
x = self.model.conv1(x)
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.cpu().numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i == 30:
im_path = '../generated/layer_vis_1' + \
'_f' + str(self.selected_filter) + '.jpg'
save_image(self.created_image, im_path)
def generate_inverted_image_specific_layer(self,
input_image,
img_size,
target_layer=3):
# Generate a random image which we will optimize
opt_img = Variable(1e-1 * torch.randn(1, 3, img_size, img_size),
requires_grad=True)
# Define optimizer for previously created image
optimizer = SGD([opt_img], lr=1e4, momentum=0.9)
# Get the output from the model after a forward pass until target_layer
# with the input image (real image, NOT the randomly generated one)
input_image_layer_output = \
self.get_output_from_specific_layer_average(input_image, target_layer)
# Alpha regularization parametrs
# Parameter alpha, which is actually sixth norm
alpha_reg_alpha = 6
# The multiplier, lambda alpha
alpha_reg_lambda = 1e-7
# Total variation regularization parameters
# Parameter beta, which is actually second norm
tv_reg_beta = 2
# The multiplier, lambda beta
tv_reg_lambda = 1e-8
for i in range(201):
optimizer.zero_grad()
# Get the output from the model after a forward pass until target_layer
# with the generated image (randomly generated one, NOT the real image)
output = self.get_output_from_specific_layer(opt_img, target_layer)
# Calculate euclidian loss
euc_loss = 1e-1 * self.euclidian_loss(
input_image_layer_output.detach(), output)
# Calculate alpha regularization
reg_alpha = alpha_reg_lambda * self.alpha_norm(
opt_img, alpha_reg_alpha)
# Calculate total variation regularization
reg_total_variation = tv_reg_lambda * self.total_variation_norm(
opt_img, tv_reg_beta)
# Sum all to optimize
loss = euc_loss + reg_alpha + reg_total_variation
# Step
loss.backward()
optimizer.step()
# Generate image every 5 iterations
if i % 5 == 0:
print('Iteration:', str(i), 'Loss:', loss.data.numpy())
recreated_im = recreate_image(opt_img)
im_path = '../generated/Inv_Image_Layer_' + str(target_layer) + \
'_Iteration_' + str(i) + '.jpg'
save_image(recreated_im, im_path)
# Reduce learning rate every 40 iterations
if i % 40 == 0:
for param_group in optimizer.param_groups:
param_group['lr'] *= 1 / 10
def visualise_layer_without_hooks(self):
# Process image and return variable
# Generate a random image
tmp = torch.from_numpy(
np.random.uniform(150, 180, (1, 3, 32, 112, 112))).float()
for i in range(self.frame_num):
if self.video is not None:
random_image = self.video[i, ...]
else:
random_image = np.uint8(
np.random.uniform(150, 180, (112, 112, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
tmp[0, :, i, ...] = processed_image[0, ...]
processed_image = Variable(tmp, requires_grad=True).to(self.device)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, self.iter_num + 1):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
x = layer(x)
if isinstance(layer, nn.MaxPool3d):
x = x[0]
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
# print(processed_image.size())
if i % self.iter_num == 0:
for j in range(self.frame_num):
out_img = processed_image[:, :, j,
...] # self.selected_frame
self.created_image = recreate_image(out_img)
# Save image
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_frame' + str(j) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_without_hooks(self,plots_path):
# Process image and return variable
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.05, weight_decay=1e-6)
for i in range(1, 101):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
kl = 0
for index, layer in enumerate(self.model):
# Forward pass layer by layer
if hasattr(layer, 'convprobforward') and callable(layer.convprobforward):
x, _kl, = layer.convprobforward(x)
kl += _kl
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
conv = 0
for index, layer in enumerate(self.model):
if hasattr(layer, 'convprobforward') and callable(layer.convprobforward):
if conv == self.selected_layer:
self.conv_output = x[0, self.selected_filter]
print(conv)
# print(self.conv_output)
# print(self.conv_output.shape)
break;
conv = conv + 1
# self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
# beta = 2 ** (1- (1 + 1)) / (2 ** 1 - 1)
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
im_path = plots_path + 'layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_without_hooks(self,
optim=None,
iterations=30,
save_only_last=True):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, self.mean,
self.std, False)
# Define optimizer for the image
if optim == None:
optimizer = Adam([self.processed_image], lr=0.1, weight_decay=1e-6)
else:
optimizer = optim
for i in range(1, iterations + 1):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.layers):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
if i % 10 == 0:
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image,
self.mean, self.std)
# Save image
if not save_only_last:
if i % 5 == 0:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
else:
if i == iterations:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_with_hooks(self,
optim=None,
iterations=30,
save_only_last=True):
# Hook the selected layer
self.hook_layer()
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, self.mean,
self.std, False)
# Define optimizer for the image
if optim == None:
optimizer = Adam([self.processed_image], lr=0.1, weight_decay=1e-6)
else:
optimizer = optim
for i in range(1, iterations + 1):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.layers):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
if i % 10 == 0:
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image,
self.mean, self.std)
# Save image
if not save_only_last:
if i % 5 == 0:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
else:
if i == iterations:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def deconv_visualization(model, pic, png_dir, demode):
pic = pic[None,:,:,:]
pic = pic.cuda()
x = model(pic)
x = x.cpu().detach().numpy()
x = x.squeeze(0)
x = np.transpose(x, (1,2,0))
x = normalization(x)
x = preprocess_image(x, resize_im=False)
x = recreate_image(x)
if not os.path.exists('./deconv/'+ png_dir):
os.makedirs('./deconv/'+ png_dir)
im_path = './deconv/'+ png_dir+ '/layer_vis_' + str(demode) + '.jpg'
save_image(x, im_path)
def filter_visualization(model, selected_layer, selected_filter, png_dir):
for name, param in net.named_parameters():
if name == selected_layer + '.weight':
x = param
x = x[selected_filter,:,:,:]
x = x.cpu().detach().numpy()
x = x.transpose(1,2,0)
x = normalization(x)
x = preprocess_image(x, resize_im=False)
x = recreate_image(x)
if not os.path.exists('./filter/'+ png_dir):
os.makedirs('./filter/'+ png_dir)
im_path = './filter/'+ png_dir+ '/layer_vis_' + str(selected_layer) + \
'_f' + str(selected_filter) + '_iter' + str(i) + '.jpg'
save_image(x, im_path)
def vis_layer(encorder, decorder, pic, png_dir, demode=1, index=1):
"""
visualing the layer deconv result
"""
pic = pic[None,:,:,:]
pic = pic.cuda()
encorder_out, indices = encorder(pic)
num_feat = encorder_out.shape[1]
if demode==1:
activation_num = (encorder_out.shape[2]*encorder_out.shape[3])//10
else :
activation_num = (encorder_out.shape[2]*encorder_out.shape[3])//2
# set other feature map activations to zero
new_feat_map = encorder_out.clone()
# choose the max activations map
for i in range(0, num_feat):
choose_map = new_feat_map[0, i, :, :]
# print(choose_map)
map_clone = choose_map.clone()
new_map = torch.zeros(choose_map.shape, device='cuda')
for j in range(activation_num):
activation = torch.max(map_clone)
new_map = torch.where(map_clone==activation,
map_clone,
new_map
)
map_clone= torch.where(map_clone==activation,
torch.zeros(map_clone.shape, device='cuda'),
map_clone
)
new_feat_map[0, i, :, :] = new_map
deconv_output = decorder(new_feat_map, indices)
x = deconv_output.cpu().detach().numpy()
x = x.squeeze(0)
x = np.transpose(x, (1,2,0))
x = normalization(x)
x = preprocess_image(x, resize_im=False)
x = recreate_image(x)
if not os.path.exists('./deconv/'+ png_dir):
os.makedirs('./deconv/'+ png_dir)
im_path = './deconv/'+ png_dir+ '/layer_vis_' + str(demode) +'_' + str(index)+ '.jpg'
save_image(x, im_path)
def visualise_layer_with_hooks(self,plots_path):
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (227, 227, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
vi = GaussianVariationalInference(torch.nn.CrossEntropyLoss())
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 101):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
kl = 0
for index, layer in enumerate(self.model):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
if hasattr(layer, 'convprobforward') and callable(layer.convprobforward):
x, _kl, = layer.convprobforward(x)
kl += _kl
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
# loss = -torch.mean(self.conv_output)
loss = vi(outputs, y, kl, beta)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
im_path = plots_path + 'layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
print('saving image at:' + im_path)
save_image(self.created_image, im_path)
def generate(self):
initial_learning_rate = self.lr
loss = 10
i = 0
while loss >= self.min_loss:
# Process image and return variable
self.processed_image = preprocess_image(self.created_image,
self.mean, self.std, False)
if self.optim == None:
optimizer = Adam([self.processed_image],
lr=initial_learning_rate,
weight_decay=1e-6)
else:
optimizer = self.optim
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
if loss > class_loss.data.numpy():
loss = class_loss.data.numpy()
if i % 25 == 0:
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image,
self.mean, self.std)
if i % 25 == 0:
# Save image
im_path = '../generated/' + str(
self.name) + '_iteration_' + str(i) + '.jpg'
save_image(self.created_image, im_path)
i += 1
im_path = '../generated/' + str(self.name) + '_iteration_final.jpg'
save_image(self.created_image, im_path)
return self.processed_image
def visualise_layer_without_hooks(self):
# Process image and return variable
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss[0].data.numpy()[0]))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 30 == 0:
im_path = self.path_to_output_files + '/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def generate(self):
initial_learning_rate = 6
#---# self.processed_image = preprocess_image(self.created_image, False)
self.processed_image = transforms.functional.to_tensor(
self.created_image).to(self.device)
self.processed_image.unsqueeze_(0)
# Convert to Pytorch variable
# Define optimizer for the image
for i in range(1, self.niter + 1):
self.processed_image = Variable(self.processed_image,
requires_grad=True)
optimizer = Adam([self.processed_image],
lr=initial_learning_rate,
weight_decay=1e2)
#optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Process image and return variable
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[
0, self.target_class] #+self.processed_image.sum()/50
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.cpu().numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.processed_image = torch.clamp(self.processed_image.detach(),
min=0,
max=1)
self.created_image = recreate_image(self.processed_image.cpu())
if i % 10 == 0:
# Save image
im_path = 'generated/c_specific_iteration_' + str(i) + '.png'
save_image(self.created_image, im_path)
def visualise_layer_with_hooks(self):
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
print("image", random_image)
# random_image = Image.open("dog.jpg")
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.model):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def generate(self, iterations=150):
"""Generates class specific image
Keyword Arguments:
iterations {int} -- Total iterations for gradient ascent (default: {150})
Returns:
np.ndarray -- Final maximally activated class image
"""
initial_learning_rate = 6
for i in range(1, iterations):
# Process image and return variable
self.processed_image = preprocess_image(self.created_image, False)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
if i % 10 == 0 or i == iterations-1:
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
if i % 10 == 0 or i == iterations-1:
# Save image
im_path = '../generated/class_'+str(self.target_class)+'/c_'+str(self.target_class)+'_'+'iter_'+str(i)+'.png'
save_image(self.created_image, im_path)
return self.processed_image
def visualise_layer_without_hooks(self):
# Process image and return variable
processed_image = self.pic[None,:,:,:]
processed_image = processed_image.cuda()
processed_image = Variable(processed_image, requires_grad=True)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=self.lr, weight_decay=1e-6)
for i in range(1, 201):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for name, module in self.model._modules.items():
# Forward pass layer by layer
x = module(x)
if name == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
self.conv_output = x[0, :]
# self.conv_output = x[0, self.selected_filter ]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.cpu().numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Save image
if i % 200 == 0:
# Recreate image
processed_image = processed_image.cpu()
self.created_image = recreate_image(processed_image)
im_path = './generated/'+ self.png_dir+ '/layer_vis_' + str(self.selected_layer) +'_'+str(self.index)+ '.jpg'
save_image(self.created_image, im_path)
def visualise_layer_with_hooks(self):
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image, where the image is a variable
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
#for i in range(1, 31):
for i in range(1, 2): # TODO: TEMP FOR DEBUGGING AND LEARNING
optimizer.zero_grad(
) # Zero the generated gradients for this iteration
# Assign create image to a variable to move forward in the model
x = processed_image
# Keep forward passing till reach the layer of interest
# then, will trigger the hooked function call
for index, layer in enumerate(self.model):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Now, have the convolution output at the layer of interest for the output depth of that layer
# Want to minimize mean of the output of selected layer's filter
# THe output are real numbers
# The higher the value (more positive) => More activation
# If you take the mean or sum, it just accounts for all position
# Now if you minimize the negative of the mean, it's same as maximizing
# Therefore, it's about maximizing the output
# It's about finding the input image that maximizes the output!
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward pass from the current loss tensor
loss.backward(
) # Generate the gradients using a single backward pass
# Update image
optimizer.step(
) # Update the parameters using the generated gradients
# Save image
if (i % 5 == 0) or (i == 1):
# Recreate image into a PIL tensor
self.created_image = recreate_image(processed_image)
im_path = '../generated/layer_vis_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg'
save_image(self.created_image, im_path)
def generate(self, blur_freq=5, blur_rad=1, wd=0.0001, clipping_value=0.1):
"""Generates class specific image with enhancements to improve image quality.
See https://arxiv.org/abs/1506.06579 for details on each argument's effect on output quality.
Play around with combinations of arguments. Besides the defaults, this combination has produced good images:
blur_freq=6, blur_rad=0.8, wd = 0.05
Keyword Arguments:
iterations {int} -- Total iterations for gradient ascent (default: {150})
blur_freq {int} -- Frequency of Gaussian blur effect, in iterations (default: {6})
blur_rad {float} -- Radius for gaussian blur, passed to PIL.ImageFilter.GaussianBlur() (default: {0.8})
wd {float} -- Weight decay value for Stochastic Gradient Ascent (default: {0.05})
clipping_value {None or float} -- Value for gradient clipping (default: {0.1})
Returns:
np.ndarray -- Final maximally activated class image
"""
initial_learning_rate = 6
for i in tqdm(range(1, self.iterations)):
# Process image and return variable
#implement gaussian blurring every ith iteration
#to improve output
if i % blur_freq == 0:
self.processed_image = preprocess_and_blur_image(
self.created_image, False, blur_rad)
else:
self.processed_image = preprocess_and_blur_image(
self.created_image, False)
if use_cuda:
self.processed_image = self.processed_image.cuda()
# Define optimizer for the image - use weight decay to add regularization
# in SGD, wd = 2 * L2 regularization (https://bbabenko.github.io/weight-decay/)
optimizer = SGD([self.processed_image],
lr=initial_learning_rate,
weight_decay=wd)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, self.target_class]
if i in np.linspace(0, self.iterations, 10, dtype=int):
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.cpu().numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
if clipping_value:
torch.nn.utils.clip_grad_norm(self.model.parameters(),
clipping_value)
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image.cpu())
if i in np.linspace(0, self.iterations, 10, dtype=int):
# Save image
im_path = f'generated_class/class_{self.class_name}/c_{self.class_name}_iter_{i}_loss_{class_loss.data.cpu().numpy()}.jpg'
save_image(self.created_image, im_path)
print("Saved image at " + im_path)
#save final image
im_path = f'generated_class/class_{self.class_name}/c_{self.class_name}_iter_{i}_loss_{class_loss.data.cpu().numpy()}.jpg'
save_image(self.created_image, im_path)
#write file with regularization details
with open(f'generated_class/class_{self.class_name}/run_details.txt',
'w') as f:
f.write(f'Iterations: {self.iterations}\n')
f.write(f'Blur freq: {blur_freq}\n')
f.write(f'Blur radius: {blur_rad}\n')
f.write(f'Weight decay: {wd}\n')
f.write(f'Clip value: {clipping_value}\n')
#rename folder path with regularization details for easy access
os.rename(
f'generated_class/class_{self.class_name}',
f'generated_class/class_{self.class_name}_blurfreq_{blur_freq}_blurrad_{blur_rad}_wd{wd}'
)
return self.processed_image
# net2 = net2.cuda()
# encorder = ResNet_encorder(demode=2)
# encorder = encorder.cuda()
# decorder = ResNet_decorder2(demode=2)
# decorder = decorder.cuda()
# params=net.state_dict()
# for k,v in params.items():
# print(k)
# print(v)
for index in range(16):
trainloader, classes = data_prepare()
pic = get_picture(trainloader)
x = pic.cpu()[None, :, :, :]
x = recreate_image(x)
save_image(x, 'orginal_'+str(index)+'.jpg')
encorder = ResNet_encorder(demode=1)
encorder = encorder.cuda()
decorder = ResNet_decorder2(demode=1)
decorder = decorder.cuda()
vis_layer(encorder, decorder, pic, 'conv1', demode=1, index=index)
#define layer to visualize
cnn_name = 'conv1'
lr = 0.01
layer_vis = CNNLayerVisualization(net, cnn_name, 0, pic, lr, cnn_name, index)
layer_vis.visualise_layer_without_hooks()
encorder = ResNet_encorder(demode=2)
encorder = encorder.cuda()
decorder = ResNet_decorder2(demode=2)
decorder = decorder.cuda()