def generate(self, target_class):
initial_learning_rate = 6
created_image = np.uint8(np.random.uniform(0, 255, (224, 224, 3)))
for i in range(1, 150):
# Process image and return variable
self.processed_image = preprocess_image(created_image, False)
# Define optimizer for the image
optimizer = SGD([self.processed_image], lr=initial_learning_rate)
# Forward
output = self.model(self.processed_image)
# Target specific class
class_loss = -output[0, target_class]
print('Iteration:', str(i), 'Loss',
"{0:.2f}".format(class_loss.data.numpy()))
# Zero grads
self.model.zero_grad()
# Backward
class_loss.backward()
# Update image
optimizer.step()
# Recreate image
created_image = recreate_image(self.processed_image)
if i % 10 == 0:
# Save image
im_path = os.path.join(
self.dst, 'c_specific_iteration_' + str(i) + '.jpg')
save_image(created_image, im_path)
return self.processed_image
def after_step(self, rbm, trainer, i):
it = i + 1
save = it in self.expt.save_after
display = it in self.expt.show_after
if save:
if self.expt.save_particles:
storage.dump(trainer.fantasy_particles, self.expt.pcd_particles_file(it))
storage.dump(rbm, self.expt.rbm_file(it))
if hasattr(trainer, 'avg_rbm'):
storage.dump(trainer.avg_rbm, self.expt.avg_rbm_file(it))
storage.dump(time.time() - self.t0, self.expt.time_file(it))
if 'particles' in self.subset and (save or display):
fig = rbm_vis.show_particles(rbm, trainer.fantasy_particles, self.expt.dataset, display=display,
figtitle='PCD particles ({} updates)'.format(it))
if display:
pylab.gcf().canvas.draw()
if save:
misc.save_image(fig, self.expt.pcd_particles_figure_file(it))
if 'gibbs_chains' in self.subset and (save or display):
fig = diagnostics.show_chains(rbm, trainer.fantasy_particles, self.expt.dataset, display=display,
figtitle='Gibbs chains (iteration {})'.format(it))
if save:
misc.save_image(fig, self.expt.gibbs_chains_figure_file(it))
if 'objective' in self.subset:
self.log_prob_tracker.update(rbm, trainer.fantasy_particles)
if display:
pylab.gcf().canvas.draw()
def dream(self, filename):
created_image = Image.open(filename).convert('RGB')
# Process image and return variable
self.processed_image = preprocess_image(created_image, True)
# Define optimizer for the image
# Earlier layers need higher learning rates to visualize whereas layer layers need less
optimizer = SGD([self.processed_image], lr=12, weight_decay=1e-4)
for i in range(1, 251):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = self.processed_image
for index, layer in enumerate(self.features):
# Forward
x = layer(x)
# Only need to forward until we the selected layer is reached
if index == self.selected_layer:
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:',
"{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(self.processed_image)
# Save image every 20 iteration
if i % 10 == 0:
print(created_image.size)
im_path = os.path.join(self.dst, 'ddream_l' + str(self.selected_layer) + \
'_f' + str(self.selected_filter) + '_iter' + str(i) + '.jpg')
save_image(self.created_image, im_path)
def save_figures(expt):
"""Save visualizations of the particles."""
if isinstance(expt, str):
expt = get_experiment(expt)
tr_expt = get_training_expt(expt)
storage.ensure_directory(expt.figures_dir())
for it in tr_expt.save_after:
for avg in AVG_VALS:
print 'Iteration', it
try:
rbm = load_rbm(expt, it, avg)
except:
continue
final_states = storage.load(expt.final_states_file(it, avg))
gibbs_states = storage.load(expt.gibbs_states_file(it, avg))
fig = rbm_vis.show_particles(rbm, final_states, expt.dataset)
misc.save_image(fig, expt.final_states_figure_file(it, avg))
fig = rbm_vis.show_particles(rbm, gibbs_states, expt.dataset)
misc.save_image(fig, expt.gibbs_states_figure_file(it, avg))
print_log_probs(expt, open(expt.log_probs_text_file(), 'w'))
def generate_inverted_image_specific_layer(self,
input_image,
img_size,
target_layer=3):
# Generate a random image which we will optimize
opt_img = Variable(1e-1 * torch.randn(1, 3, img_size, img_size),
requires_grad=True)
# Define optimizer for previously created image
optimizer = SGD([opt_img], lr=1e4, momentum=0.9)
# Get the output from the model after a forward pass until target_layer
# with the input image (real image, NOT the randomly generated one)
input_image_layer_output = \
self.get_output_from_specific_layer(input_image, target_layer)
# Alpha regularization parametrs
# Parameter alpha, which is actually sixth norm
alpha_reg_alpha = 6
# The multiplier, lambda alpha
alpha_reg_lambda = 1e-7
# Total variation regularization parameters
# Parameter beta, which is actually second norm
tv_reg_beta = 2
# The multiplier, lambda beta
tv_reg_lambda = 1e-8
for i in range(201):
optimizer.zero_grad()
# Get the output from the model after a forward pass until target_layer
# with the generated image (randomly generated one, NOT the real image)
output = self.get_output_from_specific_layer(opt_img, target_layer)
# Calculate euclidian loss
euc_loss = 1e-1 * self.euclidian_loss(
input_image_layer_output.detach(), output)
# Calculate alpha regularization
reg_alpha = alpha_reg_lambda * self.alpha_norm(
opt_img, alpha_reg_alpha)
# Calculate total variation regularization
reg_total_variation = tv_reg_lambda * self.total_variation_norm(
opt_img, tv_reg_beta)
# Sum all to optimize
loss = euc_loss + reg_alpha + reg_total_variation
# Step
loss.backward()
optimizer.step()
# Generate image every 5 iterations
if i % 5 == 0:
print('Iteration:', str(i), 'Loss:', loss.data.numpy())
recreated_im = recreate_image(opt_img)
im_path = os.path.join(self.dst, 'Inv_Image_Layer_' + str(target_layer) + \
'_Iteration_' + str(i) + '.jpg')
save_image(recreated_im, im_path)
# Reduce learning rate every 40 iterations
if i % 40 == 0:
for param_group in optimizer.param_groups:
param_group['lr'] *= 1 / 10
def after_step(self, rbm, trainer, i):
it = i + 1
save = it in self.expt.save_after
display = it in self.expt.show_after
if save:
if self.expt.save_particles:
storage.dump(trainer.fantasy_particles,
self.expt.pcd_particles_file(it))
storage.dump(rbm, self.expt.rbm_file(it))
if hasattr(trainer, 'avg_rbm'):
storage.dump(trainer.avg_rbm, self.expt.avg_rbm_file(it))
storage.dump(time.time() - self.t0, self.expt.time_file(it))
if 'particles' in self.subset and (save or display):
fig = rbm_vis.show_particles(
rbm,
trainer.fantasy_particles,
self.expt.dataset,
display=display,
figtitle='PCD particles ({} updates)'.format(it))
if display:
pylab.gcf().canvas.draw()
if save:
misc.save_image(fig, self.expt.pcd_particles_figure_file(it))
if 'gibbs_chains' in self.subset and (save or display):
fig = diagnostics.show_chains(
rbm,
trainer.fantasy_particles,
self.expt.dataset,
display=display,
figtitle='Gibbs chains (iteration {})'.format(it))
if save:
misc.save_image(fig, self.expt.gibbs_chains_figure_file(it))
if 'objective' in self.subset:
self.log_prob_tracker.update(rbm, trainer.fantasy_particles)
if display:
pylab.gcf().canvas.draw()
def visualise_layer_without_hooks(self):
# Process image and return variable
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.features):
# Forward pass layer by layer
x = layer(x)
if index == self.selected_layer:
# Only need to forward until the selected layer is reached
# Now, x is the output of the selected layer
break
# Here, we get the specific filter from the output of the convolution operation
# x is a tensor of shape 1x512x28x28.(For layer 17)
# So there are 512 unique filter outputs
# Following line selects a filter from 512 filters so self.conv_output will become
# a tensor of shape 28x28
self.conv_output = x[0, self.selected_filter]
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
filename = os.path.join(self.path, 'layer_visual_%d_%d_iter_%d.jpg' + self.selected_layer, self.selected_filter, i)
save_image(self.created_image, filename)
def visualise_layer_with_hooks(self):
'''
img
'''
# Hook the selected layer
self.hook_layer()
# Generate a random image
random_image = np.uint8(np.random.uniform(150, 180, (224, 224, 3)))
# Process image and return variable
processed_image = preprocess_image(random_image, False)
# Define optimizer for the image
optimizer = Adam([processed_image], lr=0.1, weight_decay=1e-6)
for i in range(1, 31):
optimizer.zero_grad()
# Assign create image to a variable to move forward in the model
x = processed_image
for index, layer in enumerate(self.features):
# Forward pass layer by layer
# x is not used after this point because it is only needed to trigger
# the forward hook function
x = layer(x)
# Only need to forward until the selected layer is reached
if index == self.selected_layer:
# (forward hook function triggered)
break
# Loss function is the mean of the output of the selected layer/filter
# We try to minimize the mean of the output of that specific filter
loss = -torch.mean(self.conv_output)
print('Iteration:', str(i), 'Loss:', "{0:.2f}".format(loss.data.numpy()))
# Backward
loss.backward()
# Update image
optimizer.step()
# Recreate image
self.created_image = recreate_image(processed_image)
# Save image
if i % 5 == 0:
filename = os.path.join(self.path, 'layer_visual_%d_%d_iter_%d.jpg' % (self.selected_layer, self.selected_filter, i))
save_image(self.created_image, filename)
def _save_image(self, image, path, epoch):
directory = os.path.join(self.args.save_path, 'images',
'epoch_{}'.format(epoch))
save_path = os.path.join(directory, os.path.basename(path))
mkdir(directory)
save_image(image.data.cpu(), save_path)
