def sampling( self, condition_net, image_encoder, image_net, image_generator, edge_detector,
gen_in_layer, gen_out_layer, start_code, content_layer,
n_iters, lr, lr_end, threshold,
layer, conditions, mask=None, input_image=None, #units=None, xy=0,
epsilon1=1, epsilon2=1, epsilon3=1e-10,
mask_epsilon=1e-6, edge_epsilon=1e-8,
style_epsilon=1e-8, content_epsilon=1e-8,
output_dir=None, reset_every=0, save_every=1):
# Get the input and output sizes
image_shape = condition_net.blobs['data'].data.shape
generator_output_shape = image_generator.blobs[gen_out_layer].data.shape
encoder_input_shape = image_encoder.blobs['data'].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
image_size = util.get_image_size(image_shape)
generator_output_size = util.get_image_size(generator_output_shape)
encoder_input_size = util.get_image_size(encoder_input_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft = self.compute_topleft(image_size, generator_output_size)
topleft_DAE = self.compute_topleft(encoder_input_size, generator_output_size)
src = image_generator.blobs[gen_in_layer] # the input feature layer of the generator
# Make sure the layer size and initial vector size match
assert src.data.shape == start_code.shape
use_style_transfer= style_epsilon != 0 or content_epsilon !=0
# setup style transfer
if input_image is not None and use_style_transfer:
style_transfer = StyleTransfer(image_net, style_weight=style_epsilon, content_weight=content_epsilon)
style_transfer.init_image(input_image)
elif input_image is None:
# TODO setup loading the vector components
raise NotImplementedError('input image must not be None')
elif not use_style_transfer:
print('not using style transfer')
# Variables to store the best sample
last_xx = np.zeros(image_shape) # best image
last_prob = -sys.maxint # highest probability
h = start_code.copy()
condition_idx = 0
list_samples = []
i = 0
while True:
step_size = lr + ((lr_end - lr) * i) / n_iters
condition = conditions[condition_idx] # Select a class
# 1. Compute the epsilon1 term ---
# : compute gradient d log(p(h)) / dh per DAE results in Alain & Bengio 2014
d_prior = self.h_autoencoder_grad(h=h, encoder=image_generator, decoder=image_encoder, gen_out_layer=gen_out_layer, topleft=topleft_DAE, mask=mask, input_image=input_image)
# 2. Compute the epsilon2 term ---
# Push the code through the generator to get an image x
image_generator.blobs["feat"].data[:] = h
generated = image_generator.forward()
x = generated[gen_out_layer].copy() # 256x256
# Crop from 256x256 to 227x227
cropped_x_nomask = x[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]]
if mask is not None:
cropped_x = mask * input_image + (1 - mask) * cropped_x_nomask
else:
cropped_x = cropped_x_nomask
# Forward pass the image x to the condition net up to an unit k at the given layer
# Backprop the gradient through the condition net to the image layer to get a gradient image
d_condition_x, prob, info = self.forward_backward_from_x_to_condition(net=condition_net, end=layer, image=cropped_x, condition=condition)
if mask is not None:
generated_image = (1 - mask) * d_condition_x
else:
generated_image = d_condition_x
d_edge = self.get_edge_gradient(input_image, generated_image, edge_detector)
d_condition_x = epsilon2 * generated_image + edge_epsilon * d_edge
if use_style_transfer:
d_condition_x += style_transfer.get_gradient(generated_image)
if mask is not None:
d_condition_x += mask_epsilon * (mask) * (input_image - cropped_x_nomask)
# Put the gradient back in the 256x256 format
d_condition_x256 = np.zeros_like(x)
d_condition_x256[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]] = d_condition_x.copy()
# Backpropagate the above gradient all the way to h (through generator)
# This gradient 'd_condition' is d log(p(y|h)) / dh (the epsilon2 term in Eq. 11 in the paper)
d_condition = self.backward_from_x_to_h(generator=image_generator, diff=d_condition_x256, start=gen_in_layer, end=gen_out_layer)
# if i % 10 == 0:
# self.print_progress(i, info, condition, prob, d_condition)
# 3. Compute the epsilon3 term ---
noise = np.zeros_like(h)
if epsilon3 > 0:
noise = np.random.normal(0, epsilon3, h.shape) # Gaussian noise
d_h = epsilon1 * d_prior
d_h += d_condition
d_h += noise
h += step_size/np.abs(d_h).mean() * d_h
h = np.clip(h, a_min=0, a_max=30) # Keep the code within a realistic range
# Reset the code every N iters (for diversity when running a long sampling chain)
if reset_every > 0 and i % reset_every == 0 and i > 0:
h = np.random.normal(0, 1, h.shape)
# Experimental: For sample diversity, it's a good idea to randomly pick epsilon1 as well
epsilon1 = np.random.uniform(low=1e-6, high=1e-2)
# Save every sample
last_xx = cropped_x.copy()
last_prob = prob
# Filter samples based on threshold or every N iterations
if save_every > 0 and i % save_every == 0 and prob > threshold:
name = "%s/samples/%05d.jpg" % (output_dir, i)
label = self.get_label(condition)
if mask is not None:
image = last_xx * mask + (1 - mask) * input_image
else:
image = last_xx
# TODO check why this wasn't the case
# list_samples.append( (last_xx.copy(), name, label) )
list_samples.append( (image.copy(), name, label) )
# Stop if grad is 0
if norm(d_h) == 0:
print(" d_h is 0")
break
# Randomly sample a class every N iterations
if i > 0 and i % n_iters == 0:
condition_idx += 1
if condition_idx == len(conditions):
break
i += 1 # Next iter
# returning the last sample
print( "-------------------------")
print("Last sample: prob [%s] " % last_prob)
# if mask is not None:
# return last_xx * mask + (1 - mask) * input_image, list_samples
return last_xx, list_samples
