def sampling( self, condition_net, image_encoder, image_net, image_generator, edge_detector, gen_in_layer, gen_out_layer, start_code, content_layer, n_iters, lr, lr_end, threshold, layer, conditions, mask=None, input_image=None, #units=None, xy=0, epsilon1=1, epsilon2=1, epsilon3=1e-10, mask_epsilon=1e-6, edge_epsilon=1e-8, style_epsilon=1e-8, content_epsilon=1e-8, output_dir=None, reset_every=0, save_every=1): # Get the input and output sizes image_shape = condition_net.blobs['data'].data.shape generator_output_shape = image_generator.blobs[gen_out_layer].data.shape encoder_input_shape = image_encoder.blobs['data'].data.shape # Calculate the difference between the input image of the condition net # and the output image from the generator image_size = util.get_image_size(image_shape) generator_output_size = util.get_image_size(generator_output_shape) encoder_input_size = util.get_image_size(encoder_input_shape) # The top left offset to crop the output image to get a 227x227 image topleft = self.compute_topleft(image_size, generator_output_size) topleft_DAE = self.compute_topleft(encoder_input_size, generator_output_size) src = image_generator.blobs[gen_in_layer] # the input feature layer of the generator # Make sure the layer size and initial vector size match assert src.data.shape == start_code.shape use_style_transfer= style_epsilon != 0 or content_epsilon !=0 # setup style transfer if input_image is not None and use_style_transfer: style_transfer = StyleTransfer(image_net, style_weight=style_epsilon, content_weight=content_epsilon) style_transfer.init_image(input_image) elif input_image is None: # TODO setup loading the vector components raise NotImplementedError('input image must not be None') elif not use_style_transfer: print('not using style transfer') # Variables to store the best sample last_xx = np.zeros(image_shape) # best image last_prob = -sys.maxint # highest probability h = start_code.copy() condition_idx = 0 list_samples = [] i = 0 while True: step_size = lr + ((lr_end - lr) * i) / n_iters condition = conditions[condition_idx] # Select a class # 1. Compute the epsilon1 term --- # : compute gradient d log(p(h)) / dh per DAE results in Alain & Bengio 2014 d_prior = self.h_autoencoder_grad(h=h, encoder=image_generator, decoder=image_encoder, gen_out_layer=gen_out_layer, topleft=topleft_DAE, mask=mask, input_image=input_image) # 2. Compute the epsilon2 term --- # Push the code through the generator to get an image x image_generator.blobs["feat"].data[:] = h generated = image_generator.forward() x = generated[gen_out_layer].copy() # 256x256 # Crop from 256x256 to 227x227 cropped_x_nomask = x[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]] if mask is not None: cropped_x = mask * input_image + (1 - mask) * cropped_x_nomask else: cropped_x = cropped_x_nomask # Forward pass the image x to the condition net up to an unit k at the given layer # Backprop the gradient through the condition net to the image layer to get a gradient image d_condition_x, prob, info = self.forward_backward_from_x_to_condition(net=condition_net, end=layer, image=cropped_x, condition=condition) if mask is not None: generated_image = (1 - mask) * d_condition_x else: generated_image = d_condition_x d_edge = self.get_edge_gradient(input_image, generated_image, edge_detector) d_condition_x = epsilon2 * generated_image + edge_epsilon * d_edge if use_style_transfer: d_condition_x += style_transfer.get_gradient(generated_image) if mask is not None: d_condition_x += mask_epsilon * (mask) * (input_image - cropped_x_nomask) # Put the gradient back in the 256x256 format d_condition_x256 = np.zeros_like(x) d_condition_x256[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]] = d_condition_x.copy() # Backpropagate the above gradient all the way to h (through generator) # This gradient 'd_condition' is d log(p(y|h)) / dh (the epsilon2 term in Eq. 11 in the paper) d_condition = self.backward_from_x_to_h(generator=image_generator, diff=d_condition_x256, start=gen_in_layer, end=gen_out_layer) # if i % 10 == 0: # self.print_progress(i, info, condition, prob, d_condition) # 3. Compute the epsilon3 term --- noise = np.zeros_like(h) if epsilon3 > 0: noise = np.random.normal(0, epsilon3, h.shape) # Gaussian noise d_h = epsilon1 * d_prior d_h += d_condition d_h += noise h += step_size/np.abs(d_h).mean() * d_h h = np.clip(h, a_min=0, a_max=30) # Keep the code within a realistic range # Reset the code every N iters (for diversity when running a long sampling chain) if reset_every > 0 and i % reset_every == 0 and i > 0: h = np.random.normal(0, 1, h.shape) # Experimental: For sample diversity, it's a good idea to randomly pick epsilon1 as well epsilon1 = np.random.uniform(low=1e-6, high=1e-2) # Save every sample last_xx = cropped_x.copy() last_prob = prob # Filter samples based on threshold or every N iterations if save_every > 0 and i % save_every == 0 and prob > threshold: name = "%s/samples/%05d.jpg" % (output_dir, i) label = self.get_label(condition) if mask is not None: image = last_xx * mask + (1 - mask) * input_image else: image = last_xx # TODO check why this wasn't the case # list_samples.append( (last_xx.copy(), name, label) ) list_samples.append( (image.copy(), name, label) ) # Stop if grad is 0 if norm(d_h) == 0: print(" d_h is 0") break # Randomly sample a class every N iterations if i > 0 and i % n_iters == 0: condition_idx += 1 if condition_idx == len(conditions): break i += 1 # Next iter # returning the last sample print( "-------------------------") print("Last sample: prob [%s] " % last_prob) # if mask is not None: # return last_xx * mask + (1 - mask) * input_image, list_samples return last_xx, list_samples