def get_code(generator, classifier, batch_size):
'''
Generate <batch_size> h's corresponding to images generated by generator.
Sample uniformly from h distribution, feed through generator and classifier, and if prob_highest_class > 55% then choose h.
return h and its corresponding class
'''
gen_in = settings.generator_in_layer
gen_out = settings.generator_out_layer
h_shape = generator.blobs[gen_in].data.shape
# Get the input and output sizes
image_shape = classifier.blobs['data'].data.shape
generator_output_shape = generator.blobs[gen_out].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
image_size = util.get_image_size(image_shape)
generator_output_size = util.get_image_size(generator_output_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft = util.compute_topleft(image_size, generator_output_size)
h_list = []
class_list = []
i = 0
print ("starting to generate h's")
while len(h_list) < batch_size:
print ("round %d" %i)
# Sample h from uniform distribution
h = np.random.normal(0, 1, h_shape)
# Push h through Generator to get image
generator.blobs[gen_in].data[:] = h
generated = generator.forward()
x = generated[gen_out].copy() # 256x256
# Crop from 256x256 to 227x227
cropped_x = x[:,:,topleft[0]:topleft[0]+image_size[0], topleft[1]:topleft[1]+image_size[1]]
cropped_x_copy = cropped_x.copy()
softmax_output = classifier.forward(data=cropped_x_copy, end='prob')
probs = softmax_output['prob'][0]
best_class = probs.argmax() # highest probability unit
print (probs[best_class])
if probs[best_class] > 55:
h_list.append(h)
class_list.append(best_class)
i +=1
return h
def load_images():
for fname in os.listdir(config.gamedir + '/img'):
if fname.endswith(('.jpg', '.png')):
tag = fname[:-4]
fname = 'img/' + fname
imageinfo[tag] = {'path': fname,
'size': get_image_size(os.path.join(config.gamedir, fname))}
imageinfo[tag]['ratio'] = imageinfo[tag]['size'][0]/imageinfo[tag]['size'][1]
renpy.image(tag, renpy.display.im.Scale(fname, imageinfo[tag]['ratio']*550, 550))
imageinfo[tag]['scaled'] = renpy.display.im.Scale(fname, imageinfo[tag]['ratio']*550, 550)
def blurred(self, src, dest):
dest = os.path.splitext(dest)[0] + '@thumb' + os.path.splitext(dest)[1]
resolution = settings.resolution_blurred
image_size = util.get_image_size(src)
command = 'convert '+ src + ' '
command += '-thumbnail ' + 'x'.join([str(x) for x in resolution]) + '^ '
command += dest
if os.path.isfile(dest):
return
os.system(command)
def blurred(self, src, dest):
dest = os.path.splitext(dest)[0] + '@thumb' + os.path.splitext(dest)[1]
resolution = settings.resolution_blurred
image_size = util.get_image_size(src)
command = 'convert ' + src + ' '
command += '-thumbnail ' + 'x'.join([str(x)
for x in resolution]) + '^ '
command += dest
if os.path.isfile(dest):
return
os.system(command)
def resize(self, src, dest, hidpi=False):
if hidpi:
resolution = settings.resolution_hidpi
dest = os.path.splitext(dest)[0] + '@2x' + os.path.splitext(dest)[1]
else:
resolution = settings.resolution
image_size = util.get_image_size(src)
if image_size[0] < resolution[0] or image_size[1] < resolution[1]:
print('! image "' + src + '" is too small; blurring will result when scaled up')
command = 'convert '+ src + ' '
command += '-resize ' + 'x'.join([str(x) for x in resolution]) + '^ '
command += dest
if os.path.isfile(dest):
return
os.system(command)
def on_page(self, page, page_length=20):
"""Gets the requested page of samples.
Args:
page: the page number
page_length: the page length
Returns:
the list of sample dicts for the page
"""
state = fos.StateDescriptionWithDerivables.from_dict(self.state)
if state.view is not None:
view = state.view
elif state.dataset is not None:
view = state.dataset.view()
else:
return []
view = view.skip((page - 1) * page_length).limit(page_length + 1)
samples = [
json.loads(
json_util.dumps(s.to_mongo_dict()), parse_constant=lambda c: c
)
for s in view
]
more = False
if len(samples) > page_length:
samples = samples[:page_length]
more = page + 1
results = [{"sample": s} for s in samples]
for r in results:
w, h = get_image_size(r["sample"]["filepath"])
r["width"] = w
r["height"] = h
return {"results": results, "more": more}
def resize(self, src, dest, hidpi=False):
if hidpi:
resolution = settings.resolution_hidpi
dest = os.path.splitext(dest)[0] + '@2x' + os.path.splitext(
dest)[1]
else:
resolution = settings.resolution
image_size = util.get_image_size(src)
if image_size[0] < resolution[0] or image_size[1] < resolution[1]:
print('! image "' + src +
'" is too small; blurring will result when scaled up')
command = 'convert ' + src + ' '
command += '-resize ' + 'x'.join([str(x) for x in resolution]) + '^ '
command += dest
if os.path.isfile(dest):
return
os.system(command)
def sampling(
self,
condition_net,
image_encoder,
image_generator,
gen_in_layer,
gen_out_layer,
start_code,
n_iters,
lr,
lr_end,
threshold,
layer,
conditions, #units=None, xy=0,
epsilon1=1,
epsilon2=1,
epsilon3=1e-10,
output_dir=None,
reset_every=0,
save_every=1):
# Get the input and output sizes
image_shape = condition_net.blobs['data'].data.shape
generator_output_shape = image_generator.blobs[
gen_out_layer].data.shape
encoder_input_shape = image_encoder.blobs['data'].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
image_size = util.get_image_size(image_shape)
generator_output_size = util.get_image_size(generator_output_shape)
encoder_input_size = util.get_image_size(encoder_input_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft = self.compute_topleft(image_size, generator_output_size)
topleft_DAE = self.compute_topleft(encoder_input_size,
generator_output_size)
src = image_generator.blobs[
gen_in_layer] # the input feature layer of the generator
# Make sure the layer size and initial vector size match
assert src.data.shape == start_code.shape
# Variables to store the best sample
last_xx = np.zeros(image_shape) # best image
last_prob = -sys.maxint # highest probability
h = start_code.copy()
condition_idx = 0
list_samples = []
i = 0
while True:
step_size = lr + ((lr_end - lr) * i) / n_iters
condition = conditions[condition_idx] # Select a class
# 1. Compute the epsilon1 term ---
# compute gradient d log(p(h)) / dh per DAE results in Alain & Bengio 2014
d_prior = self.h_autoencoder_grad(h=h,
encoder=image_generator,
decoder=image_encoder,
gen_out_layer=gen_out_layer,
topleft=topleft_DAE)
# 2. Compute the epsilon2 term ---
# Push the code through the generator to get an image x
image_generator.blobs["feat"].data[:] = h
generated = image_generator.forward()
x = generated[gen_out_layer].copy() # 256x256
# Crop from 256x256 to 227x227
cropped_x = x[:, :, topleft[0]:topleft[0] + image_size[0],
topleft[1]:topleft[1] + image_size[1]]
# Forward pass the image x to the condition net up to an unit k at the given layer
# Backprop the gradient through the condition net to the image layer to get a gradient image
d_condition_x, prob, info = self.forward_backward_from_x_to_condition(
net=condition_net,
end=layer,
image=cropped_x,
condition=condition)
# Put the gradient back in the 256x256 format
d_condition_x256 = np.zeros_like(x)
d_condition_x256[:, :, topleft[0]:topleft[0] + image_size[0],
topleft[1]:topleft[1] +
image_size[1]] = d_condition_x.copy()
# Backpropagate the above gradient all the way to h (through generator)
# This gradient 'd_condition' is d log(p(y|h)) / dh (the epsilon2 term in Eq. 11 in the paper)
d_condition = self.backward_from_x_to_h(generator=image_generator,
diff=d_condition_x256,
start=gen_in_layer,
end=gen_out_layer)
self.print_progress(i, info, condition, prob, d_condition)
# 3. Compute the epsilon3 term ---
noise = np.zeros_like(h)
if epsilon3 > 0:
noise = np.random.normal(0, epsilon3,
h.shape) # Gaussian noise
# Update h according to Eq.11 in the paper
d_h = epsilon1 * d_prior + epsilon2 * d_condition + noise
h += step_size / np.abs(d_h).mean() * d_h
h = np.clip(h, a_min=0,
a_max=30) # Keep the code within a realistic range
# Reset the code every N iters (for diversity when running a long sampling chain)
if reset_every > 0 and i % reset_every == 0 and i > 0:
h = np.random.normal(0, 1, h.shape)
# Experimental: For sample diversity, it's a good idea to randomly pick epsilon1 as well
epsilon1 = np.random.uniform(low=1e-6, high=1e-2)
# Save every sample
last_xx = cropped_x.copy()
last_prob = prob
# Filter samples based on threshold or every N iterations
if save_every > 0 and i % save_every == 0 and prob > threshold:
name = "%s/samples/%05d.jpg" % (output_dir, i)
label = self.get_label(condition)
list_samples.append((last_xx.copy(), name, label))
# Stop if grad is 0
if norm(d_h) == 0:
print " d_h is 0"
break
# Randomly sample a class every N iterations
if i > 0 and i % n_iters == 0:
condition_idx += 1
if condition_idx == len(conditions):
break
i += 1 # Next iter
# returning the last sample
print "-------------------------"
print "Last sample: prob [%s] " % last_prob
return last_xx, list_samples
def h_sampling(
self,
condition_net,
image_encoder,
image_generator,
gen_in_layer,
gen_out_layer,
start_code,
n_iters,
lr,
lr_end,
threshold,
layer,
conditions, #units=None, xy=0,
epsilon1=1,
epsilon2=1,
epsilon3=1e-10,
inpainting=None, # in-painting args
output_dir=None,
reset_every=0,
save_every=1):
'''
The architecture is such that x <- h -> c
Therefore unlike the usual sampling from h -> x -> c which results in images with dim: 227x227,
our sample method results in images with dim: 256x256
'''
# Get the input and output sizes
generator_output_shape = image_generator.blobs[
gen_out_layer].data.shape
encoder_input_shape = image_encoder.blobs['data'].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
generator_output_size = util.get_image_size(generator_output_shape)
encoder_input_size = util.get_image_size(encoder_input_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft_DAE = util.compute_topleft(encoder_input_size,
generator_output_size)
src = image_generator.blobs[
gen_in_layer] # the input feature layer of the generator
# Make sure the layer size and initial vector size match
assert src.data.shape == start_code.shape
# Variables to store the best sample
last_xx = np.zeros(generator_output_shape) # best image
last_prob = -sys.maxint # highest probability
h = start_code.copy()
h_shape = h.shape
# Adam Parameters
mom1 = 0.9
mom2 = 0.999
eps = 1e-8
t = 1
m_t = np.zeros(h_shape)
v_t = np.zeros(h_shape)
condition_idx = 0
list_samples = []
i = 0
# for d_h plots
d_prior_mins = []
d_prior_maxs = []
d_condition_mins = []
d_condition_maxs = []
boundary_points = []
while True:
#step_size = lr + ((lr_end - lr) * i) / n_iters
condition = conditions[condition_idx] # Select a class
# 1. Compute the epsilon1 term ---
# compute gradient d log(p(h)) / dh per DAE results in Alain & Bengio 2014
d_prior = self.h_autoencoder_grad(h=h,
encoder=image_generator,
decoder=image_encoder,
gen_out_layer=gen_out_layer,
topleft=topleft_DAE,
inpainting=inpainting)
# 2. Compute the epsilon2 term ---
# Push the code through the generator to get an image x
image_generator.blobs["feat"].data[:] = h
generated = image_generator.forward()
x = generated[gen_out_layer].copy() # 256x256
# Forward pass the latent code h to the condition net up to an unit k at the given layer
# Backprop the gradient through the condition net to the latent layer to get a gradient latent code h
d_condition, prob, info = self.forward_backward_from_h_to_condition(
net=condition_net, end=layer, h_code=h, condition=condition)
self.print_progress(i, info, condition, prob, d_condition)
# 3. Compute the epsilon3 term ---
noise = np.zeros_like(h)
if epsilon3 > 0:
noise = np.random.normal(0, epsilon3,
h.shape) # Gaussian noise
# Update h according to Eq.11 in the paper
d_h = epsilon1 * d_prior + epsilon2 * d_condition + noise
d_prior_mins.append(min(d_prior[0]))
d_prior_maxs.append(max(d_prior[0]))
d_condition_mins.append(min(d_condition[0]))
d_condition_maxs.append(max(d_condition[0]))
################ Adam ################
m_t = mom1 * m_t + (1 - mom1) * d_h
v_t = mom2 * v_t + (1 - mom2) * (d_h**2)
m_t_hat = m_t / (1 - mom1**t)
v_t_hat = v_t / (1 - mom2**t)
step_size = lr
t += 1
#h += step_size*m_t_hat/((np.sqrt(v_t_hat) + eps)*(np.abs(d_h).mean()))
h += step_size / np.abs(d_h).mean() * d_h
h = np.clip(h, a_min=0,
a_max=30) # Keep the code within a realistic range
# stochastic clipping
#h[h>30] = np.random.uniform(0, 30)
#h[h<0] = np.random.uniform(0, 30)
boundary_points.append(
np.count_nonzero(h == 30) + np.count_nonzero(h == 0))
# Reset the code every N iters (for diversity when running a long sampling chain)
if reset_every > 0 and i % reset_every == 0 and i > 0:
h = np.random.normal(0, 1, h.shape)
# Experimental: For sample diversity, it's a good idea to randomly pick epsilon1 as well
epsilon1 = np.random.uniform(low=1e-6, high=1e-2)
# Save every sample
last_xx = x.copy()
last_prob = prob
# Filter samples based on threshold or every N iterations
if save_every > 0 and i % save_every == 0 and prob > threshold:
name = "%s/samples/%05d.jpg" % (output_dir, i)
label = self.get_label(condition)
list_samples.append((last_xx.copy(), name, label))
# Stop if grad is 0
if norm(d_h) == 0:
print " d_h is 0"
break
# Randomly sample a class every N iterations
if i > 0 and i % n_iters == 0:
condition_idx += 1
if condition_idx == len(conditions):
break
i += 1 # Next iter
# returning the last sample
print "-------------------------"
print "Last sample: prob [%s] " % last_prob
return last_xx, list_samples, h, np.array(d_prior_mins), np.array(
d_prior_maxs), np.array(d_condition_mins), np.array(
d_condition_maxs), np.array(boundary_points)
"./nets/generator/noiseless/generator_batchsize_128.prototxt",
settings.generator_weights, caffe.TEST)
#generator = caffe.Net(settings.generator_definition, settings.generator_weights, caffe.TEST)
gen_in = settings.generator_in_layer
gen_out = settings.generator_out_layer
h_shape = generator.blobs[gen_in].data.shape
# Get the input and output sizes
image_shape = encoder.blobs['data'].data.shape
generator_output_shape = generator.blobs[gen_out].data.shape
# Calculate the difference between the input image of the condition net
# and the output image from the generator
image_size = util.get_image_size(image_shape)
generator_output_size = util.get_image_size(generator_output_shape)
# The top left offset to crop the output image to get a 227x227 image
topleft = util.compute_topleft(image_size, generator_output_size)
image_mean = scipy.io.loadmat('misc/ilsvrc_2012_mean.mat')[
'image_mean'] # (256, 256, 3)
image_mean = np.expand_dims(np.transpose(image_mean, (2, 0, 1)), 0)
#image_mean = np.repeat(image_mean, 10, axis=0)
import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torchvision.datasets as datasets
# Define the codec and create VideoWriter object.The output is stored in 'outpy.avi' file.
capture_video = False
edit = True
if capture_video == True:
out = cv2.VideoWriter('../videos/seal_vid2_rotated.mp4',
cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10,
(600, 600))
while True:
try:
check, frame = video.read()
print(check) # prints true as long as the webcam is running
print(frame) # prints matrix values of each frame
# cv2.namedWindow('image', cv2.WINDOW_NORMAL)
height, width, _ = util.get_image_size(frame)
print "Height of iamge is %d " % height
print "Width of iamge is %d " % width
if edit == True:
# frame = imutils.rotate(frame,90)
resize_height = int(height * 0.8)
resize_width = int(width * 0.8)
frame = cv2.resize(frame, (resize_width, resize_height))
cv2.imshow("Capturing", frame)
if capture_video == True:
out.write(frame)
key = cv2.waitKey(1)
if key == ord('s'):
cv2.imwrite(filename='../images/test_fixture2_sec1.jpg', img=frame)
video.release()
img_new = cv2.imread('../images/test_fixture2_sec1.jpg',
