def main():
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--units',
metavar='units',
type=str,
help='an unit to visualize e.g. [0, 999]')
parser.add_argument('--n_iters',
metavar='iter',
type=int,
default=10,
help='Number of sampling steps per each unit')
parser.add_argument(
'--threshold',
metavar='w',
type=float,
default=-1.0,
nargs='?',
help='The probability threshold to decide whether to keep an image')
parser.add_argument(
'--save_every',
metavar='save_iter',
type=int,
default=1,
help='Save a sample every N iterations. 0 to disable saving')
parser.add_argument('--reset_every',
metavar='reset_iter',
type=int,
default=0,
help='Reset the code every N iterations')
parser.add_argument('--lr',
metavar='lr',
type=float,
default=2.0,
nargs='?',
help='Learning rate')
parser.add_argument('--lr_end',
metavar='lr',
type=float,
default=-1.0,
nargs='?',
help='Ending Learning rate')
parser.add_argument('--epsilon1',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Prior')
parser.add_argument('--epsilon2',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Condition')
parser.add_argument('--epsilon3',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Noise')
parser.add_argument('--epsilon4',
metavar='lr',
type=float,
default=0.0,
nargs='?',
help='Context')
parser.add_argument('--seed',
metavar='n',
type=int,
default=0,
nargs='?',
help='Random seed')
parser.add_argument('--xy',
metavar='n',
type=int,
default=0,
nargs='?',
help='Spatial position for conv units')
parser.add_argument('--opt_layer',
metavar='s',
type=str,
help='Layer at which we optimize a code')
parser.add_argument('--act_layer',
metavar='s',
type=str,
default="fc8",
help='Layer at which we activate a neuron')
parser.add_argument('--init_file',
metavar='s',
type=str,
default="None",
help='Init image')
parser.add_argument('--write_labels',
action='store_true',
default=False,
help='Write class labels to images')
parser.add_argument('--output_dir',
metavar='b',
type=str,
default=".",
help='Output directory for saving results')
parser.add_argument('--net_weights',
metavar='b',
type=str,
default=settings.encoder_weights,
help='Weights of the net being visualized')
parser.add_argument('--net_definition',
metavar='b',
type=str,
default=settings.encoder_definition,
help='Definition of the net being visualized')
args = parser.parse_args()
# Default to constant learning rate
if args.lr_end < 0:
args.lr_end = args.lr
# summary
print "-------------"
print " units: %s xy: %s" % (args.units, args.xy)
print " n_iters: %s" % args.n_iters
print " reset_every: %s" % args.reset_every
print " save_every: %s" % args.save_every
print " threshold: %s" % args.threshold
print " epsilon1: %s" % args.epsilon1
print " epsilon2: %s" % args.epsilon2
print " epsilon3: %s" % args.epsilon3
print " epsilon4: %s" % args.epsilon4
print " start learning rate: %s" % args.lr
print " end learning rate: %s" % args.lr_end
print " seed: %s" % args.seed
print " opt_layer: %s" % args.opt_layer
print " act_layer: %s" % args.act_layer
print " init_file: %s" % args.init_file
print "-------------"
print " output dir: %s" % args.output_dir
print " net weights: %s" % args.net_weights
print " net definition: %s" % args.net_definition
print "-------------"
# encoder and generator for images
encoder = caffe.Net(settings.encoder_definition, settings.encoder_weights,
caffe.TEST)
generator = caffe.Net(settings.generator_definition,
settings.generator_weights, caffe.TEST)
# condition network, here an image classification net
net = caffe.Classifier(
args.net_definition,
args.net_weights,
mean=np.float32([104.0, 117.0, 123.0]), # ImageNet mean
channel_swap=(
2, 1,
0)) # the reference model has channels in BGR order instead of RGB
h_net = caffe.Net("./nets/h_classifier/h_classifier.prototxt",
"./nets/h_classifier/h_classifier.caffemodel",
caffe.TEST)
# Fix the seed
np.random.seed(args.seed)
# Sampler for class-conditional generation
sampler = ClassConditionalSampler()
inpainting = None
if args.init_file != "None":
# Pre-compute masks if we want to perform inpainting
if args.epsilon4 > 0:
mask, neg = util.get_mask()
else:
neg = None
# Get the code for the masked image
start_code, start_image = get_code(encoder=encoder,
path=args.init_file,
layer=args.opt_layer,
mask=neg)
# Package settings for in-painting experiments
if args.epsilon4 > 0:
inpainting = {
"mask": mask,
"mask_neg": neg,
"image": start_image,
"epsilon4": args.epsilon4
}
print "Loaded init code: ", start_code.shape
else:
# shape of the code being optimized
shape = generator.blobs[settings.generator_in_layer].data.shape
start_code = np.random.normal(0, 1, shape)
print ">>", np.min(start_code), np.max(start_code)
# Separate the dash-separated list of units into numbers
conditions = [{
"unit": int(u),
"xy": args.xy
} for u in args.units.split("_")]
# Optimize a code via gradient ascent
output_image, list_samples, last_h, d_prior_norms, d_condition_norms, boundary_points, h_norms = sampler.sampling(
condition_net=net,
image_encoder=encoder,
image_generator=generator,
gen_in_layer=settings.generator_in_layer,
gen_out_layer=settings.generator_out_layer,
start_code=start_code,
n_iters=args.n_iters,
lr=args.lr,
lr_end=args.lr_end,
threshold=args.threshold,
layer=args.act_layer,
conditions=conditions,
epsilon1=args.epsilon1,
epsilon2=args.epsilon2,
epsilon3=args.epsilon3,
inpainting=inpainting,
output_dir=args.output_dir,
reset_every=args.reset_every,
save_every=args.save_every)
#################### send h through the h_net to verify class probability ####################
probs = h_net.forward(fc6=last_h, end='prob')
class_prob = probs['prob'][0][conditions[0]["unit"]]
print("class probability is " + str(class_prob))
##############################################################################################
#################### Plot gradients vs. num_iters ####################
# plot the gradients
plt.subplot(3, 1, 1) #subplot(nrows, ncols, plot_number)
x1 = np.linspace(0, args.n_iters, args.n_iters + 1, endpoint=True)
plt.title('d_prior and d_condition')
plt.plot(x1,
d_prior_norms,
color="blue",
linewidth=2.0,
linestyle="--",
label='d_prior norms')
plt.plot(x1,
d_condition_norms,
color="red",
linewidth=2.0,
linestyle="--",
label='d_condition norms')
plt.legend()
plt.subplot(3, 1, 2)
x2 = np.linspace(0, args.n_iters, args.n_iters + 1, endpoint=True)
plt.title('d_prior (scaled by eps1=' + '%.0e' % Decimal(args.epsilon1) +
') and d_condition (scaled by eps2=' +
'%.0e' % Decimal(args.epsilon2) + ')')
plt.plot(x2,
d_condition_norms * args.epsilon2,
color="red",
linewidth=2.0,
linestyle="--",
label='d_condition norms')
plt.plot(x2,
d_prior_norms * args.epsilon1,
color="blue",
linewidth=2.0,
linestyle="--",
label='d_prior norms (scaled)')
plt.legend()
plt.subplot(3, 1, 3)
x3 = np.linspace(25, args.n_iters, args.n_iters + 1 - 25, endpoint=True)
plt.title('d_prior (scaled by eps1=' + '%.0e' % Decimal(args.epsilon1) +
') and d_condition (scaled by eps2=' +
'%.0e' % Decimal(args.epsilon2) + ') from n_iter=25')
plt.plot(x3,
d_condition_norms[25:] * args.epsilon2,
color="red",
linewidth=2.0,
linestyle="--",
label='d_condition norms')
plt.plot(x3,
d_prior_norms[25:] * args.epsilon1,
color="blue",
linewidth=2.0,
linestyle="--",
label='d_prior norms (scaled)')
plt.xlabel('num iters')
plt.legend()
# for i in xrange(args.n_iters):
# if i % 20 == 0:
# plt.annotate('(%s, %s)' %(i, d_condition_norms[i]), xy=(i, d_condition_norms[i]+ 20), textcoords='data')
#plt.annotate('(%s, %s)' %(i, d_condition_mins[i]), xy=(i, d_condition_mins[i] - 0.0005), textcoords='data')
# plt.title('% of boundary points')
# plt.plot(boundary_points/float(start_code.shape[1])*100)
# plt.xlabel('num iters')
# plt.title('norm of h')
# plt.plot(h_norms)
# plt.xlabel('num iters')
plt.show()
#plt.savefig("%s/gradients_plt.png")#, dpi=72)
####################################################################
# Output image
filename = "%s/%04d_%04d_%s_h_%s_%s_%s_%s__%s.jpg" % (
args.output_dir, conditions[0]["unit"], args.n_iters, args.lr,
str(args.epsilon1), str(args.epsilon2), str(
args.epsilon3), str(args.epsilon4), args.seed)
if inpainting != None:
output_image = util.stitch(start_image, output_image)
# Save the final image
util.save_image(output_image, filename)
print "%s/%s" % (os.getcwd(), filename)
# Write labels to images
print "Saving images..."
for p in list_samples:
img, name, label = p
util.save_image(img, name)
if args.write_labels:
util.write_label_to_img(name, label)
def main():
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--units',
metavar='units',
type=str,
help='an unit to visualize e.g. [0, 999]')
parser.add_argument('--n_iters',
metavar='iter',
type=int,
default=10,
help='Number of sampling steps per each unit')
parser.add_argument(
'--threshold',
metavar='w',
type=float,
default=-1.0,
nargs='?',
help='The probability threshold to decide whether to keep an image')
parser.add_argument(
'--save_every',
metavar='save_iter',
type=int,
default=1,
help='Save a sample every N iterations. 0 to disable saving')
parser.add_argument('--reset_every',
metavar='reset_iter',
type=int,
default=0,
help='Reset the code every N iterations')
parser.add_argument('--lr',
metavar='lr',
type=float,
default=2.0,
nargs='?',
help='Learning rate')
parser.add_argument('--lr_end',
metavar='lr',
type=float,
default=-1.0,
nargs='?',
help='Ending Learning rate')
parser.add_argument('--epsilon2',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Ending Learning rate')
parser.add_argument('--epsilon1',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Ending Learning rate')
parser.add_argument('--epsilon3',
metavar='lr',
type=float,
default=1.0,
nargs='?',
help='Ending Learning rate')
parser.add_argument('--seed',
metavar='n',
type=int,
default=0,
nargs='?',
help='Random seed')
parser.add_argument('--xy',
metavar='n',
type=int,
default=0,
nargs='?',
help='Spatial position for conv units')
parser.add_argument('--opt_layer',
metavar='s',
type=str,
help='Layer at which we optimize a code')
parser.add_argument('--act_layer',
metavar='s',
type=str,
default="fc8",
help='Layer at which we activate a neuron')
parser.add_argument('--init_file',
metavar='s',
type=str,
default="None",
help='Init image')
parser.add_argument('--write_labels',
action='store_true',
default=False,
help='Write class labels to images')
parser.add_argument('--output_dir',
metavar='b',
type=str,
default=".",
help='Output directory for saving results')
parser.add_argument('--net_weights',
metavar='b',
type=str,
default=settings.encoder_weights,
help='Weights of the net being visualized')
parser.add_argument('--net_definition',
metavar='b',
type=str,
default=settings.encoder_definition,
help='Definition of the net being visualized')
args = parser.parse_args()
# Default to constant learning rate
if args.lr_end < 0:
args.lr_end = args.lr
# summary
print "-------------"
print " units: %s xy: %s" % (args.units, args.xy)
print " n_iters: %s" % args.n_iters
print " reset_every: %s" % args.reset_every
print " save_every: %s" % args.save_every
print " threshold: %s" % args.threshold
print " epsilon1: %s" % args.epsilon1
print " epsilon2: %s" % args.epsilon2
print " epsilon3: %s" % args.epsilon3
print " start learning rate: %s" % args.lr
print " end learning rate: %s" % args.lr_end
print " seed: %s" % args.seed
print " opt_layer: %s" % args.opt_layer
print " act_layer: %s" % args.act_layer
print " init_file: %s" % args.init_file
print "-------------"
print " output dir: %s" % args.output_dir
print " net weights: %s" % args.net_weights
print " net definition: %s" % args.net_definition
print "-------------"
# encoder and generator for images
encoder = caffe.Net(settings.encoder_definition, settings.encoder_weights,
caffe.TEST)
generator = caffe.Net(settings.generator_definition,
settings.generator_weights, caffe.TEST)
# condition network, here an image classification net
net = caffe.Classifier(
args.net_definition,
args.net_weights,
mean=np.float32([104.0, 117.0, 123.0]), # ImageNet mean
channel_swap=(
2, 1,
0)) # the reference model has channels in BGR order instead of RGB
# Fix the seed
np.random.seed(args.seed)
if args.init_file != "None":
start_code, start_image = get_code(encoder=encoder,
path=args.init_file,
layer=args.opt_layer)
print "Loaded start code: ", start_code.shape
else:
# shape of the code being optimized
shape = generator.blobs[settings.generator_in_layer].data.shape
start_code = np.random.normal(0, 1, shape)
print ">>", np.min(start_code), np.max(start_code)
# Separate the dash-separated list of units into numbers
conditions = [{
"unit": int(u),
"xy": args.xy
} for u in args.units.split("_")]
# Optimize a code via gradient ascent
sampler = ClassConditionalSampler()
output_image, list_samples = sampler.sampling(
condition_net=net,
image_encoder=encoder,
image_generator=generator,
gen_in_layer=settings.generator_in_layer,
gen_out_layer=settings.generator_out_layer,
start_code=start_code,
n_iters=args.n_iters,
lr=args.lr,
lr_end=args.lr_end,
threshold=args.threshold,
layer=args.act_layer,
conditions=conditions,
epsilon1=args.epsilon1,
epsilon2=args.epsilon2,
epsilon3=args.epsilon3,
output_dir=args.output_dir,
reset_every=args.reset_every,
save_every=args.save_every)
# Output image
filename = "%s/%s_%04d_%04d_%s_h_%s_%s_%s__%s.jpg" % (
args.output_dir, args.act_layer, conditions[0]["unit"], args.n_iters,
args.lr, str(args.epsilon1), str(args.epsilon2), str(
args.epsilon3), args.seed)
# Save the final image
util.save_image(output_image, filename)
print "%s/%s" % (os.getcwd(), filename)
# Write labels to images
print "Saving images..."
for p in list_samples:
img, name, label = p
util.save_image(img, name)
if args.write_labels:
util.write_label_to_img(name, label)
def main():
parser = argparse.ArgumentParser(description='Process some integers.')
parser.add_argument('--sentence', metavar='w', type=str, default="", nargs='?', help='Sentence to condition on')
parser.add_argument('--n_iters', metavar='iter', type=int, default=10, help='Number of sampling steps per each unit')
parser.add_argument('--threshold', metavar='w', type=float, default=-1.0, nargs='?', help='The probability threshold to decide whether to keep an image')
parser.add_argument('--save_every', metavar='save_iter', type=int, default=1, help='Save a sample every N iterations. 0 to disable saving')
parser.add_argument('--reset_every', metavar='reset_iter', type=int, default=0, help='Reset the code every N iterations')
parser.add_argument('--lr', metavar='lr', type=float, default=2.0, nargs='?', help='Learning rate')
parser.add_argument('--lr_end', metavar='lr', type=float, default=-1.0, nargs='?', help='Ending Learning rate')
parser.add_argument('--epsilon2', metavar='lr', type=float, default=1.0, nargs='?', help='Ending Learning rate')
parser.add_argument('--epsilon1', metavar='lr', type=float, default=1.0, nargs='?', help='Ending Learning rate')
parser.add_argument('--epsilon3', metavar='lr', type=float, default=1.0, nargs='?', help='Ending Learning rate')
parser.add_argument('--seed', metavar='n', type=int, default=0, nargs='?', help='Random seed')
parser.add_argument('--xy', metavar='n', type=int, default=0, nargs='?', help='Spatial position for conv units')
parser.add_argument('--opt_layer', metavar='s', type=str, help='Layer at which we optimize a code')
parser.add_argument('--act_layer', metavar='s', type=str, default="fc8", help='Layer at which we activate a neuron')
parser.add_argument('--init_file', metavar='s', type=str, default="None", help='Init image')
parser.add_argument('--write_labels', action='store_true', default=False, help='Write class labels to images')
parser.add_argument('--output_dir', metavar='b', type=str, default=".", help='Output directory for saving results')
parser.add_argument('--net_weights', metavar='b', type=str, default=settings.encoder_weights, help='Weights of the net being visualized')
parser.add_argument('--net_definition', metavar='b', type=str, default=settings.encoder_definition, help='Definition of the net being visualized')
parser.add_argument('--captioner_definition', metavar='b', type=str, help='Definition of the net being visualized')
args = parser.parse_args()
# Default to constant learning rate
if args.lr_end < 0:
args.lr_end = args.lr
# summary
print "-------------"
print " sentence: %s" % args.sentence
print " n_iters: %s" % args.n_iters
print " reset_every: %s" % args.reset_every
print " save_every: %s" % args.save_every
print " threshold: %s" % args.threshold
print " epsilon1: %s" % args.epsilon1
print " epsilon2: %s" % args.epsilon2
print " epsilon3: %s" % args.epsilon3
print " start learning rate: %s" % args.lr
print " end learning rate: %s" % args.lr_end
print " seed: %s" % args.seed
print " opt_layer: %s" % args.opt_layer
print " act_layer: %s" % args.act_layer
print " init_file: %s" % args.init_file
print "-------------"
print " output dir: %s" % args.output_dir
print " net weights: %s" % args.net_weights
print " net definition: %s" % args.net_definition
print " captioner definition: %s" % args.captioner_definition
print "-------------"
# encoder and generator for images
encoder = caffe.Net(settings.encoder_definition, settings.encoder_weights, caffe.TEST)
generator = caffe.Net(settings.generator_definition, settings.generator_weights, caffe.TEST)
# condition network, here an image classification net
# this LRCN image captioning net has 1 binary weights but 2 definitions: 1 for feature extractor (AlexNet), 1 for LSTM
net = caffe.Net(args.net_definition, args.net_weights, caffe.TEST)
# Fix the seed
np.random.seed(args.seed)
# Split the sentence into words
words = args.sentence.split("_")
sentence = util.convert_words_into_numbers(settings.vocab_file, words)
# Condition here is the sentence
conditions = [ { "sentence": sentence, "readable": args.sentence.replace("_", " ")} ]
# Optimize a code via gradient ascent
sampler = CaptionConditionalSampler(args.captioner_definition, args.net_weights)
if args.init_file != "None":
start_code, start_image = sampler.get_code(encoder=encoder, path=args.init_file, layer=args.opt_layer)
print "Loaded start code: ", start_code.shape
else:
# shape of the code being optimized
shape = generator.blobs[settings.generator_in_layer].data.shape
start_code = np.random.normal(0, 1, shape)
output_image, list_samples = sampler.sampling( condition_net=net, image_encoder=encoder, image_generator=generator,
gen_in_layer=settings.generator_in_layer, gen_out_layer=settings.generator_out_layer, start_code=start_code,
n_iters=args.n_iters, lr=args.lr, lr_end=args.lr_end, threshold=args.threshold,
layer=args.act_layer, conditions=conditions,
epsilon1=args.epsilon1, epsilon2=args.epsilon2, epsilon3=args.epsilon3,
output_dir=args.output_dir,
reset_every=args.reset_every, save_every=args.save_every)
# Output image
filename = "%s/%s_%04d_%s_h_%s_%s_%s__%s.jpg" % (
args.output_dir,
args.act_layer,
args.n_iters,
args.lr,
str(args.epsilon1),
str(args.epsilon2),
str(args.epsilon3),
args.seed
)
# Save the final image
util.save_image(output_image, filename)
print "%s/%s" % (os.getcwd(), filename)
# Write labels to images
print "Saving images..."
for p in list_samples:
img, name, label = p
util.save_image(img, name)
if args.write_labels:
util.write_label_to_img(name, label)