def assert_gradient_ascent(objective,
model,
batch=None,
alpha=False,
shape=None):
with tf.Graph().as_default() as graph, tf.Session() as sess:
shape = shape or [1, 32, 32, 3]
t_input = param.image(shape[1], h=shape[2], batch=batch, alpha=alpha)
if alpha:
t_input = transform.collapse_alpha_random()(t_input)
model.import_graph(t_input, scope="import", forget_xy_shape=True)
def T(layer):
if layer == "input":
return t_input
if layer == "labels":
return model.labels
return graph.get_tensor_by_name("import/%s:0" % layer)
loss_t = objective(T)
opt_op = tf.train.AdamOptimizer(0.1).minimize(-loss_t)
tf.global_variables_initializer().run()
start_value = sess.run([loss_t])
for _ in range(NUM_STEPS):
_ = sess.run([opt_op])
end_value, = sess.run([loss_t])
print(start_value, end_value)
assert start_value < end_value
def render_icons(
directions,
model,
layer,
size=80,
n_steps=128,
verbose=False,
S=None,
num_attempts=3,
cossim=True,
alpha=False,
):
model.load_graphdef()
image_attempts = []
loss_attempts = []
depth = 4 if alpha else 3
batch = len(directions)
input_shape = (batch, size, size, depth)
# Render two attempts, and pull the one with the lowest loss score.
for attempt in range(num_attempts):
# Render an image for each activation vector
param_f = lambda: param.image(size,
batch=len(directions),
fft=True,
decorrelate=True,
alpha=alpha)
if cossim is True:
obj_list = [
direction_neuron_cossim_S(layer, v, batch=n, S=S)
for n, v in enumerate(directions)
]
else:
obj_list = [
direction_neuron_S(layer, v, batch=n, S=S)
for n, v in enumerate(directions)
]
obj_list += [objectives.penalize_boundary_complexity(input_shape, w=5)]
obj = objectives.Objective.sum(obj_list)
# holy mother of transforms
transforms = [
transform.pad(16, mode='constant'),
transform.jitter(4),
transform.jitter(4),
transform.jitter(8),
transform.jitter(8),
transform.jitter(8),
transform.random_scale(0.998**n for n in range(20, 40)),
transform.random_rotate(
chain(range(-20, 20), range(-10, 10), range(-5, 5), 5 * [0])),
transform.jitter(2),
transform.crop_or_pad_to(size, size)
]
if alpha:
transforms.append(transform.collapse_alpha_random())
# This is the tensorflow optimization process
# print("attempt: ", attempt)
with tf.Graph().as_default(), tf.Session() as sess:
learning_rate = 0.05
losses = []
trainer = tf.train.AdamOptimizer(learning_rate)
T = render.make_vis_T(model, obj, param_f, trainer, transforms)
vis_op, t_image = T("vis_op"), T("input")
losses_ = [obj_part(T) for obj_part in obj_list]
tf.global_variables_initializer().run()
for i in range(n_steps):
loss, _ = sess.run([losses_, vis_op])
losses.append(loss)
# if i % 100 == 0:
# print(i)
img = t_image.eval()
img_rgb = img[:, :, :, :3]
if alpha:
# print("alpha true")
k = 0.8
bg_color = 0.0
img_a = img[:, :, :, 3:]
img_merged = img_rgb * (
(1 - k) + k * img_a) + bg_color * k * (1 - img_a)
image_attempts.append(img_merged)
else:
# print("alpha false")
image_attempts.append(img_rgb)
loss_attempts.append(losses[-1])
# Use only the icons with the lowest loss
loss_attempts = np.asarray(loss_attempts)
loss_final = []
image_final = []
# print("merging best scores from attempts...")
for i, d in enumerate(directions):
# note, this should be max, it is not a traditional loss
mi = np.argmax(loss_attempts[:, i])
image_final.append(image_attempts[mi][i])
return (image_final, loss_final)
def render_icons(
directions,
model,
layer,
size=80,
n_steps=128,
verbose=False,
S=None,
num_attempts=2,
cossim=True,
alpha=False,
):
image_attempts = []
loss_attempts = []
# Render two attempts, and pull the one with the lowest loss score.
for attempt in range(num_attempts):
# Render an image for each activation vector
param_f = lambda: param.image(
size, batch=directions.shape[0], fft=True, decorrelate=True, alpha=alpha
)
if cossim is True:
obj_list = [
direction_neuron_cossim_S(layer, v, batch=n, S=S)
for n, v in enumerate(directions)
]
else:
obj_list = [
direction_neuron_S(layer, v, batch=n, S=S)
for n, v in enumerate(directions)
]
obj = objectives.Objective.sum(obj_list)
transforms = transform.standard_transforms
if alpha:
transforms.append(transform.collapse_alpha_random())
# This is the tensorflow optimization process
print("attempt: ", attempt)
with tf.Graph().as_default(), tf.Session() as sess:
learning_rate = 0.05
losses = []
trainer = tf.train.AdamOptimizer(learning_rate)
T = render.make_vis_T(model, obj, param_f, trainer, transforms)
vis_op, t_image = T("vis_op"), T("input")
losses_ = [obj_part(T) for obj_part in obj_list]
tf.global_variables_initializer().run()
for i in range(n_steps):
loss, _ = sess.run([losses_, vis_op])
losses.append(loss)
if i % 100 == 0:
print(i)
img = t_image.eval()
img_rgb = img[:, :, :, :3]
if alpha:
print("alpha true")
k = 0.8
bg_color = 0.0
img_a = img[:, :, :, 3:]
img_merged = img_rgb * ((1 - k) + k * img_a) + bg_color * k * (
1 - img_a
)
image_attempts.append(img_merged)
else:
print("alpha false")
image_attempts.append(img_rgb)
loss_attempts.append(losses[-1])
# Use only the icons with the lowest loss
loss_attempts = np.asarray(loss_attempts)
loss_final = []
image_final = []
print("merging best scores from attempts...")
for i, d in enumerate(directions):
# note, this should be max, it is not a traditional loss
mi = np.argmax(loss_attempts[:, i])
image_final.append(image_attempts[mi][i])
return (image_final, loss_final)
def render_facet(model,
neuron_obj,
layers,
style_attrs,
strength=(0.1, 0.3),
l2_weight=10.0,
resolution=128,
alpha=False):
def mean_alpha():
def inner(T):
input_t = T("input")
return tf.sqrt(tf.reduce_mean(input_t[..., 3:]**2))
return objectives.Objective(inner)
standard_transforms = [
transform.pad(2, mode='constant', constant_value=.5),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.random_scale([0.995**n for n in range(-5, 80)] +
[0.998**n for n in 2 * list(range(20, 40))]),
transform.random_rotate(
list(range(-20, 20)) + list(range(-10, 10)) + list(range(-5, 5)) +
5 * [0]),
transform.jitter(2),
transform.crop_or_pad_to(resolution, resolution)
]
if alpha:
standard_transforms.append(transform.collapse_alpha_random())
param_f = lambda: param.image(resolution, batch=9, alpha=True)
else:
param_f = lambda: param.image(resolution, batch=9)
optimizer = tf.train.AdamOptimizer(0.02)
ultimate_layer = [
n.name for n in model.graph_def.node if "image_block_4" in n.name
][-1]
obj = vector(ultimate_layer, neuron_obj)
facetsp = [(5 / len(layers)) * attr(obj, style, [layer], strength)
for style, layer in list(zip(style_attrs, layers))]
for facetp in facetsp:
obj = obj + facetp
obj = obj + l2_weight * l2()
if alpha:
obj -= mean_alpha()
obj -= 1e2 * objectives.blur_alpha_each_step()
data = render.render_vis(model,
obj,
param_f,
transforms=standard_transforms,
optimizer=optimizer,
thresholds=(1024 * 4, ))
return data
def _main(
objective_f,
model,
layer,
channel_indices,
alpha=False,
negative=False,
decorrelation_matrix=None,
n_steps=128,
parallel_transforms=16,
lr=0.05,
override_gradients=True,
):
if not isinstance(channel_indices, (tuple, list)):
channel_indices = [channel_indices]
if isinstance(layer, Layer):
layer_name = layer.name
elif isinstance(layer, str):
layer_name = layer
else:
raise ValueError("layer argument can be either a Layer object or str")
sign = -1 if negative else 1
batch = len(channel_indices)
w, h, _ = model.image_shape
with tf.Graph().as_default(), tf.Session() as sess:
# Transformation Robustness
transforms = transform.standard_transforms + [transform.crop_or_pad_to(w, h)]
if alpha:
transforms += [transform.collapse_alpha_random()]
transform_f = render.make_transform_f(transforms)
# Parameterization
image_t = param.image(
w, h, fft=True, decorrelate=True, alpha=alpha, batch=batch
)
param_t = transform_f(image_t)
# Gradient Overrides
if override_gradients:
with overrides.relu_overrides():
T = render.import_model(model, param_t, image_t)
else:
T = render.import_model(model, param_t, image_t)
# Objective
if decorrelation_matrix is None:
objs = [
sign * objective_f(layer_name, i, batch=b)
for b, i in enumerate(channel_indices)
]
else:
raise NotImplementedError
reported_obj = tf.stack([o(T) for o in objs])
obj = sum(objs)(T)
if alpha:
obj *= 1.0 - tf.reduce_mean(image_t[..., -1])
obj -= 0.1 * objectives.blur_alpha_each_step()(T)
# Optimization
optimization = tf.train.AdamOptimizer(lr).minimize(-obj)
tf.global_variables_initializer().run()
losses = np.zeros((batch, n_steps))
for step in range(n_steps):
_, loss = sess.run([optimization, reported_obj])
losses[:, step] = loss
# Evaluation
visualization = image_t.eval()
if batch == 1:
return (visualization, losses)
else:
return list(zip(visualization, losses))