def test_integration(decorrelate, fft, inceptionv1):
obj = objectives.neuron("mixed3a_pre_relu", 0)
param_f = lambda: param.image(16, decorrelate=decorrelate, fft=fft)
rendering = render.render_vis(
inceptionv1,
obj,
param_f=param_f,
thresholds=(1, 2),
verbose=False,
transforms=[],
)
start_image = rendering[0]
end_image = rendering[-1]
objective_f = objectives.neuron("mixed3a", 177)
param_f = lambda: param.image(64, decorrelate=decorrelate, fft=fft)
rendering = render.render_vis(
inceptionv1,
objective_f,
param_f,
verbose=False,
thresholds=(0, 64),
use_fixed_seed=True,
)
start_image, end_image = rendering
assert (start_image != end_image).any()
def render_activation_grid_very_naive(self,
img,
layer="mixed4d",
W=42,
n_steps=256):
# Get the activations
with tf.Graph().as_default(), tf.Session() as sess:
t_input = tf.placeholder("float32", [None, None, None, 3])
T = render.import_model(self.model, t_input, t_input)
acts = T(layer).eval({t_input: img[None]})[0]
acts_flat = acts.reshape([-1] + [acts.shape[2]])
# Render an image for each activation vector
def param_f():
return param.image(W, batch=acts_flat.shape[0])
obj = objectives.Objective.sum([
objectives.direction(layer, v, batch=n)
for n, v in enumerate(acts_flat)
])
thresholds = (n_steps // 2, n_steps)
vis_imgs = render.render_vis(self.model,
obj,
param_f,
thresholds=thresholds)[-1]
# Combine the images and display the resulting grid
vis_imgs_ = vis_imgs.reshape(list(acts.shape[:2]) + [W, W, 3])
vis_imgs_cropped = vis_imgs_[:, :, 2:-2, 2:-2, :]
show(np.hstack(np.hstack(vis_imgs_cropped)))
return vis_imgs_cropped
def vis_traditional(
self,
feature_list=None,
*,
transforms=[transform.jitter(2)],
l2_coeff=0.0,
l2_layer_name=None,
):
if feature_list is None:
feature_list = list(range(self.acts_reduced.shape[-1]))
try:
feature_list = list(feature_list)
except TypeError:
feature_list = [feature_list]
obj = sum([
objectives.direction_neuron(self.layer_name,
self.channel_dirs[feature],
batch=feature)
for feature in feature_list
])
if l2_coeff != 0.0:
assert (
l2_layer_name is not None
), "l2_layer_name must be specified if l2_coeff is non-zero"
obj -= objectives.L2(l2_layer_name) * l2_coeff
param_f = lambda: param.image(64, batch=len(feature_list))
return render.render_vis(self.model,
obj,
param_f=param_f,
transforms=transforms)[-1]
def render_atlas_tile(model,op_name,directions,icon_size=45,n_steps=127,transforms_amount=1,cossim_pow=0,L2_amount=2):
transforms_options = [
[
transform.jitter(2)
],
[
transform.pad(12, mode="constant", constant_value=.5),
transform.jitter(8),
transform.random_scale([1 + (i - 5) / 50. for i in range(11)]),
transform.random_rotate(list(range(-10, 11)) + 5 * [0]),
transform.jitter(4),
],
[
transform.pad(2, mode='constant', constant_value=.5),
transform.jitter(4),
transform.jitter(4),
transform.jitter(8),
transform.jitter(8),
transform.jitter(8),
transform.random_scale([0.995**n for n in range(-5,80)] + [0.998**n for n in 2*list(range(20,40))]),
transform.random_rotate(list(range(-20,20))+list(range(-10,10))+list(range(-5,5))+5*[0]),
transform.jitter(2),
],
]
param_f = lambda: param.image(icon_size, batch=directions.shape[0])
obj = objectives.Objective.sum(
[objectives.direction_neuron(op_name, v, batch=n, cossim_pow=cossim_pow)
for n,v in enumerate(directions)
]) - L2_amount * objectives.L2("input", 0.5) * objectives.L2("input", 0.5)
thresholds=(n_steps//2, n_steps)
vis_imgs = render.render_vis(model, obj, param_f, transforms=transforms_options[transforms_amount], thresholds=thresholds, verbose=False)[-1]
return vis_imgs
def neuron_groups(img, filename, layer, n_groups=10, attr_classes=None, filenumber=0):
# Compute activations
dirname = '../images/' + filename+'/'
if attr_classes is None:
attr_classes = []
with tf.Graph().as_default(), tf.Session():
t_input = tf.placeholder_with_default(img, [None, None, 3])
T = render.import_model(model, t_input, t_input)
acts = T(layer).eval()
# We'll use ChannelReducer (a wrapper around scikit learn's factorization tools)
# to apply Non-Negative Matrix factorization (NMF).
nmf = ChannelReducer(n_groups, "NMF")
spatial_factors = nmf.fit_transform(acts)[0].transpose(2, 0, 1).astype("float32")
channel_factors = nmf._reducer.components_.astype("float32")
# Let's organize the channels based on their horizontal position in the image
x_peak = np.argmax(spatial_factors.max(1), 1)
ns_sorted = np.argsort(x_peak)
spatial_factors = spatial_factors[ns_sorted]
channel_factors = channel_factors[ns_sorted]
# And create a feature visualziation of each group
param_f = lambda: param.image(80, batch=n_groups)
obj = sum(objectives.direction(layer, channel_factors[i], batch=i)
for i in range(n_groups))
group_icons = render.render_vis(model, obj, param_f, verbose=False)[-1]
# We'd also like to know about attribution
# First, let's turn each group into a vector over activations
group_vecs = [spatial_factors[i, ..., None] * channel_factors[i]
for i in range(n_groups)]
attrs = np.asarray([raw_class_group_attr(img, layer, attr_class, group_vecs)
for attr_class in attr_classes])
print(
attrs
)
try:
os.mkdir(dirname )
except Exception as e:
print(e)
# Let's render the visualization!
finally:
with open(dirname + '/attrs.txt', 'w') as f_w:
f_w.write(str(attrs))
for index, icon in enumerate(group_icons):
imgdata=to_image_url(icon)
print(imgdata)
imgdata = base64.b64decode(str(imgdata))
print(imgdata)
with open(dirname + str(index) + '.png', 'wb') as f_jpg:
f_jpg.write(imgdata)
def __call__(self, layer_name, channel_index):
obj = objectives.channel(layer_name, channel_index)
image = render.render_vis(self.model,
obj,
param_f=self.param_f,
thresholds=[self.threshold],
use_fixed_seed=True,
verbose=False)
return np.array(image[0][0])
def start(self):
self.image = None
self._doRun(True)
obj = objectives.channel(self.layer_id, self.unit)
self.image = render.render_vis(self.model, obj)
#self.image = render.render_vis(self.model, self.unit_id)
self._doRun(False)
def neuron_groups(model, img, layer, n_groups=6, attr_classes=[]):
# Compute activations
with tf.Graph().as_default(), tf.Session():
t_input = tf.placeholder_with_default(img, [None, None, 3])
T = render.import_model(model, t_input, t_input)
acts = T(layer).eval()
# We'll use ChannelReducer (a wrapper around scikit learn's factorization tools)
# to apply Non-Negative Matrix factorization (NMF).
nmf = ChannelReducer(n_groups, "PCA")
print(layer, n_groups)
spatial_factors = nmf.fit_transform(acts)[0].transpose(2, 0, 1).astype("float32")
channel_factors = nmf._reducer.components_.astype("float32")
# Let's organize the channels based on their horizontal position in the image
x_peak = np.argmax(spatial_factors.max(1), 1)
ns_sorted = np.argsort(x_peak)
spatial_factors = spatial_factors[ns_sorted]
channel_factors = channel_factors[ns_sorted]
# And create a feature visualziation of each group
param_f = lambda: param.image(80, batch=n_groups)
obj = sum(objectives.direction(layer, channel_factors[i], batch=i)
for i in range(n_groups))
group_icons = render.render_vis(model, obj, param_f, verbose=False)[-1]
# We'd also like to know about attribution
#
# First, let's turn each group into a vector over activations
group_vecs = [spatial_factors[i, ..., None] * channel_factors[i]
for i in range(n_groups)]
attrs = np.asarray([raw_class_group_attr(img, layer, attr_class, model, group_vecs)
for attr_class in attr_classes])
gray_scale_groups = [skimage.color.rgb2gray(icon) for icon in group_icons]
# Let's render the visualization!
data = {
"img": _image_url(img),
"n_groups": n_groups,
"spatial_factors": [_image_url(factor[..., None] / np.percentile(spatial_factors, 99) * [1, 0, 0]) for factor in
spatial_factors],
"group_icons": [_image_url(icon) for icon in gray_scale_groups]
}
# with open('ng.pickle', 'wb') as handle:
# pickle.dump(data, handle, protocol=pickle.HIGHEST_PROTOCOL)
# with open('./svelte_python/ng.pickle', 'rb') as p_file:
# data = pickle.load(p_file)
generate_html('neuron_groups', data)
def test_integration(decorrelate, fft):
obj = objectives.neuron("mixed3a_pre_relu", 0)
param_f = lambda: param.image(16, decorrelate=decorrelate, fft=fft)
rendering = render.render_vis(model,
obj,
param_f=param_f,
thresholds=(1, 2),
verbose=False,
transforms=[])
start_image = rendering[0]
end_image = rendering[-1]
assert (start_image != end_image).any()
def render_feature(cppn_f=lambda: image_cppn(84),
optimizer=tf.train.AdamOptimizer(0.001),
objective=objectives.channel('noname', 0),
transforms=[]):
vis = render.render_vis(m,
objective,
param_f=cppn_f,
optimizer=optimizer,
transforms=transforms,
thresholds=[2**i for i in range(5, 10)],
verbose=False)
#show(vis)
return vis
def start_multi(self):
self.image = None
self._doRun(True)
logger.info("!!! running all:")
for unit in range(self.layer_units):
self.unit = unit
self.notify_observers(EngineChange(unit_changed=True))
logger.info(f"!!! running unit {unit}")
obj = objectives.channel(self.layer_id, unit)
self.image = render.render_vis(self.model, obj)
if not self.running:
break
self._doRun(True)
self._doRun(False)
def visualization(learning_rate, neuron, channel, contrast, NRO_IMG, SAVE_P):
LEARNING_RATE = learning_rate
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
obj = objectives.neuron(neuron, channel)
imgs = render.render_vis(model, obj,
optimizer=optimizer,
transforms=[],
param_f=lambda: param.image(256, fft=True, decorrelate=True, init_val=NRO_IMG), # 256 es el tamanio de la imagen
thresholds=(0,2), verbose=False)
# Note that we're doubling the image scale to make artifacts more obvious
plt.figure()
plt.imshow(imgs[0][0])
plt.axis('off')
contraste = contrast # Mover este numero hasta ver algo razonable
plt.imshow(contraste*(imgs[1][0]-imgs[0][0]) + 0.5)
plt.savefig(SAVE_P, bbox_inches='tight')
def test_integration_any_channels():
inceptionv1 = InceptionV1()
objectives_f = [
objectives.deepdream("mixed4a_pre_relu"),
objectives.channel("mixed4a_pre_relu", 360),
objectives.neuron("mixed3a", 177)
]
params_f = [
lambda: param.grayscale_image_rgb(128),
lambda: arbitrary_channels_to_rgb(128, channels=10)
]
for objective_f in objectives_f:
for param_f in params_f:
rendering = render.render_vis(
inceptionv1,
objective_f,
param_f,
verbose=False,
thresholds=(0, 64),
use_fixed_seed=True,
)
start_image, end_image = rendering
assert (start_image != end_image).any()
def convert(model, inputs):
print(inputs['z'])
print(inputs['layer'])
num_neurons = {'mixed3a':255, 'mixed4a':507, 'mixed5a':831}
start = time.time()
#set up parameters
layer = inputs['layer'].lower()
neuron = int(np.clip(np.abs(np.float(inputs['z'])) * 1000, 0, num_neurons[layer]))
steps = int(inputs['steps'])
#start rendering the images
param_f = lambda: param.image(512, decorrelate=True)
output = render.render_vis(model, layer+":"+str(neuron),
param_f, thresholds=(steps,), verbose = False)
#logging
elapsed = time.time() - start
print(f'layer: {layer} neuron {neuron}: steps: {steps} time: {elapsed}')
#output results
image = output[0].squeeze() * 255
return {'image': image.astype('uint8')}
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "3"
import numpy as np
import tensorflow as tf
assert tf.__version__.startswith('1')
import lucid.modelzoo.vision_models as models
from lucid.misc.io import show
import lucid.optvis.objectives as objectives
import lucid.optvis.param as param
import lucid.optvis.render as render
import lucid.optvis.transform as transform
# Let's import a model from the Lucid modelzoo!
model = models.InceptionV1()
model.load_graphdef()
# Visualizing a neuron is easy!
_ = render.render_vis(model, "mixed4a_pre_relu:476")
print(1)
input_name = 'input_1'
network = FrozenNetwork()
network.load_graphdef()
if LAYER == "-":
images = []
layers = []
for l in sys.stdin:
layers.append(l.strip())
for layer in layers:
for i in range(COLUMNS):
obj = objectives.channel(layer, i)
renders = render.render_vis(network, obj)
assert len(renders) == 1
image = renders[0]
assert len(image) == 1
image = image[0]
images.append(image)
images = np.array(images)
height, width = 128, 128
rows = len(layers)
print(images.shape)
assert images.shape == (rows * COLUMNS, 128, 128, 3)
grid = (images.reshape(rows, COLUMNS, height, width,
3).swapaxes(1, 2).reshape(height * rows,
width * COLUMNS, 3))
scipy.misc.imsave(OUTPUT_PREFIX + ".png", grid)
sys.exit()
for op in graph.get_operations():
print(op.name, op.values()[0].shape)
LEARNING_RATE = 0.05
optimizer = tf.train.AdamOptimizer(LEARNING_RATE)
# objective = "mixed4b_pre_relu:452"
# objective = "mixed3b_pre_relu:10"
objective = "mixed5b_pre_relu:1"
thresholds = (1, 32, 128, 256) # (1, 32, 128, 256, 2048)
imgs = render.render_vis(
model,
objective,
optimizer=optimizer,
transforms=[],
param_f=lambda: param.image(64, fft=False, decorrelate=False),
thresholds=thresholds,
verbose=True)
fig([imgs])
JITTER = 1
ROTATE = 5
SCALE = 1.1
transforms = [
transform.pad(2 * JITTER),
transform.jitter(JITTER),
transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]),
transform.random_rotate(range(-ROTATE, ROTATE + 1))
download = "wget --output-document hello.jpg '"+url1+"'"
os.system(download)
#os.system('wget --output-document auto.jpg {}'.format(url1))
print(i)
with open('hello.jpg', 'r+b') as f:
with Image.open(f) as image:
cover = resizeimage.resize_cover(image, [200, 190])
cover.save('images/auto'+str(i)+'.jpg', image.format)
# image optimisation
ImageOps.equalize( Image.open("images/auto"+str(i)+".jpg")).save("images/auto"+str(i)+".jpg")
os.system('rm hello.jpg')
content_image = load('images/auto'+str(i)+'.jpg')[...,:3]
style_image = load("style/style2.png")[..., :3] # choose a style
param_f = lambda: style_transfer_param(content_image, style_image)
content_obj = 100 * activation_difference(content_layers, difference_to=CONTENT_INDEX)
content_obj.description = "Content Loss"
style_obj = activation_difference(style_layers, transform_f=gram_matrix, difference_to=STYLE_INDEX)
style_obj.description = "Style Loss"
objective = - content_obj - style_obj
vis = render.render_vis(model, objective, param_f=param_f, thresholds=[512], verbose=False, print_objectives=[content_obj, style_obj])[-1]
savepath = "new/" + data['Keywords'][i] + data['Year'][i] +"_num_"+ str(i) + ".jpg"
print("s")
save(vis[0], savepath)
print("end")
except:
continue
class FrozenNetwork(Model):
model_path = MODEL_PATH
image_shape = [224, 224, 3]
image_value_range = (0, 1)
input_name = 'input_1'
network = FrozenNetwork()
network.load_graphdef()
#for layer in network:
# print(layer.get_shape())
pixels = 224
param_f = lambda: param.image(pixels, fft=True, decorrelate=True)
#obj_test = objectives.channel(LAYER, NEURON_INDEX).get_shape()
#print(obj_test)
obj = objectives.channel(LAYER, NEURON_INDEX)
images = render.render_vis(network, obj, param_f, thresholds=(1024, ))
assert len(images) == 1
image = images[0]
assert len(image) == 1
image = image[0]
out_filename = LAYER.replace(
"/", "-") + "_" + str(NEURON_INDEX) + "_" + MODEL_PATH.zfill(10) + ".png"
scipy.misc.imsave(out_filename, image)
from lucid.modelzoo.vision_base import Model
class FrozenNetwork(Model):
model_path = network_protobuf_path
image_shape = [256, 256, 3]
image_value_range = (0, 1)
input_name = 'input_1'
network = FrozenNetwork()
network.load_graphdef()
obj = objectives.channel(layer_name, neuron_index)
param_f = lambda: param.image(512, fft=True, decorrelate=True)
renders = render.render_vis(network, obj, param_f, thresholds=(2024, ))
last_image_file = sorted(glob.glob("projection/out/*step*.png"))[-1]
stylegan_render = imageio.imread(last_image_file)
lucid_render = renders[0][0]
lucid_render = (np.clip(lucid_render, 0, 1) * 255).astype(np.uint8)
h, w = lucid_render.shape[:2]
canvas = PIL.Image.new('RGB', (w * 2, h), 'white')
canvas.paste(Image.fromarray(lucid_render), (0, 0))
canvas.paste(
Image.fromarray(stylegan_render).resize((w, h), PIL.Image.LANCZOS), (w, 0))
canvas.save("projection/combined_%s_%03d.png" %
(layer_name.split("/")[0], neuron_index))
def neuron_groups(imglist, filenamelist, layer, n_groups=6, attr_classes=None):
# Compute activations
filename = ''
for f in filenamelist:
filename += f
with open('result/' + filename + '.html', 'a') as f:
f.write('''<!DOCTYPE html>
<html>
<head >
<title>%s</title>
<script src='GroupWidget_1cb0e0d.js'></script>
</head>
<body>''' % (filename))
for key, img in enumerate(imglist):
if attr_classes is None:
attr_classes = []
with tf.Graph().as_default(), tf.Session():
t_input = tf.placeholder_with_default(img, [None, None, 3])
T = render.import_model(model, t_input, t_input)
acts = T(layer).eval()
# We'll use ChannelReducer (a wrapper around scikit learn's factorization tools)
# to apply Non-Negative Matrix factorization (NMF).
nmf = ChannelReducer(n_groups, "NMF")
spatial_factors = nmf.fit_transform(acts)[0].transpose(
2, 0, 1).astype("float32")
channel_factors = nmf._reducer.components_.astype("float32")
# Let's organize the channels based on their horizontal position in the image
x_peak = np.argmax(spatial_factors.max(1), 1)
ns_sorted = np.argsort(x_peak)
spatial_factors = spatial_factors[ns_sorted]
channel_factors = channel_factors[ns_sorted]
# And create a feature visualziation of each group
param_f = lambda: param.image(80, batch=n_groups)
obj = sum(
objectives.direction(layer, channel_factors[i], batch=i)
for i in range(n_groups))
group_icons = render.render_vis(model, obj, param_f, verbose=False)[-1]
# We'd also like to know about attribution
# First, let's turn each group into a vector over activations
group_vecs = [
spatial_factors[i, ..., None] * channel_factors[i]
for i in range(n_groups)
]
attrs = np.asarray([
raw_class_group_attr(img, layer, attr_class, group_vecs)
for attr_class in attr_classes
])
print(attrs)
# Let's render the visualization!
with open('result/' + filename + '.html', 'a') as f:
f.write(''' <main%s></main%s>
<script>
var app = new GroupWidget_1cb0e0d({
target: document.querySelector( 'main%s' ),''' %
(key, key, key))
f.write('''data: {''')
f.write('"img":"%s",\n' % str(_image_url(img)))
f.write('"n_groups"' + ":" + str(n_groups) + ',\n')
f.write('"spatial_factors"' + ":" + str([
_image_url(factor[..., None] /
np.percentile(spatial_factors, 99) * [1, 0, 0])
for factor in spatial_factors
]) + ',\n')
f.write('"group_icons"' + ":" +
str([_image_url(icon) for icon in group_icons]) + ',\n')
f.write('''} });''')
f.write('''</script>''')
with open('result/' + filename + '.html', 'a') as f:
f.write('''</body></html >''')
print(filename)
# # Compute attribution backwards from each positin in layer2
# attrs = []
# for i in range(acts2.shape[1]):
# attrs_ = []
# for j in range(acts2.shape[2]):
# grad = t_grad.eval({n_x: i, n_y: j, T(layer1): acts1})
# # linear approximation of imapct
# attr = np.sum(acts1 * grad, -1)[0]
# attrs_.append(attr)
# attrs.append(attrs_)
# return np.asarray(attrs)
#
# def orange_blue(a,b,clip=False):
# if clip:
# a,b = np.maximum(a,0), np.maximum(b,0)
# arr = np.stack([a, (a + b)/2., b], -1)
# arr /= 1e-2 + np.abs(arr).max()/1.5
# arr += 0.3
# return arr
#
# img = load("https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png")
#
# attrs = raw_spatial_spatial_attr(img, "mixed4d", "mixed5a", override=None)
# attrs = attrs / attrs.max()
_ = render.render_vis(model, "")
img = np.reshape(_, [128, 128, 3])
plt.imshow(img)
plt.show()
model = models.VGG16_caffe()
model.load_graphdef()
model.show_graph()
"""## Visualize Neuron
See the [lucid tutorial](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/tutorial.ipynb) to learn more.
We pick `InceptionV4/InceptionV4/Mixed_6b/concat` from above, and chose to focus on unit 0.
"""
model = models.VGG16_caffe()
model.load_graphdef()
_ = render.render_vis(model, "conv1_1/conv1_1:0")
"""## Caricature
See the [inversion and caricature notebook](https://colab.research.google.com/github/tensorflow/lucid/blob/master/notebooks/misc/feature_inversion_caricatures.ipynb) to learn more.
"""
from lucid.recipes.caricature import feature_inversion
img = load("https://storage.googleapis.com/lucid-static/building-blocks/examples/dog_cat.png")
model = models.VGG16_caffe()
model.load_graphdef()
result = feature_inversion(img, model, "conv1_1/conv1_1", n_steps=512, cossim_pow=0.0)
show(result)
print('loading model')
model = models.InceptionV1()
model.load_graphdef()
print('calculating')
neuron = ("mixed4a_pre_relu", 476)
version = 7
size = 64 #resulting image cube haa dimensions size X size X size X 3
def param_f3d(size):
temp = imageCube(size)
return tf.concat([
temp,
tf.transpose(temp, [1, 0, 2, 3]),
tf.transpose(temp, [2, 1, 0, 3])
], 0)
objective = objectives.channel(*neuron)
image_cube = render.render_vis(
model,
objective,
lambda: param_f3d(size),
transforms=transform.standard_transforms,
thresholds=(512, )) # threshold number of steps.
#I used 4096
image_cube = np.array(image_cube)[:, :size] #image cube
np.save(f"featureCube{size}_{version}.npy", image_cube)
def render_facet(model,
neuron_obj,
layers,
style_attrs,
strength=(0.1, 0.3),
l2_weight=10.0,
resolution=128,
alpha=False):
def mean_alpha():
def inner(T):
input_t = T("input")
return tf.sqrt(tf.reduce_mean(input_t[..., 3:]**2))
return objectives.Objective(inner)
standard_transforms = [
transform.pad(2, mode='constant', constant_value=.5),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.jitter(4),
transform.random_scale([0.995**n for n in range(-5, 80)] +
[0.998**n for n in 2 * list(range(20, 40))]),
transform.random_rotate(
list(range(-20, 20)) + list(range(-10, 10)) + list(range(-5, 5)) +
5 * [0]),
transform.jitter(2),
transform.crop_or_pad_to(resolution, resolution)
]
if alpha:
standard_transforms.append(transform.collapse_alpha_random())
param_f = lambda: param.image(resolution, batch=9, alpha=True)
else:
param_f = lambda: param.image(resolution, batch=9)
optimizer = tf.train.AdamOptimizer(0.02)
ultimate_layer = [
n.name for n in model.graph_def.node if "image_block_4" in n.name
][-1]
obj = vector(ultimate_layer, neuron_obj)
facetsp = [(5 / len(layers)) * attr(obj, style, [layer], strength)
for style, layer in list(zip(style_attrs, layers))]
for facetp in facetsp:
obj = obj + facetp
obj = obj + l2_weight * l2()
if alpha:
obj -= mean_alpha()
obj -= 1e2 * objectives.blur_alpha_each_step()
data = render.render_vis(model,
obj,
param_f,
transforms=standard_transforms,
optimizer=optimizer,
thresholds=(1024 * 4, ))
return data
input_name = 'input'
def show_image(image):
html = ""
data_url = _image_url(image)
html += '<img width=\"100\" style=\"margin: 10px\" src=\"' + data_url + '\">'
with open("img.html", "w") as f:
f.write(html)
_display_html(html)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument('--crop_size', type=int, default=128)
parser.add_argument('--model_file', type=str, default='lucid_model.pb')
args = parser.parse_args()
model = LucidModel()
model.model_path = args.model_file
model.image_shape = [args.crop_size, args.crop_size, 3]
print("Nodes in graph:")
for node in model.graph_def.node:
print(node.name)
print("=" * 30)
obj = objectives.channel("prediction/Conv2D", 0) - objectives.channel("prediction/Conv2D", 0)
res = render.render_vis(model, obj, transforms=[])
show_image(res)
def neuron_groups(img_name,
layers,
model,
attr_classes,
factorization_methods,
flag1,
flag_read_attr=False,
iter_num=100,
SG_path=False,
labels=None,
pos_flag=1,
thres_explained_var=0.7,
vis_random_seed=0,
image_size=0,
debug_flag=0):
img = load(img_name)
# img = load("./data/doghead224.jpeg")
# img = load("./data/cathead224.jpeg")
# img = resize(img, (224, 224, 3), order=1, mode='constant', anti_aliasing=False).astype(np.float32)
for attr_class in attr_classes:
root_directory = create_root_dir(img_name, attr_class, flag1)
if flag1 == "Shap":
AM_list_L, logit_list, channel_attr_list, kept_channel_list_L \
= compute_shap(img, model, attr_class, layers,
flag1, flag_read_attr=flag_read_attr,
iter_num=iter_num, labels=labels, save_directory=root_directory)
elif flag1 == "IGSG":
AM_list_L, logit_list, channel_attr_list, kept_channel_list_L \
= compute_igsg(img, model, attr_class, layers,
flag1, flag_read_attr=flag_read_attr,
iter_num=iter_num, SG_path=SG_path,
labels=labels, save_directory=root_directory)
else:
continue
print_result_from_logit(logit_list, labels)
for i_pos_neg in range(pos_flag):
AM_list = AM_list_L[i_pos_neg]
kept_channel_list = kept_channel_list_L[i_pos_neg]
if debug_flag == 1:
debug_show_AM_plus_img(AM_list, img, model)
if i_pos_neg < 1:
reverse_suffix = '_pos'
else:
reverse_suffix = '_neg'
for layer_index, AM in enumerate(AM_list):
layer = layers[layer_index]
channel_attr = channel_attr_list[layer_index]
kept_channel_index = kept_channel_list[layer_index]
for factorization_method in factorization_methods:
spatial_factors, channel_factors, n_groups = \
decompose_AM_get_group_num(factorization_method, AM, thres_explained_var)
# if debug_flag == 1:
# decompose_AM_with_UMAP(AM, n_groups)
channel_factors_max_index, channel_shap, short_index, long_index, \
n_groups = map_shap_attr_to_long(channel_factors, channel_attr, kept_channel_index)
# AM = np.squeeze(AM)
spatial_factors = weight_AM2spatial_factor(
AM, spatial_factors, n_groups, short_index,
kept_channel_index, channel_attr, i_pos_neg)
# If the attribution is negative, channel_shap should be multiply -1
if i_pos_neg > 0:
channel_shap = channel_shap * -1
# Sorting based on sum of Shapley values
ns_sorted = get_sort_groups_with_shap_scores(channel_shap)
every_group_attr_sorted, spatial_factors, channel_shap, n_groups =\
sort_groups_features(ns_sorted, spatial_factors, channel_shap, n_groups)
no_slash_layer_name = ''.join(layer.split('/'))
save_directory = create_factorization_dir(
root_directory, factorization_method,
no_slash_layer_name, reverse_suffix, n_groups)
gen_spatial_heat_maps(85, n_groups, spatial_factors,
save_directory, attr_class,
factorization_method,
no_slash_layer_name, img, AM, model)
gen_info_txt(channel_shap, n_groups, save_directory,
factorization_method, attr_class,
every_group_attr_sorted)
# Using feature attributions times activation maps as loss function to update visualization image
channel_shap = channel_shap.astype("float32")
obj = sum(
utils.dot_attr_actmaps(layer, channel_shap[i], batch=i)
for i in range(n_groups))
'''
For feature visualization, the library "lucid" will be useful because
it has implements many loss functions of different literatures, image processing operators,
and collected several pretrained tensorflow network.
'''
transforms = [
transform.pad(12),
transform.jitter(8),
transform.random_scale(
[n / 100. for n in range(80, 120)]),
transform.random_rotate(
list(range(-10, 10)) + list(range(-5, 5)) +
10 * list(range(-2, 2))),
transform.jitter(2)
]
# image parameterization with shared params for aligned optimizing images
def interpolate_f():
unique = fft_image(
(n_groups, image_size, image_size, 3),
random_seed=vis_random_seed)
shared = [
lowres_tensor(
(n_groups, image_size, image_size, 3),
(1, image_size // 2, image_size // 2, 3),
random_seed=vis_random_seed),
lowres_tensor(
(n_groups, image_size, image_size, 3),
(1, image_size // 4, image_size // 4, 3),
random_seed=vis_random_seed),
lowres_tensor(
(n_groups, image_size, image_size, 3),
(1, image_size // 8, image_size // 8, 3),
random_seed=vis_random_seed),
lowres_tensor(
(n_groups, image_size, image_size, 3),
(2, image_size // 8, image_size // 8, 3),
random_seed=vis_random_seed),
lowres_tensor(
(n_groups, image_size, image_size, 3),
(1, image_size // 16, image_size // 16, 3),
random_seed=vis_random_seed),
lowres_tensor(
(n_groups, image_size, image_size, 3),
(2, image_size // 16, image_size // 16, 3),
random_seed=vis_random_seed)
]
return utils.to_valid_rgb(unique + sum(shared),
decorrelate=True)
group_icons = render.render_vis(
model,
objective_f=obj,
param_f=interpolate_f,
transforms=transforms,
verbose=False,
relu_gradient_override=False)[-1]
save_imgs(group_icons, save_directory, attr_class,
factorization_method, no_slash_layer_name)
print("Layer {} and class {} -> finished".format(
layer, attr_class))
import tensorflow as tf
from lucid.modelzoo.vision_base import Model
import warnings
import lucid.optvis.render as render
warnings.filterwarnings('ignore')
graph_file = 'nasnet_mobile_graphdef.pb'
graph_def = tf.GraphDef()
with open(graph_file, 'rb') as f:
graph_def.ParseFromString(f.read())
for node in graph_def.node:
print(node.name)
class NasNetMobile(Model):
model_path = 'nasnet_mobile_graphdef_frozen.pb.modelzoo'
image_shape = [224, 224, 3]
image_value_range = (0, 1)
input_name = 'input'
if __name__ == '__main__':
nasnet = NasNetMobile()
_ = render.render_vis(nasnet, 'cell_0/cell_output/concat:0')
network = FrozenNetwork()
network.load_graphdef()
if LAYER == "-":
height, width = 144, 144
images = []
layers = []
for l in sys.stdin:
layers.append(l.strip())
for layer in layers:
for i in range(COLUMNS):
param_f = lambda: param.image(height, fft=True, decorrelate=True)
obj = objectives.channel(layer, i)
renders = render.render_vis(network,
obj,
param_f,
thresholds=(2048, ))
assert len(renders) == 1
image = renders[0]
assert len(image) == 1
image = image[0]
images.append(image)
images = np.array(images)
rows = len(layers)
print(images.shape)
assert images.shape == (rows * COLUMNS, height, width, 3)
grid = (images.reshape(rows, COLUMNS, height, width,
3).swapaxes(1, 2).reshape(height * rows,
width * COLUMNS, 3))
scipy.misc.imsave(OUTPUT_PREFIX + ".png", grid)
sys.exit()
from lucid.misc.io import show
import lucid.optvis.objectives as objectives
import lucid.optvis.param as param
import lucid.optvis.render as render
import lucid.optvis.transform as transform
#import lucid.optvis.optimizer as transfor
import tensorflow as tf
import matplotlib.pyplot as plt
model = models.InceptionV1()
model.load_graphdef()
neuron1 = ('mixed4b_pre_relu', 452)
C = lambda neuron: objectives.channel(*neuron)
out = render.render_vis(model, C(neuron1))
plt.imshow(out[0][0])
JITTER = 1
ROTATE = 5
SCALE = 1.1
transforms = [
transform.pad(2 * JITTER),
transform.jitter(JITTER),
transform.random_scale([SCALE**(n / 10.) for n in range(-10, 11)]),
transform.random_rotate(range(-ROTATE, ROTATE + 1))
]
imgs = render.render_vis(model,
"mixed4b_pre_relu:452",
