def runDiversitywithTransforms(layerName,
layerNeuron,
transforms=None,
imageSize=256,
batch=4,
weight=1e2):
'''
Function to run Lucent neuron diversity optimisation for a given Layer and Neuron (Channel) in a PyTorch CNN.
This function uses image augmentation transforms to help improve the clarity and resolution of the produced neuron maximisations.
'''
if transforms == None:
transforms = [
transform.pad(16),
transform.jitter(8),
transform.random_scale([n / 100. for n in range(80, 120)]),
transform.random_rotate(
list(range(-10, 10)) + list(range(-5, 5)) +
10 * list(range(-2, 2))),
transform.jitter(2),
]
batch_param_f = lambda: param.image(imageSize, batch=batch)
obj = objectives.channel(
layerName, layerNeuron) - weight * objectives.diversity(layerName)
_ = render.render_vis(model_,
obj,
batch_param_f,
transforms=transforms,
show_inline=True)
def lucid(self,
layer_n_channel,
size=224,
thresholds=[512],
progress=False):
layer_n_channel = self._correct_layer_n_channel(layer_n_channel)
transforms = transform.standard_transforms.copy()
transforms.append(lambda x: x * 255 - 117)
def param_f():
return param.image(224, fft=True, decorrelate=True)
try:
img = render.render_vis(self,
layer_n_channel,
show_image=False,
preprocess=False,
progress=progress,
transforms=transforms,
thresholds=thresholds,
param_f=param_f)[0][0]
except AssertionError:
raise AssertionError(
'Invalid layer {}. Retrieve the list of layers with `model.layer_info`.'
.format(layer_n_channel))
img = (img * 255).astype(np.uint8)
return img
def runDiversity(layerName, layerNeuron, imageSize=256, batch=4, weight=1e2):
'''
Function to run Lucent neuron diversity optimisation for a given Layer and Neuron (Channel) in a PyTorch CNN.
'''
batch_param_f = lambda: param.image(imageSize, batch=batch)
obj = objectives.channel(
layerName, layerNeuron) - weight * objectives.diversity(layerName)
_ = render.render_vis(model_, obj, batch_param_f, show_inline=True)
def visualize_grad_fourier(model, obj, path, device):
model.to(device).eval()
param_f = lambda: param.image(128, fft=True, decorrelate=False)
_ = render.render_vis(model,
obj,
param_f,
transforms=[],
save_image=True,
image_name=path,
show_image=False)
def visualize_jitter(model, obj, path, device):
model.to(device).eval()
jitter_only = [transform.jitter(8)]
param_f = lambda: param.image(512, fft=False, decorrelate=True)
_ = render.render_vis(model,
obj,
param_f,
transforms=jitter_only,
save_image=True,
image_name=path,
show_image=False)
def visualize_cppn(model, obj, path, device):
model.to(device).eval()
cppn_param_f = lambda: param.cppn(64)
# We initialize an optimizer with lower learning rate for CPPN
cppn_opt = lambda params: torch.optim.Adam(params, 5e-3)
_ = render.render_vis(model,
obj,
cppn_param_f,
cppn_opt,
transforms=[],
save_image=True,
image_name=path,
show_image=False)
def main():
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = inceptionv1(pretrained=True)
model.to(device).eval()
CPPN = False
SPATIAL_DECORRELATION = True
CHANNEL_DECORRELATION = True
if CPPN:
# CPPN parameterization
param_f = lambda: param.cppn(224)
opt = lambda params: torch.optim.Adam(params, 5e-3)
# Some objectives work better with CPPN than others
obj = "mixed4d_3x3_bottleneck_pre_relu_conv:139"
else:
param_f = lambda: param.image(
224, fft=SPATIAL_DECORRELATION, decorrelate=CHANNEL_DECORRELATION)
opt = lambda params: torch.optim.Adam(params, 5e-2)
obj = "mixed4a:476"
render.render_vis(model, obj, param_f, opt)
def visualize_neuron(layer,
neuron,
model,
activations,
dataset,
top_k=100,
with_cropped=True,
with_lucent=True):
if with_cropped:
plot_neuron_max_activations(activations,
dataset,
layer,
neuron,
top_k=top_k)
plot_neuron_max_activations(activations,
dataset,
layer,
neuron,
top_k=top_k,
cropped=False)
if with_lucent:
render.render_vis(model, f'{layer}:{neuron}', show_inline=True)
def test_integration(inceptionv1_model, decorrelate, fft):
obj = "mixed3a_1x1_pre_relu_conv:0"
param_f = lambda: param.image(224, decorrelate=decorrelate, fft=fft)
optimizer = lambda params: torch.optim.Adam(params, lr=0.1)
rendering = render.render_vis(
inceptionv1_model,
obj,
param_f,
optimizer=optimizer,
thresholds=(1, 2),
verbose=True,
show_inline=True,
)
start_image, end_image = rendering
assert (start_image != end_image).any()
def visualize_layer(model, layer, path, device):
model.to(device).eval()
if not os.path.exists(f'{path}/'):
os.makedirs(f'{path}/')
layer_name = f'features_{layer}'
image_name = f"{path}/{layer_name}.jpg"
_ = render.render_vis(model,
layer_name,
fixed_image_size=64,
save_image=True,
image_name=image_name.replace(':', '_'),
show_image=False)
def visualize_filter_fc(model, layer, filters, path, class_labels, device):
model.to(device).eval()
if not os.path.exists(f'{path}/'):
os.makedirs(f'{path}/')
for i in range(len(filters)):
layer_name = f'classifier_{layer}:{filters[i]}'
image_name = f"{path}/{layer_name}_{class_labels[filters[i]]}.jpg"
_ = render.render_vis(model,
layer_name,
fixed_image_size=64,
thresholds=(2048, ),
save_image=True,
image_name=image_name.replace(':', '_'),
show_image=False)
def visualize_filter(model, layer, filters, path, device):
model.to(device).eval()
if not os.path.exists(f'{path}/'):
os.makedirs(f'{path}/')
param_f = lambda: param.image(128, fft=False, decorrelate=False)
for i in range(len(filters)):
layer_name = f'features_{layer}:{filters[i]}'
image_name = f"{path}/{layer_name}.jpg"
_ = render.render_vis(model,
layer_name,
param_f,
save_image=True,
image_name=image_name.replace(':', '_'),
show_image=False)
def gen_visualization(model, image_name, objective, parametrizer, optimizer,
transforms, image_size, neuron, params):
if neuron:
full_image_path = params[
'prepped_model_path'] + '/visualizations/images/neuron/' + image_name
else:
full_image_path = params[
'prepped_model_path'] + '/visualizations/images/channel/' + image_name
if parametrizer is None:
parametrizer = lambda: param.image(image_size)
print('generating featviz with objective: %s' % str(objective))
_ = render.render_vis(model,
objective,
parametrizer,
optimizer,
transforms=transforms,
save_image=True,
image_name=full_image_path,
show_inline=True)
def visualize_filter_fc2(model, layer, filters, path, class_labels, device):
model.to(device).eval()
if not os.path.exists(f'{path}/'):
os.makedirs(f'{path}/')
param_f = lambda: param.image(64, fft=False, decorrelate=False)
for i in range(len(filters)):
layer_name = f'classifier_{layer}:{filters[i]}'
image_name = f"{path}/{layer_name}_{class_labels[filters[i]]}.jpg"
_ = render.render_vis(model,
layer_name,
param_f,
fixed_image_size=64,
save_image=True,
image_name=image_name.replace(':', '_'),
show_image=False)
def visualize_multiple_neurons(model, neuron_names, save_path, device):
model.to(device).eval()
param_f = lambda: param.image(512, batch=1)
neuron1 = neuron_names[0]
print(neuron1)
obj = objectives.channel(neuron_names[0][0], neuron_names[0][1])
for i in range(1, len(neuron_names)):
obj += objectives.channel(neuron_names[i][0], neuron_names[i][1])
_ = render.render_vis(model,
obj,
param_f,
save_image=True,
image_name=f'{save_path}_placeholder.jpg',
show_image=False)
def visualize_diversity_fc(model, layer, filter, path, batch_size, device):
model.to(device).eval()
if not os.path.exists(f'{path}/'):
os.makedirs(f'{path}/')
batch_param_f = lambda: param.image(128, batch=batch_size)
layer_name = f'fc_{layer}:{filter}'
obj = objectives.channel(
f"fc_{layer}", filter) - 1e2 * objectives.diversity(f"fc_{layer}")
image_name = f"{path}/{layer_name}_diversity.jpg"
_ = render.render_vis(model,
obj,
batch_param_f,
save_image=True,
image_name=image_name.replace(':', '_'),
show_image=False)
def feature_inversion(model,
device,
img,
layer=None,
n_steps=512,
cossim_pow=0.0):
# Convert image to torch.tensor and scale image
img = torch.tensor(np.transpose(img, [2, 0, 1])).to(device)
upsample = torch.nn.Upsample(224)
img = upsample(img)
obj = objectives.Objective.sum([
1.0 * dot_compare(layer, cossim_pow=cossim_pow),
objectives.blur_input_each_step(),
])
# Initialize parameterized input and stack with target image
# to be accessed in the objective function
params, image_f = param.image(224)
def stacked_param_f():
return params, lambda: torch.stack([image_f()[0], img])
transforms = [
transform.pad(8, mode='constant', constant_value=.5),
transform.jitter(8),
transform.random_scale([0.9, 0.95, 1.05, 1.1] + [1] * 4),
transform.random_rotate(list(range(-5, 5)) + [0] * 5),
transform.jitter(2),
]
_ = render.render_vis(model,
obj,
stacked_param_f,
transforms=transforms,
thresholds=(n_steps, ),
show_image=False,
progress=False)
def render_activation_grid_very_naive(
img,
model,
layer="main_net_0_ops_(1, 2)_ops_(0, 1)_op",
cell_image_size=48,
n_steps=1024):
# First wee need, to normalize and resize the image
img = torch.tensor(np.transpose(img, [2, 0, 1])).to(device)
normalize = (transform.preprocess_inceptionv1() if model._get_name()
== "InceptionV1" else transform.normalize())
transforms = [
normalize,
torch.nn.Upsample(size=224, mode="bilinear", align_corners=True),
]
transforms_f = transform.compose(transforms)
# shape: (1, 3, original height of img, original width of img)
img = img.unsqueeze(0)
# shape: (1, 3, 224, 224)
img = transforms_f(img)
# Here we compute the activations of the layer `layer` using `img` as input
# shape: (layer_channels, layer_height, layer_width), the shape depends on the layer
acts = get_layer(model, layer, img)[0]
layer_channels, layer_height, layer_width = acts.shape
# for each position `(y, x)` in the feature map `acts`, we optimize an image
# to match with the features `acts[:, y, x]`
# This means that the total number of cells (which is the batch size here)
# in the grid is layer_height*layer_width.
nb_cells = layer_height * layer_width
# Parametrization of the of each cell in the grid
param_f = lambda: param.image(cell_image_size, batch=nb_cells)
params, image_f = param_f()
obj = objectives.Objective.sum([
# for each position in `acts`, maximize the dot product between the activations
# `acts` at the position (y, x) and the features of the corresponding
# cell image on our 'grid'. The activations at (y, x) is a vector of size
# `layer_channels` (this depends on the `layer`). The features
# of the corresponding cell on our grid is a tensor of shape
# (layer_channels, cell_layer_height, cell_layer_width).
# Note that cell_layer_width != layer_width and cell_layer_height != layer_weight
# because the cell image size is smaller than the image size.
# With `dot_compare`, we maximize the dot product between
# cell_activations[y_cell, x_xcell] and acts[y,x] (both of size `layer_channels`)
# for each possible y_cell and x_cell, then take the average to get a single
# number. Check `dot_compare for more details.`
dot_compare(layer,
acts[:, y:y + 1, x:x + 1],
batch=x + y * layer_width)
for i, (
x,
y) in enumerate(product(range(layer_width), range(layer_height)))
])
results = render.render_vis(
model,
obj,
param_f,
thresholds=(n_steps, ),
progress=True,
fixed_image_size=cell_image_size,
show_image=False,
)
# shape: (layer_height*layer_width, cell_image_size, cell_image_size, 3)
imgs = results[-1] # last step results
# shape: (layer_height*layer_width, 3, cell_image_size, cell_image_size)
imgs = imgs.transpose((0, 3, 1, 2))
imgs = torch.from_numpy(imgs)
imgs = imgs[:, :, 2:-2, 2:-2]
# turn imgs into a a grid
grid = torchvision.utils.make_grid(imgs,
nrow=int(np.sqrt(nb_cells)),
padding=0)
grid = grid.permute(1, 2, 0)
grid = grid.numpy()
render.show(grid)
return imgs
def fetch_deepviz_img_for_subgraph(model, layer_name, within_id, targetid,
viz_folder, params):
model = set_across_model(model, 'target_node', None)
objective_str = layer_name + ':' + str(targetid)
neuron = params['deepviz_neuron']
#generate objective
#objective = gen_objective(targetid,model,params,neuron=neuron)
print('generating feature_viz objective string for %s' % targetid)
if not params['deepviz_neuron']:
#return layer_name+':'+str(within_id)
objective = objectives.channel(layer_name, int(within_id))
else:
objective = objectives.neuron(layer_name, int(within_id))
file_path = viz_folder + '/images.csv'
parametrizer = params['deepviz_param']
optimizer = params['deepviz_optim']
transforms = params['deepviz_transforms']
image_size = params['deepviz_image_size']
param_str = object_2_str(parametrizer, "params['deepviz_param']=")
optimizer_str = object_2_str(optimizer, "params['deepviz_optim']=")
transforms_str = object_2_str(transforms, "params['deepviz_transforms']=")
df = pd.read_csv(file_path, dtype=str)
df_sel = df.loc[(df['targetid'] == str(targetid))
& (df['objective'] == objective_str) &
(df['parametrizer'] == param_str) &
(df['optimizer'] == optimizer_str) &
(df['transforms'] == transforms_str) &
(df['neuron'] == str(neuron))]
if len(df_sel) == 0:
print('deepviz image not found for %s, generating . . .' % targetid)
#image_name = 'deepviz_'+str(targetid)+'_'+objective+'_'+str(time.time())+'.jpg'
image_name = str(targetid) + '_' + objective_str + '_' + str(
time.time()) + '.jpg'
#gen_visualization(model,image_name,objective,parametrizer,optimizer,transforms,image_size,neuron,params)
if neuron:
full_image_path = viz_folder + '/neuron/' + image_name
else:
full_image_path = viz_folder + '/channel/' + image_name
if parametrizer is None:
parametrizer = lambda: param.image(image_size)
print('generating featviz with objective: %s' % str(objective_str))
_ = render.render_vis(model,
objective,
parametrizer,
optimizer,
transforms=transforms,
save_image=True,
image_name=full_image_path,
show_inline=True)
with open(file_path, 'a') as csv:
csv.write(','.join([
image_name,
str(targetid), objective_str, param_str, optimizer_str,
transforms_str,
str(neuron)
]) + '\n')
else:
print('found pre-generated image')
image_name = df_sel.iloc[0]['image_name']
if neuron:
return 'neuron/' + image_name
else:
return 'channel/' + image_name
def render_activation_grid(img,
model,
device,
layer="mixed4d",
cell_image_size=60,
n_groups=6,
n_steps=1024,
batch_size=64,
img_crop_size=224,
img_resize=512,
img_transforms=None):
'''
Activation Grid renderer from the Lucent library, modified for more general PyTorch models.
This version has been adapted to allow image transforms from thr Torchvision.transforms module to be used.
This allows bespoke image transformations, to be applied to the input image, to match those on which the network under investigation has been trained with.
Currently, the default is resizing and cropping to the input size expected by the network, and standard scaled using the ImageNet coefficients.
This work uses the Lucent library, which borrow heavily from the Google Brain Lucide library for Tensorflow.
https://github.com/greentfrapp/lucent
Inputs:
Required inputs:
img Pil image object.
model The torch.nn model object containing the CNN trained model.
device The device the computations are to be run. GPU STRONGLY ADVISED!.
layer The name of the layer that the spatial activations will be analysed from.
Optional arguments:
See Lucent descriptions execept for following.
img_crop_size The default transform image crop size, which should match the expected input size of the network under investigation.
img_resize The size the image should be resized to before cropping. Usually a good idea to keep it to the 2s-compliment size that's larger to the input size.
e.g. for an input size of 224, a resize of 256 would suffice. For 299, then 512 would be a good choice.
img_transforms An object of Torchvision.transforms modules, composed into a object using Torchvision.transforms.Compose(), as shown in the defaults.
'''
# First wee need, to normalize and resize the image.
# Bespoke image transforms can be implemented and passed through the img_transforms argument.
if img_transforms is None:
img_transforms = transforms.Compose([
transforms.Resize(img_resize),
transforms.CenterCrop(img_crop_size),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
img = img_transforms(img).to(device)
img = img.unsqueeze(0)
# shape: (1, 3, 224, 224)
#img = transforms_f(img)
# Here we compute the activations of the layer `layer` using `img` as input
# shape: (layer_channels, layer_height, layer_width), the shape depends on the layer
acts = get_layer(model, layer, img)[0]
# shape: (layer_height, layer_width, layer_channels)
acts = acts.permute(1, 2, 0)
# shape: (layer_height*layer_width, layer_channels)
acts = acts.view(-1, acts.shape[-1])
acts_np = acts.cpu().numpy()
nb_cells = acts.shape[0]
# negative matrix factorization `NMF` is used to reduce the number
# of channels to n_groups. This will be used as the following.
# Each cell image in the grid is decomposed into a sum of
# (n_groups+1) images. First, each cell has its own set of parameters
# this is what is called `cells_params` (see below). At the same time, we have
# a of group of images of size 'n_groups', which also have their own image parametrized
# by `groups_params`. The resulting image for a given cell in the grid
# is the sum of its own image (parametrized by `cells_params`)
# plus a weighted sum of the images of the group. Each each image from the group
# is weighted by `groups[cell_index, group_idx]`. Basically, this is a way of having
# the possibility to make cells with similar activations have a similar image, because
# cells with similar activations will have a similar weighting for the elements
# of the group.
if n_groups > 0:
reducer = ChannelReducer(n_groups, "NMF")
groups = reducer.fit_transform(acts_np)
groups /= groups.max(0)
else:
groups = np.zeros([])
# shape: (layer_height*layer_width, n_groups)
groups = torch.from_numpy(groups)
# Parametrization of the images of the groups (we have 'n_groups' groups)
groups_params, groups_image_f = param.fft_image(
[n_groups, 3, cell_image_size, cell_image_size])
# Parametrization of the images of each cell in the grid (we have 'layer_height*layer_width' cells)
cells_params, cells_image_f = param.fft_image(
[nb_cells, 3, cell_image_size, cell_image_size])
# First, we need to construct the images of the grid
# from the parameterizations
def image_f():
groups_images = groups_image_f()
cells_images = cells_image_f()
X = []
for i in range(nb_cells):
x = 0.7 * cells_images[i] + 0.5 * sum(
groups[i, j] * groups_images[j] for j in range(n_groups))
X.append(x)
X = torch.stack(X)
return X
# make sure the images are between 0 and 1
image_f = param.to_valid_rgb(image_f, decorrelate=True)
# After constructing the cells images, we sample randomly a mini-batch of cells
# from the grid. This is to prevent memory overflow, especially if the grid
# is large.
def sample(image_f, batch_size):
def f():
X = image_f()
inds = torch.randint(0, len(X), size=(batch_size, ))
inputs = X[inds]
# HACK to store indices of the mini-batch, because we need them
# in objective func. Might be better ways to do that
sample.inds = inds
return inputs
return f
image_f_sampled = sample(image_f, batch_size=batch_size)
# Now, we define the objective function
def objective_func(model):
# shape: (batch_size, layer_channels, cell_layer_height, cell_layer_width)
pred = model(layer)
# use the sampled indices from `sample` to get the corresponding targets
target = acts[sample.inds].to(pred.device)
# shape: (batch_size, layer_channels, 1, 1)
target = target.view(target.shape[0], target.shape[1], 1, 1)
dot = (pred * target).sum(dim=1).mean()
return -dot
obj = objectives.Objective(objective_func)
def param_f():
# We optimize the parametrizations of both the groups and the cells
params = list(groups_params) + list(cells_params)
return params, image_f_sampled
results = render.render_vis(
model,
obj,
param_f,
thresholds=(n_steps, ),
show_image=False,
progress=True,
fixed_image_size=cell_image_size,
)
# shape: (layer_height*layer_width, 3, grid_image_size, grid_image_size)
imgs = image_f()
imgs = imgs.cpu().data
imgs = imgs[:, :, 2:-2, 2:-2]
# turn imgs into a a grid
grid = torchvision.utils.make_grid(imgs,
nrow=int(np.sqrt(nb_cells)),
padding=0)
grid = grid.permute(1, 2, 0)
grid = grid.numpy()
render.show(grid)
return imgs
import torch
import lucent
from torchvision.models import vgg16
from lucent.optvis import render, param, transform
from lucent.modelzoo import util
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = vgg16(pretrained=True)
print(util.get_model_layers(model))
model.to(device).eval()
obj = "features_22:8" # a ResNet50 layer and channel
render.render_vis(model, obj)
def render_activation_grid_less_naive(
img,
model,
layer="main_net_0_ops_(1, 2)_ops_(0, 1)_op",
cell_image_size=60,
n_groups=6,
n_steps=1024,
batch_size=64,
):
# First wee need, to normalize and resize the image
img = torch.tensor(np.transpose(img, [2, 0, 1])).to(device)
normalize = (transform.preprocess_inceptionv1() if model._get_name()
== "InceptionV1" else transform.normalize())
transforms = transform.standard_transforms.copy() + [
normalize,
torch.nn.Upsample(size=224, mode="bilinear", align_corners=True),
]
transforms_f = transform.compose(transforms)
# shape: (1, 3, original height of img, original width of img)
img = img.unsqueeze(0)
# shape: (1, 3, 224, 224)
img = transforms_f(img)
# Here we compute the activations of the layer `layer` using `img` as input
# shape: (layer_channels, layer_height, layer_width), the shape depends on the layer
acts = get_layer(model, layer, img)[0]
# shape: (layer_height, layer_width, layer_channels)
acts = acts.permute(1, 2, 0)
# shape: (layer_height*layer_width, layer_channels)
acts = acts.view(-1, acts.shape[-1])
acts_np = acts.cpu().numpy()
nb_cells = acts.shape[0]
# negative matrix factorization `NMF` is used to reduce the number
# of channels to n_groups. This will be used as the following.
# Each cell image in the grid is decomposed into a sum of
# (n_groups+1) images. First, each cell has its own set of parameters
# this is what is called `cells_params` (see below). At the same time, we have
# a of group of images of size 'n_groups', which also have their own image parametrized
# by `groups_params`. The resulting image for a given cell in the grid
# is the sum of its own image (parametrized by `cells_params`)
# plus a weighted sum of the images of the group. Each each image from the group
# is weighted by `groups[cell_index, group_idx]`. Basically, this is a way of having
# the possibility to make cells with similar activations have a similar image, because
# cells with similar activations will have a similar weighting for the elements
# of the group.
if n_groups > 0:
reducer = ChannelReducer(n_groups, "NMF")
groups = reducer.fit_transform(acts_np)
groups /= groups.max(0)
else:
groups = np.zeros([])
# shape: (layer_height*layer_width, n_groups)
groups = torch.from_numpy(groups)
# Parametrization of the images of the groups (we have 'n_groups' groups)
groups_params, groups_image_f = param.fft_image(
[n_groups, 3, cell_image_size, cell_image_size])
# Parametrization of the images of each cell in the grid (we have 'layer_height*layer_width' cells)
cells_params, cells_image_f = param.fft_image(
[nb_cells, 3, cell_image_size, cell_image_size])
# First, we need to construct the images of the grid
# from the parameterizations
def image_f():
groups_images = groups_image_f()
cells_images = cells_image_f()
X = []
for i in range(nb_cells):
x = 0.7 * cells_images[i] + 0.5 * sum(
groups[i, j] * groups_images[j] for j in range(n_groups))
X.append(x)
X = torch.stack(X)
return X
# make sure the images are between 0 and 1
image_f = param.to_valid_rgb(image_f, decorrelate=True)
# After constructing the cells images, we sample randomly a mini-batch of cells
# from the grid. This is to prevent memory overflow, especially if the grid
# is large.
def sample(image_f, batch_size):
def f():
X = image_f()
inds = torch.randint(0, len(X), size=(batch_size, ))
inputs = X[inds]
# HACK to store indices of the mini-batch, because we need them
# in objective func. Might be better ways to do that
sample.inds = inds
return inputs
return f
image_f_sampled = sample(image_f, batch_size=batch_size)
# Now, we define the objective function
def objective_func(model):
# shape: (batch_size, layer_channels, cell_layer_height, cell_layer_width)
pred = f.relu(model(layer))
# use the sampled indices from `sample` to get the corresponding targets
target = acts[sample.inds].to(pred.device)
# shape: (batch_size, layer_channels, 1, 1)
target = target.view(target.shape[0], target.shape[1], 1, 1)
dot = (pred * target).sum(dim=1).mean()
return -dot
obj = objectives.Objective(objective_func)
def param_f():
# We optimize the parametrizations of both the groups and the cells
params = list(groups_params) + list(cells_params)
return params, image_f_sampled
results = render.render_vis(
model,
obj,
param_f,
thresholds=(n_steps, ),
show_image=False,
progress=True,
fixed_image_size=cell_image_size,
)
# shape: (layer_height*layer_width, 3, grid_image_size, grid_image_size)
imgs = image_f()
imgs = imgs.cpu().data
imgs = imgs[:, :, 2:-2, 2:-2]
# turn imgs into a a grid
grid = torchvision.utils.make_grid(imgs,
nrow=int(np.sqrt(nb_cells)),
padding=0)
grid = grid.permute(1, 2, 0)
grid = grid.numpy()
render.show(grid)
return imgs
def test_render_vis(inceptionv1_model):
thresholds = (1, 2)
imgs = render.render_vis(inceptionv1_model, "mixed4a:0", thresholds=thresholds, show_image=False)
assert len(imgs) == len(thresholds)
assert imgs[0].shape == (1, 128, 128, 3)
