def runDiversitywithTransforms(layerName, layerNeuron, transforms=None, imageSize=256, batch=4, weight=1e2): ''' Function to run Lucent neuron diversity optimisation for a given Layer and Neuron (Channel) in a PyTorch CNN. This function uses image augmentation transforms to help improve the clarity and resolution of the produced neuron maximisations. ''' if transforms == None: transforms = [ transform.pad(16), transform.jitter(8), transform.random_scale([n / 100. for n in range(80, 120)]), transform.random_rotate( list(range(-10, 10)) + list(range(-5, 5)) + 10 * list(range(-2, 2))), transform.jitter(2), ] batch_param_f = lambda: param.image(imageSize, batch=batch) obj = objectives.channel( layerName, layerNeuron) - weight * objectives.diversity(layerName) _ = render.render_vis(model_, obj, batch_param_f, transforms=transforms, show_inline=True)
def runDiversity(layerName, layerNeuron, imageSize=256, batch=4, weight=1e2): ''' Function to run Lucent neuron diversity optimisation for a given Layer and Neuron (Channel) in a PyTorch CNN. ''' batch_param_f = lambda: param.image(imageSize, batch=batch) obj = objectives.channel( layerName, layerNeuron) - weight * objectives.diversity(layerName) _ = render.render_vis(model_, obj, batch_param_f, show_inline=True)
def visualize_grad_fourier(model, obj, path, device): model.to(device).eval() param_f = lambda: param.image(128, fft=True, decorrelate=False) _ = render.render_vis(model, obj, param_f, transforms=[], save_image=True, image_name=path, show_image=False)
def visualize_jitter(model, obj, path, device): model.to(device).eval() jitter_only = [transform.jitter(8)] param_f = lambda: param.image(512, fft=False, decorrelate=True) _ = render.render_vis(model, obj, param_f, transforms=jitter_only, save_image=True, image_name=path, show_image=False)
def assert_gradient_descent(objective, model): params, image = param.image(224, batch=2) optimizer = torch.optim.Adam(params, lr=0.05) T = render.hook_model(model, image) objective_f = objectives.as_objective(objective) model(image()) start_value = objective_f(T) for _ in range(NUM_STEPS): optimizer.zero_grad() model(image()) loss = objective_f(T) loss.backward() optimizer.step() end_value = objective_f(T) assert start_value > end_value
def test_integration(inceptionv1_model, decorrelate, fft): obj = "mixed3a_1x1_pre_relu_conv:0" param_f = lambda: param.image(224, decorrelate=decorrelate, fft=fft) optimizer = lambda params: torch.optim.Adam(params, lr=0.1) rendering = render.render_vis( inceptionv1_model, obj, param_f, optimizer=optimizer, thresholds=(1, 2), verbose=True, show_inline=True, ) start_image, end_image = rendering assert (start_image != end_image).any()
def visualize_filter(model, layer, filters, path, device): model.to(device).eval() if not os.path.exists(f'{path}/'): os.makedirs(f'{path}/') param_f = lambda: param.image(128, fft=False, decorrelate=False) for i in range(len(filters)): layer_name = f'features_{layer}:{filters[i]}' image_name = f"{path}/{layer_name}.jpg" _ = render.render_vis(model, layer_name, param_f, save_image=True, image_name=image_name.replace(':', '_'), show_image=False)
def gen_visualization(model, image_name, objective, parametrizer, optimizer, transforms, image_size, neuron, params): if neuron: full_image_path = params[ 'prepped_model_path'] + '/visualizations/images/neuron/' + image_name else: full_image_path = params[ 'prepped_model_path'] + '/visualizations/images/channel/' + image_name if parametrizer is None: parametrizer = lambda: param.image(image_size) print('generating featviz with objective: %s' % str(objective)) _ = render.render_vis(model, objective, parametrizer, optimizer, transforms=transforms, save_image=True, image_name=full_image_path, show_inline=True)
def visualize_multiple_neurons(model, neuron_names, save_path, device): model.to(device).eval() param_f = lambda: param.image(512, batch=1) neuron1 = neuron_names[0] print(neuron1) obj = objectives.channel(neuron_names[0][0], neuron_names[0][1]) for i in range(1, len(neuron_names)): obj += objectives.channel(neuron_names[i][0], neuron_names[i][1]) _ = render.render_vis(model, obj, param_f, save_image=True, image_name=f'{save_path}_placeholder.jpg', show_image=False)
def visualize_filter_fc2(model, layer, filters, path, class_labels, device): model.to(device).eval() if not os.path.exists(f'{path}/'): os.makedirs(f'{path}/') param_f = lambda: param.image(64, fft=False, decorrelate=False) for i in range(len(filters)): layer_name = f'classifier_{layer}:{filters[i]}' image_name = f"{path}/{layer_name}_{class_labels[filters[i]]}.jpg" _ = render.render_vis(model, layer_name, param_f, fixed_image_size=64, save_image=True, image_name=image_name.replace(':', '_'), show_image=False)
def visualize_diversity_fc(model, layer, filter, path, batch_size, device): model.to(device).eval() if not os.path.exists(f'{path}/'): os.makedirs(f'{path}/') batch_param_f = lambda: param.image(128, batch=batch_size) layer_name = f'fc_{layer}:{filter}' obj = objectives.channel( f"fc_{layer}", filter) - 1e2 * objectives.diversity(f"fc_{layer}") image_name = f"{path}/{layer_name}_diversity.jpg" _ = render.render_vis(model, obj, batch_param_f, save_image=True, image_name=image_name.replace(':', '_'), show_image=False)
def feature_inversion(model, device, img, layer=None, n_steps=512, cossim_pow=0.0): # Convert image to torch.tensor and scale image img = torch.tensor(np.transpose(img, [2, 0, 1])).to(device) upsample = torch.nn.Upsample(224) img = upsample(img) obj = objectives.Objective.sum([ 1.0 * dot_compare(layer, cossim_pow=cossim_pow), objectives.blur_input_each_step(), ]) # Initialize parameterized input and stack with target image # to be accessed in the objective function params, image_f = param.image(224) def stacked_param_f(): return params, lambda: torch.stack([image_f()[0], img]) transforms = [ transform.pad(8, mode='constant', constant_value=.5), transform.jitter(8), transform.random_scale([0.9, 0.95, 1.05, 1.1] + [1] * 4), transform.random_rotate(list(range(-5, 5)) + [0] * 5), transform.jitter(2), ] _ = render.render_vis(model, obj, stacked_param_f, transforms=transforms, thresholds=(n_steps, ), show_image=False, progress=False)
def main(): device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model = inceptionv1(pretrained=True) model.to(device).eval() CPPN = False SPATIAL_DECORRELATION = True CHANNEL_DECORRELATION = True if CPPN: # CPPN parameterization param_f = lambda: param.cppn(224) opt = lambda params: torch.optim.Adam(params, 5e-3) # Some objectives work better with CPPN than others obj = "mixed4d_3x3_bottleneck_pre_relu_conv:139" else: param_f = lambda: param.image( 224, fft=SPATIAL_DECORRELATION, decorrelate=CHANNEL_DECORRELATION) opt = lambda params: torch.optim.Adam(params, 5e-2) obj = "mixed4a:476" render.render_vis(model, obj, param_f, opt)
def render_activation_grid_very_naive( img, model, layer="main_net_0_ops_(1, 2)_ops_(0, 1)_op", cell_image_size=48, n_steps=1024): # First wee need, to normalize and resize the image img = torch.tensor(np.transpose(img, [2, 0, 1])).to(device) normalize = (transform.preprocess_inceptionv1() if model._get_name() == "InceptionV1" else transform.normalize()) transforms = [ normalize, torch.nn.Upsample(size=224, mode="bilinear", align_corners=True), ] transforms_f = transform.compose(transforms) # shape: (1, 3, original height of img, original width of img) img = img.unsqueeze(0) # shape: (1, 3, 224, 224) img = transforms_f(img) # Here we compute the activations of the layer `layer` using `img` as input # shape: (layer_channels, layer_height, layer_width), the shape depends on the layer acts = get_layer(model, layer, img)[0] layer_channels, layer_height, layer_width = acts.shape # for each position `(y, x)` in the feature map `acts`, we optimize an image # to match with the features `acts[:, y, x]` # This means that the total number of cells (which is the batch size here) # in the grid is layer_height*layer_width. nb_cells = layer_height * layer_width # Parametrization of the of each cell in the grid param_f = lambda: param.image(cell_image_size, batch=nb_cells) params, image_f = param_f() obj = objectives.Objective.sum([ # for each position in `acts`, maximize the dot product between the activations # `acts` at the position (y, x) and the features of the corresponding # cell image on our 'grid'. The activations at (y, x) is a vector of size # `layer_channels` (this depends on the `layer`). The features # of the corresponding cell on our grid is a tensor of shape # (layer_channels, cell_layer_height, cell_layer_width). # Note that cell_layer_width != layer_width and cell_layer_height != layer_weight # because the cell image size is smaller than the image size. # With `dot_compare`, we maximize the dot product between # cell_activations[y_cell, x_xcell] and acts[y,x] (both of size `layer_channels`) # for each possible y_cell and x_cell, then take the average to get a single # number. Check `dot_compare for more details.` dot_compare(layer, acts[:, y:y + 1, x:x + 1], batch=x + y * layer_width) for i, ( x, y) in enumerate(product(range(layer_width), range(layer_height))) ]) results = render.render_vis( model, obj, param_f, thresholds=(n_steps, ), progress=True, fixed_image_size=cell_image_size, show_image=False, ) # shape: (layer_height*layer_width, cell_image_size, cell_image_size, 3) imgs = results[-1] # last step results # shape: (layer_height*layer_width, 3, cell_image_size, cell_image_size) imgs = imgs.transpose((0, 3, 1, 2)) imgs = torch.from_numpy(imgs) imgs = imgs[:, :, 2:-2, 2:-2] # turn imgs into a a grid grid = torchvision.utils.make_grid(imgs, nrow=int(np.sqrt(nb_cells)), padding=0) grid = grid.permute(1, 2, 0) grid = grid.numpy() render.show(grid) return imgs
def fetch_deepviz_img_for_subgraph(model, layer_name, within_id, targetid, viz_folder, params): model = set_across_model(model, 'target_node', None) objective_str = layer_name + ':' + str(targetid) neuron = params['deepviz_neuron'] #generate objective #objective = gen_objective(targetid,model,params,neuron=neuron) print('generating feature_viz objective string for %s' % targetid) if not params['deepviz_neuron']: #return layer_name+':'+str(within_id) objective = objectives.channel(layer_name, int(within_id)) else: objective = objectives.neuron(layer_name, int(within_id)) file_path = viz_folder + '/images.csv' parametrizer = params['deepviz_param'] optimizer = params['deepviz_optim'] transforms = params['deepviz_transforms'] image_size = params['deepviz_image_size'] param_str = object_2_str(parametrizer, "params['deepviz_param']=") optimizer_str = object_2_str(optimizer, "params['deepviz_optim']=") transforms_str = object_2_str(transforms, "params['deepviz_transforms']=") df = pd.read_csv(file_path, dtype=str) df_sel = df.loc[(df['targetid'] == str(targetid)) & (df['objective'] == objective_str) & (df['parametrizer'] == param_str) & (df['optimizer'] == optimizer_str) & (df['transforms'] == transforms_str) & (df['neuron'] == str(neuron))] if len(df_sel) == 0: print('deepviz image not found for %s, generating . . .' % targetid) #image_name = 'deepviz_'+str(targetid)+'_'+objective+'_'+str(time.time())+'.jpg' image_name = str(targetid) + '_' + objective_str + '_' + str( time.time()) + '.jpg' #gen_visualization(model,image_name,objective,parametrizer,optimizer,transforms,image_size,neuron,params) if neuron: full_image_path = viz_folder + '/neuron/' + image_name else: full_image_path = viz_folder + '/channel/' + image_name if parametrizer is None: parametrizer = lambda: param.image(image_size) print('generating featviz with objective: %s' % str(objective_str)) _ = render.render_vis(model, objective, parametrizer, optimizer, transforms=transforms, save_image=True, image_name=full_image_path, show_inline=True) with open(file_path, 'a') as csv: csv.write(','.join([ image_name, str(targetid), objective_str, param_str, optimizer_str, transforms_str, str(neuron) ]) + '\n') else: print('found pre-generated image') image_name = df_sel.iloc[0]['image_name'] if neuron: return 'neuron/' + image_name else: return 'channel/' + image_name
def generate_images(args): use_cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device("cuda" if use_cuda else "cpu") # Initialize arguments based on dataset chosen if args.dataset == "first5_mnist": args.output_size = 5 args.input_channel = 1 elif args.dataset == "last5_mnist": args.output_size = 5 args.input_channel = 1 elif args.dataset == "mnist": args.output_size = 10 args.input_channel = 1 elif args.dataset == "cifar10": args.output_size = 10 args.input_channel = 3 elif args.dataset == "fmnist": args.output_size = 10 args.input_channel = 1 elif args.dataset == "kmnist": args.output_size = 10 args.input_channel = 1 if args.arch == "resnet18": arch = ResNet18 elif args.arch == "lenet5": arch = LeNet5 elif args.arch == "lenet5_halfed": arch = LeNet5Halfed print(f"\nDataset: {args.dataset}") print(f"Arch: {args.arch}") print(f"Size: {args.size}\n") for i in range(len(args.seeds)): print(f"Iteration {i+1}, Seed {args.seeds[i]}") np.random.seed(args.seeds[i]) torch.manual_seed(args.seeds[i]) torch.cuda.manual_seed_all(args.seeds[i]) torch.backends.cudnn.deterministic = True # Load model model = arch(input_channel=args.input_channel, output_size=args.output_size).to(device) model.load_state_dict( torch.load( args.model_dir + f"{args.dataset}_{args.arch}_{args.seeds[i]}", map_location=torch.device(device), )) model.eval() # Generate images of each class for label in range(args.output_size): # Initialize number of images generated from each class, the last class fills the remaining if label == args.output_size - 1: nb_images = int(args.size / args.output_size + args.size % args.output_size) else: nb_images = int(args.size / args.output_size) # Create the directory for saving if it does not exist create_op_dir(args.data_dir + "Synthetic " + args.dataset + "/" + str(args.seeds[i]) + "/" + str(label) + "/") for idx in range(nb_images): param_f = lambda: param.image(32, decorrelate=True, fft=True, channels=args.input_channel) transforms = [ transform.pad(4), transform.jitter(2), transform.random_scale( [1 + (i - 5) / 50.0 for i in range(11)]), transform.random_rotate(list(range(-5, 6)) + 5 * [0]), transform.jitter(2), ] render_vis( model=model, objective_f="labels:" + str(label), param_f=param_f, transforms=transforms, preprocess=False, thresholds=(512, ), save_image=True, image_name=os.path.dirname(os.path.abspath(__file__)) + "/cache/data/" + "Synthetic " + args.dataset + "/" + str(args.seeds[i]) + "/" # + args.data_dir + "Synthetic " + args.dataset + "/" + str(args.seeds[i]) + "/" + str(label) + "/image" + str(idx) + ".png", )
def render_vis( model, objective_f, param_f=None, optimizer=None, transforms=None, thresholds=(512, ), verbose=False, preprocess=True, progress=True, show_image=True, save_image=False, image_name=None, show_inline=False, fixed_image_size=None, ): if param_f is None: param_f = lambda: param.image(128) # param_f is a function that should return two things # params - parameters to update, which we pass to the optimizer # image_f - a function that returns an image as a tensor params, image_f = param_f() if optimizer is None: optimizer = lambda params: torch.optim.Adam(params, lr=5e-2) optimizer = optimizer(params) if transforms is None: transforms = transform.standard_transforms transforms = transforms.copy() if preprocess: if model._get_name() == "InceptionV1": # Original Tensorflow InceptionV1 takes input range [-117, 138] transforms.append(transform.preprocess_inceptionv1()) else: # Assume we use normalization for torchvision.models # See https://pytorch.org/docs/stable/torchvision/models.html transforms.append(transform.normalize()) # Upsample images smaller than 224 image_shape = image_f().shape if fixed_image_size is not None: new_size = fixed_image_size elif image_shape[2] < 224 or image_shape[3] < 224: new_size = 224 else: new_size = None if new_size: transforms.append( torch.nn.Upsample(size=new_size, mode="bilinear", align_corners=True)) transform_f = transform.compose(transforms) hook = hook_model(model, image_f) objective_f = objectives.as_objective(objective_f) if verbose: model(transform_f(image_f())) print("Initial loss: {:.3f}".format(objective_f(hook))) images = [] try: for i in tqdm(range(1, max(thresholds) + 1), disable=(not progress)): optimizer.zero_grad() try: model.encode_image(transform_f(image_f())) except RuntimeError as ex: if i == 1: # Only display the warning message # on the first iteration, no need to do that # every iteration warnings.warn( "Some layers could not be computed because the size of the " "image is not big enough. It is fine, as long as the non" "computed layers are not used in the objective function" f"(exception details: '{ex}')") loss = objective_f(hook) loss.backward() optimizer.step() if i in thresholds: image = tensor_to_img_array(image_f()) if verbose: print("Loss at step {}: {:.3f}".format( i, objective_f(hook))) if show_inline: show(image) images.append(image) except KeyboardInterrupt: print("Interrupted optimization at step {:d}.".format(i)) if verbose: print("Loss at step {}: {:.3f}".format(i, objective_f(hook))) images.append(tensor_to_img_array(image_f())) if save_image: export(image_f(), image_name) if show_inline: show(tensor_to_img_array(image_f())) elif show_image: view(image_f()) return images
def render_vis(model, objective_f, param_f=None, optimizer=None, transforms=None, thresholds=(512, ), verbose=False, preprocess=True, progress=True, show_image=True, save_image=False, image_name=None, show_inline=False): if param_f is None: param_f = lambda: param.image(128) # param_f is a function that should return two things # params - parameters to update, which we pass to the optimizer # image_f - a function that returns an image as a tensor params, image_f = param_f() if optimizer is None: optimizer = lambda params: torch.optim.Adam(params, lr=5e-2) optimizer = optimizer(params) if transforms is None: transforms = transform.standard_transforms.copy() if preprocess: if model._get_name() == "InceptionV1": # Original Tensorflow InceptionV1 takes input range [-117, 138] transforms.append(transform.preprocess_inceptionv1()) else: # Assume we use normalization for torchvision.models # See https://pytorch.org/docs/stable/torchvision/models.html transforms.append(transform.normalize()) # Upsample images smaller than 224 image_shape = image_f().shape if image_shape[2] < 224 or image_shape[3] < 224: transforms.append( torch.nn.Upsample(size=224, mode='bilinear', align_corners=True)) transform_f = transform.compose(transforms) hook = hook_model(model, image_f) objective_f = objectives.as_objective(objective_f) if verbose: model(transform_f(image_f())) print("Initial loss: {:.3f}".format(objective_f(hook))) images = [] try: for i in tqdm(range(1, max(thresholds) + 1), disable=(not progress)): optimizer.zero_grad() model(transform_f(image_f())) loss = objective_f(hook) loss.backward() optimizer.step() if i in thresholds: image = tensor_to_img_array(image_f()) if verbose: print("Loss at step {}: {:.3f}".format( i, objective_f(hook))) if show_inline: show(image) images.append(image) except KeyboardInterrupt: print("Interrupted optimization at step {:d}.".format(i)) if verbose: print("Loss at step {}: {:.3f}".format(i, objective_f(hook))) images.append(tensor_to_img_array(image_f())) if save_image: export(image_f(), image_name) if show_inline: show(tensor_to_img_array(image_f())) elif show_image: view(image_f()) return images
def param_f(): return param.image(224, fft=True, decorrelate=True)
def test_hook_model(inceptionv1_model): _, image_f = param.image(224) hook = render.hook_model(inceptionv1_model, image_f) inceptionv1_model(image_f()) assert hook("input").shape == (1, 3, 224, 224) assert hook("labels").shape == (1, 1008)