def __init__(self,
model_unit,
max_step=100,
imgsize=(227, 227),
corner=(0, 0),
savedir="",
explabel="",
backend="torch",
GAN="fc6"):
self.recording = []
self.scores_all = []
self.codes_all = []
self.generations = []
self.pref_unit = model_unit
self.backend = backend
if backend == "caffe":
self.CNNmodel = CNNmodel(model_unit[0]) # 'caffe-net'
elif backend == "torch":
if model_unit[0] == 'caffe-net': # `is` won't work here!
self.CNNmodel = CNNmodel_Torch(model_unit[0])
else: # AlexNet, VGG, ResNet, DENSE and anything else
self.CNNmodel = TorchScorer(model_unit[0])
else:
raise NotImplementedError
self.CNNmodel.select_unit(model_unit)
# Allow them to choose from multiple optimizers, substitute generator.visualize and render
if GAN == "fc6" or GAN == "fc7" or GAN == "fc8":
from GAN_utils import upconvGAN
self.G = upconvGAN(name=GAN).cuda()
self.render_tsr = self.G.visualize_batch_np # this output tensor
self.render = self.G.render
# self.G = Generator(name=GAN)
# self.render = self.G.render
if GAN == "fc8":
self.code_length = 1000
else:
self.code_length = 4096
elif GAN == "BigGAN":
from BigGAN_Evolution import BigGAN_embed_render
self.render = BigGAN_embed_render
self.code_length = 256 # 128 # 128d Class Embedding code or 256d full code could be used.
else:
raise NotImplementedError
self.optimizer = CholeskyCMAES(self.code_length,
population_size=None,
init_sigma=init_sigma,
init_code=np.zeros(
[1, self.code_length]),
Aupdate_freq=Aupdate_freq,
maximize=True,
random_seed=None,
optim_params={})
self.max_steps = max_step
self.corner = corner # up left corner of the image
self.imgsize = imgsize # size of image, allowing showing CNN resized image
self.savedir = savedir
self.explabel = explabel
self.Perturb_vec = []
help='EPS of finite differencing HVP operator, will only be '
'used when method is `ForwardIter`')
args = parser.parse_args(
) # ["--dataset", "pasu", '--method', "BP", '--idx_rg', '0', '50', '--EPS', '1E-2'])
#%%
from FeatLinModel import FeatLinModel, get_model_layers
import models # from PerceptualSimilarity
model_squ = models.PerceptualLoss(model='net-lin',
net='squeeze',
use_gpu=1,
gpu_ids=[0])
model_squ.requires_grad_(False).cuda()
from GAN_utils import upconvGAN
if args.GAN in ["fc6", "fc7", "fc8"]:
G = upconvGAN(args.GAN)
elif args.GAN == "fc6_shfl":
G = upconvGAN("fc6")
SD = G.state_dict()
shuffled_SD = {}
for name, Weight in SD.items():
idx = torch.randperm(Weight.numel())
W_shuf = Weight.view(-1)[idx].view(Weight.shape)
shuffled_SD[name] = W_shuf
G.load_state_dict(shuffled_SD)
elif args.GAN == "fc6_shfl_fix":
G = upconvGAN("fc6", pretrained=False)
G.load_state_dict(
torch.load(
"/scratch/binxu/torch/checkpoints/upconvGAN_fc6_shuffle.pt"))
G.requires_grad_(False).cuda() # this notation is incorrect in older pytorch
Python_dir = r"C:\Users\Poncelab-ML2a\Documents\Python"
elif os.environ['COMPUTERNAME'] == 'DESKTOP-MENSD6S':
Python_dir = r"E:\Github_Projects"
elif os.environ['COMPUTERNAME'] == 'DESKTOP-9DDE2RH':
Python_dir = r"D:\Github"
sys.path.append(join(Python_dir,"Visual_Neuro_InSilico_Exp"))
sys.path.append(join(Python_dir,"PerceptualSimilarity"))
import torch
from GAN_utils import upconvGAN
from GAN_hvp_operator import GANHVPOperator, compute_hessian_eigenthings
import models # from PerceptualSimilarity folder
from build_montages import build_montages
# model_vgg = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=1, gpu_ids=[0])
model_squ = models.PerceptualLoss(model='net-lin', net='squeeze', use_gpu=1, gpu_ids=[0])
G = upconvGAN(name="fc6")
G.cuda()
model_squ.cuda()
for param in model_squ.parameters():
param.requires_grad_(False)
for param in G.parameters():
param.requires_grad_(False)
#%% Test code for hessian eigendecomposition
#t0 = time()
#feat = torch.randn((1, 4096), dtype=torch.float32).requires_grad_(False).cuda()
#eigenvals, eigenvecs = compute_hessian_eigenthings(G, feat, model_squ,
# num_eigenthings=300, mode="lanczos", use_gpu=True,)
#print(time() - t0,"\n") # 81.02 s
#%% Load the codes from the Backup folder
import os
from scipy.io import loadmat
ax2.scatter(generations, norms_all, s=5, color="magenta", label="all", alpha=0.2)
ax2.set_ylabel("L2 Norm", color="red", fontsize=14)
plt.legend()
plt.title("Optimization Trajectory of Score\n" + title_str)
plt.legend()
if show:
plt.show()
return figh
#%% Select GAN
BGAN = BigGAN.from_pretrained("biggan-deep-256")
BGAN.eval().cuda()
for param in BGAN.parameters():
param.requires_grad_(False)
G = BigGAN_wrapper(BGAN)
#%%
G = upconvGAN("fc6")
G.eval().cuda()
for param in G.parameters():
param.requires_grad_(False)
#%%
# net = tv.alexnet(pretrained=True)
from insilico_Exp import TorchScorer, ExperimentEvolve
scorer = TorchScorer("alexnet")
scorer.select_unit(("alexnet", "fc6", 2))
#%%
imgs = G.visualize(torch.randn(3, 256).cuda()).cpu()
scores = scorer.score_tsr(imgs)
#%%
"""
This code majoyly compare our implementation of Cholesky CMAES with the cma from official package
import torch
from GAN_utils import upconvGAN
G = upconvGAN()
paramnum = 0
for param in G.G.parameters():
print(param.shape)
paramnum += torch.prod(param.shape)
#%% How to compress subsequent layer using
import numpy as np
from GAN_utils import upconvGAN
import torch
from GAN_hvp_operator import compute_hessian_eigenthings, get_full_hessian, GANForwardHVPOperator, GANHVPOperator
import matplotlib.pylab as plt
import matplotlib
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
from os.path import join
# import seaborn as sns
#%%
G = upconvGAN()
G.G.requires_grad_(False)
layernames = [name for name, _ in G.G.named_children()]
#%%
def Hess_hook(module, fea_in, fea_out):
print("hooker on %s" % module.__class__)
ref_feat = fea_out.detach().clone()
ref_feat.requires_grad_(False)
L2dist = torch.pow(fea_out - ref_feat, 2).sum()
L2dist_col.append(L2dist)
return None
#%%
feat = torch.randn(4096, requires_grad=True)
archdir = r"E:\OneDrive - Washington University in St. Louis\HessNetArchit"
#%%
layernames = [name for name, _ in G.G.named_children()]
'''
Experimental Code to generate samples in selected PC space from an experiment
depending on `utils` for code loading things
'''
from scipy.io import loadmat
import os
import numpy as np
import re
from sklearn.linear_model import LinearRegression
from sklearn.decomposition import PCA
import matplotlib.pylab as plt
from utils import generator
import utils
from GAN_utils import upconvGAN
G = upconvGAN("fc6").cuda()
#%%
'''
Input the experimental backup folder containing the mat codes files.
'''
backup_dir = r"N:\Stimuli\2021-ProjectPFC\2021-Evolutions\2021-06-30-Caos-01\2021-06-30-10-52-27"
newimg_dir = os.path.join(backup_dir, "PC_imgs")
#%%
os.makedirs(newimg_dir, exist_ok=True)
print("Save new images to folder %s", newimg_dir)
#%%
print("Loading the codes from experiment folder %s", backup_dir)
codes_all, generations = utils.load_codes_mat(backup_dir)
generations = np.array(generations)
print("Shape of code", codes_all.shape)
"""
This Code try to create a shuffled control for what we found for Hessian.
"""
from GAN_utils import upconvGAN
import torch
from torchvision.utils import make_grid
from torchvision.transforms import ToPILImage
G = upconvGAN("fc6",pretrained=True)
SD = G.state_dict()
#%% Shuffle the weight matrix of each layer of GAN
shuffled_SD = {}
for name, Weight in SD.items():
idx = torch.randperm(Weight.numel())
W_shuf = Weight.view(-1)[idx].view(Weight.shape)
shuffled_SD[name] = W_shuf
#%%
torch.save(shuffled_SD, "upconvGAN_fc6_shuffle.pt")
# print(name, Weight.shape, Weight.mean().item(), Weight.std().item())
#%%
G_sf = upconvGAN("fc6")
G_sf.load_state_dict(torch.load("upconvGAN_fc6_shuffle.pt"))
#%%
img = G_sf.visualize(torch.randn(10, 4096))
ToPILImage()(make_grid(img[:,:])).show()
#%%
W = SD['G.defc7.weight']
idx = torch.randperm(W.numel())
W_shuf = W.view(-1)[idx].view(W.shape)
#%%
with torch.no_grad():
