def get_official_model_weights():
# Initializing the pytorch_model
# pytorch_model.model.net.module.net: vgg16
# pytorch_model.model.net.module.lins: linear layer
pytorch_model = models.PerceptualLoss(model='net-lin',
net='vgg',
use_gpu=True)
# extract pytorch tensors
vgg_model_weights = OrderedDict()
for k, v in pytorch_model.model.net.module.net.named_parameters():
vgg_model_weights[k] = v.data
model_path = './models/weights/v0.1/vgg.pth'
lin_model_weights = torch.load(model_path)
# convert to numpy arrays
vgg_model_weights_np = OrderedDict({
name: tensor.cpu().data.numpy()
for name, tensor in vgg_model_weights.items()
})
lin_model_weights_np = OrderedDict({
name: tensor.cpu().data.numpy()
for name, tensor in lin_model_weights.items()
})
return vgg_model_weights_np, lin_model_weights_np
def similarity(request):
use_gpu = False
spatial = False
model = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=use_gpu,
spatial=spatial)
img1_filename = request.POST['img1'].filename
img1_input_file = request.POST['img1'].file
img1_file_path = os.path.join('/tmp', img1_filename)
with open(img1_file_path, 'wb') as img1_output_file:
shutil.copyfileobj(img1_input_file, img1_output_file)
img1tensor = util.im2tensor(util.load_image(img1_file_path))
img2_filename = request.POST['img2'].filename
img2_input_file = request.POST['img2'].file
img2_file_path = os.path.join('/tmp', img2_filename)
with open(img2_file_path, 'wb') as img2_output_file:
shutil.copyfileobj(img2_input_file, img2_output_file)
img2tensor = util.im2tensor(util.load_image(img2_file_path))
d = model.forward(img1tensor, img2tensor)
return Response('Distancia: %.3f' % d)
def precompute(self):
self.FL_w = FeatureLoss(self.args.vgg_weight_dir,
self.args.vgg_layer_weight_w)
for p in self.FL_w.parameters():
p.requires_grad = False
self.FL_n = FeatureLoss(self.args.vgg_weight_dir,
self.args.vgg_layer_weight_n)
for p in self.FL_n.parameters():
p.requires_grad = False
self.FL_wn = FeatureLoss(self.args.vgg_weight_dir,
self.args.vgg_layer_weight_wn)
for p in self.FL_wn.parameters():
p.requires_grad = False
self.SL = StyleLoss()
for p in self.SL.parameters():
p.requires_grad = False
self.LPIPS = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=True,
gpu_ids=[0])
self.Laplacian = kornia.filters.Laplacian(3, normalized=False)
if self.args.embed_tex:
self.albedo_ref_pix, self.normal_ref_pix, self.rough_ref_pix, self.specular_ref_pix, \
self.albedo_ref_vgg, self.normal_ref_vgg, self.rough_ref_vgg, self.specular_ref_vgg = \
self.eval_texture_vgg(self.textures_ref)
self.fl_albedo_ref_w = self.FL_w(self.albedo_ref_vgg)
self.fl_normal_ref_w = self.FL_w(self.normal_ref_vgg)
self.fl_rough_ref_w = self.FL_w(self.rough_ref_vgg)
self.fl_specular_ref_w = self.FL_w(self.specular_ref_vgg)
self.fl_albedo_ref_n = self.FL_n(self.albedo_ref_vgg)
self.fl_normal_ref_n = self.FL_n(self.normal_ref_vgg)
self.fl_rough_ref_n = self.FL_n(self.rough_ref_vgg)
self.fl_specular_ref_n = self.FL_n(self.specular_ref_vgg)
self.fl_albedo_ref_wn = self.FL_wn(self.albedo_ref_vgg)
self.fl_normal_ref_wn = self.FL_wn(self.normal_ref_vgg)
self.fl_rough_ref_wn = self.FL_wn(self.rough_ref_vgg)
self.fl_specular_ref_wn = self.FL_wn(self.specular_ref_vgg)
else:
self.rendered_ref_pix = self.rendered_ref
self.rendered_ref_vgg = self.eval_render_vgg(
self.rendered_ref, self.args.jittering)
if self.args.jittering:
self.sl_rendered_ref = self.SL(self.rendered_ref_vgg)
else:
self.fl_rendered_ref_w = self.FL_w(self.rendered_ref_vgg)
self.fl_rendered_ref_n = self.FL_n(self.rendered_ref_vgg)
self.fl_rendered_ref_wn = self.FL_wn(self.rendered_ref_vgg)
def main():
args = get_trainer_args()
def load_videos(path):
print("Loading trajectories from {}".format(path))
if not path.endswith('.npy'):
raise ValueError("Can only read in .npy files!")
seqs = np.load(path)
assert len(seqs.shape) == 5 # need [batch, T, C, H, W] input data
assert seqs.shape[
2] == 3 # assume 3-channeled seq with channel in last dim
seqs = torch.Tensor(seqs)
if args.use_gpu: seqs = seqs.cuda()
return seqs # range [-1, 1]
gt_seqs = load_videos(args.gt)
pred_seqs = load_videos(args.pred)
print('shape: ', gt_seqs.shape)
assert gt_seqs.shape == pred_seqs.shape
n_seqs, time, c, h, w = gt_seqs.shape
n_batches = int(np.floor(n_seqs / args.batch_size))
# import pdb; pdb.set_trace()
# get sequence mask (for sequences with variable length
mask = 1 - torch.all(torch.all(torch.all(
(gt_seqs + 1.0).abs() < 1e-6, dim=-1),
dim=-1),
dim=-1) # check for black images
mask2 = 1 - torch.all(torch.all(torch.all((gt_seqs).abs() < 1e-6, dim=-1),
dim=-1),
dim=-1) # check for gray images
mask = mask * mask2
# Initializing the model
model = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=args.use_gpu)
# run forward pass to compute LPIPS distances
distances = []
for b in range(n_batches):
x, y = gt_seqs[b * args.batch_size:(b + 1) *
args.batch_size], pred_seqs[b *
args.batch_size:(b + 1) *
args.batch_size]
lpips_dist = batch_apply((x, y), model, separate_arguments=True)
distances.append(lpips_dist)
distances = torch.cat(distances)
mean_distance = distances[mask].mean()
print("LPIPS distance: {}".format(mean_distance))
def compute_dists_pair(use_gpu, dir, out):
## Initializing the model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu)
dists = []
# crawl directories
# f = open(out,'w')
# category_files = os.listdir(opt.dir)
# print('category_file', category_files)
i = 0
for dir_path_class, dir_name_class, file_names_class in os.walk(dir):
# print('1',dir_path_class)
# print('2',dir_name_class)
# print('3',file_names_class)
dists_category = []
for (ff, file0) in enumerate(file_names_class[:-1]):
if ff < 10:
# print('here',file0)
# print(os.path.exists(os.path.join(dir_path_class,file0)))
img0 = util.im2tensor(
util.load_image(os.path.join(
dir_path_class, file0))) # RGB image from [-1,1]
if (use_gpu):
img0 = img0.cuda()
for (gg, file1) in enumerate(file_names_class[ff + 1:]):
img1 = util.im2tensor(
util.load_image(os.path.join(dir_path_class, file1)))
if (use_gpu):
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1).item()
dists_category.append(dist01)
else:
# print('continue')
dists_category_mean = np.mean(np.array(dists_category))
print('{}_cateogory {}_samples lpips is {}'.format(
i, len(dists_category), dists_category_mean))
dists.append(dists_category_mean)
break
i = i + 1
# if i > 5:
# break
# print('(%s, %s): %.3f'%(file0,file1,dist01))
# f.writelines('(%s, %s): %.3f'%(file0,file1,dist01))
dist_mean = np.mean(np.array(dists))
print('Mean: %.3f' % dist_mean)
return dists, dist_mean
def __init__(self, net='vgg', use_gpu=True, precision='float'):
""" LPIPS loss with spatial weighting """
super(PerceptualLoss, self).__init__()
self.lpips = models.PerceptualLoss(model='net-lin',
net=net,
spatial=True,
use_gpu=use_gpu)
if use_gpu:
self.lpips = nn.DataParallel(self.lpips).cuda()
if precision == 'half':
self.lpips.half()
elif precision == 'float':
self.lpips.float()
elif precision == 'double':
self.lpips.double()
return
def __init__(self, net='vgg', use_gpu=True, lpips_dir=None):
""" LPIPS loss with spatial weighting """
super(PerceptualLoss, self).__init__()
lpips_dir = self.download(lpips_dir=lpips_dir)
# consider removing from path after importing
sys.path.insert(1, lpips_dir)
with HiddenPrints():
import models
self.lpips = models.PerceptualLoss(model='net-lin',
net=net,
spatial=True,
use_gpu=use_gpu)
self.lpips = nn.DataParallel(self.lpips).cuda()
return
else:
BGAN = BigGAN.from_pretrained("biggan-deep-256")
sys.path.append(r"D:\Github\PerceptualSimilarity")
sys.path.append(r"E:\Github_Projects\PerceptualSimilarity")
Hpath = r"E:\OneDrive - Washington University in St. Louis\Hessian_summary\BigGAN\H_avg_1000cls.npz"
if args.dataset == "BigGAN_rnd":
imgfolder = r"E:\Cluster_Backup\Datasets\BigGAN_rnd"
savedir = r"E:\OneDrive - Washington University in St. Louis\BigGAN_invert\BasinCMA\BigGAN_rnd"
elif args.dataset == "ImageNet":
imgfolder = r"E:\Cluster_Backup\Datasets\ImageTranslation\GAN_real\B\train"
savedir = r"E:\OneDrive - Washington University in St. Louis\BigGAN_invert\BasinCMA\ImageNet"
# savedir = r"E:\Cluster_Backup\BigGAN_invert\ImageNet"
#%%
import models # from PerceptualSimilarity folder
ImDist = models.PerceptualLoss(model='net-lin', net=args.ImDist, use_gpu=1, gpu_ids=[0])
for param in ImDist.parameters():
param.requires_grad_(False)
def L1loss(target, img):
return (img - target).abs().sum(axis=1).mean(axis=1)
# alpha = 5 # relative weight
BGAN.cuda()
BGAN.eval()
for param in BGAN.parameters():
param.requires_grad_(False)
#%% Load the precomputed Hessian and Form them as basis matrix
data = np.load(Hpath)
evc_clas = torch.from_numpy(data['eigvects_clas_avg'])#.cuda()
evc_nois = torch.from_numpy(data['eigvects_nois_avg'])#.cuda()
evc_all = torch.from_numpy(data['eigvects_avg']).cuda()
def evaluation():
parser = config_parser()
args = parser.parse_args()
# Load data
if args.dataset_type == 'llff':
target_idx = args.target_idx
images, poses, bds, render_poses = load_nvidia_data(args.datadir,
args.start_frame, args.end_frame,
args.factor,
target_idx=target_idx,
recenter=True, bd_factor=.9,
spherify=args.spherify,
final_height=args.final_height)
hwf = poses[0,:3,-1]
poses = poses[:,:3,:4]
print('Loaded llff', images.shape, render_poses.shape, hwf, args.datadir)
# if not isinstance(i_test, list):
i_test = []
i_val = [] #i_test
i_train = np.array([i for i in np.arange(int(images.shape[0])) if
(i not in i_test and i not in i_val)])
print('DEFINING BOUNDS')
if args.no_ndc:
near = np.percentile(bds[:, 0], 5) * 0.9 #np.ndarray.min(bds) #* .9
far = np.percentile(bds[:, 1], 95) * 1.1 #np.ndarray.max(bds) #* 1.
else:
near = 0.
far = 1.
print('NEAR FAR', near, far)
else:
print('ONLY SUPPORT LLFF!!!!!!!!')
sys.exit()
# Cast intrinsics to right types
H, W, focal = hwf
H, W = int(H), int(W)
hwf = [H, W, focal]
# Create log dir and copy the config file
basedir = args.basedir
args.expname = args.expname + '_F%02d-%02d'%(args.start_frame,
args.end_frame)
expname = args.expname
os.makedirs(os.path.join(basedir, expname), exist_ok=True)
f = os.path.join(basedir, expname, 'args.txt')
with open(f, 'w') as file:
for arg in sorted(vars(args)):
attr = getattr(args, arg)
file.write('{} = {}\n'.format(arg, attr))
if args.config is not None:
f = os.path.join(basedir, expname, 'config.txt')
with open(f, 'w') as file:
file.write(open(args.config, 'r').read())
# Create nerf model
render_kwargs_train, render_kwargs_test, \
start, grad_vars, optimizer = create_nerf(args)
global_step = start
bds_dict = {
'near' : near,
'far' : far,
}
render_kwargs_train.update(bds_dict)
render_kwargs_test.update(bds_dict)
num_img = float(images.shape[0])
poses = torch.Tensor(poses).to(device)
with torch.no_grad():
model = models.PerceptualLoss(model='net-lin',net='alex',
use_gpu=True,version=0.1)
total_psnr = 0.
total_ssim = 0.
total_lpips = 0.
count = 0.
total_psnr_dy = 0.
total_ssim_dy = 0.
total_lpips_dy = 0.
t = time.time()
# for each time step
for img_i in i_train:
img_idx_embed = img_i/num_img * 2. - 1.0
# for each target viewpoint
for camera_i in range(0, 12):
print(time.time() - t)
t = time.time()
print(img_i, camera_i)
if img_i % 12 == camera_i:
continue
c2w = poses[camera_i]
ret = render(img_idx_embed, 0, False,
num_img,
H, W, focal,
chunk=1024*16, c2w=c2w[:3,:4],
**render_kwargs_test)
rgb = ret['rgb_map_ref'].cpu().numpy()#.append(ret['rgb_map_ref'].cpu().numpy())
gt_img_path = os.path.join(args.datadir,
'mv_images',
'%05d'%img_i,
'cam%02d.jpg'%(camera_i + 1))
# print('gt_img_path ', gt_img_path)
gt_img = cv2.imread(gt_img_path)[:, :, ::-1]
gt_img = cv2.resize(gt_img,
dsize=(rgb.shape[1], rgb.shape[0]),
interpolation=cv2.INTER_AREA)
gt_img = np.float32(gt_img) / 255
psnr = skimage.measure.compare_psnr(gt_img, rgb)
ssim = skimage.measure.compare_ssim(gt_img, rgb,
multichannel=True)
gt_img_0 = im2tensor(gt_img).cuda()
rgb_0 = im2tensor(rgb).cuda()
lpips = model.forward(gt_img_0, rgb_0)
lpips = lpips.item()
print(psnr, ssim, lpips)
total_psnr += psnr
total_ssim += ssim
total_lpips += lpips
count += 1
dynamic_mask_path = os.path.join(args.datadir,
'mv_masks',
'%05d'%img_i,
'cam%02d.png'%(camera_i + 1))
print(dynamic_mask_path)
dynamic_mask = np.float32(cv2.imread(dynamic_mask_path) > 1e-3)#/255.
dynamic_mask = cv2.resize(dynamic_mask,
dsize=(rgb.shape[1], rgb.shape[0]),
interpolation=cv2.INTER_NEAREST)
dynamic_mask_0 = torch.Tensor(dynamic_mask[:, :, :, np.newaxis].transpose((3, 2, 0, 1)))
dynamic_ssim = calculate_ssim(gt_img,
rgb,
dynamic_mask)
dynamic_psnr = calculate_psnr(gt_img,
rgb,
dynamic_mask)
dynamic_lpips = model.forward(gt_img_0,
rgb_0,
dynamic_mask_0).item()
total_psnr_dy += dynamic_psnr
total_ssim_dy += dynamic_ssim
total_lpips_dy += dynamic_lpips
mean_psnr = total_psnr / count
mean_ssim = total_ssim / count
mean_lpips = total_lpips / count
print('mean_psnr ', mean_psnr)
print('mean_ssim ', mean_ssim)
print('mean_lpips ', mean_lpips)
mean_psnr_dy = total_psnr_dy / count
mean_ssim_dy = total_ssim_dy / count
mean_lpips_dy= total_lpips_dy / count
print('mean_psnr dy', mean_psnr_dy)
print('mean_ssim dy', mean_ssim_dy)
print('mean_lpips dy', mean_lpips_dy)
def main():
# handle command line arguments
ap = argparse.ArgumentParser()
ap.add_argument('-d0', '--dir0', required=True, type=str, default='./imgs/ex_dir0',
help='Reference images directory')
ap.add_argument('-m0', '--mask0', required=False, type=str, default='*.png',
help='file mask for files in d0 (e.g. .png)')
ap.add_argument('-d1', '--dir1', required=True, type=str, default='./imgs/ex_dir1', help='Other images directory')
ap.add_argument('-m1', '--mask1', required=False, type=str, default='*.png',
help='file mask for files in d1 (e.g. .png)')
ap.add_argument('-o', '--out', required=False, type=str, default='./results/',
help='Distance files (.csv) directory')
ap.add_argument('--use_gpu', action='store_true', help='Flag to use GPU to compute distance')
ap.add_argument('-w', '--weights', default='./models/face_models/mmod_human_face_detector.dat',
help='path to face model file')
ap.add_argument('-H', '--hog_detector', required=False, action='store_true', help='use HOG detector')
ap.add_argument('-A', '--haar_detector', required=False, action='store_true', help='use Haar detector')
ap.add_argument('-C', '--cnn_detector', required=False, action='store_true', help='use CNN-based detector')
ap.add_argument('-N', '--no_face_detector', required=False, action='store_true',
help='Do NOT perform face detection. Compute quality over whole image')
ap.add_argument('-v', '--verbose', required=False, help='verbose output', action='store_true')
args = ap.parse_args()
print("Input dir0:", args.dir0)
print("Input dir0 file mask:", args.mask0)
print("Input dir1:", args.dir1)
print("Input dir1 file mask:", args.mask1)
print("Output dir: ", args.out)
if not os.path.exists(args.out):
os.makedirs(args.out)
if args.use_gpu:
print("Compute LPIPS similarity using GPU (fast)")
else:
print("Compute LPIPS similarity using CPU (slow)")
if args.no_face_detector:
print("Do NOT perform face detection. Compute frame quality over the whole image.")
file_det_name = "-no-face"
else:
if args.hog_detector:
print("Find faces using DLib HOG detector")
if args.haar_detector:
print("Find faces using OpenCV Haar detector")
if args.cnn_detector:
print("Find faces using DLib CNN detector")
if not args.hog_detector and not args.haar_detector and not args.cnn_detector:
args.haar_detector = True
print("No face detector selected. Using default OpenCV Haar detector")
file_det_name = ""
# it is expected that directories contain same number of files that can be sorted so that one is associated with the
# file in the corresponding position in the other directory
dir0_files = []
for file in os.listdir(args.dir0):
if fnmatch.fnmatch(file, args.mask0):
dir0_files.append(file)
dir0_files.sort()
print("Dir 0 (ref images) contains " + str(len(dir0_files)) + " " + args.mask0 + " files.")
dir1_files = []
for file in os.listdir(args.dir1):
if fnmatch.fnmatch(file, args.mask1):
dir1_files.append(file)
dir1_files.sort()
print("Dir 1 (mod images) contains " + str(len(dir1_files)) + " " + args.mask1 + " files.")
# Initializing face detector
fd = FaceDetector()
# Initializing LPIPS perceptual quality model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=args.use_gpu)
brisque_model = cv2.quality_QualityBRISQUE.create(BRISQUE_MODEL_FILE, BRISQUE_RANGE_FILE)
# process all files
num_all_files = len(dir0_files)
# open files and write headers
out_file_base_name = "all_files-" + os.path.basename(args.dir0) + "-" + os.path.basename(
args.dir1) + file_det_name
print("Base file name:" + out_file_base_name)
out_file_LPIPS_all = open(
os.path.join(args.out, out_file_base_name + "-LPIPS.csv"), 'w')
out_file_MSSIM_RGB_all = open(
os.path.join(args.out, out_file_base_name + "-MSSIM-RGB.csv"), 'w')
out_file_MSSIM_Y_all = open(
os.path.join(args.out, out_file_base_name + "-MSSIM-Y.csv"), 'w')
out_file_BRISQUE_all = open(
os.path.join(args.out, out_file_base_name + "-BRISQUE.csv"), 'w')
out_file_BRISQUE_all.writelines('files, BRISQUE score ref, BRISQUE score mod\n')
out_file_MSSIM_RGB_all.writelines('files, SSIM R, SSIM G, SSIM B\n')
out_file_MSSIM_Y_all.writelines('files, SSIM Y\n')
out_file_LPIPS_all.writelines('files, LPIPS distance\n')
processed_file = 1
for reference_image_name, modified_image_name in zip(dir0_files, dir1_files):
if processed_file % 50 == 0:
print("Processing ref. image " + str(processed_file) + "/" + str(num_all_files))
# open reference and modified images
ref_image = cv2.imread(os.path.join(args.dir0, reference_image_name))
mod_image = cv2.imread(os.path.join(args.dir1, modified_image_name))
if ref_image is None or mod_image is None:
# print("Skipping images: {} and {}".format(reference_image_name, modified_image_name))
continue
filename_all = os.path.splitext(reference_image_name)[0] + "-" + os.path.splitext(modified_image_name)[0]
if args.no_face_detector: # compute quality over whole image. No face detection
ref_face_regions = [ref_image]
mod_face_regions = [mod_image]
else: # perform face detection. use face regions to compute quality
fd.process_frame(ref_image, use_cnn=args.cnn_detector, use_hog=args.hog_detector,
use_haar=args.haar_detector)
detected_faces = fd.convert_dlib_rectangles_to_opencv_faces(fd.cnn_faces) + \
fd.convert_dlib_rectangles_to_opencv_faces(fd.hog_faces) + \
fd.convert_dlib_rectangles_to_opencv_faces(fd.haar_faces)
resized_faces = resize_64x64_multiple_faces_list(detected_faces)
ref_face_regions = get_face_regions(ref_image, resized_faces)
mod_face_regions = get_face_regions(mod_image, resized_faces)
for ref_face_region, mod_face_region in zip(ref_face_regions, mod_face_regions):
MSSIM_Y_dist = compute_MSSIM(cv2.cvtColor(ref_face_region, cv2.COLOR_RGB2GRAY),
cv2.cvtColor(mod_face_region, cv2.COLOR_RGB2GRAY))
MSSIM_RGB_dist = compute_MSSIM(ref_face_region, mod_face_region)
BRISQUE_score_ref = brisque_model.compute(ref_face_region)
BRISQUE_score_mod = brisque_model.compute(mod_face_region)
print("{}, {:.6f}, {:.6f}".format(filename_all, BRISQUE_score_ref[0], BRISQUE_score_mod[0]),
file=out_file_BRISQUE_all)
print('{}, {:.6f}'.format(filename_all, round(MSSIM_Y_dist[0] * 100, 2)), file=out_file_MSSIM_Y_all)
print('{}, {:.6f}, {:.6f}, {:.6f}'.format(filename_all, round(MSSIM_RGB_dist[2] * 100, 2),
round(MSSIM_RGB_dist[1] * 100, 2),
round(MSSIM_RGB_dist[0] * 100, 2)), file=out_file_MSSIM_RGB_all)
if args.no_face_detector:
ref_face_blocks = get_64x64_full_image_regions(ref_face_region)
mod_face_blocks = get_64x64_full_image_regions(mod_face_region)
else:
ref_face_blocks = get_64x64_face_regions(ref_face_region)
mod_face_blocks = get_64x64_face_regions(mod_face_region)
LPIPS_dist = 0
for ref_face_block, mod_face_block in zip(ref_face_blocks, mod_face_blocks):
img0 = util.im2tensor(cv2.cvtColor(ref_face_block, cv2.COLOR_BGR2RGB)) # RGB image from [-1,1]
img1 = util.im2tensor(cv2.cvtColor(mod_face_block, cv2.COLOR_BGR2RGB))
if args.use_gpu:
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
LPIPS_dist += model.forward(img0, img1)
LPIPS_dist /= len(ref_face_blocks)
out_file_LPIPS_all.writelines('%s, %.6f\n' % (filename_all, LPIPS_dist))
if DEBUG:
processed_image = draw_faces(ref_image.copy(), cnn_faces=fd.cnn_faces, hog_faces=fd.hog_faces,
haar_faces=fd.haar_faces)
cv2.imshow('Processed image', processed_image)
out_img_filename = "/tmp/" + reference_image_name + ".jpg"
cv2.imwrite(out_img_filename, processed_image)
deb_face_count = 0
ref_face_regions = get_face_regions(ref_image, resized_faces)
for ref_detected_face in ref_face_regions:
out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str(deb_face_count) + ".jpg"
cv2.imwrite(out_img_filename, ref_detected_face)
ref_face_blocks = get_64x64_face_regions(ref_detected_face)
deb_block_count = 0
for ref_face_block in ref_face_blocks:
out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str(
deb_face_count) + "_-_block_" + str(deb_block_count) + ".jpg"
cv2.imwrite(out_img_filename, ref_face_block)
deb_block_count += 1
deb_face_count += 1
if cv2.waitKey(1) == 27:
break # esc to quit
processed_file += 1
if DEBUG:
cv2.destroyAllWindows()
out_file_LPIPS_all.close()
out_file_MSSIM_RGB_all.close()
out_file_MSSIM_Y_all.close()
out_file_BRISQUE_all.close()
# ===================forward=====================
output = model(img)
loss = criterion(output, img)
# ===================backward====================
optimizer.zero_grad()
loss.backward()
optimizer.step()
# ===================log========================
writer.add_scalar('loss', loss, epoch)
if epoch % 5 == 0:
writer.add_images('reconstruct', output, epoch)
perceptual_loss = PerceptualSimilarity.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=True,
gpu_ids=[0])
L2_loss = nn.MSELoss(reduction='none')
total_auc = 0
total_acc = 0
total_image = 0
for index, (img, mask) in enumerate(test_loader):
img = Variable(img).cuda()
mask = Variable(mask).cuda()
# 取得 model 輸出
output = model(img)
# 計算 dif (相似度以及 L2)
dif = perceptual_loss.forward(output, img)
from time import time
from importlib import reload
import re
import numpy as np
import matplotlib
matplotlib.use('Agg') # if you dont want image show up
import matplotlib.pylab as plt
import torch
sys.path.append("D:\Github\pytorch-caffe")
from caffenet import *
import utils
from utils import generator
from insilico_Exp import CNNmodel
sys.path.append(r"D:\Github\PerceptualSimilarity")
import models
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=1, gpu_ids=[0])
#%%
hess_dir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\Artiphysiology\Hessian"
output_dir = r"D:\Generator_DB_Windows\data\with_CNN\hessian"
def perturb_images_sphere(cent_vec, perturb_vec, PC2_ang_step = 18, PC3_ang_step = 18):
sphere_norm = np.linalg.norm(cent_vec)
vectors = np.zeros((3, cent_vec.size))
vectors[ 0, :] = cent_vec / sphere_norm
vectors[1:3, :] = perturb_vec
img_list = []
for j in range(-5, 6):
for k in range(-5, 6):
theta = PC2_ang_step * j / 180 * np.pi
phi = PC3_ang_step * k / 180 * np.pi
code_vec = np.array([[np.cos(theta) * np.cos(phi),
np.sin(theta) * np.cos(phi),
def main():
dataSize = 128
batchSize = 8
# imageSize = 32
imageSize = 64
# discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth'
# discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth'
discCheckpointPath = None
gpu = torch.device('cuda')
# imageDataset = CatDataset(
# imageSubdirPath=r'E:\data\cat-vs-dog\cat',
# transform=transforms.Compose(
# [
# transforms.Resize((imageSize, imageSize)),
# transforms.ToTensor(),
# transforms.Normalize([0.5], [0.5])
# ]
# )
# )
imageDataset = datasets.CIFAR10(root=r'e:\data\images\cifar10', download=True,
transform=transforms.Compose([
transforms.Resize((imageSize, imageSize)),
transforms.ToTensor(),
# transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
transforms.Normalize([0.5], [0.5]),
]))
# For now we normalize the vectors to have norm 1, but don't make sure
# that the data has certain mean/std.
pointDataset = AuthorDataset(
jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json'
)
imageLoader = DataLoader(imageDataset, batch_size=batchSize, sampler=InfiniteSampler(imageDataset))
pointLoader = DataLoader(pointDataset, batch_size=batchSize, sampler=InfiniteSampler(pointDataset))
# Generate a random distance matrix.
# # Make a matrix with positive values.
# distancesCpu = np.clip(np.random.normal(0.5, 1.0 / 3, (dataSize, dataSize)), 0, 1)
# # Make it symmetrical.
# distancesCpu = np.matmul(distancesCpu, distancesCpu.T)
# Generate random points and compute distances, guaranteeing that the triangle rule isn't broken.
# randomPoints = generate_points(dataSize)
# distancesCpu = scipy.spatial.distance_matrix(randomPoints, randomPoints, p=2)
# catImagePath = os.path.expandvars(r'${DEV_METAPHOR_DATA_PATH}/cats/cat.247.jpg')
# catImage = skimage.transform.resize(imageio.imread(catImagePath), (64, 64), 1).transpose(2, 0, 1)
# imagesInitCpu = np.clip(np.random.normal(0.5, 0.5 / 3, (dataSize, 3, imageSize, imageSize)), 0, 1)
# imagesInitCpu = np.clip(np.tile(catImage, (dataSize, 1, 1, 1)) + np.random.normal(0., 0.5 / 6, (dataSize, 3, 64, 64)), 0, 1)
# images = torch.tensor(imagesInitCpu, requires_grad=True, dtype=torch.float32, device=gpu)
scale = torch.tensor(4.0, requires_grad=True, dtype=torch.float32, device=gpu)
lossModel = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu)
bceLoss = torch.nn.BCELoss()
# discriminator = Discriminator(imageSize, 3)
discriminator = Discriminator(3, 64, 1)
if discCheckpointPath:
discriminator.load_state_dict(torch.load(discCheckpointPath))
else:
discriminator.init_params()
discriminator = discriminator.to(gpu)
generator = Generator(nz=pointDataset[0][0].shape[0], ngf=64)
generator.init_params()
generator = generator.to(gpu)
# todo init properly, if training
# discriminator.apply(weights_init_normal)
# optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999))
optimizerScale = torch.optim.Adam([scale], lr=0.001)
optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))
# optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999))
optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))
import matplotlib.pyplot as plt
fig, axes = plt.subplots(nrows=2 * 2, ncols=batchSize // 2)
fig2 = plt.figure()
ax2 = fig2.add_subplot(1, 1, 1)
outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S'))
os.makedirs(outPath)
imageIter = iter(imageLoader)
pointIter = iter(pointLoader)
for batchIndex in range(10000):
imageBatchReal, _ = next(imageIter) # type: Tuple(torch.Tensor, Any)
imageBatchReal = imageBatchReal.to(gpu)
# imageBatchReal = torch.tensor(realImageBatchCpu, device=gpu)
# noinspection PyTypeChecker
# randomIndices = np.random.randint(0, dataSize, batchSize).tolist() # type: List[int]
# # randomIndices = list(range(dataSize)) # type: List[int]
# distanceBatch = torch.tensor(distancesCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu)
# imageBatchFake = images[randomIndices].contiguous()
vectorBatch, _ = next(pointIter)
vectorBatch = vectorBatch.to(gpu)
distanceBatch = l2_sqr_dist_matrix(vectorBatch) # In-batch vector distances.
imageBatchFake = generator(vectorBatch[:, :, None, None].float())
# todo It's possible to compute this more efficiently, but would require re-implementing lpips.
distImages = lossModel.forward(imageBatchFake.repeat(repeats=(batchSize, 1, 1, 1)).contiguous(),
imageBatchFake.repeat_interleave(repeats=batchSize, dim=0).contiguous(), normalize=True)
distPredMat = distImages.reshape((batchSize, batchSize))
lossDist = torch.sum((distanceBatch - distPredMat * scale) ** 2) # MSE
discPred = discriminator(imageBatchFake)
lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu))
lossGen = lossDist + 1.0 * lossRealness
optimizerGen.zero_grad()
optimizerScale.zero_grad()
lossGen.backward()
optimizerGen.step()
optimizerScale.step()
lossDiscReal = bceLoss(discriminator(imageBatchReal), torch.ones(imageBatchReal.shape[0], device=gpu))
lossDiscFake = bceLoss(discriminator(imageBatchFake.detach()), torch.zeros(imageBatchFake.shape[0], device=gpu))
lossDisc = (lossDiscFake + lossDiscReal) / 2
# lossDisc = torch.tensor(0)
optimizerDisc.zero_grad()
lossDisc.backward()
optimizerDisc.step()
# with torch.no_grad():
# # todo We're clamping all the images every batch, can we clamp only the ones updated?
# # images = torch.clamp(images, 0, 1) # For some reason this was making the training worse.
# images.data = torch.clamp(images.data, 0, 1)
if batchIndex % 100 == 0:
msg = 'iter {}, loss gen {:.3f}, loss dist {:.3f}, loss real {:.3f}, loss disc {:.3f}, scale: {:.3f}'.format(
batchIndex, lossGen.item(), lossDist.item(), lossRealness.item(), lossDisc.item(), scale.item()
)
print(msg)
def gpu_images_to_numpy(images):
imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1)
imagesNumpy = (imagesNumpy + 1) / 2
return imagesNumpy
# print(discPred.tolist())
imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake)
imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal)
for i, ax in enumerate(axes.flatten()[:batchSize]):
ax.imshow(imageBatchFakeCpu[i])
for i, ax in enumerate(axes.flatten()[batchSize:]):
ax.imshow(imageBatchRealCpu[i])
fig.suptitle(msg)
with torch.no_grad():
points = np.asarray([pointDataset[i][0] for i in range(200)], dtype=np.float32)
images = gpu_images_to_numpy(generator(torch.tensor(points[..., None, None], device=gpu)))
authorVectorsProj = umap.UMAP(n_neighbors=5, random_state=1337).fit_transform(points)
plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2)
fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png'))
fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png'))
plt.close(fig)
plt.close(fig2)
with torch.no_grad():
imageNumber = 48
points = np.asarray([pointDataset[i][0] for i in range(imageNumber)], dtype=np.float32)
imagesGpu = generator(torch.tensor(points[..., None, None], device=gpu))
# Compute LPIPS distances, batch to avoid memory issues.
bs = 8
assert imageNumber % bs == 0
distImages = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0]))
for i in range(imageNumber // bs):
startA, endA = i * bs, (i + 1) * bs
imagesA = imagesGpu[startA:endA]
for j in range(imageNumber // bs):
startB, endB = j * bs, (j + 1) * bs
imagesB = imagesGpu[startB:endB]
distBatch = lossModel.forward(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(),
imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(),
normalize=True).cpu().numpy()
distImages[startA:endA, startB:endB] = distBatch.reshape((bs, bs))
# Move to the CPU and append an alpha channel for rendering.
images = gpu_images_to_numpy(imagesGpu)
images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images]
distPoints = l2_sqr_dist_matrix(torch.tensor(points, dtype=torch.double)).numpy()
assert np.abs(distPoints - distPoints.T).max() < 1e-5
distPoints = np.minimum(distPoints, distPoints.T) # Remove rounding errors, guarantee symmetry.
config = DistanceMatrixConfig()
config.dataRange = (0., 4.)
render_distance_matrix(os.path.join(outPath, f'dist_point_{batchIndex}.png'),
distPoints,
images,
config)
assert np.abs(distImages - distImages.T).max() < 1e-5
distImages = np.minimum(distImages, distImages.T) # Remove rounding errors, guarantee symmetry.
config = DistanceMatrixConfig()
config.dataRange = (0., 1.)
render_distance_matrix(os.path.join(outPath, f'dist_images_{batchIndex}.png'),
distImages,
images,
config)
torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))
torch.save(discriminator.state_dict(), os.path.join(outPath, 'disc_{}.pth'.format(batchIndex)))
"""
Use the fc6 fc7 GANs implemented in pure python from GAN_utils, and hessian package in pytorch.
to compute the Hessian and eigen decomposition.
"""
import numpy as np
import torch
import torch.optim as optim
import torch.nn.functional as F
from hessian import hessian
import sys
sys.path.append(r"D:\Github\PerceptualSimilarity")
sys.path.append(r"E:\Github_Projects\PerceptualSimilarity")
import models # from PerceptualSimilarity folder
model_vgg = models.PerceptualLoss(model='net-lin',
net='vgg',
use_gpu=1,
gpu_ids=[0])
from GAN_utils import upconvGAN
G = upconvGAN("fc6")
#%%
def sim_hessian_computation(z, percept_loss, savepath=None):
"""
Depending on Generator imported from caffe to pytorch.
Depending on Hessian, and autograd
:param z: vector to compute hessian at
:param percept_loss: the model from PerceptualSimilarity package
:return: H: Hessian Matrix
eigval: eigen decomposition, eigen values
def main():
dataSize = 128
batchSize = 8
# imageSize = 32
imageSize = 64
initWithCats = True
# discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth'
# discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth'
discCheckpointPath = None
gpu = torch.device('cuda')
imageRootPath = r'E:\data\cat-vs-dog\cat'
catDataset = CatDataset(
imageSubdirPath=imageRootPath,
transform=transforms.Compose([
transforms.Resize((imageSize, imageSize)),
# torchvision.transforms.functional.to_grayscale,
transforms.ToTensor(),
# transforms.Lambda(lambda x: torch.reshape(x, x.shape[1:])),
transforms.Normalize([0.5], [0.5])
]))
sampler = InfiniteSampler(catDataset)
catLoader = DataLoader(catDataset, batch_size=batchSize, sampler=sampler)
# Generate a random distance matrix.
# # Make a matrix with positive values.
# distancesCpu = np.clip(np.random.normal(0.5, 1.0 / 3, (dataSize, dataSize)), 0, 1)
# # Make it symmetrical.
# distancesCpu = np.matmul(distancesCpu, distancesCpu.T)
# Generate random points and compute distances, guaranteeing that the triangle rule isn't broken.
randomPoints = generate_points(dataSize)
distancesCpu = scipy.spatial.distance_matrix(randomPoints,
randomPoints,
p=2)
if initWithCats:
imagePaths = random.choices(glob.glob(os.path.join(imageRootPath,
'*')),
k=dataSize)
catImages = []
for p in imagePaths:
image = skimage.transform.resize(imageio.imread(p),
(imageSize, imageSize),
1).transpose(2, 0, 1)
catImages.append(image)
imagesInitCpu = np.asarray(catImages)
else:
imagesInitCpu = np.clip(
np.random.normal(0.5, 0.5 / 3,
(dataSize, 3, imageSize, imageSize)), 0, 1)
images = torch.tensor(imagesInitCpu,
requires_grad=True,
dtype=torch.float32,
device=gpu)
scale = torch.tensor(1.0,
requires_grad=True,
dtype=torch.float32,
device=gpu)
lossModel = models.PerceptualLoss(model='net-lin', net='vgg',
use_gpu=True).to(gpu)
lossBce = torch.nn.BCELoss()
# discriminator = Discriminator(imageSize, 3)
discriminator = Discriminator(3, 64, 1)
if discCheckpointPath:
discriminator.load_state_dict(torch.load(discCheckpointPath))
else:
discriminator.init_params()
discriminator = discriminator.to(gpu)
optimizerImages = torch.optim.Adam([images, scale],
lr=1e-3,
betas=(0.9, 0.999))
# optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999))
optimizerDisc = torch.optim.Adam(discriminator.parameters(),
lr=0.0002,
betas=(0.5, 0.999))
import matplotlib.pyplot as plt
fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2)
fig2 = plt.figure()
ax2 = fig2.add_subplot(1, 1, 1)
outPath = os.path.join(
'images',
datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S'))
os.makedirs(outPath)
catIter = iter(catLoader)
for batchIndex in range(10000):
realImageBatch, _ = next(catIter) # type: Tuple(torch.Tensor, Any)
realImageBatch = realImageBatch.to(gpu)
# realImageBatch = torch.tensor(realImageBatchCpu, device=gpu)
# noinspection PyTypeChecker
randomIndices = np.random.randint(
0, dataSize, batchSize).tolist() # type: List[int]
# randomIndices = list(range(dataSize)) # type: List[int]
distanceBatch = torch.tensor(
distancesCpu[randomIndices, :][:, randomIndices],
dtype=torch.float32,
device=gpu)
imageBatch = images[randomIndices].contiguous()
distPred = lossModel.forward(
imageBatch.repeat(repeats=(batchSize, 1, 1, 1)).contiguous(),
imageBatch.repeat_interleave(repeats=batchSize,
dim=0).contiguous(),
normalize=True)
distPredMat = distPred.reshape((batchSize, batchSize))
lossDist = torch.sum((distanceBatch - distPredMat * scale)**2) # MSE
discPred = discriminator(imageBatch)
lossRealness = lossBce(discPred,
torch.ones(imageBatch.shape[0], 1, device=gpu))
lossImages = lossDist + 100.0 * lossRealness # todo
# lossImages = lossRealness # todo
optimizerImages.zero_grad()
lossImages.backward()
optimizerImages.step()
lossDiscReal = lossBce(
discriminator(realImageBatch),
torch.ones(realImageBatch.shape[0], 1, device=gpu))
lossDiscFake = lossBce(discriminator(imageBatch.detach()),
torch.zeros(imageBatch.shape[0], 1, device=gpu))
lossDisc = (lossDiscFake + lossDiscReal) / 2
# lossDisc = torch.tensor(0)
optimizerDisc.zero_grad()
lossDisc.backward()
optimizerDisc.step()
with torch.no_grad():
# todo We're clamping all the images every batch, can we do clamp only the ones updated?
# images = torch.clamp(images, 0, 1) # For some reason this was making the training worse.
images.data = torch.clamp(images.data, 0, 1)
if batchIndex % 100 == 0:
msg = 'iter {}, loss images {:.3f}, loss dist {:.3f}, loss real {:.3f}, loss disc {:.3f}, scale: {:.3f}'.format(
batchIndex, lossImages.item(), lossDist.item(),
lossRealness.item(), lossDisc.item(), scale.item())
print(msg)
# print(discPred.tolist())
imageBatchCpu = imageBatch.cpu().data.numpy().transpose(0, 2, 3, 1)
for i, ax in enumerate(axes.flatten()):
ax.imshow(imageBatchCpu[i])
fig.suptitle(msg)
imagesAllCpu = images.cpu().data.numpy().transpose(0, 2, 3, 1)
plot_image_scatter(ax2,
randomPoints,
imagesAllCpu,
downscaleRatio=2)
fig.savefig(
os.path.join(outPath, 'batch_{}.png'.format(batchIndex)))
fig2.savefig(
os.path.join(outPath, 'scatter_{}.png'.format(batchIndex)))
(opt.dataset, test_data_size))
model = create_model(opt, _isTrain=False)
model.switch_to_eval()
torch.backends.cudnn.enabled = True
torch.backends.cudnn.benchmark = True
global_step = 0
total_l1 = 0.
total_psnr = 0.
total_lpips = 0.
count = 0.
lpips_model = models.PerceptualLoss(model='net',
net='alex',
use_gpu=True,
gpu_ids=[1])
for i, data in enumerate(test_dataset):
global_step = global_step + 1
print('global_step', global_step)
targets = data
model.set_input(targets)
l1_error, psnr, lpips, valid = model.evaluate_all(lpips_model)
print('l1_error ', l1_error)
print('psnr ', psnr)
print('lpips ', lpips)
from util import util
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-p0', '--path0', type=str, default='./imgs/ex_ref.png')
parser.add_argument('-p1', '--path1', type=str, default='./imgs/ex_p0.png')
parser.add_argument('-v', '--version', type=str, default='0.1')
parser.add_argument('--use_gpu',
action='store_true',
help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
model = models.PerceptualLoss(model='net-lin',
net='vgg',
use_gpu=opt.use_gpu,
version=opt.version)
# Load images
img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.path1))
if (opt.use_gpu):
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1)
print('Distance: %.3f' % dist01)
import models
from util import util
import os
import torch
import numpy as np
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('-p0','--path0', type=str, default='./imgs/ex_ref.png')
parser.add_argument('-p1','--path1', type=str, default='./imgs/ex_p0.png')
parser.add_argument('-v','--version', type=str, default='0.1')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
model = models.PerceptualLoss(model='net-lin',net='alex',use_gpu=opt.use_gpu,version='0.1')
# Load images
img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.path1))
if(opt.use_gpu):
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0,img1)
print('Distance: %.3f'%dist01)
outputs = './experiment1'
default=
'/media/yaurehman2/Seagate Backup Plus Drive/dataset/Video/test/input/DAVIS/'
)
parser.add_argument('-i',
'--input',
type=str,
default='./Yuzhi_txt/DAVIS_test_imagelist.txt')
parser.add_argument('-o', '--out', type=str, default='./LBVTC_ECCV16.txt')
parser.add_argument('--use_gpu',
action='store_true',
help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=opt.use_gpu)
def text_readlines(filename):
# Try to read a txt file and return a list.Return [] if there was a mistake.
try:
file = open(filename, 'r')
except IOError:
error = []
return error
content = file.readlines()
# This for loop deletes the EOF (like \n)
for i in range(len(content)):
content[i] = content[i][:len(content[i]) - 1]
file.close()
return content
import torch
from util import util
import models
from models import dist_model as dm
from IPython import embed
use_gpu = False # Whether to use GPU
spatial = False # Return a spatial map of perceptual distance.
# Linearly calibrated models (LPIPS)
model = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=use_gpu,
spatial=spatial)
# Can also set net = 'squeeze' or 'vgg'
# Off-the-shelf uncalibrated networks
# model = models.PerceptualLoss(model='net', net='alex', use_gpu=use_gpu, spatial=spatial)
# Can also set net = 'squeeze' or 'vgg'
# Low-level metrics
# model = models.PerceptualLoss(model='L2', colorspace='Lab', use_gpu=use_gpu)
# model = models.PerceptualLoss(model='ssim', colorspace='RGB', use_gpu=use_gpu)
## Example usage with dummy tensors
dummy_im0 = torch.Tensor(1, 3, 64,
64) # image should be RGB, normalized to [-1,1]
dummy_im1 = torch.Tensor(1, 3, 64, 64)
dist = model.forward(dummy_im0, dummy_im1)
## Example usage with images
# from models import dist_model
# model = dist_model.DistModel()
from os.path import join
import models
import util.util as util
import matplotlib.pylab as plt
use_gpu = True
fig_outdir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\ImageDiffMetric"
#%%
net_name = 'squeeze'
SpatialDist = models.PerceptualLoss(model='net-lin',
net=net_name,
colorspace='rgb',
spatial=True,
use_gpu=True,
gpu_ids=[0])
PerceptLoss = models.PerceptualLoss(model='net-lin',
net=net_name,
colorspace='rgb',
spatial=False,
use_gpu=True,
gpu_ids=[0])
#%%
imgdir = r"\\storage1.ris.wustl.edu\crponce\Active\Stimuli\2019-06-Evolutions\beto-191212a\backup_12_12_2019_10_47_39"
file0 = "block048_thread000_gen_gen047_001896.jpg"
file1 = "block048_thread000_gen_gen047_001900.jpg"
img0_ = util.load_image(join(imgdir, file0))
img1_ = util.load_image(join(imgdir, file1))
img0 = util.im2tensor(img0_) # RGB image from [-1,1]
if (use_gpu):
img0 = img0.cuda()
import torch.optim as optim
import torch.nn.functional as F
# from cv2 import imread, imwrite
import matplotlib
matplotlib.use('Agg') # if you dont want image show up
import matplotlib.pylab as plt
import sys
sys.path.append("D:\Github\pytorch-caffe")
sys.path.append("D:\Github\pytorch-receptive-field")
from caffenet import *
from hessian import hessian
#% Set up PerceptualLoss judger
sys.path.append(r"D:\Github\PerceptualSimilarity")
import models # from PerceptualSimilarity folder
model = models.PerceptualLoss(model='net-lin',
net='squeeze',
use_gpu=1,
gpu_ids=[0])
model_vgg = models.PerceptualLoss(model='net-lin',
net='vgg',
use_gpu=1,
gpu_ids=[0])
model_alex = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=1,
gpu_ids=[0])
#%%
hess_dir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\Artiphysiology\Hessian"
output_dir = r"D:\Generator_DB_Windows\data\with_CNN\hessian"
output_dir = join(output_dir, "rand_spot")
os.makedirs(output_dir)
unit_arr = [
import os
from util import util
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--cfg', type=str, default='pretrain')
parser.add_argument('--ninput', type=int, default=106)
parser.add_argument('--nsample', type=int, default=50)
parser.add_argument('--gpu_index', type=int, default=0)
opt = parser.parse_args()
device = torch.device('cuda', index=opt.gpu_index) if torch.cuda.is_available() else torch.device('cpu')
torch.set_grad_enabled(False)
## Initializing the model
model = models.PerceptualLoss(model='net-lin',net='alex', use_gpu=opt.gpu_index >= 0 and torch.cuda.is_available(), gpu_ids=[opt.gpu_index])
results_dir = os.path.expanduser('~/results/diverse_gan/bicyclegan/%s/test/images' % opt.cfg)
print(results_dir)
t_LPIPS = 0.0
for i in range(opt.ninput):
imgs = []
for j in range(opt.nsample):
img_name = 'input_%03d_random_sample%02d.png' % (i, j + 1)
img_path = os.path.join(results_dir, img_name)
# print(img_path)
img = util.im2tensor(util.load_image(img_path))
imgs.append(img)
LPIPS = 0.0
use_gpu = False
ref_path = './imgs/ex_ref.png'
pred_path = './imgs/ex_p1.png'
ref_img = scipy.misc.imread(ref_path).transpose(2, 0, 1) / 255.
pred_img = scipy.misc.imread(pred_path).transpose(2, 0, 1) / 255.
# Torchify
ref = Variable(torch.FloatTensor(ref_img).unsqueeze(0), requires_grad=False)
pred = Variable(torch.FloatTensor(pred_img).unsqueeze(0), requires_grad=True)
if (use_gpu):
ref = ref.cuda()
pred = pred.cuda()
loss_fn = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=use_gpu)
optimizer = torch.optim.Adam([
pred,
], lr=1e-3, betas=(0.9, 0.999))
import matplotlib.pyplot as plt
plt.ion()
fig = plt.figure(1)
ax = fig.add_subplot(131)
ax.imshow(ref_img.transpose(1, 2, 0))
ax.set_title('reference')
ax = fig.add_subplot(133)
ax.imshow(pred_img.transpose(1, 2, 0))
ax.set_title('orig pred')
for i in range(1000):
def main():
dataSize = 32
batchSize = 8
elpipsBatchSize = 1
# imageSize = 32
imageSize = 64
nz = 100
# discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth'
discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth'
genCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netG_epoch_24.pth'
gpu = torch.device('cuda')
# For now we normalize the vectors to have norm 1, but don't make sure
# that the data has certain mean/std.
pointDataset = AuthorDataset(
jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json'
)
# Take top N points.
points = np.asarray([pointDataset[i][0] for i in range(dataSize)])
distPointsCpu = l2_sqr_dist_matrix(torch.tensor(points)).numpy()
latents = torch.tensor(np.random.normal(0.0, 1.0, (dataSize, nz)),
requires_grad=True, dtype=torch.float32, device=gpu)
scale = torch.tensor(2.7, requires_grad=True, dtype=torch.float32, device=gpu) # todo Re-check!
bias = torch.tensor(0.0, requires_grad=True, dtype=torch.float32, device=gpu) # todo Re-check!
lpips = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu)
# lossModel = lpips
config = elpips.Config()
config.batch_size = elpipsBatchSize # Ensemble size for ELPIPS.
config.set_scale_levels_by_image_size(imageSize, imageSize)
lossModel = elpips.ElpipsMetric(config, lpips).to(gpu)
discriminator = Discriminator(3, 64, 1)
if discCheckpointPath:
discriminator.load_state_dict(torch.load(discCheckpointPath))
else:
discriminator.init_params()
discriminator = discriminator.to(gpu)
generator = Generator(nz=nz, ngf=64)
if genCheckpointPath:
generator.load_state_dict(torch.load(genCheckpointPath))
else:
generator.init_params()
generator = generator.to(gpu)
# optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999))
optimizerScale = torch.optim.Adam([scale, bias], lr=0.001)
# optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999))
# optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999))
# optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999))
optimizerLatents = torch.optim.Adam([latents], lr=5e-3, betas=(0.9, 0.999))
fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2)
fig2 = plt.figure()
ax2 = fig2.add_subplot(1, 1, 1)
outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S'))
os.makedirs(outPath)
summaryWriter = SummaryWriter(outPath)
for batchIndex in range(10000):
# noinspection PyTypeChecker
randomIndices = np.random.randint(0, dataSize, batchSize).tolist() # type: List[int]
# # randomIndices = list(range(dataSize)) # type: List[int]
distTarget = torch.tensor(distPointsCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu)
latentsBatch = latents[randomIndices]
imageBatchFake = generator(latentsBatch[:, :, None, None].float())
# todo It's possible to compute this more efficiently, but would require re-implementing lpips.
# For now, compute the full BSxBS matrix row-by-row to avoid memory issues.
lossDistTotal = torch.tensor(0.0, device=gpu)
distanceRows = []
for iRow in range(batchSize):
distPredFlat = lossModel(imageBatchFake[iRow].repeat(repeats=(batchSize, 1, 1, 1)).contiguous(),
imageBatchFake, normalize=True)
distPred = distPredFlat.reshape((1, batchSize))
distanceRows.append(distPred)
lossDist = torch.sum((distTarget[iRow] - (distPred * scale + bias)) ** 2) # MSE
lossDistTotal += lossDist
lossDistTotal /= batchSize * batchSize # Compute the mean.
distPredFull = torch.cat(distanceRows, dim=0)
# print('{} - {} || {} - {}'.format(
# torch.min(distPred).item(),
# torch.max(distPred).item(),
# torch.min(distTarget).item(),
# torch.max(distTarget).item()
# ))
# discPred = discriminator(imageBatchFake)
# lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu))
# lossGen = lossDist + 1.0 * lossRealness
lossLatents = lossDistTotal
# optimizerGen.zero_grad()
# optimizerScale.zero_grad()
# lossGen.backward()
# optimizerGen.step()
# optimizerScale.step()
optimizerLatents.zero_grad()
# optimizerScale.zero_grad()
lossLatents.backward()
optimizerLatents.step()
# optimizerScale.step()
# with torch.no_grad():
# # todo We're clamping all the images every batch, can we clamp only the ones updated?
# # images = torch.clamp(images, 0, 1) # For some reason this was making the training worse.
# images.data = torch.clamp(images.data, 0, 1)
if batchIndex % 100 == 0:
msg = 'iter {} loss dist {:.3f} scale: {:.3f} bias: {:.3f}'.format(batchIndex, lossDistTotal.item(), scale.item(), bias.item())
print(msg)
summaryWriter.add_scalar('loss-dist', lossDistTotal.item(), global_step=batchIndex)
def gpu_images_to_numpy(images):
imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1)
imagesNumpy = (imagesNumpy + 1) / 2
return imagesNumpy
# print(discPred.tolist())
imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake)
# imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal)
for iCol, ax in enumerate(axes.flatten()[:batchSize]):
ax.imshow(imageBatchFakeCpu[iCol])
fig.suptitle(msg)
with torch.no_grad():
images = gpu_images_to_numpy(generator(latents[..., None, None]))
authorVectorsProj = umap.UMAP(n_neighbors=min(5, dataSize), random_state=1337).fit_transform(points)
plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2)
fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png'))
fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png'))
plt.close(fig)
plt.close(fig2)
with torch.no_grad():
imagesGpu = generator(latents[..., None, None])
imageNumber = imagesGpu.shape[0]
# Compute LPIPS distances, batch to avoid memory issues.
bs = min(imageNumber, 8)
assert imageNumber % bs == 0
distPredEval = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0]))
for iCol in range(imageNumber // bs):
startA, endA = iCol * bs, (iCol + 1) * bs
imagesA = imagesGpu[startA:endA]
for j in range(imageNumber // bs):
startB, endB = j * bs, (j + 1) * bs
imagesB = imagesGpu[startB:endB]
distBatchEval = lossModel(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(),
imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(),
normalize=True).cpu().numpy()
distPredEval[startA:endA, startB:endB] = distBatchEval.reshape((bs, bs))
distPredEval = (distPredEval * scale.item() + bias.item())
# Move to the CPU and append an alpha channel for rendering.
images = gpu_images_to_numpy(imagesGpu)
images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images]
distPoints = distPointsCpu
assert np.abs(distPoints - distPoints.T).max() < 1e-5
distPoints = np.minimum(distPoints, distPoints.T) # Remove rounding errors, guarantee symmetry.
config = DistanceMatrixConfig()
config.dataRange = (0., 4.)
_, pointIndicesSorted = render_distance_matrix(
os.path.join(outPath, f'dist_point_{batchIndex}.png'),
distPoints,
images,
config=config
)
# print(np.abs(distPredFlat - distPredFlat.T).max())
# assert np.abs(distPredFlat - distPredFlat.T).max() < 1e-5
# todo The symmetry doesn't hold for E-LPIPS, since it's stochastic.
distPredEval = np.minimum(distPredEval, distPredEval.T) # Remove rounding errors, guarantee symmetry.
config = DistanceMatrixConfig()
config.dataRange = (0., 4.)
render_distance_matrix(
os.path.join(outPath, f'dist_images_{batchIndex}.png'),
distPredEval,
images,
config=config
)
config = DistanceMatrixConfig()
config.dataRange = (0., 4.)
render_distance_matrix(
os.path.join(outPath, f'dist_images_aligned_{batchIndex}.png'),
distPredEval,
images,
predefinedOrder=pointIndicesSorted,
config=config
)
fig, axes = plt.subplots(ncols=2)
axes[0].matshow(distTarget.cpu().numpy(), vmin=0, vmax=4)
axes[1].matshow(distPredFull.cpu().numpy() * scale.item(), vmin=0, vmax=4)
fig.savefig(os.path.join(outPath, f'batch_dist_{batchIndex}.png'))
plt.close(fig)
surveySize = 30
fig, axes = plt.subplots(nrows=3, ncols=surveySize, figsize=(surveySize, 3))
assert len(images) == dataSize
allIndices = list(range(dataSize))
with open(os.path.join(outPath, f'survey_{batchIndex}.txt'), 'w') as file:
for iCol in range(surveySize):
randomIndices = random.sample(allIndices, k=3)
leftToMid = distPointsCpu[randomIndices[0], randomIndices[1]]
rightToMid = distPointsCpu[randomIndices[2], randomIndices[1]]
correctAnswer = 'left' if leftToMid < rightToMid else 'right'
file.write("{}\t{}\t{}\t{}\t{}\n".format(iCol, correctAnswer, leftToMid, rightToMid,
str(tuple(randomIndices))))
for iRow in (0, 1, 2):
axes[iRow][iCol].imshow(images[randomIndices[iRow]])
fig.savefig(os.path.join(outPath, f'survey_{batchIndex}.png'))
plt.close(fig)
torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))
torch.save(discriminator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex)))
summaryWriter.close()
default=[0, 50],
nargs="+",
help='range of index of vectors to use')
parser.add_argument(
'--EPS',
type=float,
default=1E-2,
help='EPS of finite differencing HVP operator, will only be '
'used when method is `ForwardIter`')
args = parser.parse_args(
) # ["--dataset", "pasu", '--method', "BP", '--idx_rg', '0', '50', '--EPS', '1E-2'])
#%%
from FeatLinModel import FeatLinModel, get_model_layers
import models # from PerceptualSimilarity
model_squ = models.PerceptualLoss(model='net-lin',
net='squeeze',
use_gpu=1,
gpu_ids=[0])
model_squ.requires_grad_(False).cuda()
from GAN_utils import upconvGAN
if args.GAN in ["fc6", "fc7", "fc8"]:
G = upconvGAN(args.GAN)
elif args.GAN == "fc6_shfl":
G = upconvGAN("fc6")
SD = G.state_dict()
shuffled_SD = {}
for name, Weight in SD.items():
idx = torch.randperm(Weight.numel())
W_shuf = Weight.view(-1)[idx].view(Weight.shape)
shuffled_SD[name] = W_shuf
G.load_state_dict(shuffled_SD)
plt.imshow(grid_img.permute(1, 2, 0))
# plt.savefig("trackimage_0_40.png", dpi=600)
# files.download("trackimage_0_40.png")
"""# Test your networks
Did you remember to cut out a test set? If not you really should, test on images your network has never seen.
#### LPIPS Metrics
"""
!git clone https://github.com/richzhang/PerceptualSimilarity
!cd PerceptualSimilarity
import models
criterion_lpips = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=True, gpu_ids=[0])
# Instantiate the LPIPS metric
criterion_pixel = nn.MSELoss().cuda()
generator.eval()
discriminator.eval()
mse = np.empty(len(testloader))
lpips = np.empty(len(testloader))
for i, images in enumerate(testloader):
lr = F.interpolate(images, size=(16,16))
lr_full = F.interpolate(lr, size=(64,64), mode="bilinear", align_corners=False)
