def get_official_model_weights(): # Initializing the pytorch_model # pytorch_model.model.net.module.net: vgg16 # pytorch_model.model.net.module.lins: linear layer pytorch_model = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True) # extract pytorch tensors vgg_model_weights = OrderedDict() for k, v in pytorch_model.model.net.module.net.named_parameters(): vgg_model_weights[k] = v.data model_path = './models/weights/v0.1/vgg.pth' lin_model_weights = torch.load(model_path) # convert to numpy arrays vgg_model_weights_np = OrderedDict({ name: tensor.cpu().data.numpy() for name, tensor in vgg_model_weights.items() }) lin_model_weights_np = OrderedDict({ name: tensor.cpu().data.numpy() for name, tensor in lin_model_weights.items() }) return vgg_model_weights_np, lin_model_weights_np
def similarity(request): use_gpu = False spatial = False model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial) img1_filename = request.POST['img1'].filename img1_input_file = request.POST['img1'].file img1_file_path = os.path.join('/tmp', img1_filename) with open(img1_file_path, 'wb') as img1_output_file: shutil.copyfileobj(img1_input_file, img1_output_file) img1tensor = util.im2tensor(util.load_image(img1_file_path)) img2_filename = request.POST['img2'].filename img2_input_file = request.POST['img2'].file img2_file_path = os.path.join('/tmp', img2_filename) with open(img2_file_path, 'wb') as img2_output_file: shutil.copyfileobj(img2_input_file, img2_output_file) img2tensor = util.im2tensor(util.load_image(img2_file_path)) d = model.forward(img1tensor, img2tensor) return Response('Distancia: %.3f' % d)
def precompute(self): self.FL_w = FeatureLoss(self.args.vgg_weight_dir, self.args.vgg_layer_weight_w) for p in self.FL_w.parameters(): p.requires_grad = False self.FL_n = FeatureLoss(self.args.vgg_weight_dir, self.args.vgg_layer_weight_n) for p in self.FL_n.parameters(): p.requires_grad = False self.FL_wn = FeatureLoss(self.args.vgg_weight_dir, self.args.vgg_layer_weight_wn) for p in self.FL_wn.parameters(): p.requires_grad = False self.SL = StyleLoss() for p in self.SL.parameters(): p.requires_grad = False self.LPIPS = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=True, gpu_ids=[0]) self.Laplacian = kornia.filters.Laplacian(3, normalized=False) if self.args.embed_tex: self.albedo_ref_pix, self.normal_ref_pix, self.rough_ref_pix, self.specular_ref_pix, \ self.albedo_ref_vgg, self.normal_ref_vgg, self.rough_ref_vgg, self.specular_ref_vgg = \ self.eval_texture_vgg(self.textures_ref) self.fl_albedo_ref_w = self.FL_w(self.albedo_ref_vgg) self.fl_normal_ref_w = self.FL_w(self.normal_ref_vgg) self.fl_rough_ref_w = self.FL_w(self.rough_ref_vgg) self.fl_specular_ref_w = self.FL_w(self.specular_ref_vgg) self.fl_albedo_ref_n = self.FL_n(self.albedo_ref_vgg) self.fl_normal_ref_n = self.FL_n(self.normal_ref_vgg) self.fl_rough_ref_n = self.FL_n(self.rough_ref_vgg) self.fl_specular_ref_n = self.FL_n(self.specular_ref_vgg) self.fl_albedo_ref_wn = self.FL_wn(self.albedo_ref_vgg) self.fl_normal_ref_wn = self.FL_wn(self.normal_ref_vgg) self.fl_rough_ref_wn = self.FL_wn(self.rough_ref_vgg) self.fl_specular_ref_wn = self.FL_wn(self.specular_ref_vgg) else: self.rendered_ref_pix = self.rendered_ref self.rendered_ref_vgg = self.eval_render_vgg( self.rendered_ref, self.args.jittering) if self.args.jittering: self.sl_rendered_ref = self.SL(self.rendered_ref_vgg) else: self.fl_rendered_ref_w = self.FL_w(self.rendered_ref_vgg) self.fl_rendered_ref_n = self.FL_n(self.rendered_ref_vgg) self.fl_rendered_ref_wn = self.FL_wn(self.rendered_ref_vgg)
def main(): args = get_trainer_args() def load_videos(path): print("Loading trajectories from {}".format(path)) if not path.endswith('.npy'): raise ValueError("Can only read in .npy files!") seqs = np.load(path) assert len(seqs.shape) == 5 # need [batch, T, C, H, W] input data assert seqs.shape[ 2] == 3 # assume 3-channeled seq with channel in last dim seqs = torch.Tensor(seqs) if args.use_gpu: seqs = seqs.cuda() return seqs # range [-1, 1] gt_seqs = load_videos(args.gt) pred_seqs = load_videos(args.pred) print('shape: ', gt_seqs.shape) assert gt_seqs.shape == pred_seqs.shape n_seqs, time, c, h, w = gt_seqs.shape n_batches = int(np.floor(n_seqs / args.batch_size)) # import pdb; pdb.set_trace() # get sequence mask (for sequences with variable length mask = 1 - torch.all(torch.all(torch.all( (gt_seqs + 1.0).abs() < 1e-6, dim=-1), dim=-1), dim=-1) # check for black images mask2 = 1 - torch.all(torch.all(torch.all((gt_seqs).abs() < 1e-6, dim=-1), dim=-1), dim=-1) # check for gray images mask = mask * mask2 # Initializing the model model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=args.use_gpu) # run forward pass to compute LPIPS distances distances = [] for b in range(n_batches): x, y = gt_seqs[b * args.batch_size:(b + 1) * args.batch_size], pred_seqs[b * args.batch_size:(b + 1) * args.batch_size] lpips_dist = batch_apply((x, y), model, separate_arguments=True) distances.append(lpips_dist) distances = torch.cat(distances) mean_distance = distances[mask].mean() print("LPIPS distance: {}".format(mean_distance))
def compute_dists_pair(use_gpu, dir, out): ## Initializing the model model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu) dists = [] # crawl directories # f = open(out,'w') # category_files = os.listdir(opt.dir) # print('category_file', category_files) i = 0 for dir_path_class, dir_name_class, file_names_class in os.walk(dir): # print('1',dir_path_class) # print('2',dir_name_class) # print('3',file_names_class) dists_category = [] for (ff, file0) in enumerate(file_names_class[:-1]): if ff < 10: # print('here',file0) # print(os.path.exists(os.path.join(dir_path_class,file0))) img0 = util.im2tensor( util.load_image(os.path.join( dir_path_class, file0))) # RGB image from [-1,1] if (use_gpu): img0 = img0.cuda() for (gg, file1) in enumerate(file_names_class[ff + 1:]): img1 = util.im2tensor( util.load_image(os.path.join(dir_path_class, file1))) if (use_gpu): img1 = img1.cuda() # Compute distance dist01 = model.forward(img0, img1).item() dists_category.append(dist01) else: # print('continue') dists_category_mean = np.mean(np.array(dists_category)) print('{}_cateogory {}_samples lpips is {}'.format( i, len(dists_category), dists_category_mean)) dists.append(dists_category_mean) break i = i + 1 # if i > 5: # break # print('(%s, %s): %.3f'%(file0,file1,dist01)) # f.writelines('(%s, %s): %.3f'%(file0,file1,dist01)) dist_mean = np.mean(np.array(dists)) print('Mean: %.3f' % dist_mean) return dists, dist_mean
def __init__(self, net='vgg', use_gpu=True, precision='float'): """ LPIPS loss with spatial weighting """ super(PerceptualLoss, self).__init__() self.lpips = models.PerceptualLoss(model='net-lin', net=net, spatial=True, use_gpu=use_gpu) if use_gpu: self.lpips = nn.DataParallel(self.lpips).cuda() if precision == 'half': self.lpips.half() elif precision == 'float': self.lpips.float() elif precision == 'double': self.lpips.double() return
def __init__(self, net='vgg', use_gpu=True, lpips_dir=None): """ LPIPS loss with spatial weighting """ super(PerceptualLoss, self).__init__() lpips_dir = self.download(lpips_dir=lpips_dir) # consider removing from path after importing sys.path.insert(1, lpips_dir) with HiddenPrints(): import models self.lpips = models.PerceptualLoss(model='net-lin', net=net, spatial=True, use_gpu=use_gpu) self.lpips = nn.DataParallel(self.lpips).cuda() return
else: BGAN = BigGAN.from_pretrained("biggan-deep-256") sys.path.append(r"D:\Github\PerceptualSimilarity") sys.path.append(r"E:\Github_Projects\PerceptualSimilarity") Hpath = r"E:\OneDrive - Washington University in St. Louis\Hessian_summary\BigGAN\H_avg_1000cls.npz" if args.dataset == "BigGAN_rnd": imgfolder = r"E:\Cluster_Backup\Datasets\BigGAN_rnd" savedir = r"E:\OneDrive - Washington University in St. Louis\BigGAN_invert\BasinCMA\BigGAN_rnd" elif args.dataset == "ImageNet": imgfolder = r"E:\Cluster_Backup\Datasets\ImageTranslation\GAN_real\B\train" savedir = r"E:\OneDrive - Washington University in St. Louis\BigGAN_invert\BasinCMA\ImageNet" # savedir = r"E:\Cluster_Backup\BigGAN_invert\ImageNet" #%% import models # from PerceptualSimilarity folder ImDist = models.PerceptualLoss(model='net-lin', net=args.ImDist, use_gpu=1, gpu_ids=[0]) for param in ImDist.parameters(): param.requires_grad_(False) def L1loss(target, img): return (img - target).abs().sum(axis=1).mean(axis=1) # alpha = 5 # relative weight BGAN.cuda() BGAN.eval() for param in BGAN.parameters(): param.requires_grad_(False) #%% Load the precomputed Hessian and Form them as basis matrix data = np.load(Hpath) evc_clas = torch.from_numpy(data['eigvects_clas_avg'])#.cuda() evc_nois = torch.from_numpy(data['eigvects_nois_avg'])#.cuda() evc_all = torch.from_numpy(data['eigvects_avg']).cuda()
def evaluation(): parser = config_parser() args = parser.parse_args() # Load data if args.dataset_type == 'llff': target_idx = args.target_idx images, poses, bds, render_poses = load_nvidia_data(args.datadir, args.start_frame, args.end_frame, args.factor, target_idx=target_idx, recenter=True, bd_factor=.9, spherify=args.spherify, final_height=args.final_height) hwf = poses[0,:3,-1] poses = poses[:,:3,:4] print('Loaded llff', images.shape, render_poses.shape, hwf, args.datadir) # if not isinstance(i_test, list): i_test = [] i_val = [] #i_test i_train = np.array([i for i in np.arange(int(images.shape[0])) if (i not in i_test and i not in i_val)]) print('DEFINING BOUNDS') if args.no_ndc: near = np.percentile(bds[:, 0], 5) * 0.9 #np.ndarray.min(bds) #* .9 far = np.percentile(bds[:, 1], 95) * 1.1 #np.ndarray.max(bds) #* 1. else: near = 0. far = 1. print('NEAR FAR', near, far) else: print('ONLY SUPPORT LLFF!!!!!!!!') sys.exit() # Cast intrinsics to right types H, W, focal = hwf H, W = int(H), int(W) hwf = [H, W, focal] # Create log dir and copy the config file basedir = args.basedir args.expname = args.expname + '_F%02d-%02d'%(args.start_frame, args.end_frame) expname = args.expname os.makedirs(os.path.join(basedir, expname), exist_ok=True) f = os.path.join(basedir, expname, 'args.txt') with open(f, 'w') as file: for arg in sorted(vars(args)): attr = getattr(args, arg) file.write('{} = {}\n'.format(arg, attr)) if args.config is not None: f = os.path.join(basedir, expname, 'config.txt') with open(f, 'w') as file: file.write(open(args.config, 'r').read()) # Create nerf model render_kwargs_train, render_kwargs_test, \ start, grad_vars, optimizer = create_nerf(args) global_step = start bds_dict = { 'near' : near, 'far' : far, } render_kwargs_train.update(bds_dict) render_kwargs_test.update(bds_dict) num_img = float(images.shape[0]) poses = torch.Tensor(poses).to(device) with torch.no_grad(): model = models.PerceptualLoss(model='net-lin',net='alex', use_gpu=True,version=0.1) total_psnr = 0. total_ssim = 0. total_lpips = 0. count = 0. total_psnr_dy = 0. total_ssim_dy = 0. total_lpips_dy = 0. t = time.time() # for each time step for img_i in i_train: img_idx_embed = img_i/num_img * 2. - 1.0 # for each target viewpoint for camera_i in range(0, 12): print(time.time() - t) t = time.time() print(img_i, camera_i) if img_i % 12 == camera_i: continue c2w = poses[camera_i] ret = render(img_idx_embed, 0, False, num_img, H, W, focal, chunk=1024*16, c2w=c2w[:3,:4], **render_kwargs_test) rgb = ret['rgb_map_ref'].cpu().numpy()#.append(ret['rgb_map_ref'].cpu().numpy()) gt_img_path = os.path.join(args.datadir, 'mv_images', '%05d'%img_i, 'cam%02d.jpg'%(camera_i + 1)) # print('gt_img_path ', gt_img_path) gt_img = cv2.imread(gt_img_path)[:, :, ::-1] gt_img = cv2.resize(gt_img, dsize=(rgb.shape[1], rgb.shape[0]), interpolation=cv2.INTER_AREA) gt_img = np.float32(gt_img) / 255 psnr = skimage.measure.compare_psnr(gt_img, rgb) ssim = skimage.measure.compare_ssim(gt_img, rgb, multichannel=True) gt_img_0 = im2tensor(gt_img).cuda() rgb_0 = im2tensor(rgb).cuda() lpips = model.forward(gt_img_0, rgb_0) lpips = lpips.item() print(psnr, ssim, lpips) total_psnr += psnr total_ssim += ssim total_lpips += lpips count += 1 dynamic_mask_path = os.path.join(args.datadir, 'mv_masks', '%05d'%img_i, 'cam%02d.png'%(camera_i + 1)) print(dynamic_mask_path) dynamic_mask = np.float32(cv2.imread(dynamic_mask_path) > 1e-3)#/255. dynamic_mask = cv2.resize(dynamic_mask, dsize=(rgb.shape[1], rgb.shape[0]), interpolation=cv2.INTER_NEAREST) dynamic_mask_0 = torch.Tensor(dynamic_mask[:, :, :, np.newaxis].transpose((3, 2, 0, 1))) dynamic_ssim = calculate_ssim(gt_img, rgb, dynamic_mask) dynamic_psnr = calculate_psnr(gt_img, rgb, dynamic_mask) dynamic_lpips = model.forward(gt_img_0, rgb_0, dynamic_mask_0).item() total_psnr_dy += dynamic_psnr total_ssim_dy += dynamic_ssim total_lpips_dy += dynamic_lpips mean_psnr = total_psnr / count mean_ssim = total_ssim / count mean_lpips = total_lpips / count print('mean_psnr ', mean_psnr) print('mean_ssim ', mean_ssim) print('mean_lpips ', mean_lpips) mean_psnr_dy = total_psnr_dy / count mean_ssim_dy = total_ssim_dy / count mean_lpips_dy= total_lpips_dy / count print('mean_psnr dy', mean_psnr_dy) print('mean_ssim dy', mean_ssim_dy) print('mean_lpips dy', mean_lpips_dy)
def main(): # handle command line arguments ap = argparse.ArgumentParser() ap.add_argument('-d0', '--dir0', required=True, type=str, default='./imgs/ex_dir0', help='Reference images directory') ap.add_argument('-m0', '--mask0', required=False, type=str, default='*.png', help='file mask for files in d0 (e.g. .png)') ap.add_argument('-d1', '--dir1', required=True, type=str, default='./imgs/ex_dir1', help='Other images directory') ap.add_argument('-m1', '--mask1', required=False, type=str, default='*.png', help='file mask for files in d1 (e.g. .png)') ap.add_argument('-o', '--out', required=False, type=str, default='./results/', help='Distance files (.csv) directory') ap.add_argument('--use_gpu', action='store_true', help='Flag to use GPU to compute distance') ap.add_argument('-w', '--weights', default='./models/face_models/mmod_human_face_detector.dat', help='path to face model file') ap.add_argument('-H', '--hog_detector', required=False, action='store_true', help='use HOG detector') ap.add_argument('-A', '--haar_detector', required=False, action='store_true', help='use Haar detector') ap.add_argument('-C', '--cnn_detector', required=False, action='store_true', help='use CNN-based detector') ap.add_argument('-N', '--no_face_detector', required=False, action='store_true', help='Do NOT perform face detection. Compute quality over whole image') ap.add_argument('-v', '--verbose', required=False, help='verbose output', action='store_true') args = ap.parse_args() print("Input dir0:", args.dir0) print("Input dir0 file mask:", args.mask0) print("Input dir1:", args.dir1) print("Input dir1 file mask:", args.mask1) print("Output dir: ", args.out) if not os.path.exists(args.out): os.makedirs(args.out) if args.use_gpu: print("Compute LPIPS similarity using GPU (fast)") else: print("Compute LPIPS similarity using CPU (slow)") if args.no_face_detector: print("Do NOT perform face detection. Compute frame quality over the whole image.") file_det_name = "-no-face" else: if args.hog_detector: print("Find faces using DLib HOG detector") if args.haar_detector: print("Find faces using OpenCV Haar detector") if args.cnn_detector: print("Find faces using DLib CNN detector") if not args.hog_detector and not args.haar_detector and not args.cnn_detector: args.haar_detector = True print("No face detector selected. Using default OpenCV Haar detector") file_det_name = "" # it is expected that directories contain same number of files that can be sorted so that one is associated with the # file in the corresponding position in the other directory dir0_files = [] for file in os.listdir(args.dir0): if fnmatch.fnmatch(file, args.mask0): dir0_files.append(file) dir0_files.sort() print("Dir 0 (ref images) contains " + str(len(dir0_files)) + " " + args.mask0 + " files.") dir1_files = [] for file in os.listdir(args.dir1): if fnmatch.fnmatch(file, args.mask1): dir1_files.append(file) dir1_files.sort() print("Dir 1 (mod images) contains " + str(len(dir1_files)) + " " + args.mask1 + " files.") # Initializing face detector fd = FaceDetector() # Initializing LPIPS perceptual quality model model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=args.use_gpu) brisque_model = cv2.quality_QualityBRISQUE.create(BRISQUE_MODEL_FILE, BRISQUE_RANGE_FILE) # process all files num_all_files = len(dir0_files) # open files and write headers out_file_base_name = "all_files-" + os.path.basename(args.dir0) + "-" + os.path.basename( args.dir1) + file_det_name print("Base file name:" + out_file_base_name) out_file_LPIPS_all = open( os.path.join(args.out, out_file_base_name + "-LPIPS.csv"), 'w') out_file_MSSIM_RGB_all = open( os.path.join(args.out, out_file_base_name + "-MSSIM-RGB.csv"), 'w') out_file_MSSIM_Y_all = open( os.path.join(args.out, out_file_base_name + "-MSSIM-Y.csv"), 'w') out_file_BRISQUE_all = open( os.path.join(args.out, out_file_base_name + "-BRISQUE.csv"), 'w') out_file_BRISQUE_all.writelines('files, BRISQUE score ref, BRISQUE score mod\n') out_file_MSSIM_RGB_all.writelines('files, SSIM R, SSIM G, SSIM B\n') out_file_MSSIM_Y_all.writelines('files, SSIM Y\n') out_file_LPIPS_all.writelines('files, LPIPS distance\n') processed_file = 1 for reference_image_name, modified_image_name in zip(dir0_files, dir1_files): if processed_file % 50 == 0: print("Processing ref. image " + str(processed_file) + "/" + str(num_all_files)) # open reference and modified images ref_image = cv2.imread(os.path.join(args.dir0, reference_image_name)) mod_image = cv2.imread(os.path.join(args.dir1, modified_image_name)) if ref_image is None or mod_image is None: # print("Skipping images: {} and {}".format(reference_image_name, modified_image_name)) continue filename_all = os.path.splitext(reference_image_name)[0] + "-" + os.path.splitext(modified_image_name)[0] if args.no_face_detector: # compute quality over whole image. No face detection ref_face_regions = [ref_image] mod_face_regions = [mod_image] else: # perform face detection. use face regions to compute quality fd.process_frame(ref_image, use_cnn=args.cnn_detector, use_hog=args.hog_detector, use_haar=args.haar_detector) detected_faces = fd.convert_dlib_rectangles_to_opencv_faces(fd.cnn_faces) + \ fd.convert_dlib_rectangles_to_opencv_faces(fd.hog_faces) + \ fd.convert_dlib_rectangles_to_opencv_faces(fd.haar_faces) resized_faces = resize_64x64_multiple_faces_list(detected_faces) ref_face_regions = get_face_regions(ref_image, resized_faces) mod_face_regions = get_face_regions(mod_image, resized_faces) for ref_face_region, mod_face_region in zip(ref_face_regions, mod_face_regions): MSSIM_Y_dist = compute_MSSIM(cv2.cvtColor(ref_face_region, cv2.COLOR_RGB2GRAY), cv2.cvtColor(mod_face_region, cv2.COLOR_RGB2GRAY)) MSSIM_RGB_dist = compute_MSSIM(ref_face_region, mod_face_region) BRISQUE_score_ref = brisque_model.compute(ref_face_region) BRISQUE_score_mod = brisque_model.compute(mod_face_region) print("{}, {:.6f}, {:.6f}".format(filename_all, BRISQUE_score_ref[0], BRISQUE_score_mod[0]), file=out_file_BRISQUE_all) print('{}, {:.6f}'.format(filename_all, round(MSSIM_Y_dist[0] * 100, 2)), file=out_file_MSSIM_Y_all) print('{}, {:.6f}, {:.6f}, {:.6f}'.format(filename_all, round(MSSIM_RGB_dist[2] * 100, 2), round(MSSIM_RGB_dist[1] * 100, 2), round(MSSIM_RGB_dist[0] * 100, 2)), file=out_file_MSSIM_RGB_all) if args.no_face_detector: ref_face_blocks = get_64x64_full_image_regions(ref_face_region) mod_face_blocks = get_64x64_full_image_regions(mod_face_region) else: ref_face_blocks = get_64x64_face_regions(ref_face_region) mod_face_blocks = get_64x64_face_regions(mod_face_region) LPIPS_dist = 0 for ref_face_block, mod_face_block in zip(ref_face_blocks, mod_face_blocks): img0 = util.im2tensor(cv2.cvtColor(ref_face_block, cv2.COLOR_BGR2RGB)) # RGB image from [-1,1] img1 = util.im2tensor(cv2.cvtColor(mod_face_block, cv2.COLOR_BGR2RGB)) if args.use_gpu: img0 = img0.cuda() img1 = img1.cuda() # Compute distance LPIPS_dist += model.forward(img0, img1) LPIPS_dist /= len(ref_face_blocks) out_file_LPIPS_all.writelines('%s, %.6f\n' % (filename_all, LPIPS_dist)) if DEBUG: processed_image = draw_faces(ref_image.copy(), cnn_faces=fd.cnn_faces, hog_faces=fd.hog_faces, haar_faces=fd.haar_faces) cv2.imshow('Processed image', processed_image) out_img_filename = "/tmp/" + reference_image_name + ".jpg" cv2.imwrite(out_img_filename, processed_image) deb_face_count = 0 ref_face_regions = get_face_regions(ref_image, resized_faces) for ref_detected_face in ref_face_regions: out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str(deb_face_count) + ".jpg" cv2.imwrite(out_img_filename, ref_detected_face) ref_face_blocks = get_64x64_face_regions(ref_detected_face) deb_block_count = 0 for ref_face_block in ref_face_blocks: out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str( deb_face_count) + "_-_block_" + str(deb_block_count) + ".jpg" cv2.imwrite(out_img_filename, ref_face_block) deb_block_count += 1 deb_face_count += 1 if cv2.waitKey(1) == 27: break # esc to quit processed_file += 1 if DEBUG: cv2.destroyAllWindows() out_file_LPIPS_all.close() out_file_MSSIM_RGB_all.close() out_file_MSSIM_Y_all.close() out_file_BRISQUE_all.close()
# ===================forward===================== output = model(img) loss = criterion(output, img) # ===================backward==================== optimizer.zero_grad() loss.backward() optimizer.step() # ===================log======================== writer.add_scalar('loss', loss, epoch) if epoch % 5 == 0: writer.add_images('reconstruct', output, epoch) perceptual_loss = PerceptualSimilarity.PerceptualLoss(model='net-lin', net='alex', use_gpu=True, gpu_ids=[0]) L2_loss = nn.MSELoss(reduction='none') total_auc = 0 total_acc = 0 total_image = 0 for index, (img, mask) in enumerate(test_loader): img = Variable(img).cuda() mask = Variable(mask).cuda() # 取得 model 輸出 output = model(img) # 計算 dif (相似度以及 L2) dif = perceptual_loss.forward(output, img)
from time import time from importlib import reload import re import numpy as np import matplotlib matplotlib.use('Agg') # if you dont want image show up import matplotlib.pylab as plt import torch sys.path.append("D:\Github\pytorch-caffe") from caffenet import * import utils from utils import generator from insilico_Exp import CNNmodel sys.path.append(r"D:\Github\PerceptualSimilarity") import models model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=1, gpu_ids=[0]) #%% hess_dir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\Artiphysiology\Hessian" output_dir = r"D:\Generator_DB_Windows\data\with_CNN\hessian" def perturb_images_sphere(cent_vec, perturb_vec, PC2_ang_step = 18, PC3_ang_step = 18): sphere_norm = np.linalg.norm(cent_vec) vectors = np.zeros((3, cent_vec.size)) vectors[ 0, :] = cent_vec / sphere_norm vectors[1:3, :] = perturb_vec img_list = [] for j in range(-5, 6): for k in range(-5, 6): theta = PC2_ang_step * j / 180 * np.pi phi = PC3_ang_step * k / 180 * np.pi code_vec = np.array([[np.cos(theta) * np.cos(phi), np.sin(theta) * np.cos(phi),
def main(): dataSize = 128 batchSize = 8 # imageSize = 32 imageSize = 64 # discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth' # discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth' discCheckpointPath = None gpu = torch.device('cuda') # imageDataset = CatDataset( # imageSubdirPath=r'E:\data\cat-vs-dog\cat', # transform=transforms.Compose( # [ # transforms.Resize((imageSize, imageSize)), # transforms.ToTensor(), # transforms.Normalize([0.5], [0.5]) # ] # ) # ) imageDataset = datasets.CIFAR10(root=r'e:\data\images\cifar10', download=True, transform=transforms.Compose([ transforms.Resize((imageSize, imageSize)), transforms.ToTensor(), # transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)), transforms.Normalize([0.5], [0.5]), ])) # For now we normalize the vectors to have norm 1, but don't make sure # that the data has certain mean/std. pointDataset = AuthorDataset( jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json' ) imageLoader = DataLoader(imageDataset, batch_size=batchSize, sampler=InfiniteSampler(imageDataset)) pointLoader = DataLoader(pointDataset, batch_size=batchSize, sampler=InfiniteSampler(pointDataset)) # Generate a random distance matrix. # # Make a matrix with positive values. # distancesCpu = np.clip(np.random.normal(0.5, 1.0 / 3, (dataSize, dataSize)), 0, 1) # # Make it symmetrical. # distancesCpu = np.matmul(distancesCpu, distancesCpu.T) # Generate random points and compute distances, guaranteeing that the triangle rule isn't broken. # randomPoints = generate_points(dataSize) # distancesCpu = scipy.spatial.distance_matrix(randomPoints, randomPoints, p=2) # catImagePath = os.path.expandvars(r'${DEV_METAPHOR_DATA_PATH}/cats/cat.247.jpg') # catImage = skimage.transform.resize(imageio.imread(catImagePath), (64, 64), 1).transpose(2, 0, 1) # imagesInitCpu = np.clip(np.random.normal(0.5, 0.5 / 3, (dataSize, 3, imageSize, imageSize)), 0, 1) # imagesInitCpu = np.clip(np.tile(catImage, (dataSize, 1, 1, 1)) + np.random.normal(0., 0.5 / 6, (dataSize, 3, 64, 64)), 0, 1) # images = torch.tensor(imagesInitCpu, requires_grad=True, dtype=torch.float32, device=gpu) scale = torch.tensor(4.0, requires_grad=True, dtype=torch.float32, device=gpu) lossModel = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu) bceLoss = torch.nn.BCELoss() # discriminator = Discriminator(imageSize, 3) discriminator = Discriminator(3, 64, 1) if discCheckpointPath: discriminator.load_state_dict(torch.load(discCheckpointPath)) else: discriminator.init_params() discriminator = discriminator.to(gpu) generator = Generator(nz=pointDataset[0][0].shape[0], ngf=64) generator.init_params() generator = generator.to(gpu) # todo init properly, if training # discriminator.apply(weights_init_normal) # optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999)) optimizerScale = torch.optim.Adam([scale], lr=0.001) optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999)) # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999)) optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999)) import matplotlib.pyplot as plt fig, axes = plt.subplots(nrows=2 * 2, ncols=batchSize // 2) fig2 = plt.figure() ax2 = fig2.add_subplot(1, 1, 1) outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S')) os.makedirs(outPath) imageIter = iter(imageLoader) pointIter = iter(pointLoader) for batchIndex in range(10000): imageBatchReal, _ = next(imageIter) # type: Tuple(torch.Tensor, Any) imageBatchReal = imageBatchReal.to(gpu) # imageBatchReal = torch.tensor(realImageBatchCpu, device=gpu) # noinspection PyTypeChecker # randomIndices = np.random.randint(0, dataSize, batchSize).tolist() # type: List[int] # # randomIndices = list(range(dataSize)) # type: List[int] # distanceBatch = torch.tensor(distancesCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu) # imageBatchFake = images[randomIndices].contiguous() vectorBatch, _ = next(pointIter) vectorBatch = vectorBatch.to(gpu) distanceBatch = l2_sqr_dist_matrix(vectorBatch) # In-batch vector distances. imageBatchFake = generator(vectorBatch[:, :, None, None].float()) # todo It's possible to compute this more efficiently, but would require re-implementing lpips. distImages = lossModel.forward(imageBatchFake.repeat(repeats=(batchSize, 1, 1, 1)).contiguous(), imageBatchFake.repeat_interleave(repeats=batchSize, dim=0).contiguous(), normalize=True) distPredMat = distImages.reshape((batchSize, batchSize)) lossDist = torch.sum((distanceBatch - distPredMat * scale) ** 2) # MSE discPred = discriminator(imageBatchFake) lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu)) lossGen = lossDist + 1.0 * lossRealness optimizerGen.zero_grad() optimizerScale.zero_grad() lossGen.backward() optimizerGen.step() optimizerScale.step() lossDiscReal = bceLoss(discriminator(imageBatchReal), torch.ones(imageBatchReal.shape[0], device=gpu)) lossDiscFake = bceLoss(discriminator(imageBatchFake.detach()), torch.zeros(imageBatchFake.shape[0], device=gpu)) lossDisc = (lossDiscFake + lossDiscReal) / 2 # lossDisc = torch.tensor(0) optimizerDisc.zero_grad() lossDisc.backward() optimizerDisc.step() # with torch.no_grad(): # # todo We're clamping all the images every batch, can we clamp only the ones updated? # # images = torch.clamp(images, 0, 1) # For some reason this was making the training worse. # images.data = torch.clamp(images.data, 0, 1) if batchIndex % 100 == 0: msg = 'iter {}, loss gen {:.3f}, loss dist {:.3f}, loss real {:.3f}, loss disc {:.3f}, scale: {:.3f}'.format( batchIndex, lossGen.item(), lossDist.item(), lossRealness.item(), lossDisc.item(), scale.item() ) print(msg) def gpu_images_to_numpy(images): imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1) imagesNumpy = (imagesNumpy + 1) / 2 return imagesNumpy # print(discPred.tolist()) imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake) imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal) for i, ax in enumerate(axes.flatten()[:batchSize]): ax.imshow(imageBatchFakeCpu[i]) for i, ax in enumerate(axes.flatten()[batchSize:]): ax.imshow(imageBatchRealCpu[i]) fig.suptitle(msg) with torch.no_grad(): points = np.asarray([pointDataset[i][0] for i in range(200)], dtype=np.float32) images = gpu_images_to_numpy(generator(torch.tensor(points[..., None, None], device=gpu))) authorVectorsProj = umap.UMAP(n_neighbors=5, random_state=1337).fit_transform(points) plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2) fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png')) fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png')) plt.close(fig) plt.close(fig2) with torch.no_grad(): imageNumber = 48 points = np.asarray([pointDataset[i][0] for i in range(imageNumber)], dtype=np.float32) imagesGpu = generator(torch.tensor(points[..., None, None], device=gpu)) # Compute LPIPS distances, batch to avoid memory issues. bs = 8 assert imageNumber % bs == 0 distImages = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0])) for i in range(imageNumber // bs): startA, endA = i * bs, (i + 1) * bs imagesA = imagesGpu[startA:endA] for j in range(imageNumber // bs): startB, endB = j * bs, (j + 1) * bs imagesB = imagesGpu[startB:endB] distBatch = lossModel.forward(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(), imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(), normalize=True).cpu().numpy() distImages[startA:endA, startB:endB] = distBatch.reshape((bs, bs)) # Move to the CPU and append an alpha channel for rendering. images = gpu_images_to_numpy(imagesGpu) images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images] distPoints = l2_sqr_dist_matrix(torch.tensor(points, dtype=torch.double)).numpy() assert np.abs(distPoints - distPoints.T).max() < 1e-5 distPoints = np.minimum(distPoints, distPoints.T) # Remove rounding errors, guarantee symmetry. config = DistanceMatrixConfig() config.dataRange = (0., 4.) render_distance_matrix(os.path.join(outPath, f'dist_point_{batchIndex}.png'), distPoints, images, config) assert np.abs(distImages - distImages.T).max() < 1e-5 distImages = np.minimum(distImages, distImages.T) # Remove rounding errors, guarantee symmetry. config = DistanceMatrixConfig() config.dataRange = (0., 1.) render_distance_matrix(os.path.join(outPath, f'dist_images_{batchIndex}.png'), distImages, images, config) torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex))) torch.save(discriminator.state_dict(), os.path.join(outPath, 'disc_{}.pth'.format(batchIndex)))
""" Use the fc6 fc7 GANs implemented in pure python from GAN_utils, and hessian package in pytorch. to compute the Hessian and eigen decomposition. """ import numpy as np import torch import torch.optim as optim import torch.nn.functional as F from hessian import hessian import sys sys.path.append(r"D:\Github\PerceptualSimilarity") sys.path.append(r"E:\Github_Projects\PerceptualSimilarity") import models # from PerceptualSimilarity folder model_vgg = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=1, gpu_ids=[0]) from GAN_utils import upconvGAN G = upconvGAN("fc6") #%% def sim_hessian_computation(z, percept_loss, savepath=None): """ Depending on Generator imported from caffe to pytorch. Depending on Hessian, and autograd :param z: vector to compute hessian at :param percept_loss: the model from PerceptualSimilarity package :return: H: Hessian Matrix eigval: eigen decomposition, eigen values
def main(): dataSize = 128 batchSize = 8 # imageSize = 32 imageSize = 64 initWithCats = True # discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth' # discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth' discCheckpointPath = None gpu = torch.device('cuda') imageRootPath = r'E:\data\cat-vs-dog\cat' catDataset = CatDataset( imageSubdirPath=imageRootPath, transform=transforms.Compose([ transforms.Resize((imageSize, imageSize)), # torchvision.transforms.functional.to_grayscale, transforms.ToTensor(), # transforms.Lambda(lambda x: torch.reshape(x, x.shape[1:])), transforms.Normalize([0.5], [0.5]) ])) sampler = InfiniteSampler(catDataset) catLoader = DataLoader(catDataset, batch_size=batchSize, sampler=sampler) # Generate a random distance matrix. # # Make a matrix with positive values. # distancesCpu = np.clip(np.random.normal(0.5, 1.0 / 3, (dataSize, dataSize)), 0, 1) # # Make it symmetrical. # distancesCpu = np.matmul(distancesCpu, distancesCpu.T) # Generate random points and compute distances, guaranteeing that the triangle rule isn't broken. randomPoints = generate_points(dataSize) distancesCpu = scipy.spatial.distance_matrix(randomPoints, randomPoints, p=2) if initWithCats: imagePaths = random.choices(glob.glob(os.path.join(imageRootPath, '*')), k=dataSize) catImages = [] for p in imagePaths: image = skimage.transform.resize(imageio.imread(p), (imageSize, imageSize), 1).transpose(2, 0, 1) catImages.append(image) imagesInitCpu = np.asarray(catImages) else: imagesInitCpu = np.clip( np.random.normal(0.5, 0.5 / 3, (dataSize, 3, imageSize, imageSize)), 0, 1) images = torch.tensor(imagesInitCpu, requires_grad=True, dtype=torch.float32, device=gpu) scale = torch.tensor(1.0, requires_grad=True, dtype=torch.float32, device=gpu) lossModel = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu) lossBce = torch.nn.BCELoss() # discriminator = Discriminator(imageSize, 3) discriminator = Discriminator(3, 64, 1) if discCheckpointPath: discriminator.load_state_dict(torch.load(discCheckpointPath)) else: discriminator.init_params() discriminator = discriminator.to(gpu) optimizerImages = torch.optim.Adam([images, scale], lr=1e-3, betas=(0.9, 0.999)) # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999)) optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999)) import matplotlib.pyplot as plt fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2) fig2 = plt.figure() ax2 = fig2.add_subplot(1, 1, 1) outPath = os.path.join( 'images', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S')) os.makedirs(outPath) catIter = iter(catLoader) for batchIndex in range(10000): realImageBatch, _ = next(catIter) # type: Tuple(torch.Tensor, Any) realImageBatch = realImageBatch.to(gpu) # realImageBatch = torch.tensor(realImageBatchCpu, device=gpu) # noinspection PyTypeChecker randomIndices = np.random.randint( 0, dataSize, batchSize).tolist() # type: List[int] # randomIndices = list(range(dataSize)) # type: List[int] distanceBatch = torch.tensor( distancesCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu) imageBatch = images[randomIndices].contiguous() distPred = lossModel.forward( imageBatch.repeat(repeats=(batchSize, 1, 1, 1)).contiguous(), imageBatch.repeat_interleave(repeats=batchSize, dim=0).contiguous(), normalize=True) distPredMat = distPred.reshape((batchSize, batchSize)) lossDist = torch.sum((distanceBatch - distPredMat * scale)**2) # MSE discPred = discriminator(imageBatch) lossRealness = lossBce(discPred, torch.ones(imageBatch.shape[0], 1, device=gpu)) lossImages = lossDist + 100.0 * lossRealness # todo # lossImages = lossRealness # todo optimizerImages.zero_grad() lossImages.backward() optimizerImages.step() lossDiscReal = lossBce( discriminator(realImageBatch), torch.ones(realImageBatch.shape[0], 1, device=gpu)) lossDiscFake = lossBce(discriminator(imageBatch.detach()), torch.zeros(imageBatch.shape[0], 1, device=gpu)) lossDisc = (lossDiscFake + lossDiscReal) / 2 # lossDisc = torch.tensor(0) optimizerDisc.zero_grad() lossDisc.backward() optimizerDisc.step() with torch.no_grad(): # todo We're clamping all the images every batch, can we do clamp only the ones updated? # images = torch.clamp(images, 0, 1) # For some reason this was making the training worse. images.data = torch.clamp(images.data, 0, 1) if batchIndex % 100 == 0: msg = 'iter {}, loss images {:.3f}, loss dist {:.3f}, loss real {:.3f}, loss disc {:.3f}, scale: {:.3f}'.format( batchIndex, lossImages.item(), lossDist.item(), lossRealness.item(), lossDisc.item(), scale.item()) print(msg) # print(discPred.tolist()) imageBatchCpu = imageBatch.cpu().data.numpy().transpose(0, 2, 3, 1) for i, ax in enumerate(axes.flatten()): ax.imshow(imageBatchCpu[i]) fig.suptitle(msg) imagesAllCpu = images.cpu().data.numpy().transpose(0, 2, 3, 1) plot_image_scatter(ax2, randomPoints, imagesAllCpu, downscaleRatio=2) fig.savefig( os.path.join(outPath, 'batch_{}.png'.format(batchIndex))) fig2.savefig( os.path.join(outPath, 'scatter_{}.png'.format(batchIndex)))
(opt.dataset, test_data_size)) model = create_model(opt, _isTrain=False) model.switch_to_eval() torch.backends.cudnn.enabled = True torch.backends.cudnn.benchmark = True global_step = 0 total_l1 = 0. total_psnr = 0. total_lpips = 0. count = 0. lpips_model = models.PerceptualLoss(model='net', net='alex', use_gpu=True, gpu_ids=[1]) for i, data in enumerate(test_dataset): global_step = global_step + 1 print('global_step', global_step) targets = data model.set_input(targets) l1_error, psnr, lpips, valid = model.evaluate_all(lpips_model) print('l1_error ', l1_error) print('psnr ', psnr) print('lpips ', lpips)
from util import util parser = argparse.ArgumentParser( formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-p0', '--path0', type=str, default='./imgs/ex_ref.png') parser.add_argument('-p1', '--path1', type=str, default='./imgs/ex_p0.png') parser.add_argument('-v', '--version', type=str, default='0.1') parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU') opt = parser.parse_args() ## Initializing the model model = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=opt.use_gpu, version=opt.version) # Load images img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1] img1 = util.im2tensor(util.load_image(opt.path1)) if (opt.use_gpu): img0 = img0.cuda() img1 = img1.cuda() # Compute distance dist01 = model.forward(img0, img1) print('Distance: %.3f' % dist01)
import models from util import util import os import torch import numpy as np parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('-p0','--path0', type=str, default='./imgs/ex_ref.png') parser.add_argument('-p1','--path1', type=str, default='./imgs/ex_p0.png') parser.add_argument('-v','--version', type=str, default='0.1') parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU') opt = parser.parse_args() ## Initializing the model model = models.PerceptualLoss(model='net-lin',net='alex',use_gpu=opt.use_gpu,version='0.1') # Load images img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1] img1 = util.im2tensor(util.load_image(opt.path1)) if(opt.use_gpu): img0 = img0.cuda() img1 = img1.cuda() # Compute distance dist01 = model.forward(img0,img1) print('Distance: %.3f'%dist01) outputs = './experiment1'
default= '/media/yaurehman2/Seagate Backup Plus Drive/dataset/Video/test/input/DAVIS/' ) parser.add_argument('-i', '--input', type=str, default='./Yuzhi_txt/DAVIS_test_imagelist.txt') parser.add_argument('-o', '--out', type=str, default='./LBVTC_ECCV16.txt') parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU') opt = parser.parse_args() ## Initializing the model model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=opt.use_gpu) def text_readlines(filename): # Try to read a txt file and return a list.Return [] if there was a mistake. try: file = open(filename, 'r') except IOError: error = [] return error content = file.readlines() # This for loop deletes the EOF (like \n) for i in range(len(content)): content[i] = content[i][:len(content[i]) - 1] file.close() return content
import torch from util import util import models from models import dist_model as dm from IPython import embed use_gpu = False # Whether to use GPU spatial = False # Return a spatial map of perceptual distance. # Linearly calibrated models (LPIPS) model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial) # Can also set net = 'squeeze' or 'vgg' # Off-the-shelf uncalibrated networks # model = models.PerceptualLoss(model='net', net='alex', use_gpu=use_gpu, spatial=spatial) # Can also set net = 'squeeze' or 'vgg' # Low-level metrics # model = models.PerceptualLoss(model='L2', colorspace='Lab', use_gpu=use_gpu) # model = models.PerceptualLoss(model='ssim', colorspace='RGB', use_gpu=use_gpu) ## Example usage with dummy tensors dummy_im0 = torch.Tensor(1, 3, 64, 64) # image should be RGB, normalized to [-1,1] dummy_im1 = torch.Tensor(1, 3, 64, 64) dist = model.forward(dummy_im0, dummy_im1) ## Example usage with images
# from models import dist_model # model = dist_model.DistModel() from os.path import join import models import util.util as util import matplotlib.pylab as plt use_gpu = True fig_outdir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\ImageDiffMetric" #%% net_name = 'squeeze' SpatialDist = models.PerceptualLoss(model='net-lin', net=net_name, colorspace='rgb', spatial=True, use_gpu=True, gpu_ids=[0]) PerceptLoss = models.PerceptualLoss(model='net-lin', net=net_name, colorspace='rgb', spatial=False, use_gpu=True, gpu_ids=[0]) #%% imgdir = r"\\storage1.ris.wustl.edu\crponce\Active\Stimuli\2019-06-Evolutions\beto-191212a\backup_12_12_2019_10_47_39" file0 = "block048_thread000_gen_gen047_001896.jpg" file1 = "block048_thread000_gen_gen047_001900.jpg" img0_ = util.load_image(join(imgdir, file0)) img1_ = util.load_image(join(imgdir, file1)) img0 = util.im2tensor(img0_) # RGB image from [-1,1] if (use_gpu): img0 = img0.cuda()
import torch.optim as optim import torch.nn.functional as F # from cv2 import imread, imwrite import matplotlib matplotlib.use('Agg') # if you dont want image show up import matplotlib.pylab as plt import sys sys.path.append("D:\Github\pytorch-caffe") sys.path.append("D:\Github\pytorch-receptive-field") from caffenet import * from hessian import hessian #% Set up PerceptualLoss judger sys.path.append(r"D:\Github\PerceptualSimilarity") import models # from PerceptualSimilarity folder model = models.PerceptualLoss(model='net-lin', net='squeeze', use_gpu=1, gpu_ids=[0]) model_vgg = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=1, gpu_ids=[0]) model_alex = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=1, gpu_ids=[0]) #%% hess_dir = r"C:\Users\ponce\OneDrive - Washington University in St. Louis\Artiphysiology\Hessian" output_dir = r"D:\Generator_DB_Windows\data\with_CNN\hessian" output_dir = join(output_dir, "rand_spot") os.makedirs(output_dir) unit_arr = [
import os from util import util parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--cfg', type=str, default='pretrain') parser.add_argument('--ninput', type=int, default=106) parser.add_argument('--nsample', type=int, default=50) parser.add_argument('--gpu_index', type=int, default=0) opt = parser.parse_args() device = torch.device('cuda', index=opt.gpu_index) if torch.cuda.is_available() else torch.device('cpu') torch.set_grad_enabled(False) ## Initializing the model model = models.PerceptualLoss(model='net-lin',net='alex', use_gpu=opt.gpu_index >= 0 and torch.cuda.is_available(), gpu_ids=[opt.gpu_index]) results_dir = os.path.expanduser('~/results/diverse_gan/bicyclegan/%s/test/images' % opt.cfg) print(results_dir) t_LPIPS = 0.0 for i in range(opt.ninput): imgs = [] for j in range(opt.nsample): img_name = 'input_%03d_random_sample%02d.png' % (i, j + 1) img_path = os.path.join(results_dir, img_name) # print(img_path) img = util.im2tensor(util.load_image(img_path)) imgs.append(img) LPIPS = 0.0
use_gpu = False ref_path = './imgs/ex_ref.png' pred_path = './imgs/ex_p1.png' ref_img = scipy.misc.imread(ref_path).transpose(2, 0, 1) / 255. pred_img = scipy.misc.imread(pred_path).transpose(2, 0, 1) / 255. # Torchify ref = Variable(torch.FloatTensor(ref_img).unsqueeze(0), requires_grad=False) pred = Variable(torch.FloatTensor(pred_img).unsqueeze(0), requires_grad=True) if (use_gpu): ref = ref.cuda() pred = pred.cuda() loss_fn = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=use_gpu) optimizer = torch.optim.Adam([ pred, ], lr=1e-3, betas=(0.9, 0.999)) import matplotlib.pyplot as plt plt.ion() fig = plt.figure(1) ax = fig.add_subplot(131) ax.imshow(ref_img.transpose(1, 2, 0)) ax.set_title('reference') ax = fig.add_subplot(133) ax.imshow(pred_img.transpose(1, 2, 0)) ax.set_title('orig pred') for i in range(1000):
def main(): dataSize = 32 batchSize = 8 elpipsBatchSize = 1 # imageSize = 32 imageSize = 64 nz = 100 # discCheckpointPath = r'E:\projects\visus\PyTorch-GAN\implementations\dcgan\checkpoints\2020_07_10_15_53_34\disc_step4800.pth' discCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netD_epoch_24.pth' genCheckpointPath = r'E:\projects\visus\pytorch-examples\dcgan\out\netG_epoch_24.pth' gpu = torch.device('cuda') # For now we normalize the vectors to have norm 1, but don't make sure # that the data has certain mean/std. pointDataset = AuthorDataset( jsonPath=r'E:\out\scripts\metaphor-vis\authors-all.json' ) # Take top N points. points = np.asarray([pointDataset[i][0] for i in range(dataSize)]) distPointsCpu = l2_sqr_dist_matrix(torch.tensor(points)).numpy() latents = torch.tensor(np.random.normal(0.0, 1.0, (dataSize, nz)), requires_grad=True, dtype=torch.float32, device=gpu) scale = torch.tensor(2.7, requires_grad=True, dtype=torch.float32, device=gpu) # todo Re-check! bias = torch.tensor(0.0, requires_grad=True, dtype=torch.float32, device=gpu) # todo Re-check! lpips = models.PerceptualLoss(model='net-lin', net='vgg', use_gpu=True).to(gpu) # lossModel = lpips config = elpips.Config() config.batch_size = elpipsBatchSize # Ensemble size for ELPIPS. config.set_scale_levels_by_image_size(imageSize, imageSize) lossModel = elpips.ElpipsMetric(config, lpips).to(gpu) discriminator = Discriminator(3, 64, 1) if discCheckpointPath: discriminator.load_state_dict(torch.load(discCheckpointPath)) else: discriminator.init_params() discriminator = discriminator.to(gpu) generator = Generator(nz=nz, ngf=64) if genCheckpointPath: generator.load_state_dict(torch.load(genCheckpointPath)) else: generator.init_params() generator = generator.to(gpu) # optimizerImages = torch.optim.Adam([images, scale], lr=1e-2, betas=(0.9, 0.999)) optimizerScale = torch.optim.Adam([scale, bias], lr=0.001) # optimizerGen = torch.optim.Adam(generator.parameters(), lr=0.0002, betas=(0.5, 0.999)) # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=2e-4, betas=(0.9, 0.999)) # optimizerDisc = torch.optim.Adam(discriminator.parameters(), lr=0.0002, betas=(0.5, 0.999)) optimizerLatents = torch.optim.Adam([latents], lr=5e-3, betas=(0.9, 0.999)) fig, axes = plt.subplots(nrows=2, ncols=batchSize // 2) fig2 = plt.figure() ax2 = fig2.add_subplot(1, 1, 1) outPath = os.path.join('runs', datetime.datetime.today().strftime('%Y_%m_%d_%H_%M_%S')) os.makedirs(outPath) summaryWriter = SummaryWriter(outPath) for batchIndex in range(10000): # noinspection PyTypeChecker randomIndices = np.random.randint(0, dataSize, batchSize).tolist() # type: List[int] # # randomIndices = list(range(dataSize)) # type: List[int] distTarget = torch.tensor(distPointsCpu[randomIndices, :][:, randomIndices], dtype=torch.float32, device=gpu) latentsBatch = latents[randomIndices] imageBatchFake = generator(latentsBatch[:, :, None, None].float()) # todo It's possible to compute this more efficiently, but would require re-implementing lpips. # For now, compute the full BSxBS matrix row-by-row to avoid memory issues. lossDistTotal = torch.tensor(0.0, device=gpu) distanceRows = [] for iRow in range(batchSize): distPredFlat = lossModel(imageBatchFake[iRow].repeat(repeats=(batchSize, 1, 1, 1)).contiguous(), imageBatchFake, normalize=True) distPred = distPredFlat.reshape((1, batchSize)) distanceRows.append(distPred) lossDist = torch.sum((distTarget[iRow] - (distPred * scale + bias)) ** 2) # MSE lossDistTotal += lossDist lossDistTotal /= batchSize * batchSize # Compute the mean. distPredFull = torch.cat(distanceRows, dim=0) # print('{} - {} || {} - {}'.format( # torch.min(distPred).item(), # torch.max(distPred).item(), # torch.min(distTarget).item(), # torch.max(distTarget).item() # )) # discPred = discriminator(imageBatchFake) # lossRealness = bceLoss(discPred, torch.ones(imageBatchFake.shape[0], device=gpu)) # lossGen = lossDist + 1.0 * lossRealness lossLatents = lossDistTotal # optimizerGen.zero_grad() # optimizerScale.zero_grad() # lossGen.backward() # optimizerGen.step() # optimizerScale.step() optimizerLatents.zero_grad() # optimizerScale.zero_grad() lossLatents.backward() optimizerLatents.step() # optimizerScale.step() # with torch.no_grad(): # # todo We're clamping all the images every batch, can we clamp only the ones updated? # # images = torch.clamp(images, 0, 1) # For some reason this was making the training worse. # images.data = torch.clamp(images.data, 0, 1) if batchIndex % 100 == 0: msg = 'iter {} loss dist {:.3f} scale: {:.3f} bias: {:.3f}'.format(batchIndex, lossDistTotal.item(), scale.item(), bias.item()) print(msg) summaryWriter.add_scalar('loss-dist', lossDistTotal.item(), global_step=batchIndex) def gpu_images_to_numpy(images): imagesNumpy = images.cpu().data.numpy().transpose(0, 2, 3, 1) imagesNumpy = (imagesNumpy + 1) / 2 return imagesNumpy # print(discPred.tolist()) imageBatchFakeCpu = gpu_images_to_numpy(imageBatchFake) # imageBatchRealCpu = gpu_images_to_numpy(imageBatchReal) for iCol, ax in enumerate(axes.flatten()[:batchSize]): ax.imshow(imageBatchFakeCpu[iCol]) fig.suptitle(msg) with torch.no_grad(): images = gpu_images_to_numpy(generator(latents[..., None, None])) authorVectorsProj = umap.UMAP(n_neighbors=min(5, dataSize), random_state=1337).fit_transform(points) plot_image_scatter(ax2, authorVectorsProj, images, downscaleRatio=2) fig.savefig(os.path.join(outPath, f'batch_{batchIndex}.png')) fig2.savefig(os.path.join(outPath, f'scatter_{batchIndex}.png')) plt.close(fig) plt.close(fig2) with torch.no_grad(): imagesGpu = generator(latents[..., None, None]) imageNumber = imagesGpu.shape[0] # Compute LPIPS distances, batch to avoid memory issues. bs = min(imageNumber, 8) assert imageNumber % bs == 0 distPredEval = np.zeros((imagesGpu.shape[0], imagesGpu.shape[0])) for iCol in range(imageNumber // bs): startA, endA = iCol * bs, (iCol + 1) * bs imagesA = imagesGpu[startA:endA] for j in range(imageNumber // bs): startB, endB = j * bs, (j + 1) * bs imagesB = imagesGpu[startB:endB] distBatchEval = lossModel(imagesA.repeat(repeats=(bs, 1, 1, 1)).contiguous(), imagesB.repeat_interleave(repeats=bs, dim=0).contiguous(), normalize=True).cpu().numpy() distPredEval[startA:endA, startB:endB] = distBatchEval.reshape((bs, bs)) distPredEval = (distPredEval * scale.item() + bias.item()) # Move to the CPU and append an alpha channel for rendering. images = gpu_images_to_numpy(imagesGpu) images = [np.concatenate([im, np.ones(im.shape[:-1] + (1,))], axis=-1) for im in images] distPoints = distPointsCpu assert np.abs(distPoints - distPoints.T).max() < 1e-5 distPoints = np.minimum(distPoints, distPoints.T) # Remove rounding errors, guarantee symmetry. config = DistanceMatrixConfig() config.dataRange = (0., 4.) _, pointIndicesSorted = render_distance_matrix( os.path.join(outPath, f'dist_point_{batchIndex}.png'), distPoints, images, config=config ) # print(np.abs(distPredFlat - distPredFlat.T).max()) # assert np.abs(distPredFlat - distPredFlat.T).max() < 1e-5 # todo The symmetry doesn't hold for E-LPIPS, since it's stochastic. distPredEval = np.minimum(distPredEval, distPredEval.T) # Remove rounding errors, guarantee symmetry. config = DistanceMatrixConfig() config.dataRange = (0., 4.) render_distance_matrix( os.path.join(outPath, f'dist_images_{batchIndex}.png'), distPredEval, images, config=config ) config = DistanceMatrixConfig() config.dataRange = (0., 4.) render_distance_matrix( os.path.join(outPath, f'dist_images_aligned_{batchIndex}.png'), distPredEval, images, predefinedOrder=pointIndicesSorted, config=config ) fig, axes = plt.subplots(ncols=2) axes[0].matshow(distTarget.cpu().numpy(), vmin=0, vmax=4) axes[1].matshow(distPredFull.cpu().numpy() * scale.item(), vmin=0, vmax=4) fig.savefig(os.path.join(outPath, f'batch_dist_{batchIndex}.png')) plt.close(fig) surveySize = 30 fig, axes = plt.subplots(nrows=3, ncols=surveySize, figsize=(surveySize, 3)) assert len(images) == dataSize allIndices = list(range(dataSize)) with open(os.path.join(outPath, f'survey_{batchIndex}.txt'), 'w') as file: for iCol in range(surveySize): randomIndices = random.sample(allIndices, k=3) leftToMid = distPointsCpu[randomIndices[0], randomIndices[1]] rightToMid = distPointsCpu[randomIndices[2], randomIndices[1]] correctAnswer = 'left' if leftToMid < rightToMid else 'right' file.write("{}\t{}\t{}\t{}\t{}\n".format(iCol, correctAnswer, leftToMid, rightToMid, str(tuple(randomIndices)))) for iRow in (0, 1, 2): axes[iRow][iCol].imshow(images[randomIndices[iRow]]) fig.savefig(os.path.join(outPath, f'survey_{batchIndex}.png')) plt.close(fig) torch.save(generator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex))) torch.save(discriminator.state_dict(), os.path.join(outPath, 'gen_{}.pth'.format(batchIndex))) summaryWriter.close()
default=[0, 50], nargs="+", help='range of index of vectors to use') parser.add_argument( '--EPS', type=float, default=1E-2, help='EPS of finite differencing HVP operator, will only be ' 'used when method is `ForwardIter`') args = parser.parse_args( ) # ["--dataset", "pasu", '--method', "BP", '--idx_rg', '0', '50', '--EPS', '1E-2']) #%% from FeatLinModel import FeatLinModel, get_model_layers import models # from PerceptualSimilarity model_squ = models.PerceptualLoss(model='net-lin', net='squeeze', use_gpu=1, gpu_ids=[0]) model_squ.requires_grad_(False).cuda() from GAN_utils import upconvGAN if args.GAN in ["fc6", "fc7", "fc8"]: G = upconvGAN(args.GAN) elif args.GAN == "fc6_shfl": G = upconvGAN("fc6") SD = G.state_dict() shuffled_SD = {} for name, Weight in SD.items(): idx = torch.randperm(Weight.numel()) W_shuf = Weight.view(-1)[idx].view(Weight.shape) shuffled_SD[name] = W_shuf G.load_state_dict(shuffled_SD)
plt.imshow(grid_img.permute(1, 2, 0)) # plt.savefig("trackimage_0_40.png", dpi=600) # files.download("trackimage_0_40.png") """# Test your networks Did you remember to cut out a test set? If not you really should, test on images your network has never seen. #### LPIPS Metrics """ !git clone https://github.com/richzhang/PerceptualSimilarity !cd PerceptualSimilarity import models criterion_lpips = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=True, gpu_ids=[0]) # Instantiate the LPIPS metric criterion_pixel = nn.MSELoss().cuda() generator.eval() discriminator.eval() mse = np.empty(len(testloader)) lpips = np.empty(len(testloader)) for i, images in enumerate(testloader): lr = F.interpolate(images, size=(16,16)) lr_full = F.interpolate(lr, size=(64,64), mode="bilinear", align_corners=False)