def get_perceptual_distance(input_image_list, output_image_list, model_path=None, model=None, use_gpu=True): import PerceptualSimilarity.models.dist_model as dm import PerceptualSimilarity.util.util as psutil if model is None: model = dm.DistModel() model.initialize( model="net-lin", net="alex", model_path="LPIPS/PerceptualSimilarity/weights/v0.1/alex.pth", use_gpu=use_gpu) dist_scores = np.zeros((len(input_image_list), len(output_image_list))) for i, img_i in enumerate(input_image_list): for j, img_o in enumerate(output_image_list): if type(img_i) == str and type(img_o) == str: ex_i = psutil.im2tensor(psutil.load_image(img_i)) ex_o = psutil.im2tensor(psutil.load_image(img_o)) else: assert np.shape(img_i) == np.shape(img_o) ex_i = psutil.im2tensor(img_i) ex_o = psutil.im2tensor(img_o) dist_scores[i, j] = model.forward(ex_i, ex_o)[0] return dist_scores, np.mean(dist_scores), np.std(dist_scores)
def test_seg2vid(gt_imgs, flows): flow_wrapper = flowwrapper() predictions = [] total_MSE_loss = 0 total_LPIPS_loss = 0 MSE_loss = nn.MSELoss() LPIPS_loss = dm.DistModel() LPIPS_loss.initialize(model='net-lin', net='vgg', use_gpu=True, spatial=True) for gt, flow in zip(gt_imgs, flows): frame0 = torch.from_numpy(gt[0]).unsqueeze(0) frame4 = torch.from_numpy(gt[4]).unsqueeze(0) flow = torch.from_numpy(flow).unsqueeze(0) seg2vid_preds = [ torch.unsqueeze(flow_wrapper(frame0, flow[:, :, i, :, :]), 1) for i in range(4) ] pred = seg2vid_preds[0][0] total_MSE_loss += MSE_loss(pred, frame4) total_LPIPS_loss += LPIPS_loss.forward(pred, frame4) predictions.append(seg2vid_preds) return predictions, (total_MSE_loss / len(gt_imgs)).data, (total_LPIPS_loss / len(gt_imgs)).mean()
def __init__( self, model='net-lin', net='alex', colorspace='rgb', spatial=False, use_gpu=True, gpu_ids=[0], version='0.1' ): # VGG using our perceptually-learned weights (LPIPS metric) # def __init__(self, model='net', net='vgg', use_gpu=True): # "default" way of using VGG as a perceptual loss super(PerceptualLoss, self).__init__() print('Setting up Perceptual loss...') self.use_gpu = use_gpu self.spatial = spatial self.gpu_ids = gpu_ids self.model = dist_model.DistModel() self.model.initialize(model=model, net=net, use_gpu=use_gpu, colorspace=colorspace, spatial=self.spatial, gpu_ids=gpu_ids, version=version) print('...[%s] initialized' % self.model.name()) print('...Done')
def run_test(model, scale_factor, is_consecutive, gen_inter_frames, gt_imgs, flows): predictions = [ ] # a list of lists of images (could be [[img], [img]] or [[img, img], [img, img]]) total_MSE_loss = 0 total_LPIPS_loss = 0 MSE_loss = nn.MSELoss() LPIPS_loss = dm.DistModel() LPIPS_loss.initialize(model='net-lin', net='vgg', use_gpu=True, spatial=True) model.cuda() for gt, flow in zip(gt_imgs, flows): flow = convert_flow(flow, scale_factor, is_consecutive) flow = flow.cuda() dyan_out = model.forward(flow) flow_predicitons = dyan_out.data.resize(4, 2, 256, 256) frame0 = torch.from_numpy(gt[0]) # first frame if is_consecutive and gen_inter_frames: # loop through four optical flows output_frames = [] warped_frame = frame0.unsqueeze(0).cuda() for i in range(4): current_flow = flow_predicitons[i].unsqueeze(0).cuda() warped_frame = warp(warped_frame, current_flow) gt_frame = torch.from_numpy(gt[i + 1]).unsqueeze(0).cuda() if i == 3: total_MSE_loss += MSE_loss(warped_frame, gt_frame) total_LPIPS_loss += LPIPS_loss.forward( warped_frame, gt_frame) output_frames.append(warped_frame) else: frame_indx = 3 if is_consecutive else 0 input_frame = torch.from_numpy( np.expand_dims(gt[frame_indx], axis=0)).cuda() current_flow = flow_predicitons[3].unsqueeze(0).cuda() warped_frame = warp(input_frame, current_flow) gt_frame = torch.from_numpy(gt[4]).unsqueeze(0).cuda() total_MSE_loss += MSE_loss(warped_frame, gt_frame) total_LPIPS_loss += LPIPS_loss.forward(warped_frame, gt_frame) output_frames = [warped_frame] predictions.append(output_frames) avg_mse = total_MSE_loss / len(gt_imgs) avg_lpips = (total_LPIPS_loss / len(gt_imgs)).mean() return predictions, avg_mse, avg_lpips
def create_pdist_model(network='vgg', use_gpu=True): ''' percetual similarity metric https://github.com/richzhang/PerceptualSimilarity :param network can be vgg or alex ''' ## Initializing the model model = dm.DistModel() model.initialize(model='net-lin', net=network, use_gpu=use_gpu) return model
def main(): argv = sys.argv[1:] (input,name,seed,perplexity,gpu) = getArgs(argv) makeModelReadyImages(input,name) makePairsList(input,name) data=pd.read_csv("./datatables/%s_pairs_list.csv" % name) if gpu.lower() in ('yes', 'true', 't', 'y', '1'): gpu=True elif gpu.lower() in ('no', 'false', 'f', 'n', '0'): gpu=False use_gpu = gpu # Whether to use GPU spatial = False # Return a spatial map of perceptual distance. # Optional args spatial_shape and spatial_order control output shape and resampling filter: see DistModel.initialize() for details. ## Initializing the model model = dm.DistModel() # Linearly calibrated models #model.initialize(model='net-lin',net='squeeze',use_gpu=use_gpu,spatial=spatial) model.initialize(model='net-lin',net='alex',use_gpu=use_gpu,spatial=spatial) #model.initialize(model='net-lin',net='vgg',use_gpu=use_gpu,spatial=spatial) # Off-the-shelf uncalibrated networks #model.initialize(model='net',net='squeeze',use_gpu=use_gpu) #model.initialize(model='net',net='alex',use_gpu=use_gpu) #model.initialize(model='net',net='vgg',use_gpu=use_gpu) # Low-level metrics #model.initialize(model='l2',colorspace='Lab') #model.initialize(model='ssim',colorspace='RGB') print('Model [%s] initialized'%model.name()) ## Example usage with images dist=[] for index, row in data.iterrows(): model_input_dir= "./model_ready_images/%s" % (name) img_1_path="%s/%s" % (model_input_dir, row['img1']) img_2_path="%s/%s" % (model_input_dir, row['img2']) img_1=util.load_image(img_1_path) img_2=util.load_image(img_2_path) ex_ref = util.im2tensor(img_1) ex_p1 = util.im2tensor(img_2) ex_d0 = model.forward(ex_ref,ex_p1) dist.append(ex_d0) print(ex_d0) data.distance=dist data.to_csv("./datatables/output_%s_data.csv" % name) makeEmb(name,perplexity,seed) emb_path="./datatables/%s_emb.csv" % name data=pd.read_csv(emb_path) visualize_scatter_with_images(name=name,data=data) sys.exit()
def LPIPS(ims1, ims2, scale): model = dm.DistModel() model.initialize(model="net-lin", net="alex", use_gpu=True, version="0.1") mean_distance = 0.0 for im1, im2 in zip(ims1, ims2): im1 = im1[scale:-scale, scale:-scale] im2 = im2[scale:-scale, scale:-scale] im1 = torch.from_numpy(im1).permute(2, 0, 1).unsqueeze(0) im2 = torch.from_numpy(im2).permute(2, 0, 1).unsqueeze(0) # LPIPS needs [-1, 1] range tensor im1 = im1 * 2 - 1 im2 = im2 * 2 - 1 mean_distance += model.forward(im1, im2)[0] / len(ims1) return mean_distance
def main(args): torch.manual_seed(123) np.random.seed(123) data_loader = model_utils.build_loaders(args) # change to image model = model_utils.build_all_model(args) # CNN, GCN, Encoder, Decoder # 'LPIPS' metric = dm.DistModel() metric.initialize(model='net-lin', net='alex', use_gpu=True, version='0.1') save_name = os.path.join(args.checkpoint.split('.')[0], '%s_%s' % (args.test_mod, args.test_split)) evaluator = Evaluator(save_name, args, name=args.dataset, metric=metric) cnt = 0 for batch in data_loader: print('Hallucinating... %d / %d' % (cnt, len(data_loader)), batch['index'][0]) quan(batch, model, evaluator, args) cnt += 1 evaluator.draw_save_error() evaluator.draw_save_pix()
def get_df_from_logs(log_dir, lpips_compute=False, ignore_error=True, use_gpu=True): # lpips_compute can take a very longggg time, only set to true if you really want it~ from tqdm import tqdm import PerceptualSimilarity.models.dist_model as dm df_list = [] if lpips_compute: model = dm.DistModel() model_path = "/notebooks/psangkloy3/amit_cadgan//LPIPS/PerceptualSimilarity/weights/v0.1/squeeze.pth" model.initialize(model="net-lin", net="squeeze", model_path=model_path, use_gpu=use_gpu) else: model = None with tqdm(total=len(os.listdir(log_dir)), file=sys.stdout) as pbar: for i, log in enumerate(os.listdir(log_dir)): pbar.set_description("processed: %d" % (1 + i)) pbar.update(1) try: df_list.append( get_df_from_log(os.path.join(log_dir, log), lpips_compute=lpips_compute, model=model)) except Exception as e: # dirty hack, old log doesn't have images/metadata, eventually we shouldn't need this if ignore_error: pass else: logging.error(e) raise if len(df_list) == 0: return [] else: return pd.concat(df_list, ignore_index=True)
def __init__(self, model='net', net='alex', use_gpu=True): print('Setting up Perceptual loss..') self.model = dist_model.DistModel() self.model.initialize(model=model, net=net, use_gpu=True) print('Done')
# import sys; sys.path += ['models'] import torch from PerceptualSimilarity.util import util from PerceptualSimilarity.models import dist_model as dm from IPython import embed use_gpu = True # Whether to use GPU spatial = False # Return a spatial map of perceptual distance. # Optional args spatial_shape and spatial_order control output shape and resampling filter: see DistModel.initialize() for details. ## Initializing the model model = dm.DistModel() # Linearly calibrated models #model.initialize(model='net-lin',net='squeeze',use_gpu=use_gpu,spatial=spatial) model.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial) #model.initialize(model='net-lin',net='vgg',use_gpu=use_gpu,spatial=spatial) # Off-the-shelf uncalibrated networks #model.initialize(model='net',net='squeeze',use_gpu=use_gpu) #model.initialize(model='net',net='alex',use_gpu=use_gpu) #model.initialize(model='net',net='vgg',use_gpu=use_gpu) # Low-level metrics # model.initialize(model='l2',colorspace='Lab') # model.initialize(model='ssim',colorspace='RGB') print('Model [%s] initialized' % model.name()) ## Example usage with dummy tensors dummy_im0 = torch.Tensor(1, 3, 64, 64) # image should be RGB, normalized to [-1,1]
# LSiM from LSIM.distance_model import * from LSIM.metrics import * ## MODEL INITIALIZATION use_gpu = True modelLSiM = DistanceModel(baseType="lsim", isTrain=False, useGPU=use_gpu) modelLSiM.load("Models/LSiM.pth") modelL2 = Metric("L2") modelSSIM = Metric("SSIM") modelLPIPS = dm.DistModel() modelLPIPS.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=False) print() ## DISTANCE COMPUTATION ref = imageio.imread("Images/plumeReference.png")[...,:3] plumeA = imageio.imread("Images/plumeA.png")[...,:3] plumeB = imageio.imread("Images/plumeB.png")[...,:3] distA_LSiM = modelLSiM.computeDistance(ref, plumeA, interpolateTo=224, interpolateOrder=0) distB_LSiM = modelLSiM.computeDistance(ref, plumeB, interpolateTo=224, interpolateOrder=0) distA_L2 = modelL2.computeDistance(ref, plumeA) distB_L2 = modelL2.computeDistance(ref, plumeB)
os.environ["CUDA_VISIBLE_DEVICES"] = "0" useGPU = True batch = 8 workerThreads = 4 cutOffIndex = 10 #only compare first image to all others (instead of every pair) for better performance #loadFile= "Results/distances.npy" loadFile = "" saveDir = "Results/" mode = "spearman" #mode = "pearson" #mode = "pearsonMean" # MODEL AND DATA INITIALIZATION model1 = lpipsModel.DistModel() model1.initialize(model='net-lin', net='alex', use_gpu=useGPU, spatial=False) model2 = DistanceModel(baseType="alex", featureDistance="L2", frozenLayers=[0, 1, 2, 3, 4], normMode="normUnit", isTrain=False, useGPU=useGPU) model2.load("Models/Experimental/Alex_InitRandom.pth") model3 = DistanceModel(baseType="alex", featureDistance="L2", frozenLayers=[0, 1, 2, 3, 4], normMode="normUnit", isTrain=False, useGPU=useGPU)