def get_perceptual_distance(input_image_list,
output_image_list,
model_path=None,
model=None,
use_gpu=True):
import PerceptualSimilarity.models.dist_model as dm
import PerceptualSimilarity.util.util as psutil
if model is None:
model = dm.DistModel()
model.initialize(
model="net-lin",
net="alex",
model_path="LPIPS/PerceptualSimilarity/weights/v0.1/alex.pth",
use_gpu=use_gpu)
dist_scores = np.zeros((len(input_image_list), len(output_image_list)))
for i, img_i in enumerate(input_image_list):
for j, img_o in enumerate(output_image_list):
if type(img_i) == str and type(img_o) == str:
ex_i = psutil.im2tensor(psutil.load_image(img_i))
ex_o = psutil.im2tensor(psutil.load_image(img_o))
else:
assert np.shape(img_i) == np.shape(img_o)
ex_i = psutil.im2tensor(img_i)
ex_o = psutil.im2tensor(img_o)
dist_scores[i, j] = model.forward(ex_i, ex_o)[0]
return dist_scores, np.mean(dist_scores), np.std(dist_scores)
def test_seg2vid(gt_imgs, flows):
flow_wrapper = flowwrapper()
predictions = []
total_MSE_loss = 0
total_LPIPS_loss = 0
MSE_loss = nn.MSELoss()
LPIPS_loss = dm.DistModel()
LPIPS_loss.initialize(model='net-lin',
net='vgg',
use_gpu=True,
spatial=True)
for gt, flow in zip(gt_imgs, flows):
frame0 = torch.from_numpy(gt[0]).unsqueeze(0)
frame4 = torch.from_numpy(gt[4]).unsqueeze(0)
flow = torch.from_numpy(flow).unsqueeze(0)
seg2vid_preds = [
torch.unsqueeze(flow_wrapper(frame0, flow[:, :, i, :, :]), 1)
for i in range(4)
]
pred = seg2vid_preds[0][0]
total_MSE_loss += MSE_loss(pred, frame4)
total_LPIPS_loss += LPIPS_loss.forward(pred, frame4)
predictions.append(seg2vid_preds)
return predictions, (total_MSE_loss /
len(gt_imgs)).data, (total_LPIPS_loss /
len(gt_imgs)).mean()
def __init__(
self,
model='net-lin',
net='alex',
colorspace='rgb',
spatial=False,
use_gpu=True,
gpu_ids=[0],
version='0.1'
): # VGG using our perceptually-learned weights (LPIPS metric)
# def __init__(self, model='net', net='vgg', use_gpu=True): # "default" way of using VGG as a perceptual loss
super(PerceptualLoss, self).__init__()
print('Setting up Perceptual loss...')
self.use_gpu = use_gpu
self.spatial = spatial
self.gpu_ids = gpu_ids
self.model = dist_model.DistModel()
self.model.initialize(model=model,
net=net,
use_gpu=use_gpu,
colorspace=colorspace,
spatial=self.spatial,
gpu_ids=gpu_ids,
version=version)
print('...[%s] initialized' % self.model.name())
print('...Done')
def run_test(model, scale_factor, is_consecutive, gen_inter_frames, gt_imgs,
flows):
predictions = [
] # a list of lists of images (could be [[img], [img]] or [[img, img], [img, img]])
total_MSE_loss = 0
total_LPIPS_loss = 0
MSE_loss = nn.MSELoss()
LPIPS_loss = dm.DistModel()
LPIPS_loss.initialize(model='net-lin',
net='vgg',
use_gpu=True,
spatial=True)
model.cuda()
for gt, flow in zip(gt_imgs, flows):
flow = convert_flow(flow, scale_factor, is_consecutive)
flow = flow.cuda()
dyan_out = model.forward(flow)
flow_predicitons = dyan_out.data.resize(4, 2, 256, 256)
frame0 = torch.from_numpy(gt[0]) # first frame
if is_consecutive and gen_inter_frames:
# loop through four optical flows
output_frames = []
warped_frame = frame0.unsqueeze(0).cuda()
for i in range(4):
current_flow = flow_predicitons[i].unsqueeze(0).cuda()
warped_frame = warp(warped_frame, current_flow)
gt_frame = torch.from_numpy(gt[i + 1]).unsqueeze(0).cuda()
if i == 3:
total_MSE_loss += MSE_loss(warped_frame, gt_frame)
total_LPIPS_loss += LPIPS_loss.forward(
warped_frame, gt_frame)
output_frames.append(warped_frame)
else:
frame_indx = 3 if is_consecutive else 0
input_frame = torch.from_numpy(
np.expand_dims(gt[frame_indx], axis=0)).cuda()
current_flow = flow_predicitons[3].unsqueeze(0).cuda()
warped_frame = warp(input_frame, current_flow)
gt_frame = torch.from_numpy(gt[4]).unsqueeze(0).cuda()
total_MSE_loss += MSE_loss(warped_frame, gt_frame)
total_LPIPS_loss += LPIPS_loss.forward(warped_frame, gt_frame)
output_frames = [warped_frame]
predictions.append(output_frames)
avg_mse = total_MSE_loss / len(gt_imgs)
avg_lpips = (total_LPIPS_loss / len(gt_imgs)).mean()
return predictions, avg_mse, avg_lpips
def create_pdist_model(network='vgg', use_gpu=True):
'''
percetual similarity metric
https://github.com/richzhang/PerceptualSimilarity
:param network can be vgg or alex
'''
## Initializing the model
model = dm.DistModel()
model.initialize(model='net-lin', net=network, use_gpu=use_gpu)
return model
def main():
argv = sys.argv[1:]
(input,name,seed,perplexity,gpu) = getArgs(argv)
makeModelReadyImages(input,name)
makePairsList(input,name)
data=pd.read_csv("./datatables/%s_pairs_list.csv" % name)
if gpu.lower() in ('yes', 'true', 't', 'y', '1'):
gpu=True
elif gpu.lower() in ('no', 'false', 'f', 'n', '0'):
gpu=False
use_gpu = gpu # Whether to use GPU
spatial = False # Return a spatial map of perceptual distance.
# Optional args spatial_shape and spatial_order control output shape and resampling filter: see DistModel.initialize() for details.
## Initializing the model
model = dm.DistModel()
# Linearly calibrated models
#model.initialize(model='net-lin',net='squeeze',use_gpu=use_gpu,spatial=spatial)
model.initialize(model='net-lin',net='alex',use_gpu=use_gpu,spatial=spatial)
#model.initialize(model='net-lin',net='vgg',use_gpu=use_gpu,spatial=spatial)
# Off-the-shelf uncalibrated networks
#model.initialize(model='net',net='squeeze',use_gpu=use_gpu)
#model.initialize(model='net',net='alex',use_gpu=use_gpu)
#model.initialize(model='net',net='vgg',use_gpu=use_gpu)
# Low-level metrics
#model.initialize(model='l2',colorspace='Lab')
#model.initialize(model='ssim',colorspace='RGB')
print('Model [%s] initialized'%model.name())
## Example usage with images
dist=[]
for index, row in data.iterrows():
model_input_dir= "./model_ready_images/%s" % (name)
img_1_path="%s/%s" % (model_input_dir, row['img1'])
img_2_path="%s/%s" % (model_input_dir, row['img2'])
img_1=util.load_image(img_1_path)
img_2=util.load_image(img_2_path)
ex_ref = util.im2tensor(img_1)
ex_p1 = util.im2tensor(img_2)
ex_d0 = model.forward(ex_ref,ex_p1)
dist.append(ex_d0)
print(ex_d0)
data.distance=dist
data.to_csv("./datatables/output_%s_data.csv" % name)
makeEmb(name,perplexity,seed)
emb_path="./datatables/%s_emb.csv" % name
data=pd.read_csv(emb_path)
visualize_scatter_with_images(name=name,data=data)
sys.exit()
def LPIPS(ims1, ims2, scale):
model = dm.DistModel()
model.initialize(model="net-lin", net="alex", use_gpu=True, version="0.1")
mean_distance = 0.0
for im1, im2 in zip(ims1, ims2):
im1 = im1[scale:-scale, scale:-scale]
im2 = im2[scale:-scale, scale:-scale]
im1 = torch.from_numpy(im1).permute(2, 0, 1).unsqueeze(0)
im2 = torch.from_numpy(im2).permute(2, 0, 1).unsqueeze(0)
# LPIPS needs [-1, 1] range tensor
im1 = im1 * 2 - 1
im2 = im2 * 2 - 1
mean_distance += model.forward(im1, im2)[0] / len(ims1)
return mean_distance
def main(args):
torch.manual_seed(123)
np.random.seed(123)
data_loader = model_utils.build_loaders(args) # change to image
model = model_utils.build_all_model(args) # CNN, GCN, Encoder, Decoder
# 'LPIPS'
metric = dm.DistModel()
metric.initialize(model='net-lin', net='alex', use_gpu=True, version='0.1')
save_name = os.path.join(args.checkpoint.split('.')[0], '%s_%s' % (args.test_mod, args.test_split))
evaluator = Evaluator(save_name, args, name=args.dataset, metric=metric)
cnt = 0
for batch in data_loader:
print('Hallucinating... %d / %d' % (cnt, len(data_loader)), batch['index'][0])
quan(batch, model, evaluator, args)
cnt += 1
evaluator.draw_save_error()
evaluator.draw_save_pix()
def get_df_from_logs(log_dir,
lpips_compute=False,
ignore_error=True,
use_gpu=True):
# lpips_compute can take a very longggg time, only set to true if you really want it~
from tqdm import tqdm
import PerceptualSimilarity.models.dist_model as dm
df_list = []
if lpips_compute:
model = dm.DistModel()
model_path = "/notebooks/psangkloy3/amit_cadgan//LPIPS/PerceptualSimilarity/weights/v0.1/squeeze.pth"
model.initialize(model="net-lin",
net="squeeze",
model_path=model_path,
use_gpu=use_gpu)
else:
model = None
with tqdm(total=len(os.listdir(log_dir)), file=sys.stdout) as pbar:
for i, log in enumerate(os.listdir(log_dir)):
pbar.set_description("processed: %d" % (1 + i))
pbar.update(1)
try:
df_list.append(
get_df_from_log(os.path.join(log_dir, log),
lpips_compute=lpips_compute,
model=model))
except Exception as e:
# dirty hack, old log doesn't have images/metadata, eventually we shouldn't need this
if ignore_error:
pass
else:
logging.error(e)
raise
if len(df_list) == 0:
return []
else:
return pd.concat(df_list, ignore_index=True)
def __init__(self, model='net', net='alex', use_gpu=True):
print('Setting up Perceptual loss..')
self.model = dist_model.DistModel()
self.model.initialize(model=model, net=net, use_gpu=True)
print('Done')
# import sys; sys.path += ['models']
import torch
from PerceptualSimilarity.util import util
from PerceptualSimilarity.models import dist_model as dm
from IPython import embed
use_gpu = True # Whether to use GPU
spatial = False # Return a spatial map of perceptual distance.
# Optional args spatial_shape and spatial_order control output shape and resampling filter: see DistModel.initialize() for details.
## Initializing the model
model = dm.DistModel()
# Linearly calibrated models
#model.initialize(model='net-lin',net='squeeze',use_gpu=use_gpu,spatial=spatial)
model.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial)
#model.initialize(model='net-lin',net='vgg',use_gpu=use_gpu,spatial=spatial)
# Off-the-shelf uncalibrated networks
#model.initialize(model='net',net='squeeze',use_gpu=use_gpu)
#model.initialize(model='net',net='alex',use_gpu=use_gpu)
#model.initialize(model='net',net='vgg',use_gpu=use_gpu)
# Low-level metrics
# model.initialize(model='l2',colorspace='Lab')
# model.initialize(model='ssim',colorspace='RGB')
print('Model [%s] initialized' % model.name())
## Example usage with dummy tensors
dummy_im0 = torch.Tensor(1, 3, 64,
64) # image should be RGB, normalized to [-1,1]
# LSiM
from LSIM.distance_model import *
from LSIM.metrics import *
## MODEL INITIALIZATION
use_gpu = True
modelLSiM = DistanceModel(baseType="lsim", isTrain=False, useGPU=use_gpu)
modelLSiM.load("Models/LSiM.pth")
modelL2 = Metric("L2")
modelSSIM = Metric("SSIM")
modelLPIPS = dm.DistModel()
modelLPIPS.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=False)
print()
## DISTANCE COMPUTATION
ref = imageio.imread("Images/plumeReference.png")[...,:3]
plumeA = imageio.imread("Images/plumeA.png")[...,:3]
plumeB = imageio.imread("Images/plumeB.png")[...,:3]
distA_LSiM = modelLSiM.computeDistance(ref, plumeA, interpolateTo=224, interpolateOrder=0)
distB_LSiM = modelLSiM.computeDistance(ref, plumeB, interpolateTo=224, interpolateOrder=0)
distA_L2 = modelL2.computeDistance(ref, plumeA)
distB_L2 = modelL2.computeDistance(ref, plumeB)
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
useGPU = True
batch = 8
workerThreads = 4
cutOffIndex = 10 #only compare first image to all others (instead of every pair) for better performance
#loadFile= "Results/distances.npy"
loadFile = ""
saveDir = "Results/"
mode = "spearman"
#mode = "pearson"
#mode = "pearsonMean"
# MODEL AND DATA INITIALIZATION
model1 = lpipsModel.DistModel()
model1.initialize(model='net-lin', net='alex', use_gpu=useGPU, spatial=False)
model2 = DistanceModel(baseType="alex",
featureDistance="L2",
frozenLayers=[0, 1, 2, 3, 4],
normMode="normUnit",
isTrain=False,
useGPU=useGPU)
model2.load("Models/Experimental/Alex_InitRandom.pth")
model3 = DistanceModel(baseType="alex",
featureDistance="L2",
frozenLayers=[0, 1, 2, 3, 4],
normMode="normUnit",
isTrain=False,
useGPU=useGPU)