def similarity(request):
use_gpu = False
spatial = False
model = models.PerceptualLoss(model='net-lin',
net='alex',
use_gpu=use_gpu,
spatial=spatial)
img1_filename = request.POST['img1'].filename
img1_input_file = request.POST['img1'].file
img1_file_path = os.path.join('/tmp', img1_filename)
with open(img1_file_path, 'wb') as img1_output_file:
shutil.copyfileobj(img1_input_file, img1_output_file)
img1tensor = util.im2tensor(util.load_image(img1_file_path))
img2_filename = request.POST['img2'].filename
img2_input_file = request.POST['img2'].file
img2_file_path = os.path.join('/tmp', img2_filename)
with open(img2_file_path, 'wb') as img2_output_file:
shutil.copyfileobj(img2_input_file, img2_output_file)
img2tensor = util.im2tensor(util.load_image(img2_file_path))
d = model.forward(img1tensor, img2tensor)
return Response('Distancia: %.3f' % d)
def compute_metric(dir1, dir2, mode):
files1 = os.listdir(dir1)
files2 = os.listdir(dir2)
img_files1 = sorted([
file for file in files1
if file.endswith('.png') or file.endswith('.jpg')
])
img_files2 = sorted([
file for file in files2
if file.endswith('.png') or file.endswith('.jpg')
])
assert len(img_files1) == len(img_files2)
vals = numpy.empty(len(img_files1))
if mode == 'perceptual':
global model
for ind in range(len(img_files1)):
if mode == 'ssim' or mode == 'l2':
img1 = skimage.img_as_float(
skimage.io.imread(os.path.join(dir1, img_files1[ind])))
img2 = skimage.img_as_float(
skimage.io.imread(os.path.join(dir2, img_files2[ind])))
if mode == 'ssim':
vals[ind] = skimage.measure.compare_ssim(img1,
img2,
datarange=img2.max() -
img2.min(),
multichannel=True)
else:
vals[ind] = numpy.mean((img1 - img2)**2) * 255.0 * 255.0
elif mode == 'perceptual':
img1 = util.im2tensor(
util.load_image(os.path.join(dir1, img_files1[ind])))
img2 = util.im2tensor(
util.load_image(os.path.join(dir2, img_files2[ind])))
vals[ind] = model.forward(img1, img2)[0]
else:
raise
filename_all = mode + '_all.txt'
filename_breakdown = mode + '_breakdown.txt'
filename_single = mode + '.txt'
numpy.savetxt(os.path.join(dir1, filename_all), vals, fmt="%f, ")
target = open(os.path.join(dir1, filename_single), 'w')
target.write("%f" % numpy.mean(vals))
target.close()
if len(img_files1) == 30:
target = open(os.path.join(dir1, filename_breakdown), 'w')
target.write("%f, %f, %f" % (numpy.mean(
vals[:5]), numpy.mean(vals[5:10]), numpy.mean(vals[10:])))
target.close()
return vals
def compute(image_file1, image_file2, gpu_flag): ## Initializing the model model = dm.DistModel() model.initialize(model='net-lin', net='squeeze', use_gpu=gpu_flag) # Load images img0 = util.im2tensor(util.load_image(image_file1)) # RGB image from [-1,1] img1 = util.im2tensor(util.load_image(image_file2)) # Compute distance dist01 = model.forward(img0, img1) return float(dist01[0])
def compute_dists_pair(use_gpu, dir, out):
## Initializing the model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=use_gpu)
dists = []
# crawl directories
# f = open(out,'w')
# category_files = os.listdir(opt.dir)
# print('category_file', category_files)
i = 0
for dir_path_class, dir_name_class, file_names_class in os.walk(dir):
# print('1',dir_path_class)
# print('2',dir_name_class)
# print('3',file_names_class)
dists_category = []
for (ff, file0) in enumerate(file_names_class[:-1]):
if ff < 10:
# print('here',file0)
# print(os.path.exists(os.path.join(dir_path_class,file0)))
img0 = util.im2tensor(
util.load_image(os.path.join(
dir_path_class, file0))) # RGB image from [-1,1]
if (use_gpu):
img0 = img0.cuda()
for (gg, file1) in enumerate(file_names_class[ff + 1:]):
img1 = util.im2tensor(
util.load_image(os.path.join(dir_path_class, file1)))
if (use_gpu):
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1).item()
dists_category.append(dist01)
else:
# print('continue')
dists_category_mean = np.mean(np.array(dists_category))
print('{}_cateogory {}_samples lpips is {}'.format(
i, len(dists_category), dists_category_mean))
dists.append(dists_category_mean)
break
i = i + 1
# if i > 5:
# break
# print('(%s, %s): %.3f'%(file0,file1,dist01))
# f.writelines('(%s, %s): %.3f'%(file0,file1,dist01))
dist_mean = np.mean(np.array(dists))
print('Mean: %.3f' % dist_mean)
return dists, dist_mean
def predict():
_logger.info('start predict...')
res = {}
errors = []
if request.method == 'POST':
if 'image' not in request.files:
errors.append('要求有image字段，类型为multipart/form-data, 对应上传文件')
else:
f = request.files['image']
file_path = './static/' + secure_filename(f.filename)
_logger.info('file_path: %s' % file_path)
f.save(file_path)
# 将image 读取并转为 tensor
img = util.im2tensor(file_path)
input_data = {'A': img, 'A_paths': file_path}
model.set_input(input_data)
model.test()
visuals = model.get_current_visuals() # 得到结果字典（fake: 图片）
result_img = util.tensor2im(visuals['fake'])
save_path = time.strftime(
'%H_%M_%S',
time.localtime()) + '_%d.bmp' % random.randint(0, 12)
if result_img is not None:
cv2.imwrite('./static/%s' % save_path, result_img)
result = '/static/%s' % save_path
if len(errors) > 0:
res['msg'] = ';'.join(errors)
res['status'] = False
else:
res['status'] = True
res['img'] = result
return Response(response=json.dumps(res),
status=200,
mimetype='application/json')
def scale_img(self, img, scale):
sz = img.size()
img2 = torch.FloatTensor(sz[0], sz[1], int(scale * sz[2]),
int(scale * sz[3]))
for i in range(0, sz[0]):
tmp = util.tensor2im(img[i])
tmp = cv2.resize(tmp, (0, 0), fx=scale, fy=scale)
iimg2 = util.im2tensor(tmp)
img2[i] = iimg2
img2 = im2.to(self.device)
return img2
def compute_perceptual(dir1, dir2):
files1 = os.listdir(dir1)
files2 = os.listdir(dir2)
img_files1 = sorted([
file for file in files1
if file.endswith('.png') or file.endswith('.jpg')
])
img_files2 = sorted([
file for file in files2
if file.endswith('.png') or file.endswith('.jpg')
])
#img_files1 = sorted([file for file in files1 if file.endswith('.png') or file.endswith('synthesized_image.jpg')])
#img_files2 = sorted([file for file in files1 if file.endswith('.png') or file.endswith('real_image.jpg')])
assert len(img_files1) == len(img_files2)
vals = numpy.empty(len(img_files1))
global model
for ind in range(len(img_files1)):
img1 = util.im2tensor(
util.load_image(os.path.join(dir1, img_files1[ind])))
img2 = util.im2tensor(
util.load_image(os.path.join(dir2, img_files2[ind])))
vals[ind] = model.forward(img1, img2)[0]
numpy.savetxt(os.path.join(dir1, 'perceptual_all.txt'), vals, fmt="%f, ")
target = open(os.path.join(dir1, 'perceptual.txt'), 'w')
target.write("%f" % numpy.mean(vals))
target.close()
if len(img_files1) == 30:
target = open(os.path.join(dir1, 'perceptual_breakdown.txt'), 'w')
target.write("%f, %f, %f" % (numpy.mean(
vals[:5]), numpy.mean(vals[5:10]), numpy.mean(vals[10:])))
target.close()
return vals
def cal_lipis(self):
for method in self.methods:
img_path_GT = os.path.join(self.folder_root, 'HR', self.dataset)
img_path_SR = os.path.join(self.folder_root, 'SR', self.dataset,
method)
lipis_List = []
img_paths_GT = os.listdir(img_path_GT)
img_paths_SR = os.listdir(img_path_SR)
for path_GT, path_SR in zip(img_paths_GT, img_paths_SR):
assert path_GT == path_SR, "Images with different name"
img_GT = util.im2tensor(
util.load_image(os.path.join(img_path_GT, path_GT)))
img_SR = util.im2tensor(
util.load_image(os.path.join(img_path_SR, path_SR)))
name = path_SR[:-4]
lipis = self.model.forward(img_GT, img_SR)[0]
print("Image {} LIPIS result: {}".format(name, lipis))
lipis_List.append(lipis)
aver_lipis = np.mean(np.asarray(lipis_List))
print("Average LIPIS result of {} in {} dataset: {}".format(
method, self.dataset, aver_lipis))
print("End")
# category_files = os.listdir(opt.dir)
# print('category_file', category_files)
i = 0
for dir_path_class, dir_name_class, file_names_class in os.walk(opt.dir):
# print('1',dir_path_class)
# print('2',dir_name_class)
# print('3',file_names_class)
print('number of each category', len(file_names_class))
print('{}_category'.format(i))
for (ff, file0) in enumerate(file_names_class[:-1]):
if ff < 10:
# print('here',file0)
# print(os.path.exists(os.path.join(dir_path_class,file0)))
img0 = util.im2tensor(
util.load_image(os.path.join(dir_path_class,
file0))) # RGB image from [-1,1]
if (opt.use_gpu):
img0 = img0.cuda()
for (gg, file1) in enumerate(file_names_class[ff + 1:]):
img1 = util.im2tensor(
util.load_image(os.path.join(dir_path_class, file1)))
if (opt.use_gpu):
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1).item()
dists.append(dist01)
else:
# print('continue')
continue
parser.add_argument('--use_gpu',
action='store_true',
help='turn on flag to use GPU')
parser.add_argument('--test_list', type=str, default='')
opt = parser.parse_args()
## Initializing the model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=opt.use_gpu)
dists = []
# Load images
with open(opt.test_list, 'r') as f:
for ll in f:
path0, path1 = ll.strip().split('\t')
print(path0, path1)
img0 = util.im2tensor(
util.load_image(os.path.join(opt.path, path0 + '.png')))
img1 = util.im2tensor(
util.load_image(os.path.join(opt.path, path1 + '.png')))
if opt.use_gpu:
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1).data.cpu().squeeze().numpy()
print('Distance: %.4f' % dist01)
dists.append(dist01)
print('Average distance: %.4f' % (sum(dists) / len(dists)))
print('Standard deviation:', np.array(dists).std())
def backward_D_B(self):
"""Calculate GAN loss for discriminator D_B"""
fake_A = self.fake_A_pool.query(self.fake_A)
self.fake_A_D = fake_A
PIL_fake_A_Jitter = self.jitter(Image.fromarray(
util.tensor2im(fake_A)))
fake_A = util.im2tensor(np.array(PIL_fake_A_Jitter))
PIL_real_A_Jitter = self.jitter(
Image.fromarray(util.tensor2im(self.real_A)))
real_A = util.im2tensor(np.array(PIL_real_A_Jitter))
toVGG = torch.cat([fake_A, real_A], 0)
out0, out1, out2, out3 = self.calculate_Features(
self.upsample_Feature(toVGG))
concat_fake_A_M = torch.cat([
self.upsample_M(out0[0, :, :, :].unsqueeze(0)),
self.upsample_M(out1[0, :, :, :].unsqueeze(0)),
self.upsample_M(out2[0, :, :, :].unsqueeze(0))
], 1)
concat_fake_A_M = torch.cat([self.upsample_M(fake_A), concat_fake_A_M],
1)
concat_real_A_M = torch.cat([
self.upsample_M(out0[1, :, :, :].unsqueeze(0)),
self.upsample_M(out1[1, :, :, :].unsqueeze(0)),
self.upsample_M(out2[1, :, :, :].unsqueeze(0))
], 1)
concat_real_A_M = torch.cat([self.upsample_M(real_A), concat_real_A_M],
1)
concat_fake_A_G = torch.cat([
self.upsample_G(out1[0, :, :, :].unsqueeze(0)),
self.upsample_G(out2[0, :, :, :].unsqueeze(0)),
self.upsample_G(out3[0, :, :, :].unsqueeze(0))
], 1)
concat_fake_A_G = torch.cat([self.upsample_G(fake_A), concat_fake_A_G],
1)
concat_real_A_G = torch.cat([
self.upsample_G(out1[1, :, :, :].unsqueeze(0)),
self.upsample_G(out2[1, :, :, :].unsqueeze(0)),
self.upsample_G(out3[1, :, :, :].unsqueeze(0))
], 1)
concat_real_A_G = torch.cat([self.upsample_G(real_A), concat_real_A_G],
1)
self.loss_D_B_M, squeeze_real, squeeze_fake = self.backward_D_basic(
self.netD_B_M, self.upsample_M(concat_real_A_M),
self.upsample_M(concat_fake_A_M))
self.squeeze_fake_M1 = self.lay1(
self.upsample_crop(squeeze_fake[:, 0:3, :]))
self.squeeze_real_M1 = self.lay1(
self.upsample_crop(squeeze_real[:, 0:3, :]))
self.loss_D_B_G, _, _ = self.backward_D_basic(
self.netD_B_G, self.upsample_G(concat_real_A_G),
self.upsample_G(concat_fake_A_G))
for i in range(len(content)):
content[i] = content[i][:len(content[i]) - 1]
file.close()
return content
files = text_readlines(opt.input)
print(len(files))
# crawl directories
f = open(opt.out, 'w')
sum_dist = 0
for file in files:
# Load images
img0 = util.im2tensor(util.load_image(opt.dir0 +
file)) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.dir1 + file))
if (opt.use_gpu):
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1)
sum_dist += dist01
print('%s: %.3f' % (file, dist01))
f.writelines('%s: %.6f\n' % (file, dist01))
sum_dist = sum_dist / len(files)
print(sum_dist)
## Initializing the model
model = dm.DistModel()
model.initialize(model='net-lin', net='alex', use_gpu=opt.use_gpu)
# crawl directories
f = open(opt.out, 'w')
files = os.listdir(opt.dir0)
print(files)
all_dist01 = []
for file in files:
if (os.path.exists(os.path.join(opt.dir1, file))):
print("a")
# Load images
img0 = util.im2tensor(util.load_image(os.path.join(
opt.dir0, file))) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(os.path.join(opt.dir1, file)))
# Compute distance
dist01 = model.forward(img0, img1)
all_dist01.append(dist01)
print('%s: %.3f' % (file, dist01))
f.writelines('%s: %.6f\n' % (file, dist01))
all_dist01 = np.asarray(all_dist01, dtype=np.float32)
print('mean: {} std: {}'.format(all_dist01.mean(), all_dist01.std()))
f.writelines('mean: {} std: {}'.format(all_dist01.mean(), all_dist01.std()))
f.close()
#model.initialize(model='net',net='squeeze',use_gpu=use_gpu)
#model.initialize(model='net',net='alex',use_gpu=use_gpu)
#model.initialize(model='net',net='vgg',use_gpu=use_gpu)
# Low-level metrics
# model.initialize(model='l2',colorspace='Lab')
# model.initialize(model='ssim',colorspace='RGB')
print('Model [%s] initialized' % model.name())
## Example usage with dummy tensors
# dummy_im0 = torch.Tensor(1,3,64,64) # image should be RGB, normalized to [-1,1]
# dummy_im1 = torch.Tensor(1,3,64,64)
# dist = model.forward(dummy_im0,dummy_im1)
## Example usage with images
ex_ref = util.im2tensor(util.load_image('./imgs/ex_ref.png'))
ex_p0 = util.im2tensor(util.load_image('./imgs/ex_p0.png'))
ex_p1 = util.im2tensor(util.load_image('./imgs/ex_p1.png'))
ex_d0 = model.forward(ex_ref, ex_p0)
ex_d1 = model.forward(ex_ref, ex_p1)
if not spatial:
print('Distances: (%.3f, %.3f)' % (ex_d0, ex_d1))
else:
print(
'Distances: (%.3f, %.3f)' % (ex_d0.mean(), ex_d1.mean())
) # The mean distance is approximately the same as the non-spatial distance
# Visualize a spatially-varying distance map between ex_p0 and ex_ref
import pylab
pylab.imshow(ex_d0)
pylab.show()
if filespath[-3:] == 'jpg':
ret.append(os.path.join(root, filespath))
return ret
imglist_dir0 = get_files(opt.dir0)
imglist_dir1 = get_files(opt.dir1)
assert len(imglist_dir0) == len(imglist_dir1)
totallen = len(imglist_dir0)
print(totallen)
# crawl directories
f = open(opt.out, 'w')
for file in range(totallen):
# Load images
img0 = util.im2tensor(util.load_image(
imglist_dir0[file])) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(imglist_dir1[file]))
if (opt.use_gpu):
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0, img1)
print('%s-th image: %.3f' % (file, dist01))
f.writelines('%s: %.6f\n' % (file, dist01))
f.close()
num = 0 score_max = 0 score_min = 1 for ii in range(10): for subdir in os.listdir(opt.dir): subdir = opt.dir + '/'+ subdir count = 0 all_file = os.listdir(subdir) for i in range(20): randp = np.random.permutation(len(all_file)) rand1 = randp[0] rand2 = randp[1] # Load images img0 = util.im2tensor(util.load_image(os.path.join(subdir,all_file[rand1]))) # RGB image from [-1,1] img1 = util.im2tensor(util.load_image(os.path.join(subdir,all_file[rand2]))) # mask = torch.zeros(img0.shape) # mask[:, :, round((224/192)* 60):round((224/192) * 140), round((224/192) * 32):round((224/192)* 168)] = 1 # img0 = img0 * mask # img1 = img1 * mask # Compute distance # dist01 = model.forward(img0[:, :, 92 :144, 48 :172], # img1[:, :, 92 :144, 48 :172]) # celebA #dist01 = model.forward(img0[:, :, 68:104, 48:154], # img1[:, :, 68:104, 48:154]) # lfw dist01 = model.forward(img0[:, :, 48:78, 24:96], img1[:, :, 48:78, 24:96]) # meglass #dist01 = model.forward(img0,img1)
import argparse
from models import dist_model as dm
from util import util
import numpy as np
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--path0', type=str, default='./imgs/ex_ref.png')
parser.add_argument('--path1', type=str, default='./imgs/ex_p0.png')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
model = dm.DistModel()
model.initialize(model='net-lin',net='vgg',use_gpu=opt.use_gpu)
# Load images
img0 = util.im2tensor(util.load_image(opt.path0), factor=1500./2.) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.path1), factor=1500./2.)
# Compute distance
dist01 = model.forward(img0,img1)
print('Distance: %.4f'%dist01)
torch.set_grad_enabled(False)
## Initializing the model
model = models.PerceptualLoss(model='net-lin',net='alex', use_gpu=opt.gpu_index >= 0 and torch.cuda.is_available(), gpu_ids=[opt.gpu_index])
results_dir = os.path.expanduser('~/results/diverse_gan/bicyclegan/%s/test/images' % opt.cfg)
print(results_dir)
t_LPIPS = 0.0
for i in range(opt.ninput):
imgs = []
for j in range(opt.nsample):
img_name = 'input_%03d_random_sample%02d.png' % (i, j + 1)
img_path = os.path.join(results_dir, img_name)
# print(img_path)
img = util.im2tensor(util.load_image(img_path))
imgs.append(img)
LPIPS = 0.0
n_pairs = 0
for p in range(1, len(imgs)):
for q in range(p):
LPIPS += model.forward(imgs[q].to(device), imgs[p].to(device)).item()
n_pairs += 1
LPIPS /= n_pairs
t_LPIPS += LPIPS
print('%d %.4f' % (i, LPIPS))
t_LPIPS /= opt.ninput
print('Total LPIPS %.4f' % t_LPIPS)
content[i] = content[i][:len(content[i])-1]
file.close()
return content
files = text_readlines(opt.input)
print(len(files))
# crawl directories
f = open(opt.out,'w')
sum_dist = 0
for file in files:
# Load images
img0 = util.load_image(opt.dir0 + file)
img1 = util.load_image(opt.dir1 + file)
img1 = cv2.resize(img1, (img0.shape[1], img0.shape[0]))
img0 = util.im2tensor(img0)
img1 = util.im2tensor(img1)
if(opt.use_gpu):
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
dist01 = model.forward(img0,img1)
sum_dist += dist01
print('%s: %.3f'%(file,dist01))
f.writelines('%s: %.6f\n'%(file,dist01))
sum_dist = sum_dist / len(files)
print(sum_dist)
# Off-the-shelf uncalibrated networks
#model.initialize(model='net',net='squeeze',use_gpu=use_gpu)
#model.initialize(model='net',net='alex',use_gpu=use_gpu)
#model.initialize(model='net',net='vgg',use_gpu=use_gpu)
# Low-level metrics
# model.initialize(model='l2',colorspace='Lab')
# model.initialize(model='ssim',colorspace='RGB')
print('Model [%s] initialized'%model.name())
## Example usage with dummy tensors
dummy_im0 = torch.Tensor(1,3,64,64) # image should be RGB, normalized to [-1,1]
dummy_im1 = torch.Tensor(1,3,64,64)
dist = model.forward(dummy_im0,dummy_im1)
## Example usage with images
ex_ref = util.im2tensor(util.load_image('./imgs/ex_ref.png'))
ex_p0 = util.im2tensor(util.load_image('./imgs/ex_p0.png'))
ex_p1 = util.im2tensor(util.load_image('./imgs/ex_p1.png'))
ex_d0 = model.forward(ex_ref,ex_p0)
ex_d1 = model.forward(ex_ref,ex_p1)
if not spatial:
print('Distances: (%.3f, %.3f)'%(ex_d0, ex_d1))
else:
print('Distances: (%.3f, %.3f)'%(ex_d0.mean(),ex_d1.mean())) # The mean distance is approximately the same as the non-spatial distance
# Visualize a spatially-varying distance map between ex_p0 and ex_ref
import pylab
pylab.imshow(ex_d0)
pylab.show()
# import sys; sys.path += ['models']
import argparse
from util import util
from models import dist_model as dm
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--path0', type=str, default='./imgs/ex_ref.png')
parser.add_argument('--path1', type=str, default='./imgs/ex_p0.png')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
model = dm.DistModel()
model.initialize(model='net-lin',net='alex',use_gpu=opt.use_gpu)
# Load images
img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.path1))
# Compute distance
dist01 = model.forward(img0,img1)
print('Distance: %.3f'%dist01)
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--dir0', type=str, default='./imgs/ex_dir0')
parser.add_argument('--dir1', type=str, default='./imgs/ex_dir1')
parser.add_argument('--out', type=str, default='./imgs/example_dists.txt')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
opt = parser.parse_args()
# embed()
## Initializing the model
model = dm.DistModel()
model.initialize(model='net-lin',net='alex',use_gpu=opt.use_gpu)
# crawl directories
f = open(opt.out,'w')
files = os.listdir(opt.dir0)
for file in files:
if(os.path.exists(os.path.join(opt.dir1,file))):
# Load images
img0 = util.im2tensor(util.load_image(os.path.join(opt.dir0,file))) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(os.path.join(opt.dir1,file)))
# Compute distance
dist01 = model.forward(img0,img1)
print('%s: %.3f'%(file,dist01))
f.writelines('%s: %.6f\n'%(file,dist01))
f.close()
def main():
# handle command line arguments
ap = argparse.ArgumentParser()
ap.add_argument('-d0', '--dir0', required=True, type=str, default='./imgs/ex_dir0',
help='Reference images directory')
ap.add_argument('-m0', '--mask0', required=False, type=str, default='*.png',
help='file mask for files in d0 (e.g. .png)')
ap.add_argument('-d1', '--dir1', required=True, type=str, default='./imgs/ex_dir1', help='Other images directory')
ap.add_argument('-m1', '--mask1', required=False, type=str, default='*.png',
help='file mask for files in d1 (e.g. .png)')
ap.add_argument('-o', '--out', required=False, type=str, default='./results/',
help='Distance files (.csv) directory')
ap.add_argument('--use_gpu', action='store_true', help='Flag to use GPU to compute distance')
ap.add_argument('-w', '--weights', default='./models/face_models/mmod_human_face_detector.dat',
help='path to face model file')
ap.add_argument('-H', '--hog_detector', required=False, action='store_true', help='use HOG detector')
ap.add_argument('-A', '--haar_detector', required=False, action='store_true', help='use Haar detector')
ap.add_argument('-C', '--cnn_detector', required=False, action='store_true', help='use CNN-based detector')
ap.add_argument('-N', '--no_face_detector', required=False, action='store_true',
help='Do NOT perform face detection. Compute quality over whole image')
ap.add_argument('-v', '--verbose', required=False, help='verbose output', action='store_true')
args = ap.parse_args()
print("Input dir0:", args.dir0)
print("Input dir0 file mask:", args.mask0)
print("Input dir1:", args.dir1)
print("Input dir1 file mask:", args.mask1)
print("Output dir: ", args.out)
if not os.path.exists(args.out):
os.makedirs(args.out)
if args.use_gpu:
print("Compute LPIPS similarity using GPU (fast)")
else:
print("Compute LPIPS similarity using CPU (slow)")
if args.no_face_detector:
print("Do NOT perform face detection. Compute frame quality over the whole image.")
file_det_name = "-no-face"
else:
if args.hog_detector:
print("Find faces using DLib HOG detector")
if args.haar_detector:
print("Find faces using OpenCV Haar detector")
if args.cnn_detector:
print("Find faces using DLib CNN detector")
if not args.hog_detector and not args.haar_detector and not args.cnn_detector:
args.haar_detector = True
print("No face detector selected. Using default OpenCV Haar detector")
file_det_name = ""
# it is expected that directories contain same number of files that can be sorted so that one is associated with the
# file in the corresponding position in the other directory
dir0_files = []
for file in os.listdir(args.dir0):
if fnmatch.fnmatch(file, args.mask0):
dir0_files.append(file)
dir0_files.sort()
print("Dir 0 (ref images) contains " + str(len(dir0_files)) + " " + args.mask0 + " files.")
dir1_files = []
for file in os.listdir(args.dir1):
if fnmatch.fnmatch(file, args.mask1):
dir1_files.append(file)
dir1_files.sort()
print("Dir 1 (mod images) contains " + str(len(dir1_files)) + " " + args.mask1 + " files.")
# Initializing face detector
fd = FaceDetector()
# Initializing LPIPS perceptual quality model
model = models.PerceptualLoss(model='net-lin', net='alex', use_gpu=args.use_gpu)
brisque_model = cv2.quality_QualityBRISQUE.create(BRISQUE_MODEL_FILE, BRISQUE_RANGE_FILE)
# process all files
num_all_files = len(dir0_files)
# open files and write headers
out_file_base_name = "all_files-" + os.path.basename(args.dir0) + "-" + os.path.basename(
args.dir1) + file_det_name
print("Base file name:" + out_file_base_name)
out_file_LPIPS_all = open(
os.path.join(args.out, out_file_base_name + "-LPIPS.csv"), 'w')
out_file_MSSIM_RGB_all = open(
os.path.join(args.out, out_file_base_name + "-MSSIM-RGB.csv"), 'w')
out_file_MSSIM_Y_all = open(
os.path.join(args.out, out_file_base_name + "-MSSIM-Y.csv"), 'w')
out_file_BRISQUE_all = open(
os.path.join(args.out, out_file_base_name + "-BRISQUE.csv"), 'w')
out_file_BRISQUE_all.writelines('files, BRISQUE score ref, BRISQUE score mod\n')
out_file_MSSIM_RGB_all.writelines('files, SSIM R, SSIM G, SSIM B\n')
out_file_MSSIM_Y_all.writelines('files, SSIM Y\n')
out_file_LPIPS_all.writelines('files, LPIPS distance\n')
processed_file = 1
for reference_image_name, modified_image_name in zip(dir0_files, dir1_files):
if processed_file % 50 == 0:
print("Processing ref. image " + str(processed_file) + "/" + str(num_all_files))
# open reference and modified images
ref_image = cv2.imread(os.path.join(args.dir0, reference_image_name))
mod_image = cv2.imread(os.path.join(args.dir1, modified_image_name))
if ref_image is None or mod_image is None:
# print("Skipping images: {} and {}".format(reference_image_name, modified_image_name))
continue
filename_all = os.path.splitext(reference_image_name)[0] + "-" + os.path.splitext(modified_image_name)[0]
if args.no_face_detector: # compute quality over whole image. No face detection
ref_face_regions = [ref_image]
mod_face_regions = [mod_image]
else: # perform face detection. use face regions to compute quality
fd.process_frame(ref_image, use_cnn=args.cnn_detector, use_hog=args.hog_detector,
use_haar=args.haar_detector)
detected_faces = fd.convert_dlib_rectangles_to_opencv_faces(fd.cnn_faces) + \
fd.convert_dlib_rectangles_to_opencv_faces(fd.hog_faces) + \
fd.convert_dlib_rectangles_to_opencv_faces(fd.haar_faces)
resized_faces = resize_64x64_multiple_faces_list(detected_faces)
ref_face_regions = get_face_regions(ref_image, resized_faces)
mod_face_regions = get_face_regions(mod_image, resized_faces)
for ref_face_region, mod_face_region in zip(ref_face_regions, mod_face_regions):
MSSIM_Y_dist = compute_MSSIM(cv2.cvtColor(ref_face_region, cv2.COLOR_RGB2GRAY),
cv2.cvtColor(mod_face_region, cv2.COLOR_RGB2GRAY))
MSSIM_RGB_dist = compute_MSSIM(ref_face_region, mod_face_region)
BRISQUE_score_ref = brisque_model.compute(ref_face_region)
BRISQUE_score_mod = brisque_model.compute(mod_face_region)
print("{}, {:.6f}, {:.6f}".format(filename_all, BRISQUE_score_ref[0], BRISQUE_score_mod[0]),
file=out_file_BRISQUE_all)
print('{}, {:.6f}'.format(filename_all, round(MSSIM_Y_dist[0] * 100, 2)), file=out_file_MSSIM_Y_all)
print('{}, {:.6f}, {:.6f}, {:.6f}'.format(filename_all, round(MSSIM_RGB_dist[2] * 100, 2),
round(MSSIM_RGB_dist[1] * 100, 2),
round(MSSIM_RGB_dist[0] * 100, 2)), file=out_file_MSSIM_RGB_all)
if args.no_face_detector:
ref_face_blocks = get_64x64_full_image_regions(ref_face_region)
mod_face_blocks = get_64x64_full_image_regions(mod_face_region)
else:
ref_face_blocks = get_64x64_face_regions(ref_face_region)
mod_face_blocks = get_64x64_face_regions(mod_face_region)
LPIPS_dist = 0
for ref_face_block, mod_face_block in zip(ref_face_blocks, mod_face_blocks):
img0 = util.im2tensor(cv2.cvtColor(ref_face_block, cv2.COLOR_BGR2RGB)) # RGB image from [-1,1]
img1 = util.im2tensor(cv2.cvtColor(mod_face_block, cv2.COLOR_BGR2RGB))
if args.use_gpu:
img0 = img0.cuda()
img1 = img1.cuda()
# Compute distance
LPIPS_dist += model.forward(img0, img1)
LPIPS_dist /= len(ref_face_blocks)
out_file_LPIPS_all.writelines('%s, %.6f\n' % (filename_all, LPIPS_dist))
if DEBUG:
processed_image = draw_faces(ref_image.copy(), cnn_faces=fd.cnn_faces, hog_faces=fd.hog_faces,
haar_faces=fd.haar_faces)
cv2.imshow('Processed image', processed_image)
out_img_filename = "/tmp/" + reference_image_name + ".jpg"
cv2.imwrite(out_img_filename, processed_image)
deb_face_count = 0
ref_face_regions = get_face_regions(ref_image, resized_faces)
for ref_detected_face in ref_face_regions:
out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str(deb_face_count) + ".jpg"
cv2.imwrite(out_img_filename, ref_detected_face)
ref_face_blocks = get_64x64_face_regions(ref_detected_face)
deb_block_count = 0
for ref_face_block in ref_face_blocks:
out_img_filename = "/tmp/" + reference_image_name + "-_face_" + str(
deb_face_count) + "_-_block_" + str(deb_block_count) + ".jpg"
cv2.imwrite(out_img_filename, ref_face_block)
deb_block_count += 1
deb_face_count += 1
if cv2.waitKey(1) == 27:
break # esc to quit
processed_file += 1
if DEBUG:
cv2.destroyAllWindows()
out_file_LPIPS_all.close()
out_file_MSSIM_RGB_all.close()
out_file_MSSIM_Y_all.close()
out_file_BRISQUE_all.close()
from util import util
from models import dist_model as dm
from IPython import embed
import os, sys
import numpy as np
use_gpu = True # Whether to use GPU
spatial = False # Return a spatial map of perceptual distance
## Initializing the model
model = dm.DistModel()
# Linearly calibrated models
model.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial)
# Low-level metrics
print('Model [%s] initialized' % model.name())
files = os.listdir(sys.argv[1])
index = np.load(sys.argv[2])
lpips = np.zeros(10)
for i in range(10):
p = index[i]
score = 0.
for j in range(1900):
img1 = util.im2tensor(util.load_image(files[j]))
img2 = util.im2tensor(util.load_image(files[p[j]]))
score += model.forward(ex_ref, ex_p0)
lpips[i] = score / 1900.
print(mp.mean(lpips), np.std(lpips))
def compute_metric(dir1, dir2, mode, mask=None, thre=0):
if mode == 'perceptual':
from models import dist_model as dm
import torch
from util import util
global model
if model is None:
cwd = os.getcwd()
os.chdir('../../PerceptualSimilarity')
model = dm.DistModel()
#model.initialize(model='net-lin',net='alex',use_gpu=True, spatial=True)
model.initialize(model='net-lin', net='alex', use_gpu=True)
print('Model [%s] initialized' % model.name())
os.chdir(cwd)
if mode.startswith('perceptual_tf'):
sys.path += ['../../lpips-tensorflow']
import lpips_tf
import tensorflow as tf
image0_ph = tf.placeholder(tf.float32, [1, None, None, 3])
image1_ph = tf.placeholder(tf.float32, [1, None, None, 3])
if mode == 'perceptual_tf':
distance_t = lpips_tf.lpips(image0_ph,
image1_ph,
model='net-lin',
net='alex')
elif mode == 'perceptual_tf_vgg':
distance_t = lpips_tf.lpips(image0_ph,
image1_ph,
model='net-lin',
net='vgg')
else:
raise
sess = tf.Session()
if mode == 'l2_with_gradient':
import demo
import tensorflow as tf
output = tf.placeholder(tf.float32, shape=[1, None, None, None])
gradient = demo.image_gradients(output)
sess = tf.Session()
files1 = os.listdir(dir1)
files2 = os.listdir(dir2)
img_files1 = sorted([
file for file in files1
if file.endswith('.png') or file.endswith('.jpg')
])
img_files2 = sorted([
file for file in files2
if file.endswith('.png') or file.endswith('.jpg')
])
if '--prefix' in sys.argv:
prefix_idx = sys.argv.index('--prefix')
prefix = sys.argv[prefix_idx + 1]
img_files2 = [file for file in img_files2 if file.startswith(prefix)]
if mask is not None:
mask_files = []
for dir in sorted(mask):
files3 = os.listdir(dir)
add_mask_files = sorted([
os.path.join(dir, file) for file in files3
if file.startswith('mask')
])
if len(add_mask_files) == 0:
files_to_dilate = sorted([
file for file in files3
if file.startswith('g_intermediates')
])
for file in files_to_dilate:
mask_arr = numpy.load(os.path.join(dir, file))
mask_arr = numpy.squeeze(mask_arr)
mask_arr = mask_arr >= thre
dilated_mask = dilation(mask_arr, disk(10))
dilated_filename = 'mask_' + file
numpy.save(os.path.join(dir, dilated_filename),
dilated_mask.astype('f'))
add_mask_files.append(os.path.join(dir, dilated_filename))
mask_files += add_mask_files
print(mask_files)
assert len(mask_files) == len(img_files1)
skip_first_n = 0
if '--skip_first_n' in sys.argv:
try:
skip_first_n = int(sys.argv[sys.argv.index('--skip_first_n') + 1])
except:
skip_first_n = 0
skip_last_n = 0
if '--skip_last_n' in sys.argv:
try:
skip_last_n = int(sys.argv[sys.argv.index('--skip_last_n') + 1])
except:
skip_last_n = 0
img_files2 = img_files2[skip_first_n:]
if skip_last_n > 0:
img_files2 = img_files2[:-skip_last_n]
assert len(img_files1) == len(img_files2)
# locate GT gradient directory
if mode == 'l2_with_gradient':
head, tail = os.path.split(dir2)
gradient_gt_dir = os.path.join(head, tail[:-3] + 'grad')
if not os.path.exists(gradient_gt_dir):
printf("dir not found,", gradient_gt_dir)
raise
gradient_gt_files = os.listdir(gradient_gt_dir)
gradient_gt_files = sorted(
[file for file in gradient_gt_files if file.endswith('.npy')])
assert len(img_files1) == len(gradient_gt_files)
vals = numpy.empty(len(img_files1))
#if mode == 'perceptual':
# global model
for ind in range(len(img_files1)):
if mode == 'ssim' or mode == 'l2' or mode == 'l2_with_gradient':
img1 = skimage.img_as_float(
skimage.io.imread(os.path.join(dir1, img_files1[ind])))
img2 = skimage.img_as_float(
skimage.io.imread(os.path.join(dir2, img_files2[ind])))
if mode == 'ssim':
#vals[ind] = skimage.measure.compare_ssim(img1, img2, datarange=img2.max()-img2.min(), multichannel=True)
metric_val = skimage.measure.compare_ssim(
img1,
img2,
datarange=img2.max() - img2.min(),
multichannel=True)
else:
#vals[ind] = numpy.mean((img1 - img2) ** 2) * 255.0 * 255.0
metric_val = ((img1 - img2)**2) * 255.0 * 255.0
if mode == 'l2_with_gradient':
metric_val = numpy.mean(metric_val, axis=2)
gradient_gt = numpy.load(
os.path.join(gradient_gt_dir, gradient_gt_files[ind]))
dx, dy = sess.run(gradient,
feed_dict={
output:
numpy.expand_dims(img1[..., ::-1],
axis=0)
})
#is_edge = skimage.feature.canny(skimage.color.rgb2gray(img1))
dx_ground = gradient_gt[:, :, :, 1:4]
dy_ground = gradient_gt[:, :, :, 4:]
edge_ground = gradient_gt[:, :, :, 0]
gradient_loss_term = numpy.mean(
(dx - dx_ground)**2.0 + (dy - dy_ground)**2.0, axis=3)
metric_val += numpy.squeeze(
0.2 * 255.0 * 255.0 * gradient_loss_term * edge_ground *
edge_ground.size / numpy.sum(edge_ground))
#if mode == 'l2' and mask is not None:
# img_diff = (img1 - img2) ** 2.0
# mask_img = numpy.load(mask_files[ind])
# img_diff *= numpy.expand_dims(mask_img, axis=2)
# vals[ind] = (numpy.sum(img_diff) / numpy.sum(mask_img * 3)) * 255.0 * 255.0
elif mode == 'perceptual':
img1 = util.im2tensor(
util.load_image(os.path.join(dir1, img_files1[ind])))
img2 = util.im2tensor(
util.load_image(os.path.join(dir2, img_files2[ind])))
#vals[ind] = numpy.mean(model.forward(img1, img2)[0])
metric_val = numpy.expand_dims(model.forward(img1, img2), axis=2)
elif mode.startswith('perceptual_tf'):
img1 = np.expand_dims(skimage.img_as_float(
skimage.io.imread(os.path.join(dir1, img_files1[ind]))),
axis=0)
img2 = np.expand_dims(skimage.img_as_float(
skimage.io.imread(os.path.join(dir2, img_files2[ind]))),
axis=0)
metric_val = sess.run(distance_t,
feed_dict={
image0_ph: img1,
image1_ph: img2
})
else:
raise
if mask is not None:
assert mode in ['l2', 'perceptual']
mask_img = numpy.load(mask_files[ind])
metric_val *= numpy.expand_dims(mask_img, axis=2)
vals[ind] = numpy.sum(metric_val) / (numpy.sum(mask_img) *
metric_val.shape[2])
else:
vals[ind] = numpy.mean(metric_val)
mode = mode + ('_mask' if mask is not None else '')
filename_all = mode + '_all.txt'
filename_breakdown = mode + '_breakdown.txt'
filename_single = mode + '.txt'
numpy.savetxt(os.path.join(dir1, filename_all), vals, fmt="%f, ")
target = open(os.path.join(dir1, filename_single), 'w')
target.write("%f" % numpy.mean(vals))
target.close()
if len(img_files1) == 30:
target = open(os.path.join(dir1, filename_breakdown), 'w')
target.write("%f, %f, %f" % (numpy.mean(
vals[:5]), numpy.mean(vals[5:10]), numpy.mean(vals[10:])))
target.close()
if mode in ['l2_with_gradient', 'perceptual_tf']:
sess.close()
return vals
import argparse
from models import dist_model as dm
from util import util
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--path0', type=str, default='./imgs/ex_ref.png')
parser.add_argument('--path1', type=str, default='./imgs/ex_p0.png')
parser.add_argument('--use_gpu', action='store_true', help='turn on flag to use GPU')
opt = parser.parse_args()
## Initializing the model
print("Initializing the model")
model = dm.DistModel()
model.initialize(model='net-lin',net='squeeze',use_gpu=opt.use_gpu)
# Load images
print("Load images")
img0 = util.im2tensor(util.load_image(opt.path0)) # RGB image from [-1,1]
img1 = util.im2tensor(util.load_image(opt.path1))
# Compute distance
print("Compute distance")
dist01 = model.forward(img0,img1)
print('Distance: %.3f'%dist01)
# model.initialize(model='net',net='squeeze',use_gpu=True)
# model.initialize(model='net',net='alex',use_gpu=True)
# model.initialize(model='net',net='vgg',use_gpu=True)
# Low-level metrics
# model.initialize(model='l2',colorspace='Lab')
# model.initialize(model='ssim',colorspace='RGB')
# print('Model [%s] initialized'%model.name())
## Example usage with dummy tensors
# dummy_im0 = torch.Tensor(1,3,64,64) # image should be RGB, normalized to [-1,1]
# dummy_im1 = torch.Tensor(1,3,64,64)
# dist = model.forward(dummy_im0,dummy_im1)
## Example usage with images
ex_ref = util.im2tensor(util.load_image('./imgs/ex_ref.png'))
ex_p0 = util.im2tensor(util.load_image('./imgs/ex_p0.png'))
ex_p1 = util.im2tensor(util.load_image('./imgs/ex_p1.png'))
# ex_d0 = model.forward(ex_ref,ex_p0)[0]
# ex_d1 = model.forward(ex_ref,ex_ref)[0]
# ---------- new model -------
model.initialize(model='net-lin', net='alex', use_gpu=True)
print('Model [%s] initialized' % model.name())
same_image = model.forward(ex_ref, ex_ref)[0]
print("comparing Same Image : ", same_image)
my_image_load = util.im2tensor(util.load_image('./imgs/my_image.PNG'))
my_image = model.forward(my_image_load, my_image_load)[0]
print("comparing Same My Image : ", same_image)
spatial=True,
use_gpu=True,
gpu_ids=[0])
PerceptLoss = models.PerceptualLoss(model='net-lin',
net=net_name,
colorspace='rgb',
spatial=False,
use_gpu=True,
gpu_ids=[0])
#%%
imgdir = r"\\storage1.ris.wustl.edu\crponce\Active\Stimuli\2019-06-Evolutions\beto-191212a\backup_12_12_2019_10_47_39"
file0 = "block048_thread000_gen_gen047_001896.jpg"
file1 = "block048_thread000_gen_gen047_001900.jpg"
img0_ = util.load_image(join(imgdir, file0))
img1_ = util.load_image(join(imgdir, file1))
img0 = util.im2tensor(img0_) # RGB image from [-1,1]
if (use_gpu):
img0 = img0.cuda()
img1 = util.im2tensor(img1_)
if (use_gpu):
img1 = img1.cuda()
#%
# Compute distance
dist01 = SpatialDist.forward(img0, img1) #.item()
dist_sum = PerceptLoss.forward(img0, img1).item()
# dists.append(dist01)
# print('(%s, %s): %.3f'%(file0,file1,dist01))
# f.writelines('(%s, %s): %.3f'%(file0,file1,dist01))
# %
plt.figure(figsize=[9, 3.5])
plt.subplot(131)
model.initialize(model='net-lin', net='alex', use_gpu=use_gpu, spatial=spatial)
#model.initialize(model='net-lin',net='vgg',use_gpu=use_gpu,spatial=spatial)
# Off-the-shelf uncalibrated networks
#model.initialize(model='net',net='squeeze',use_gpu=use_gpu)
#model.initialize(model='net',net='alex',use_gpu=use_gpu)
#model.initialize(model='net',net='vgg',use_gpu=use_gpu)
# Low-level metrics
# model.initialize(model='l2',colorspace='Lab')
# model.initialize(model='ssim',colorspace='RGB')
print('Model [%s] initialized' % model.name())
## Example usage with images
dist = []
for index, row in data.iterrows():
img_1 = util.load_image('/mnt/wham/ImageSim/females64/%s' % row['img1'])
img_2 = util.load_image('/mnt/wham/ImageSim/females64/%s' % row['img2'])
ex_ref = util.im2tensor(img_1)
ex_p1 = util.im2tensor(img_2)
ex_d0 = model.forward(ex_ref, ex_p1)
dist.append(ex_d0)
print(ex_d0)
data.distance = dist
data.to_csv("/mnt/wham/ImageSim/output_females_data.csv")
