def guided(img, guide, d=11, sigmaColor=500, color_space='RGB'):
'''
img - (H,W,3) or (H,W,1) intensity float in [0,1]
d: Diameter of each pixel neighborhood
sigmaColor: Filter sigma in the color space
sigmaSpace: Filter sigma in the coordinate space
color_space: Color space to perform filtering
'''
if color_space == 'YUV':
img = cv2.cvtColor(np.uint8(img * 255), cv2.COLOR_BGR2YUV)
img_out = guidedFilter(guide, img, radius=d, eps=sigmaColor, dDepth=-1)
img_out = np.float32(cv2.cvtColor(img_out, cv2.COLOR_YUV2BGR)) / 255.
return img_out
elif color_space == 'LAB':
img = cv2.cvtColor(np.uint8(img * 255), cv2.COLOR_BGR2LAB)
img_out = guidedFilter(guide, img, radius=d, eps=sigmaColor, dDepth=-1)
img_out = np.float32(cv2.cvtColor(img_out, cv2.COLOR_LAB2BGR)) / 255.
return img_out
else:
img_out = np.float32(
guidedFilter(np.uint8(guide * 255),
np.uint8(img * 255),
radius=d,
eps=sigmaColor,
dDepth=-1)) / 255
return img_out
def computeDisp(img_left, img_right, max_disp):
'''
The mainfunction of disparity computation
Arg(s):
img_left(np.array) : BGR image with left view field
img_right(np.array) : BGR image with right view field
max_dsip(int) : the max disparity for calculation
Return(s):
disparity(np.array) : the disparity map of final result (height, width, max_disp)
'''
img_left = img_left.astype(np.float32)
img_right = img_right.astype(np.float32)
# >>> Cost computation
start_time = time.time()
disparity_left, disparity_right = build_cost_volume(
img_left, img_right, max_disp)
end_time = time.time()
print()
print('time of build cost volume: {:.3f} s'.format(end_time - start_time))
# >>> Cost aggregation
start_time = time.time()
for disp in range(max_disp):
disparity_left[:, :, disp] = guidedFilter(
img_left.astype(np.uint8),
disparity_left[:, :, disp].astype(np.uint8), 16, 50, -1)
disparity_right[:, :, disp] = guidedFilter(
img_right.astype(np.uint8),
disparity_right[:, :, disp].astype(np.uint8), 16, 50, -1)
end_time = time.time()
print('time of cost aggregation: {:.3f} s'.format(end_time - start_time))
# >>> Disparity optimization (Winner Take All)
start_time = time.time()
disparity_left = np.argmin(disparity_left, axis=2).astype(np.uint8)
disparity_right = np.argmin(disparity_right, axis=2).astype(np.uint8)
end_time = time.time()
print('time of disparity optimization: {:.3f} s'.format(end_time -
start_time))
# >>> Disparity refinement (Left-right consistency check)
start_time = time.time()
disparity, holes = consistency_check(disparity_left, disparity_right,
max_disp)
disparity = hole_filling(disparity, holes)
disparity = weightedMedianFilter(img_left.astype(np.uint8),
disparity.astype(np.uint8), 23, 5,
cv2.ximgproc.WMF_JAC)
end_time = time.time()
print('time of disparity refinement: {:.3f} s'.format(end_time -
start_time))
return disparity.astype(np.uint8)
def denoise_image(img, denoise_option, denoise_kernel_size):
if denoise_option == 'bm3d':
denoised_image = bm3d.bm3d(img,
sigma_psd=30 / 255,
stage_arg=bm3d.BM3DStages.ALL_STAGES) # .
denoised_image = np.around(denoised_image,
0).astype(np.uint8) #convert float to uint8
elif denoise_option == 'bilat':
denoised_image = cv2.bilateralFilter(img, denoise_kernel_size, 75, 75)
elif denoise_option == 'median':
denoised_image = cv2.medianBlur(img, denoise_kernel_size)
elif denoise_option == 'guided':
r = 2
# try r=2, 4, or 8
eps = 0.4 * 0.4
# try eps=0.1^2, 0.2^2, 0.4^2
eps *= 255 * 255
# Because the intensity range of our images is [0, 255]
denoised_image = guidedFilter(img, img, r, eps)
elif denoise_option == 'gauss':
denoised_image = cv2.blur(img,
(denoise_kernel_size, denoise_kernel_size))
elif denoise_option == 'kuwahara':
denoised_image = Kuwahara(img, 13)
else:
denoised_image = None
return denoised_image
def GFPCA(pan, hs):
M, N, c = pan.shape
m, n, C = hs.shape
ratio = int(np.round(M / m))
print('get sharpening ratio: ', ratio)
assert int(np.round(M / m)) == int(np.round(N / n))
p = princomp(n_components=C)
pca_hs = p.fit_transform(np.reshape(hs, (m * n, C)))
pca_hs = np.reshape(pca_hs, (m, n, C))
pca_hs = upsample_interp23(pca_hs, ratio)
gp_hs = []
for i in range(C):
temp = guidedFilter(np.float32(pan),
np.float32(np.expand_dims(pca_hs[:, :, i], -1)),
8,
eps=0.001**2)
temp = np.expand_dims(temp, axis=-1)
gp_hs.append(temp)
gp_hs = np.concatenate(gp_hs, axis=-1)
I_GFPCA = p.inverse_transform(gp_hs)
#adjustment
I_GFPCA[I_GFPCA < 0] = 0
I_GFPCA[I_GFPCA > 1] = 1
return np.uint8(I_GFPCA * 255)
def image_slice_filter(image_path,outdir,image_name):
img = Image.open(image_path)
width, height = img.size
left = 0
top = 0
right = int(width/2)
bottom = height
#Crop and resize non rain part of image
crop_1 = (left, top, right, bottom)
image_1 = img.crop(crop_1)
image_1 = image_1.resize((64,64),Image.ANTIALIAS)
image_1.save(os.path.join(outdir+"/non_rain/",image_name+"_non_rain"+".png"))
#Crop and resize rain part of image
crop_2 = (right, top, width, bottom)
image_2 = img.crop(crop_2)
image_2 = image_2.resize((64,64),Image.ANTIALIAS)
image_2.save(os.path.join(outdir+"/rain/",image_name+"_rain"+".png"))
#Filter the rain image
img = cv2.imread(outdir+"/rain/"+image_name+"_rain"+".png")
guided = guidedFilter(img,img,15,0.2*255*255)
detail = img - guided
cv2.imwrite(outdir+"/rain/"+image_name+"_rain"+"_guided.png",guided)
cv2.imwrite(outdir+"/rain/"+image_name+"_rain"+"_detail.png",detail)
def t_matting(self, mask_out_np):
refine_t = guidedFilter(self.original_image.transpose(1, 2, 0).astype(np.float32),
mask_out_np[0].astype(np.float32), 50, 1e-4)
if self.clip:
return np.array([np.clip(refine_t, 0.1, 1)])
else:
return np.array([np.clip(refine_t, 0, 1)])
def process_opencv(self, initImg, contentImg):
'''
:param initImg: intermediate output. Either image path or PIL Image
:param contentImg: content image output. Either path or PIL Image
:return: stylized output image. PIL Image
'''
if type(initImg) == str:
init_img = cv2.imread(
initImg) # cv2.imread load image with BGR channels
init_img = init_img[2:-2, 2:-2, :]
else:
init_img = np.array(
initImg)[:, :, ::-1].copy() # convert RGB to BGR
if type(contentImg) == str:
cont_img = cv2.imread(
contentImg) # cv2.imread load image with BGR channels
else:
cont_img = np.array(
contentImg)[:, :, ::-1].copy() # convert RGB to BGR
output_img = guidedFilter(guide=cont_img,
src=init_img,
radius=self.r,
eps=self.eps)
output_img = cv2.cvtColor(output_img,
cv2.COLOR_BGR2RGB) # convert back to RGB
output_img = Image.fromarray(output_img)
return output_img
def get_mine(self, im):
im_float = im.astype('float32')
u = guidedFilter(guide=im_float,
src=im_float,
radius=self.radius,
eps=self.eps)
if self.fast_trans_estimate:
sig = cv2.boxFilter((im_float - u) * (im_float - u), -1,
self.ksize)
else:
sig = guidedFilter(guide=im_float,
src=(im_float - u) * (im_float - u),
radius=self.radius,
eps=self.eps)
return u - np.abs(np.sqrt(sig))
def skymask_refinement(self, G_pred, img):
r, eps = 20, 0.01
refined_skymask = guidedFilter(img[:, :, 2], G_pred[:, :, 0], r, eps)
refined_skymask = np.stack(
[refined_skymask, refined_skymask, refined_skymask], axis=-1)
return np.clip(refined_skymask, a_min=0, a_max=1)
def filtering(filterType, img_Eq, param1, param2):
if filterType == 'bilateral':
img_filtered= cv2.bilateralFilter(img_Eq,param1,param2,param2)
if filterType == 'anisotropic':
img_filtered = ad(img_Eq, niter=param2,step= param1, kappa=50,gamma=0.10, option=1)
img_filtered=np.uint8(img_filtered)# if not frangi does not accept img_filtered because it is float between -1 and 1
if filterType == 'guided':
img_filtered = guidedFilter(img_Eq, img_Eq, param1, param2)
return img_filtered
def testValues(guide,source,eps,space):
source = (source * 255).astype('uint8')
guide = (guide * 255).astype('uint8')
temp = guidedFilter(guide,source,radius=space,eps=eps)
temp = temp/255.0
source = source/255.0
showImage(temp)
print('source:[{},{},{}]'.format(source.min(),source.max(),source.mean()))
print('filtered:[{},{},{}]'.format(temp.min(),temp.max(),temp.mean()))
def guidedFilter3(source,guide,sigmaSpace=16, eps = 0.0001):
source = (source+1)/2
guide = (guide+1)/2
# result[:,:,0] =GuidedFilter(source[:,:,0], guide[:,:,0], sigmaSpace, eps).smooth.astype('float32')
# result[:,:,1] =GuidedFilter(source[:,:,1], guide[:,:,1], sigmaSpace, eps).smooth.astype('float32')
# result[:,:,2] =GuidedFilter(source[:,:,2], guide[:,:,2], sigmaSpace, eps).smooth.astype('float32')
source = (source * 255).astype('uint8')
guide = (guide * 255).astype('uint8')
result = guidedFilter(guide,source,sigmaSpace,eps)
result = result/255.0
# result = cv2.resize(result, (h_,w_))
return 2*result-1
def _fix_mask_2(self, thre):
""" fixing the masks using soft matting
"""
masks_np = [torch_to_np(mask) for mask in self.mask_net_outputs]
# TODO 理解为让mask平滑一些
new_mask_nps = [np.array([guidedFilter(image_np.transpose(1, 2, 0).astype(np.float32),
mask_np[0].astype(np.float32), 64, 1e-4)])
for image_np, mask_np in zip(self.images, masks_np)]
def to_bin(x):
v = np.zeros_like(x)
v[x > thre] = 1
return v
self.fixed_masks = [to_bin(m) for m in new_mask_nps]
def decomposition_NotLog(input):
# 1. Logarithm
# log_input = np.log(input + 1e-6)
# 2. Scale to [0,1]
# log_input = (log_input - log_input.min() + 1e-6) / (log_input.max() - log_input.min() + 1e-6)
# 3. Guided filter
base = guidedFilter(input.astype('single'),
input.astype('single'),
radius=3,
eps=0.1)
# 4. Decomposition
detail = input - base
return base, detail
def _fix_mask(src_image, learned_mask):
"""
fixing the masks using soft matting
:return:
"""
new_mask = guidedFilter(
src_image.transpose(1, 2, 0).astype(np.float32),
learned_mask[0].astype(np.float32),
radius=7,
eps=1e-4)
def to_bin(x):
v = np.zeros_like(x)
v[x > 0.5] = 1
return v
return to_bin(np.array([new_mask]))
def __getitem__(self, idx):
if torch.is_tensor(idx):
idx = idx.tolist()
SDR_name = osp.join(self.SDR_dir,
'{:06d}.{}'.format(idx + 1, self.file_type))
SDR_img = cv2.imread(SDR_name, cv2.IMREAD_UNCHANGED)
# transfer to YUV format from UVY
SDR_img = cv2.cvtColor(SDR_img, cv2.COLOR_BGR2RGB)
if self.phase == "val":
SDR_img = cv2.resize(SDR_img, (0, 0),
fx=1.0 / self.scale,
fy=1.0 / self.scale)
# normalize to [0, 1]
SDR_img = SDR_img.astype(np.float32)
SDR_img = SDR_img / 255.0
SDR_base = guidedFilter(guide=SDR_img, src=SDR_img, radius=5, eps=0.01)
# transform to torch tensor
SDR_img = np.moveaxis(SDR_img, -1, 0)
SDR_base = np.moveaxis(SDR_base, -1, 0)
SDR_img = torch.from_numpy(SDR_img)
SDR_base = torch.from_numpy(SDR_base)
# if `phase` != "test": then HDR images are provided
if self.phase != "test":
HDR_name = osp.join(self.HDR_dir,
'{:06d}.{}'.format(idx + 1, self.file_type))
HDR_img = cv2.imread(HDR_name, cv2.IMREAD_UNCHANGED)
HDR_img = cv2.cvtColor(HDR_img, cv2.COLOR_BGR2RGB)
HDR_img = HDR_img.astype(np.float32)
HDR_img = HDR_img / 1023.0
HDR_img = np.moveaxis(HDR_img, -1, 0)
HDR_img = torch.from_numpy(HDR_img)
else:
# mark as None
HDR_img = -1
return SDR_img, HDR_img, SDR_base
def _guided_filter_forward(_ndarr,
_radius=5,
_eps=None,
_dDepth=-1,
_scale=0.01,
_option='layer_detail'):
"""
:param _ndarr: image, _ndarr.dtype=np.float32, _ndarr.shape==(H, W, 3) where, channel order is cv2.COLOR_RGB
:return: res: res.dtype=np.float32, res.shape==(H, W, 6) where, i) layer_base, ii) layer_detail
"""
eps_min = 1e-7
ndarr_guide = cv2.cvtColor(_ndarr, cv2.COLOR_RGB2GRAY)
if _eps is None:
_eps = ((_scale * (ndarr_guide.max() - ndarr_guide.min()))**2)
if _eps == 0:
_eps = ((_scale * (255.0 - 0.0))**2)
layer_base = ximgproc.guidedFilter(guide=ndarr_guide,
src=_ndarr,
radius=_radius,
eps=_eps,
dDepth=_dDepth)
if _option == 'layer_detail':
layer_detail = _ndarr / (layer_base + eps_min)
layer_detail_max = None
elif _option == 'layer_detail_rescale':
layer_detail = _ndarr / (layer_base + eps_min)
layer_detail_max = layer_detail.max()
layer_detail = (layer_detail / (layer_detail_max + eps_min))
else:
raise NotImplementedError
res_layer = np.concatenate((layer_base, layer_detail), axis=2)
return [res_layer, layer_detail_max]
def forward(self, I, R):
'''I is the Generated Image from G1, to be filtered
R is the Composed Image, acts as the guidance image
'''
"TODO: check input shape for guided filter and change it accordingly"
to_image = transforms.ToPILImage()
to_tensor = transforms.ToTensor()
bs = I.shape[0]
res = []
for j in range(bs):
i = np.asarray(to_image(I[j].cpu()))
r = np.asarray(to_image(R[j].cpu()))
filtered = to_tensor(
PIL.Image.fromarray(
guidedFilter(guide=r,
src=i,
radius=self.radius,
eps=self.eps)))
res.append(filtered[None])
res = torch.cat(res, dim=0)
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
res = res.to(device)
return res
def doGuidedFiltering(img, mask, iters=3, maxDim=400):
r = 10
eps = 1e-4
eps *= 255 * 255
assert (img.shape[:2] == mask.shape)
oriShape = mask.shape[::-1]
maxVal = mask.max()
if maxVal > 0:
mask = (mask.astype(np.int32) * 255 // maxVal).astype(np.uint8)
if max(oriShape) > maxDim:
resizeShape = tuple(
[dim * maxDim // max(oriShape) for dim in oriShape])
else:
resizeShape = oriShape
# resizeShape = tuple([dim // s * s for dim in resizeShape])
imgResize = cv2.resize(img, resizeShape, interpolation=cv2.INTER_AREA)
maskResize = cv2.resize(mask, resizeShape, interpolation=cv2.INTER_LINEAR)
imgResize = cv2.GaussianBlur(imgResize, (3, 3), 0.5)
ret, maskResize = cv2.threshold(maskResize, 127, 255, cv2.THRESH_BINARY)
for i in range(iters):
maskGF = guidedFilter(imgResize, maskResize, r, eps)
ret, maskResize = cv2.threshold(maskGF, 127, 255, cv2.THRESH_BINARY)
maskGF = cv2.resize(maskGF, oriShape, interpolation=cv2.INTER_CUBIC)
ret, maskGF = cv2.threshold(maskGF, 255, 255, cv2.THRESH_TRUNC)
ret, maskGF = cv2.threshold(maskGF, 0, 255, cv2.THRESH_TOZERO)
#region = (mask > 0.9 * 255) | (mask < 0.1 * 255)
#maskGF[region] = mask[region]
maskGF = doMorphSmoothing(maskGF, 1, 20, maxDim)
ret, maskGF = cv2.threshold(maskGF, 127, 255, cv2.THRESH_BINARY)
return maskGF
def compute_skymask(img):
h, w, c = img.shape
imgx = cv2.resize(img, (args["in_size_w"], args["in_size_h"]))
imgx = np.array(imgx, dtype=np.float32)
imgx = torch.tensor(imgx).permute([2, 0, 1]).unsqueeze(0)
with torch.no_grad():
pred = model(imgx.to(device))
pred = torch.nn.functional.interpolate(pred, (h, w),
mode="bicubic",
align_corners=False)
pred = pred[0, :].permute([1, 2, 0])
pred = torch.cat([pred, pred, pred], dim=-1)
pred = np.array(pred.detach().cpu())
pred = np.clip(pred, a_max=1.0, a_min=0.0)
r, eps = 20, 0.01
refined_skymask = guidedFilter(img[:, :, 2], pred[:, :, 0], r, eps)
refined_skymask = np.stack(
[refined_skymask, refined_skymask, refined_skymask], axis=-1)
refined_skymask = np.clip(refined_skymask, a_min=0, a_max=1)
return refined_skymask
img_nocrack1 = img_nocrack1.astype(np.float32) / 255.0
img_crack1 = img_crack1 = cv2.imread(
"D://oezkan/Data/MASTERTHESIS_EL_start/0000000231_crack.tif", 0)
img_crack1 = img_crack1.astype(np.float32) / 255.0
img_crack2 = img_crack2 = cv2.imread(
"D://oezkan/Data/MASTERTHESIS_EL_start/0000000281_crack.tif", 0)
img_crack2 = img_crack2.astype(np.float32) / 255.0
img_crack3 = img_crack3 = cv2.imread(
"D://oezkan/Data/MASTERTHESIS_EL_start/0000001220_crack.tif", 0)
img_crack3 = img_crack3.astype(np.float32) / 255.0
start_time1 = time.time()
gf_nocrack1 = guidedFilter(img_nocrack1, img_nocrack1, 4, 0.2**2)
print(time.time() - start_time1)
start_time1 = time.time()
gf_crack1 = guidedFilter(img_crack1, img_crack1, 4, 0.2**2)
print(time.time() - start_time1)
start_time1 = time.time()
gf_crack2 = guidedFilter(img_crack2, img_crack2, 4, 0.2**2)
print(time.time() - start_time1)
start_time1 = time.time()
gf_crack3 = guidedFilter(img_crack3, img_crack3, 4, 0.2**2)
print(time.time() - start_time1)
"""
# Original images
def guidFilterTest(guide_image,src_image,radius,eps):#guide , src, radius of Guided Filter, regularization term of Guided Filter ''' if guide_image is the same as src_image, this filter will change to edge-perserving filter. ''' return guidedFilter(guide_image,src_image,radius,eps)
def guided_optimize(guides, srcs, r, eps):
Ws = [guidedFilter(guide.astype(np.float32), src.astype(np.float32), r, eps)
for guide, src in zip(guides, srcs)]
Ws = np.dstack(Ws) + 1e-12
Ws = Ws / Ws.sum(axis=2, keepdims=True)
return Ws
for j in range(0,width):
for k in [aB, aG, aR]:
t[i,j] += 255*(Ibright2[i,j] - k) / (255 - k)
t[i,j] = int(t[i,j] / 3)
t = np.array(t, dtype = np.uint8)
cv2.imwrite('Transmittance.png', t)
cv2.imshow('Transmittance',t)
cv2.waitKey(0)
# Apply filter
tFiltered = np.zeros((height,width))
srcGray = cv2.cvtColor(src,cv2.COLOR_BGR2GRAY)
t = cv2.imread('Transmittance.png', 0)
tFiltered = guidedFilter(srcGray, t, 500,200)
tFiltered = np.array(tFiltered, dtype = np.uint8)
cv2.imwrite('TransmittanceFiltered.png', tFiltered)
cv2.imshow('TransmittanceFiltered',tFiltered)
cv2.waitKey(0)
# # Step 6: Image restoration
jB = np.zeros((height,width))
jG = np.zeros((height,width))
jR = np.zeros((height,width))
for i in range(0,height):
for j in range(0,width):
jR[i,j] = (rNew[i,j] - aR) / tFiltered[i,j] + aR
jG[i,j] = 255 - int((gNew[i,j] - (255 - aG)*(255 - tFiltered[i,j])) / tFiltered[i,j])
jB[i,j] = 255 - int((bNew[i,j] - (255 - aB)*(255 - tFiltered[i,j])) / tFiltered[i,j])
# performance measure
perform_result = perf_measure(labeled_crack, img_fr_roi, img_roi)
cv2.imwrite(str(im) + '_Frangi_ad_.tif', img_fr_roi)
#save the data
with open(str(im) + '_Frangi_ad_classification_results.txt',
'w') as output:
output.write(str(perform_result))
if (index == 2):
#GUIDED
for i in range(len(param_gf[0])):
for j in range(len(param_gf[1])):
#Guided
img_filtered = guidedFilter(img_Eq, img_Eq, param_gf[0][i],
param_gf[1][j])
v.append(
img_as_ubyte(
frangi(img_filtered, (1.0, 2.1),
0.5,
0.5,
0.125,
black_ridges=True)))
img_fr = v[0]
for sigma_index in range(len(v) - 1):
img_fr = np.maximum(img_fr, v[sigma_index + 1])
_, img_thresh = cv2.threshold(img_fr, 0, 255, cv2.THRESH_BINARY)
img_fr_roi = img_thresh * img_roi
def computeDisp(Il, Ir, max_disp):
h, w, ch = Il.shape
labels = np.zeros((h, w), dtype=np.uint8)
dspMap_l = np.zeros((h, w), dtype=np.uint8)
dspMap_r = np.zeros((h, w), dtype=np.uint8)
SDI_l = np.zeros((h, w, max_disp), dtype=np.uint8)
SDI_r = np.zeros((h, w, max_disp), dtype=np.uint8)
Ilg = cv2.cvtColor(Il, cv2.COLOR_BGR2GRAY)
Irg = cv2.cvtColor(Ir, cv2.COLOR_BGR2GRAY)
# >>> Cost computation
tic = time.time()
# TODO: Compute matching cost from Il and Ir
for d in range(max_disp):
distance = np.zeros((h, w))
for y in range(h):
for x in range(w):
if x - d >= 0:
#ref_r0 = Ir[y, x-d, 0]
#ref_r1 = Ir[y, x-d, 1]
#ref_r2 = Ir[y, x-d, 2]
ref_r = Irg[y, x - d]
else:
#ref_r0 = Ir[y, x, 0]
#ref_r1 = Ir[y, x, 1]
#ref_r2 = Ir[y, x, 2]
ref_r = Irg[y, x]
#distance[y, x] = (Il[y, x, 0] - ref_r0)**2 + (Il[y, x, 1] - ref_r1)**2 + (Il[y, x, 2] - ref_r2)**2
distance[y, x] = (Ilg[y, x] - ref_r)**2
SDI_l[:, :, d] = distance
for d in range(max_disp):
distance = np.zeros((h, w))
for y in range(h):
for x in range(w):
if x + d < w:
#ref_l0 = Il[y, x+d, 0]
#ref_l1 = Il[y, x+d, 1]
#ref_l2 = Il[y, x+d, 2]
ref_l = Ilg[y, x + d]
else:
#ref_l0 = Il[y, x, 0]
#ref_l1 = Il[y, x, 1]
#ref_l2 = Il[y, x, 2]
ref_l = Ilg[y, x]
#distance[y, x] = (Ir[y, x, 0] - ref_l0)**2 + (Ir[y, x, 1] - ref_l1)**2 + (Ir[y, x, 2] - ref_l2)**2
distance[y, x] = (Irg[y, x] - ref_l)**2
SDI_r[:, :, d] = distance
toc = time.time()
print('* Elapsed time (cost computation): %f sec.' % (toc - tic))
#for d in range(max_disp):
# cv2.imwrite('sub%s.jpg'%(d), SDI_l[:, :, d])
#exit()
# >>> Cost aggregation
agg_SDI_l = np.zeros((h, w, max_disp), dtype=np.uint8)
agg_SDI_r = np.zeros((h, w, max_disp), dtype=np.uint8)
radius = 9
eps = 0.01**2
tic = time.time()
# TODO: Refine cost by aggregate nearby costs
for d in range(max_disp):
#agg_SDI_l[:, :, d] = jointBilateralFilter(Il, SDI_l[:, :, d], -1, 35, 11)
agg_SDI_l[:, :, d] = guidedFilter(Il, SDI_l[:, :, d], radius, eps)
for d in range(max_disp):
#agg_SDI_r[:, :, d] = jointBilateralFilter(Ir, SDI_r[:, :, d], -1, 35, 11)
agg_SDI_r[:, :, d] = guidedFilter(Ir, SDI_r[:, :, d], radius, eps)
toc = time.time()
print('* Elapsed time (cost aggregation): %f sec.' % (toc - tic))
#for d in range(max_disp):
# cv2.imwrite('agg%s.jpg'%(d), agg_SDI_l[:, :, d])
#exit()
# >>> Disparity optimization
tic = time.time()
# TODO: Find optimal disparity based on estimated cost. Usually winner-take-all.
for y in range(h):
for x in range(w):
min_dsp = np.inf
min_d = np.inf
for d in range(max_disp):
if agg_SDI_l[y, x, d] < min_dsp:
min_dsp = agg_SDI_l[y, x, d]
min_d = d
dspMap_l[y, x] = min_d
for y in range(h):
for x in range(w):
min_dsp = np.inf
min_d = np.inf
for d in range(max_disp):
if agg_SDI_r[y, x, d] < min_dsp:
min_dsp = agg_SDI_r[y, x, d]
min_d = d
dspMap_r[y, x] = min_d
toc = time.time()
print('* Elapsed time (disparity optimization): %f sec.' % (toc - tic))
#new_arr = ((dspMap_l - dspMap_l.min()) * (1/(dspMap_l.max() - dspMap_l.min()) * 255).astype('uint8'))
#cv2.imwrite('map_l.png', new_arr)
#new_arr = ((dspMap_r - dspMap_r.min()) * (1/(dspMap_r.max() - dspMap_r.min()) * 255).astype('uint8'))
#cv2.imwrite('map_r.png', new_arr)
#exit()
# >>> Disparity refinement
tic = time.time()
# TODO: Do whatever to enhance the disparity map
# ex: Left-right consistency check + hole filling + weighted median filtering
#LRC
consistency_left = np.zeros((h, w), dtype=np.uint8)
thres = 0
for y in range(h):
for x in range(w):
pixel_value_l = dspMap_l[y, x]
if x - pixel_value_l >= 0:
pixel_value_r = dspMap_r[y, x - pixel_value_l]
else:
pixel_value_r = dspMap_r[y, x]
if (np.abs(pixel_value_l - pixel_value_r) <= thres):
consistency_left[y, x] = pixel_value_l
else:
consistency_left[y, x] = 0
#hole-filling
Fp_l = np.zeros((h, w), dtype=np.uint8)
Fp_r = np.zeros((h, w), dtype=np.uint8)
filled = np.zeros((h, w), dtype=np.uint8)
for y in range(h):
for x in range(w):
if consistency_left[y, x] == 0:
dx = 1
while x - dx >= 0 and consistency_left[y, x - dx] == 0:
dx += 1
if x - dx < 0:
dxx = 0
while x - dx + dxx < 0 or consistency_left[y, x - dx +
dxx] == 0:
dxx += 1
Fp_l[y, x] = consistency_left[y, x - dx + dxx]
else:
Fp_l[y, x] = consistency_left[y, x - dx]
dx = 1
while x + dx < w and consistency_left[y, x + dx] == 0:
dx += 1
if x + dx >= w:
dxx = 0
while x + dx - dxx >= w or consistency_left[y, x + dx -
dxx] == 0:
dxx += 1
Fp_r[y, x] = consistency_left[y, x + dx - dxx]
else:
Fp_r[y, x] = consistency_left[y, x + dx]
else:
Fp_l[y, x] = consistency_left[y, x]
Fp_r[y, x] = consistency_left[y, x]
for y in range(h):
for x in range(w):
filled[y, x] = min(Fp_l[y, x], Fp_r[y, x])
#weighted-median filtering
labels = weightedMedianFilter(Il, filled, 15)
toc = time.time()
print('* Elapsed time (disparity refinement): %f sec.' % (toc - tic))
#consistency_left = ((consistency_left - consistency_left.min()) * (1/(consistency_left.max() - consistency_left.min()) * 255).astype('uint8'))
#cv2.imwrite('lrc.png', consistency_left)
#filled = ((filled - filled.min()) * (1/(filled.max() - filled.min()) * 255).astype('uint8'))
#cv2.imwrite('filled.png', filled)
#labels = ((labels - labels.min()) * (1/(labels.max() - labels.min()) * 255).astype('uint8'))
#cv2.imwrite('median.png', labels)
#exit()
return labels
def gf(guide, src, r=100, eps=1e-8):
return guidedFilter(guide.astype('float32'), src.astype('float32'), r, eps).astype('float64')
def Swap(source_path,
# style_path,
ref_path,
headless_path,
hue_value=260,
alpha_blend=40,
alpha_blend_color=40,
color_variation=1e-5,
hue_change=False,
morph_closing=False,
linear_color=False,
test_results=False):
# Face Matrix Transference
eps = 5e-6
eps *= 255*255
COLOUR_CORRECT_BLUR_FRAC = 0.6
LEFT_EYE_POINTS = list(range(42, 48))
RIGHT_EYE_POINTS = list(range(36, 42))
FACE_POINTS = list(range(17, 68))
LEFT_EYE_POINTS = list(range(42, 48))
RIGHT_EYE_POINTS = list(range(36, 42))
LEFT_BROW_POINTS = list(range(22, 27))
RIGHT_BROW_POINTS = list(range(17, 22))
NOSE_POINTS = list(range(27, 35))
MOUTH_POINTS = list(range(48, 61))
JAW_POINTS_ = list(range(4, 13))
ALIGN_POINTS = (JAW_POINTS_)
OVERLAY_POINTS = [
LEFT_EYE_POINTS + RIGHT_EYE_POINTS + LEFT_BROW_POINTS + RIGHT_BROW_POINTS+
NOSE_POINTS + MOUTH_POINTS+JAW_POINTS_
]
FEATHER_AMOUNT = 15
"""
source_path Path to the source face
style_path deprecated
ref_path reference template path
headless_path headless template path
"""
# Makefacebox
face_box = makeFaceBox(source_path, save=False)
# Style transfer and colortransfer
face_stylized = doStyle(Image.fromarray(face_box))
# Landmarks
# possible problem? -> landmarks need to be of the facebox
source_im = np.asarray(face_box)[...,::-1]
style_im = np.asarray(face_stylized)[...,::-1]
target_im = cv2.imread(ref_path, cv2.IMREAD_COLOR)
head_less = cv2.imread(headless_path, cv2.IMREAD_COLOR)
st_im = style_im[...,::-1]/255
hl_tm = head_less[...,::-1]/255
style_linear_ct = match_color(st_im, hl_tm, eps = color_variation)
lm = landmarksFromImage([face_box[...,::-1], target_im[...,::-1]])
source_lm = lm[0][0][0]
target_lm = lm[1][0][0]
if linear_color:
style_im = cv2.addWeighted(style_im, alpha_blend_color/100, (style_linear_ct*255).astype(np.uint8), (1-alpha_blend_color/100),0, dtype=cv2.CV_8U)
style_im = style_im[...,::-1]
#TODO[] ?fix this to do it inplace
parsing = face2parsing_maps(face_box)
# masked_face, mask_face, points_face = parsing2mask(source_im, parsing, include=[0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0])
masked_face, mask_face = parsing2mask(source_im, parsing, include = [0,1,1,1,1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0])
mask_face = guidedFilter(source_im.astype(np.float32), mask_face.astype(np.float32), 40, eps)
mask_face3d = np.array([mask_face,mask_face,mask_face]).transpose(1,2,0)
# mask_face3d = get_masked_blur(mask_face, 15, (15,15)) #<- blurs and makes 3d mask
# masked_hair, mask_hair, points_hair = parsing2mask(source_im, parsing, include=[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0])
masked_hair, mask_hair = parsing2mask(source_im, parsing, include = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0])
# Hair mask process
mask_hair_erode = cv2.erode(mask_hair, cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(3,3)), iterations = 3)
better_hair = guidedFilter(source_im.astype(np.float32), mask_hair_erode.astype(np.float32), 40, eps)
better_hair3d = np.array([better_hair,better_hair,better_hair]).transpose(1,2,0)
# mask_blur = get_masked_blur(mask, 15, (15,15))
compose_mask = better_hair3d + mask_face3d
# Mask morph operations
if morph_closing:
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,(10, 10))
compose_mask = cv2.morphologyEx(compose_mask, cv2.MORPH_CLOSE, kernel)
compose_mask = cv2.morphologyEx(compose_mask, cv2.MORPH_OPEN, kernel)
t_lm = np.matrix(target_lm[ALIGN_POINTS])
s_lm = np.matrix(source_lm[ALIGN_POINTS])
M = transformation_from_points(t_lm, s_lm)
warped_mask = warp_im(compose_mask, M, target_im.shape)
# Warped style/source image
if hue_change:
# hue filter
hsv_img = rgb2hsv(style_im/255)
rgb_img = hsv2rgb(hsv_img)
hue_mean = np.mean(hsv_img[:,:,0])*360
picked_hue = 260
if hue_mean < picked_hue:
hue_shifter = (picked_hue-hue_mean)/360
else:
hue_shifter = (hue_mean-picked_hue)/360
hue_shifter = -hue_shifter
new_hue = np.array([hsv_img[:,:,0]+hue_shifter, hsv_img[:,:,1], hsv_img[:,:,2]]).transpose(1,2,0)
new_img = hsv2rgb(new_hue)
blend_hue = cv2.addWeighted(new_img, alpha_blend/100,rgb_img, (1-alpha_blend/100),0)
warped_source_im = warp_im(blend_hue*255, M, target_im.shape)
else:
warped_source_im = warp_im(style_im, M, target_im.shape)
if test_results:
show_landmarks(lm[0], source_path, './temp/source_landmarks.jpg')
show_landmarks(lm[1], ref_path, './temp/target_landmarks.jpg')
cv2.imwrite("./temp/style_linear_ct.jpg", style_linear_ct[...,::-1]*255)
cv2.imwrite("./temp/target_im_im.jpg", target_im)
cv2.imwrite("./temp/head_less_im.jpg", head_less)
cv2.imwrite("./temp/source_im.jpg", source_im)
cv2.imwrite("./temp/style_im.jpg", style_im)
cv2.imwrite("./temp/composed_mask.jpg", compose_mask*255)
cv2.imwrite("./temp/warped_mask.jpg", warped_mask*255)
final = head_less*(1-warped_mask)+(warped_source_im*warped_mask)
print("Swap Done")
return final
import numpy as np
import cv2 as cv2
from cv2.ximgproc import guidedFilter
import os
if __name__ == '__main__':
img = cv2.imread("test_2.png")
guided = guidedFilter(img, img, 15, 0.2 * 255 * 255)
detail = img - guided
path = os.getcwd()
cv2.imwrite(path + "/test_guided.jpg", guided)
cv2.imwrite(path + "/test_detail.jpg", detail)
def test(self):
epoch = self.optimizer.get_last_epoch()
torch.set_grad_enabled(False)
self.ckp.write_log('\nEvaluation:')
self.ckp.add_log(torch.zeros(1, len(self.loader_test),
len(self.scale)))
self.model.eval()
timer_test = utility.timer()
if self.args.save_results: self.ckp.begin_background()
for idx_data, d in enumerate(self.loader_test):
for idx_scale, scale in enumerate(self.scale):
d.dataset.set_scale(idx_scale)
for lr, hr, filename in tqdm(d, ncols=80):
lr, hr = self.prepare(lr, hr)
sr = self.model(lr, idx_scale)
sr = utility.quantize(sr, self.args.rgb_range)
if self.args.guided_filtering:
sr_cpu = sr.squeeze().permute(
1, 2, 0).detach().cpu().numpy()
sr_cpu = sr_cpu.astype(np.uint8)
if self.args.guided_type == 'RGB':
guide = sr_cpu
elif self.args.guided_type == 'Gray':
guide = cv2.cvtColor(sr_cpu, cv2.COLOR_RGB2GRAY)
elif self.args.guided_type == 'Ycbcr':
guide = cv2.cvtColor(sr_cpu, cv2.COLOR_BGR2YCR_CB)
guide = guide[:, :, 0]
elif self.args.guided_type == 'HSV':
guide = cv2.cvtColor(sr_cpu, cv2.COLOR_RGB2HSV)
# sr_cpu = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
else:
guide = sr_cpu
guided_img = guidedFilter(guide, sr_cpu,
self.args.guided_radius,
self.args.guided_eps)
sr = torch.from_numpy(guided_img).permute(
2, 0, 1).unsqueeze(0).float().to(sr.device)
save_list = [sr]
self.ckp.log[
-1, idx_data, idx_scale] += 0.5 * utility.calc_psnr(
sr, hr, scale, self.args.rgb_range, dataset=d
) / 50. + 0.5 * (utility.calc_ssim(
sr, hr, scale, self.args.rgb_range) - 0.4) / 0.6
if self.args.save_gt:
save_list.extend([lr, hr])
if self.args.save_results:
self.ckp.save_results(d, filename[0], save_list, scale)
self.ckp.log[-1, idx_data, idx_scale] /= len(d)
best = self.ckp.log.max(0)
self.ckp.write_log(
'[{} x{}]\tScore: {:.6f} (Best: {:.6f} @epoch {})'.format(
d.dataset.name, scale, self.ckp.log[-1, idx_data,
idx_scale],
best[0][idx_data,
idx_scale], best[1][idx_data, idx_scale] + 1))
self.ckp.write_log('Forward: {:.2f}s\n'.format(timer_test.toc()))
self.ckp.write_log('Saving...')
if self.args.save_results:
self.ckp.end_background()
if not self.args.test_only:
self.ckp.save(self, epoch, is_best=(best[1][0, 0] + 1 == epoch))
self.ckp.write_log('Total: {:.2f}s\n'.format(timer_test.toc()),
refresh=True)
torch.set_grad_enabled(True)
