def calc_landmark_ssim_score(X, X_recon, lms, wnd_size=None):
if wnd_size is None:
wnd_size = (X_recon.shape[-1] // 16) - 1
input_images = vis.to_disp_images(X, denorm=True)
recon_images = vis.to_disp_images(X_recon, denorm=True)
data_range = 255.0 if input_images[0].dtype == np.uint8 else 1.0
nimgs = len(input_images)
nlms = len(lms[0])
scores = np.zeros((nimgs, nlms), dtype=np.float32)
for i in range(nimgs):
S = compare_ssim(input_images[i],
recon_images[i],
data_range=data_range,
multichannel=True,
full=True)[1]
S = S.mean(axis=2)
for lid in range(nlms):
x = int(lms[i, lid, 0])
y = int(lms[i, lid, 1])
t = max(0, y - wnd_size // 2)
b = min(S.shape[0] - 1, y + wnd_size // 2)
l = max(0, x - wnd_size // 2)
r = min(S.shape[1] - 1, x + wnd_size // 2)
wnd = S[t:b, l:r]
with warnings.catch_warnings():
warnings.simplefilter("ignore", category=RuntimeWarning)
scores[i, lid] = wnd.mean()
return np.nan_to_num(scores)
def calc_landmark_ncc(X, X_recon, lms):
input_images = vis.to_disp_images(X, denorm=True)
recon_images = vis.to_disp_images(X_recon, denorm=True)
nimgs = len(input_images)
nlms = len(lms[0])
wnd_size = (X_recon.shape[-1] // 16) - 1
nccs = np.zeros((nimgs, nlms), dtype=np.float32)
img_shape = input_images[0].shape
for i in range(nimgs):
for lid in range(nlms):
x = int(lms[i, lid, 0])
y = int(lms[i, lid, 1])
t = max(0, y - wnd_size // 2)
b = min(img_shape[0] - 1, y + wnd_size // 2)
l = max(0, x - wnd_size // 2)
r = min(img_shape[1] - 1, x + wnd_size // 2)
wnd1 = input_images[i][t:b, l:r]
wnd2 = recon_images[i][t:b, l:r]
ncc = ((wnd1 - wnd1.mean()) *
(wnd2 - wnd2.mean())).mean() / (wnd1.std() * wnd2.std())
nccs[i, lid] = ncc
return np.clip(np.nan_to_num(nccs), a_min=-1, a_max=1)
from csl_common.utils.common import init_random
from csl_common.utils.ds_utils import build_transform
from csl_common.vis import vis
import config
init_random(3)
path = config.get_dataset_paths('wflw')[0]
ds = WFLW(root=path,
train=False,
deterministic=True,
use_cache=False,
daug=0,
image_size=256,
transform=build_transform(deterministic=False, daug=0))
ds.filter_labels({'pose': 1, 'occlusion': 0, 'make-up': 1})
dl = td.DataLoader(ds, batch_size=10, shuffle=False, num_workers=0)
print(ds)
for data in dl:
batch = Batch(data, gpu=False)
images = vis.to_disp_images(batch.images, denorm=True)
# lms = lmutils.convert_landmarks(to_numpy(batch.landmarks), lmutils.LM98_TO_LM68)
lms = batch.landmarks
images = vis.add_landmarks_to_images(images,
lms,
draw_wireframe=False,
color=(0, 255, 0),
radius=3)
vis.vis_square(images, nCols=10, fx=1., fy=1., normalize=False)
def draw_results(X_resized,
X_recon,
levels_z=None,
landmarks=None,
landmarks_pred=None,
cs_errs=None,
ncols=15,
fx=0.5,
fy=0.5,
additional_status_text=''):
clean_images = True
if clean_images:
landmarks = None
nimgs = len(X_resized)
ncols = min(ncols, nimgs)
img_size = X_recon.shape[-1]
disp_X = vis.to_disp_images(X_resized, denorm=True)
disp_X_recon = vis.to_disp_images(X_recon, denorm=True)
# reconstruction error in pixels
l1_dists = 255.0 * to_numpy(
(X_resized - X_recon).abs().reshape(len(disp_X), -1).mean(dim=1))
# SSIM errors
ssim = np.zeros(nimgs)
for i in range(nimgs):
ssim[i] = compare_ssim(disp_X[i],
disp_X_recon[i],
data_range=1.0,
multichannel=True)
landmarks = to_numpy(landmarks)
cs_errs = to_numpy(cs_errs)
text_size = img_size / 256
text_thickness = 2
#
# Visualise resized input images and reconstructed images
#
if landmarks is not None:
disp_X = vis.add_landmarks_to_images(
disp_X,
landmarks,
draw_wireframe=False,
landmarks_to_draw=lmcfg.LANDMARKS_19)
disp_X_recon = vis.add_landmarks_to_images(
disp_X_recon,
landmarks,
draw_wireframe=False,
landmarks_to_draw=lmcfg.LANDMARKS_19)
if landmarks_pred is not None:
disp_X = vis.add_landmarks_to_images(disp_X,
landmarks_pred,
color=(1, 0, 0))
disp_X_recon = vis.add_landmarks_to_images(disp_X_recon,
landmarks_pred,
color=(1, 0, 0))
if not clean_images:
disp_X_recon = vis.add_error_to_images(disp_X_recon,
l1_dists,
format_string='{:.1f}',
size=text_size,
thickness=text_thickness)
disp_X_recon = vis.add_error_to_images(disp_X_recon,
1 - ssim,
loc='bl-1',
format_string='{:>4.2f}',
vmax=0.8,
vmin=0.2,
size=text_size,
thickness=text_thickness)
if cs_errs is not None:
disp_X_recon = vis.add_error_to_images(disp_X_recon,
cs_errs,
loc='bl-2',
format_string='{:>4.2f}',
vmax=0.0,
vmin=0.4,
size=text_size,
thickness=text_thickness)
# landmark errors
lm_errs = np.zeros(1)
if landmarks is not None:
try:
from landmarks import lmutils
lm_errs = lmutils.calc_landmark_nme_per_img(
landmarks, landmarks_pred)
disp_X_recon = vis.add_error_to_images(disp_X_recon,
lm_errs,
loc='br',
format_string='{:>5.2f}',
vmax=15,
size=img_size / 256,
thickness=2)
except:
pass
img_input = vis.make_grid(disp_X, nCols=ncols, normalize=False)
img_recon = vis.make_grid(disp_X_recon, nCols=ncols, normalize=False)
img_input = cv2.resize(img_input,
None,
fx=fx,
fy=fy,
interpolation=cv2.INTER_CUBIC)
img_recon = cv2.resize(img_recon,
None,
fx=fx,
fy=fy,
interpolation=cv2.INTER_CUBIC)
img_stack = [img_input, img_recon]
#
# Visualise hidden layers
#
VIS_HIDDEN = False
if VIS_HIDDEN:
img_z = vis.draw_z_vecs(levels_z, size=(img_size, 30), ncols=ncols)
img_z = cv2.resize(img_z,
dsize=(img_input.shape[1], img_z.shape[0]),
interpolation=cv2.INTER_NEAREST)
img_stack.append(img_z)
cs_errs_mean = np.mean(cs_errs) if cs_errs is not None else np.nan
status_bar_text = ("l1 recon err: {:.2f}px "
"ssim: {:.3f}({:.3f}) "
"lms err: {:2} {}").format(l1_dists.mean(),
cs_errs_mean,
1 - ssim.mean(),
lm_errs.mean(),
additional_status_text)
img_status_bar = vis.draw_status_bar(status_bar_text,
status_bar_width=img_input.shape[1],
status_bar_height=30,
dtype=img_input.dtype)
img_stack.append(img_status_bar)
return np.vstack(img_stack)
if __name__ == '__main__':
from csl_common.vis import vis
import torch
import config
torch.manual_seed(0)
torch.cuda.manual_seed_all(0)
dirs = config.get_dataset_paths('w300')
ds = W300(root=dirs[0],
cache_root=dirs[1],
train=False,
deterministic=True,
use_cache=False,
image_size=256,
test_split='challenging',
daug=0,
align_face_orientation=True,
crop_source='lm_ground_truth')
dl = td.DataLoader(ds, batch_size=10, shuffle=False, num_workers=0)
for data in dl:
batch = Batch(data, gpu=False)
inputs = batch.images.clone()
imgs = vis.to_disp_images(inputs, denorm=True)
imgs = vis.add_landmarks_to_images(imgs,
batch.landmarks,
radius=3,
color=(0, 255, 0))
# imgs = vis.add_landmarks_to_images(imgs, data['landmarks_of'].numpy(), color=(1,0,0))
vis.vis_square(imgs, nCols=5, fx=1, fy=1, normalize=False)