def horn_brooks(intensity_image, initial_estimate, normal_model, light_vector,
n_iters=100, c_lambda=0.001,
mapping_object=IdentityMapper()):
"""
M. Brooks and B. Horn
Shape and Source from Shading
1985
"""
from scipy.signal import convolve2d
# Ensure the light is a unit vector
light_vector = normalise_vector(light_vector)
# Equation (1): Should never be < 0 if image is properly scaled
theta_vec = np.arccos(intensity_image.as_vector())
theta_image = intensity_image.from_vector(theta_vec)
n_im = estimate_normals_from_intensity(initial_estimate, theta_image)
average_kernel = np.array([[0.0, 0.25, 0.0],
[0.25, 0.0, 0.25],
[0.0, 0.25, 0.0]])
scale_constant = (1.0 / 4.0 * c_lambda)
n_vec = n_im.as_vector(keep_channels=True)
I_vec = intensity_image.as_vector()
for i in xrange(n_iters):
n_dot_s = np.sum(n_vec * light_vector, axis=1)
# Calculate the average normal neighbourhood
n_im.from_vector_inplace(n_vec)
n_xs = convolve2d(n_im.pixels[:, :, 0], average_kernel, mode='same')
n_ys = convolve2d(n_im.pixels[:, :, 1], average_kernel, mode='same')
n_zs = convolve2d(n_im.pixels[:, :, 2], average_kernel, mode='same')
n_bar = np.concatenate([n_xs[..., None],
n_ys[..., None],
n_zs[..., None]], axis=-1)
n_bar = MaskedImage(n_bar, mask=n_im.mask).as_vector(
keep_channels=True)
rho = scale_constant * (I_vec - n_dot_s)
m = n_bar + np.dot(rho[..., None], light_vector[..., None].T)
n_im.from_vector_inplace(normalise_vector(m))
v0 = mapping_object.logmap(n_im)
# Vector of best-fit parameters
vprime = normal_model.reconstruct(v0)
nprime = mapping_object.expmap(vprime)
nprime = normalise_image(nprime)
n_vec = nprime.as_vector(keep_channels=True)
n_im.from_vector_inplace(n_vec)
return normalise_image(n_im)
def horn_brooks(intensity_image, initial_estimate, normal_model, light_vector,
n_iters=100, c_lambda=0.001,
mapping_object=IdentityMapper()):
"""
M. Brooks and B. Horn
Shape and Source from Shading
1985
"""
from scipy.signal import convolve2d
# Ensure the light is a unit vector
light_vector = normalise_vector(light_vector)
# Equation (1): Should never be < 0 if image is properly scaled
theta_vec = np.arccos(intensity_image.as_vector())
theta_image = intensity_image.from_vector(theta_vec)
n_im = estimate_normals_from_intensity(initial_estimate, theta_image)
average_kernel = np.array([[0.0, 0.25, 0.0],
[0.25, 0.0, 0.25],
[0.0, 0.25, 0.0]])
scale_constant = (1.0 / 4.0 * c_lambda)
n_vec = n_im.as_vector(keep_channels=True)
I_vec = intensity_image.as_vector()
for i in xrange(n_iters):
n_dot_s = np.sum(n_vec * light_vector, axis=1)
# Calculate the average normal neighbourhood
n_im.from_vector_inplace(n_vec)
n_xs = convolve2d(n_im.pixels[:, :, 0], average_kernel, mode='same')
n_ys = convolve2d(n_im.pixels[:, :, 1], average_kernel, mode='same')
n_zs = convolve2d(n_im.pixels[:, :, 2], average_kernel, mode='same')
n_bar = np.concatenate([n_xs[..., None],
n_ys[..., None],
n_zs[..., None]], axis=-1)
n_bar = MaskedImage(n_bar, mask=n_im.mask).as_vector(
keep_channels=True)
rho = scale_constant * (I_vec - n_dot_s)
m = n_bar + np.dot(rho[..., None], light_vector[..., None].T)
n_im.from_vector_inplace(normalise_vector(m))
v0 = mapping_object.logmap(n_im)
# Vector of best-fit parameters
vprime = normal_model.reconstruct(v0)
nprime = mapping_object.expmap(vprime)
nprime = normalise_image(nprime)
n_vec = nprime.as_vector(keep_channels=True)
n_im.from_vector_inplace(n_vec)
return normalise_image(n_im)
def mean_vector(images):
N = len(images)
avg = np.zeros_like(images[0].as_vector(keep_channels=True))
for im in images:
avg += im.as_vector(keep_channels=True)
return normalise_vector(avg / N)
def on_cone_rotation(theta_image, normal_image, s):
theta = theta_image.as_vector()
nprime = normal_image.as_vector(keep_channels=True)
# cross product and break in to row vectors
C = np.cross(nprime, s)
C = normalise_vector(C)
u = C[:, 0]
v = C[:, 1]
w = C[:, 2]
# expects |nprime| = |sec| = 1
# represents intensity and can never be < 0
d = np.squeeze(np.inner(nprime, s) / (row_norm(nprime) * row_norm(s)))
d = np.nan_to_num(d)
d[d < 0.0] = 0.0
beta = np.arccos(d)
# flip beta and theta so that it rotates along the correct axis
alpha = beta - theta
c = np.cos(alpha)
cprime = 1.0 - c
s = np.sin(alpha)
# setup structures
N = nprime.shape[0]
phi = np.zeros([N, 3, 3])
phi[:, 0, 0] = c + u ** 2 * cprime
phi[:, 0, 1] = -w * s + u * v * cprime
phi[:, 0, 2] = v * s + u * w * cprime
phi[:, 1, 0] = w * s + u * v * cprime
phi[:, 1, 1] = c + v ** 2 * cprime
phi[:, 1, 2] = -u * s + v * w * cprime
phi[:, 2, 0] = -v * s + u * w * cprime
phi[:, 2, 1] = u * s + v * w * cprime
phi[:, 2, 2] = c + w ** 2 * cprime
n = np.einsum('kjl, klm -> kj', phi, nprime[..., None])
# Normalize the result ??
n = normalise_vector(n)
return normal_image.from_vector(n)
def on_cone_rotation(theta_image, normal_image, s):
theta = theta_image.as_vector()
nprime = normal_image.as_vector(keep_channels=True)
# cross product and break in to row vectors
C = np.cross(nprime, s)
C = normalise_vector(C)
u = C[:, 0]
v = C[:, 1]
w = C[:, 2]
# expects |nprime| = |sec| = 1
# represents intensity and can never be < 0
d = np.squeeze(np.inner(nprime, s) / (row_norm(nprime) * row_norm(s)))
d = np.nan_to_num(d)
d[d < 0.0] = 0.0
beta = np.arccos(d)
# flip beta and theta so that it rotates along the correct axis
alpha = beta - theta
c = np.cos(alpha)
cprime = 1.0 - c
s = np.sin(alpha)
# setup structures
N = nprime.shape[0]
phi = np.zeros([N, 3, 3])
phi[:, 0, 0] = c + u ** 2 * cprime
phi[:, 0, 1] = -w * s + u * v * cprime
phi[:, 0, 2] = v * s + u * w * cprime
phi[:, 1, 0] = w * s + u * v * cprime
phi[:, 1, 1] = c + v ** 2 * cprime
phi[:, 1, 2] = -u * s + v * w * cprime
phi[:, 2, 0] = -v * s + u * w * cprime
phi[:, 2, 1] = u * s + v * w * cprime
phi[:, 2, 2] = c + w ** 2 * cprime
n = np.einsum('kjl, klm -> kj', phi, nprime[..., None])
# Normalize the result ??
n = normalise_vector(n)
return normal_image.from_vector(n)
def intrinsic_mean(sd_vectors, mapping_class, max_iters=20):
# Compute initial estimate (Euclidean mean of data)
mus = normalise_vector(np.mean(sd_vectors, axis=0))
for i in xrange(max_iters):
mapping_object = mapping_class(mus)
# Iteratively improve estimate of intrinsic mean
mean_tangent = np.mean(mapping_object.logmap(sd_vectors), axis=0)
mus = np.squeeze(mapping_object.expmap(mean_tangent))
return mus
def intrinsic_mean(sd_vectors, mapping_class, max_iters=20):
# Compute initial estimate (Euclidean mean of data)
mus = normalise_vector(np.mean(sd_vectors, axis=0))
for i in xrange(max_iters):
mapping_object = mapping_class(mus)
# Iteratively improve estimate of intrinsic mean
mean_tangent = np.mean(mapping_object.logmap(sd_vectors), axis=0)
mus = np.squeeze(mapping_object.expmap(mean_tangent))
return mus
def photometric_stereo(images, lights):
"""
Images is a 3 or more channel image representing the same object taken
under different lighting conditions. Lights is a n_channels x 3 matrix that
represents the direction from which each image is lit.
Only the masked pixels are recovered.
Parameters
----------
images : (M, N, C) :class:`pybug.image.MaskedNDImage`
An image where each channel is an image lit under a unique lighting
direction.
lights : (C, 3) ndarray
A matrix representing the light directions for each of the channels in
``images``.
Returns
-------
normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the components of the recovered normals.
albedo_image : (M, N, 1) :class:`pybug.image.MaskedNDImage`
A 1-channel image representing the albedo at each pixel.
"""
# Ensure the light are unit vectors
lights = normalise_vector(lights)
LL = pinv(lights)
# n_masked_pixels x n_channels
pixels = images.as_vector(keep_channels=True)
n_images = pixels.shape[1]
if n_images < 3:
raise ValueError('Photometric Stereo is undefined with less than 3 '
'input images.')
if LL.shape[1] != n_images:
raise ValueError('You must provide a light direction for each input '
'channel.')
normals = np.dot(pixels, LL.T)
magnitudes = np.sqrt((normals * normals).sum(axis=1))
albedo = magnitudes
normals[magnitudes != 0.0, :] /= magnitudes[magnitudes != 0.0][..., None]
return (images.from_vector(normals, n_channels=3),
images.from_vector(albedo, n_channels=1))
def photometric_stereo(images, lights):
"""
Images is a 3 or more channel image representing the same object taken
under different lighting conditions. Lights is a n_channels x 3 matrix that
represents the direction from which each image is lit.
Only the masked pixels are recovered.
Parameters
----------
images : (M, N, C) :class:`pybug.image.MaskedNDImage`
An image where each channel is an image lit under a unique lighting
direction.
lights : (C, 3) ndarray
A matrix representing the light directions for each of the channels in
``images``.
Returns
-------
normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the components of the recovered normals.
albedo_image : (M, N, 1) :class:`pybug.image.MaskedNDImage`
A 1-channel image representing the albedo at each pixel.
"""
# Ensure the light are unit vectors
lights = normalise_vector(lights)
LL = pinv(lights)
# n_masked_pixels x n_channels
pixels = images.as_vector(keep_channels=True)
n_images = pixels.shape[1]
if n_images < 3:
raise ValueError('Photometric Stereo is undefined with less than 3 '
'input images.')
if LL.shape[1] != n_images:
raise ValueError('You must provide a light direction for each input '
'channel.')
normals = np.dot(pixels, LL.T)
magnitudes = np.sqrt((normals * normals).sum(axis=1))
albedo = magnitudes
normals[magnitudes != 0.0, :] /= magnitudes[magnitudes != 0.0][..., None]
return (images.from_vector(normals, n_channels=3),
images.from_vector(albedo, n_channels=1))
def estimate_normals_from_intensity(average_normals, theta_image):
theta = theta_image.as_vector()
# Represents tan(phi) = sin(partial I/ partial y) / cos(partial I/ partial x)
# Where [partial I/ partial y] is the y-direction of the gradient field
average_masked_pixels = average_normals.as_vector(keep_channels=True)
n = np.sqrt(average_masked_pixels[:, 0] ** 2 + average_masked_pixels[:, 1] ** 2)
cosphi = average_masked_pixels[:, 0] / n
sinphi = average_masked_pixels[:, 1] / n
# Reset any nan-vectors to 0.0
cosphi = np.nan_to_num(cosphi)
sinphi = np.nan_to_num(sinphi)
nestimates = np.zeros_like(average_masked_pixels)
# sin(theta) * cos(phi)
nestimates[:, 0] = np.sin(theta) * cosphi
# sin(theta) * sin(phi)
nestimates[:, 1] = np.sin(theta) * sinphi
nestimates[:, 2] = np.cos(theta)
# Unit normals
nestimates = normalise_vector(nestimates)
return average_normals.from_vector(nestimates)
def estimate_normals_from_intensity(average_normals, theta_image):
theta = theta_image.as_vector()
# Represents tan(phi) = sin(partial I/ partial y) / cos(partial I/ partial x)
# Where [partial I/ partial y] is the y-direction of the gradient field
average_masked_pixels = average_normals.as_vector(keep_channels=True)
n = np.sqrt(average_masked_pixels[:, 0] ** 2 + average_masked_pixels[:, 1] ** 2)
cosphi = average_masked_pixels[:, 0] / n
sinphi = average_masked_pixels[:, 1] / n
# Reset any nan-vectors to 0.0
cosphi = np.nan_to_num(cosphi)
sinphi = np.nan_to_num(sinphi)
nestimates = np.zeros_like(average_masked_pixels)
# sin(theta) * cos(phi)
nestimates[:, 0] = np.sin(theta) * cosphi
# sin(theta) * sin(phi)
nestimates[:, 1] = np.sin(theta) * sinphi
nestimates[:, 2] = np.cos(theta)
# Unit normals
nestimates = normalise_vector(nestimates)
return average_normals.from_vector(nestimates)
# ## Calculate the Spherical feature space # <codecell> from cosine_normals import Spherical spherical_matrix = Spherical().logmap(normal_matrix) spherical_images = create_feature_space(spherical_matrix, warped_images[0], 'spherical') # <markdowncell> # ## Calculate the AEP feature space # <codecell> from vector_utils import normalise_vector mean_normals = normalise_vector(np.mean(normal_matrix, 0)) # <codecell> from logmap_utils import partial_logmap from aep import AEP aep_matrix = AEP(mean_normals).logmap(normal_matrix) aep_images = create_feature_space(aep_matrix, warped_images[0], 'aep') # <markdowncell> # ## Calculate the PGA feature space # <codecell>
# <codecell>
from cosine_normals import Spherical
spherical_matrix = Spherical().logmap(normal_matrix)
spherical_images = create_feature_space(spherical_matrix, warped_images[0],
'spherical')
# <markdowncell>
# ## Calculate the AEP feature space
# <codecell>
from vector_utils import normalise_vector
mean_normals = normalise_vector(np.mean(normal_matrix, 0))
# <codecell>
from logmap_utils import partial_logmap
from aep import AEP
aep_matrix = AEP(mean_normals).logmap(normal_matrix)
aep_images = create_feature_space(aep_matrix, warped_images[0], 'aep')
# <markdowncell>
# ## Calculate the PGA feature space
# <codecell>
def build_all_models_frgc(images, ref_frame_path, subject_id,
out_path='/vol/atlas/homes/pts08/',
transform_class=ThinPlateSplines,
square_mask=False):
print "Beginning model creation for {0}".format(subject_id)
# Build reference frame
ref_frame = mio.import_image(ref_frame_path)
labeller([ref_frame], 'PTS', ibug_68_closed_mouth)
ref_frame.crop_to_landmarks(boundary=2, group='ibug_68_closed_mouth',
label='all')
if not square_mask:
ref_frame.constrain_mask_to_landmarks(group='ibug_68_closed_mouth',
label='all')
reference_shape = ref_frame.landmarks['ibug_68_closed_mouth'].lms
# Extract all shapes
labeller(images, 'PTS', ibug_68_closed_mouth)
shapes = [img.landmarks['ibug_68_closed_mouth'].lms for img in images]
# Warp each of the images to the reference image
print "Warping all frgc shapes to reference frame of {0}".format(subject_id)
tps_transforms = [transform_class(reference_shape, shape) for shape in shapes]
warped_images = [img.warp_to(ref_frame.mask, t)
for img, t in zip(images, tps_transforms)]
# Calculate the normal matrix
print 'Extracting all normals'
normal_matrix = extract_normals(warped_images)
# Save memory by deleting all the images since we don't need them any more.
# Keep one around that we can query for it's size etc
example_image = deepcopy(warped_images[0])
del warped_images[:]
# Normals
print 'Computing normal feature space'
normal_images = create_feature_space(normal_matrix, example_image,
'normals', subject_id,
out_path=out_path)
# Spherical
print 'Computing spherical feature space'
spherical_matrix = Spherical().logmap(normal_matrix)
spherical_images = create_feature_space(spherical_matrix, example_image,
'spherical', subject_id,
out_path=out_path)
# AEP
print 'Computing AEP feature space'
mean_normals = normalise_vector(np.mean(normal_matrix, 0))
aep_matrix = AEP(mean_normals).logmap(normal_matrix)
aep_images = create_feature_space(aep_matrix, example_image, 'aep',
subject_id,
out_path=out_path)
# PGA
print 'Computing PGA feature space'
mu = intrinsic_mean(normal_matrix, PGA, max_iters=50)
pga_matrix = PGA(mu).logmap(normal_matrix)
pga_images = create_feature_space(pga_matrix, example_image, 'pga',
subject_id,
out_path=out_path)
# PCA models
n_components = 200
print 'Computing PCA models ({} components)'.format(n_components)
template = ref_frame
normal_model = PCAModel(normal_images, center=True)
normal_model.trim_components(200)
cosine_model = PCAModel(normal_images, center=False)
cosine_model.trim_components(200)
spherical_model = PCAModel(spherical_images, center=False)
spherical_model.trim_components(200)
aep_model = PCAModel(aep_images, center=False)
aep_model.trim_components(200)
pga_model = PCAModel(pga_images, center=False)
pga_model.trim_components(200)
mean_normals_image = normal_model.mean
mu_image = mean_normals_image.from_vector(mu)
# Save out models
pickle_model(out_path, subject_id, 'normal', normal_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'cosine', cosine_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'spherical', spherical_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'aep', aep_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'pga', pga_model, template,
mean_normals, intrinsic_means=mu_image)
def geometric_sfs(intensity_image, initial_estimate,
normal_model, light_vector, n_iters=100,
mapping_object=IdentityMapper()):
"""
It is assumed that the given intensity image has been pre-aligned so that
it is in correspondence with the model.
1. Calculate an initial estimate of the field of surface normals n using
(12)
2. Each normal in the estimated field n undergoes an azimuthal equidistant
projection (3) to give a vector of transformed coordinates v0.
3. The vector of best fit model parameters is given by ``b = P.T * v0 ``.
4. The vector of transformed coordinates corresponding
to the best-fit parameters is given by ``vprime = (P P.T)v0``
5. Using the inverse azimuthal equidistant projection
(4), find the off-cone best fit surface normal nprime from vprime.
6. Find the on-cone surface normal nprimeprime by rotating the
off-cone surface normal nprime using
``nprimeprime(i,j) = theta * nprime(i, j)``
7. Test for convergence. If
``sum over i,j arccos(n(i,j) . nprimeprime(i,j)) < eps``,
where eps is a predetermined threshold, then stop and return b as the
estimated model parameters and ``nprimeprime`` as rhe recovered needle
map.
8. Make ``n(i,j) = nprimeprime(i,j)`` and return to Step 2.
Parameters
----------
intensity_image : (M, N, 1) :class:`pybug.image.MaskedNDImage`
The 1-channel intensity image of a face to recover normals from.
initial_estimate : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the initial estimate of the normals
for the intensity image.
normal_model : :class:`pybug.model.linear.PCAModel`
A PCA model representing a subspace of normals.
light_vector : (3,) ndarray
A single light vector that represent the lighting direction that the
image is lit from.
n_iters : int, optional
The maximum number of iterations to perform.
Default: 100
max_error : float, optional
The maximum epsilon to test for convergence.
Default: 10^-6
mapping_object : MappingClass, optional
A class that provides both the logmap and expmap functions.
The logmap function is performed on the normals before reconstructing
from the PCA model. The expmap function is performed on the subspace
after reconstruction from the PCA model
Default: Identity mapping (no-op)
Returns
-------
normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the components of the recovered normals.
References
----------
[1] Smith, William AP, and Edwin R. Hancock.
Facial Shape-from-shading and Recognition Using Principal Geodesic
Analysis and Robust Statistics
IJCV (2008)
[2] Smith, William AP, and Edwin R. Hancock.
Recovering facial shape using a statistical model of surface normal
direction.
T-PAMI (2006)
"""
# Ensure the light is a unit vector
light_vector = normalise_vector(light_vector)
# Equation (1): Should never be < 0 if image is properly scaled
theta_vec = np.arccos(intensity_image.as_vector())
theta_image = intensity_image.from_vector(theta_vec)
n = estimate_normals_from_intensity(initial_estimate, theta_image)
for i in xrange(n_iters):
v0 = mapping_object.logmap(n)
# Vector of best-fit parameters
vprime = normal_model.reconstruct(v0)
nprime = mapping_object.expmap(vprime)
nprime = normalise_image(nprime)
# Equivalent to
# expmap(theta * logmap(expmap(vprime)) /
# row_norm(logmap(expmap(vprime))))
npp = on_cone_rotation(theta_image, nprime, light_vector)
n = npp
return normalise_image(npp)
from pga import PGA, intrinsic_mean
from aep import AEP
from cosine_normals import Spherical
from vector_utils import normalise_vector
from numpy.testing import assert_allclose
import numpy as np
n_samples = 10
n_vectors = 100
sd_vectors = normalise_vector(
np.random.uniform(low=-1.0, high=1.0, size=(n_samples, n_vectors, 3)))
base_vectors = normalise_vector(
np.random.uniform(low=-1.0, high=1.0, size=(n_vectors, 3)))
small_sd_vectors = np.array([[[0.75098051, 0.17029863, 0.6379864],
[0.50001385, 0.53529381, -0.68076919]],
[[-0.86560911, -0.41208702, -0.28443833],
[-0.63544436, 0.60226651, 0.4832034]]])
small_base_vectors = np.array([[0.67114357, -0.01134708, 0.74124055],
[-0.76751246, 0.62469913, 0.14379021]])
small_expected_pga_tangent_vectors = np.array([[[0.13036596, 0.18233274],
[-0.09662949, 1.71587233]],
[[-1.81744851, -1.68283606],
[0.32233591, -0.17535739]]])
small_expected_aep_smith_tangent_vectors = np.array(
[[[0.18451048, -0.12726506], [-1.26978438, -1.15810307]],
[[-1.71331896, 1.78874102], [-0.06747517, 0.36069066]]])
small_expected_spherical_tangent_vectors = np.array(
[[[0.97523904, 0.22115337, 0.6379864, 0.77004763],
[0.68261463, 0.73077853, -0.68076919, 0.73249799]],
[[-0.90290416, -0.42984192, -0.28443833, 0.95869434],
from pga import PGA, intrinsic_mean
from aep import AEP
from cosine_normals import Spherical
from vector_utils import normalise_vector
from numpy.testing import assert_allclose
import numpy as np
n_samples = 10
n_vectors = 100
sd_vectors = normalise_vector(
np.random.uniform(low=-1.0, high=1.0, size=(n_samples, n_vectors, 3)))
base_vectors = normalise_vector(
np.random.uniform(low=-1.0, high=1.0, size=(n_vectors, 3)))
small_sd_vectors = np.array([[[ 0.75098051, 0.17029863, 0.6379864 ],
[ 0.50001385, 0.53529381, -0.68076919]],
[[-0.86560911, -0.41208702, -0.28443833],
[-0.63544436, 0.60226651, 0.4832034 ]]])
small_base_vectors = np.array([[ 0.67114357, -0.01134708, 0.74124055],
[-0.76751246, 0.62469913, 0.14379021]])
small_expected_pga_tangent_vectors = np.array([[[ 0.13036596, 0.18233274],
[-0.09662949, 1.71587233]],
[[-1.81744851, -1.68283606],
[ 0.32233591, -0.17535739]]])
small_expected_aep_smith_tangent_vectors = np.array(
[[[ 0.18451048, -0.12726506],
[-1.26978438, -1.15810307]],
def build_all_models_frgc(images,
ref_frame_path,
subject_id,
out_path='/vol/atlas/homes/pts08/',
transform_class=ThinPlateSplines,
square_mask=False):
print "Beginning model creation for {0}".format(subject_id)
# Build reference frame
ref_frame = mio.import_image(ref_frame_path)
labeller([ref_frame], 'PTS', ibug_68_closed_mouth)
ref_frame.crop_to_landmarks(boundary=2,
group='ibug_68_closed_mouth',
label='all')
if not square_mask:
ref_frame.constrain_mask_to_landmarks(group='ibug_68_closed_mouth',
label='all')
reference_shape = ref_frame.landmarks['ibug_68_closed_mouth'].lms
# Extract all shapes
labeller(images, 'PTS', ibug_68_closed_mouth)
shapes = [img.landmarks['ibug_68_closed_mouth'].lms for img in images]
# Warp each of the images to the reference image
print "Warping all frgc shapes to reference frame of {0}".format(
subject_id)
tps_transforms = [
transform_class(reference_shape, shape) for shape in shapes
]
warped_images = [
img.warp_to(ref_frame.mask, t)
for img, t in zip(images, tps_transforms)
]
# Calculate the normal matrix
print 'Extracting all normals'
normal_matrix = extract_normals(warped_images)
# Save memory by deleting all the images since we don't need them any more.
# Keep one around that we can query for it's size etc
example_image = deepcopy(warped_images[0])
del warped_images[:]
# Normals
print 'Computing normal feature space'
normal_images = create_feature_space(normal_matrix,
example_image,
'normals',
subject_id,
out_path=out_path)
# Spherical
print 'Computing spherical feature space'
spherical_matrix = Spherical().logmap(normal_matrix)
spherical_images = create_feature_space(spherical_matrix,
example_image,
'spherical',
subject_id,
out_path=out_path)
# AEP
print 'Computing AEP feature space'
mean_normals = normalise_vector(np.mean(normal_matrix, 0))
aep_matrix = AEP(mean_normals).logmap(normal_matrix)
aep_images = create_feature_space(aep_matrix,
example_image,
'aep',
subject_id,
out_path=out_path)
# PGA
print 'Computing PGA feature space'
mu = intrinsic_mean(normal_matrix, PGA, max_iters=50)
pga_matrix = PGA(mu).logmap(normal_matrix)
pga_images = create_feature_space(pga_matrix,
example_image,
'pga',
subject_id,
out_path=out_path)
# PCA models
n_components = 200
print 'Computing PCA models ({} components)'.format(n_components)
template = ref_frame
normal_model = PCAModel(normal_images, center=True)
normal_model.trim_components(200)
cosine_model = PCAModel(normal_images, center=False)
cosine_model.trim_components(200)
spherical_model = PCAModel(spherical_images, center=False)
spherical_model.trim_components(200)
aep_model = PCAModel(aep_images, center=False)
aep_model.trim_components(200)
pga_model = PCAModel(pga_images, center=False)
pga_model.trim_components(200)
mean_normals_image = normal_model.mean
mu_image = mean_normals_image.from_vector(mu)
# Save out models
pickle_model(out_path, subject_id, 'normal', normal_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'cosine', cosine_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'spherical', spherical_model, template,
mean_normals)
pickle_model(out_path, subject_id, 'aep', aep_model, template,
mean_normals)
pickle_model(out_path,
subject_id,
'pga',
pga_model,
template,
mean_normals,
intrinsic_means=mu_image)
def geometric_sfs(intensity_image, initial_estimate,
normal_model, light_vector, n_iters=100,
mapping_object=ImageMapper(IdentityMapper())):
"""
It is assumed that the given intensity image has been pre-aligned so that
it is in correspondence with the model.
1. Calculate an initial estimate of the field of surface normals n using
(12)
2. Each normal in the estimated field n undergoes an azimuthal equidistant
projection (3) to give a vector of transformed coordinates v0.
3. The vector of best fit model parameters is given by ``b = P.T * v0 ``.
4. The vector of transformed coordinates corresponding
to the best-fit parameters is given by ``vprime = (P P.T)v0``
5. Using the inverse azimuthal equidistant projection
(4), find the off-cone best fit surface normal nprime from vprime.
6. Find the on-cone surface normal nprimeprime by rotating the
off-cone surface normal nprime using
``nprimeprime(i,j) = theta * nprime(i, j)``
7. Test for convergence. If
``sum over i,j arccos(n(i,j) . nprimeprime(i,j)) < eps``,
where eps is a predetermined threshold, then stop and return b as the
estimated model parameters and ``nprimeprime`` as rhe recovered needle
map.
8. Make ``n(i,j) = nprimeprime(i,j)`` and return to Step 2.
Parameters
----------
intensity_image : (M, N, 1) :class:`pybug.image.MaskedNDImage`
The 1-channel intensity image of a face to recover normals from.
initial_estimate : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the initial estimate of the normals
for the intensity image.
normal_model : :class:`pybug.model.linear.PCAModel`
A PCA model representing a subspace of normals.
light_vector : (3,) ndarray
A single light vector that represent the lighting direction that the
image is lit from.
n_iters : int, optional
The maximum number of iterations to perform.
Default: 100
max_error : float, optional
The maximum epsilon to test for convergence.
Default: 10^-6
mapping_object : MappingClass, optional
A class that provides both the logmap and expmap functions.
The logmap function is performed on the normals before reconstructing
from the PCA model. The expmap function is performed on the subspace
after reconstruction from the PCA model
Default: Identity mapping (no-op)
Returns
-------
normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage`
A 3-channel image representing the components of the recovered normals.
References
----------
[1] Smith, William AP, and Edwin R. Hancock.
Facial Shape-from-shading and Recognition Using Principal Geodesic
Analysis and Robust Statistics
IJCV (2008)
[2] Smith, William AP, and Edwin R. Hancock.
Recovering facial shape using a statistical model of surface normal
direction.
T-PAMI (2006)
"""
# Ensure the light is a unit vector
light_vector = normalise_vector(light_vector)
# Equation (1): Should never be < 0 if image is properly scaled
theta_vec = np.arccos(intensity_image.as_vector())
theta_image = intensity_image.from_vector(theta_vec)
n = estimate_normals_from_intensity(initial_estimate, theta_image)
for i in xrange(n_iters):
v0 = mapping_object.logmap(n)
# Vector of best-fit parameters
vprime = normal_model.reconstruct(v0)
nprime = mapping_object.expmap(vprime)
nprime = normalise_image(nprime)
# Equivalent to
# expmap(theta * logmap(expmap(vprime)) /
# row_norm(logmap(expmap(vprime))))
npp = on_cone_rotation(theta_image, nprime, light_vector)
n = npp
return normalise_image(npp)
