def horn_brooks(intensity_image, initial_estimate, normal_model, light_vector, n_iters=100, c_lambda=0.001, mapping_object=IdentityMapper()): """ M. Brooks and B. Horn Shape and Source from Shading 1985 """ from scipy.signal import convolve2d # Ensure the light is a unit vector light_vector = normalise_vector(light_vector) # Equation (1): Should never be < 0 if image is properly scaled theta_vec = np.arccos(intensity_image.as_vector()) theta_image = intensity_image.from_vector(theta_vec) n_im = estimate_normals_from_intensity(initial_estimate, theta_image) average_kernel = np.array([[0.0, 0.25, 0.0], [0.25, 0.0, 0.25], [0.0, 0.25, 0.0]]) scale_constant = (1.0 / 4.0 * c_lambda) n_vec = n_im.as_vector(keep_channels=True) I_vec = intensity_image.as_vector() for i in xrange(n_iters): n_dot_s = np.sum(n_vec * light_vector, axis=1) # Calculate the average normal neighbourhood n_im.from_vector_inplace(n_vec) n_xs = convolve2d(n_im.pixels[:, :, 0], average_kernel, mode='same') n_ys = convolve2d(n_im.pixels[:, :, 1], average_kernel, mode='same') n_zs = convolve2d(n_im.pixels[:, :, 2], average_kernel, mode='same') n_bar = np.concatenate([n_xs[..., None], n_ys[..., None], n_zs[..., None]], axis=-1) n_bar = MaskedImage(n_bar, mask=n_im.mask).as_vector( keep_channels=True) rho = scale_constant * (I_vec - n_dot_s) m = n_bar + np.dot(rho[..., None], light_vector[..., None].T) n_im.from_vector_inplace(normalise_vector(m)) v0 = mapping_object.logmap(n_im) # Vector of best-fit parameters vprime = normal_model.reconstruct(v0) nprime = mapping_object.expmap(vprime) nprime = normalise_image(nprime) n_vec = nprime.as_vector(keep_channels=True) n_im.from_vector_inplace(n_vec) return normalise_image(n_im)
def mean_vector(images): N = len(images) avg = np.zeros_like(images[0].as_vector(keep_channels=True)) for im in images: avg += im.as_vector(keep_channels=True) return normalise_vector(avg / N)
def on_cone_rotation(theta_image, normal_image, s): theta = theta_image.as_vector() nprime = normal_image.as_vector(keep_channels=True) # cross product and break in to row vectors C = np.cross(nprime, s) C = normalise_vector(C) u = C[:, 0] v = C[:, 1] w = C[:, 2] # expects |nprime| = |sec| = 1 # represents intensity and can never be < 0 d = np.squeeze(np.inner(nprime, s) / (row_norm(nprime) * row_norm(s))) d = np.nan_to_num(d) d[d < 0.0] = 0.0 beta = np.arccos(d) # flip beta and theta so that it rotates along the correct axis alpha = beta - theta c = np.cos(alpha) cprime = 1.0 - c s = np.sin(alpha) # setup structures N = nprime.shape[0] phi = np.zeros([N, 3, 3]) phi[:, 0, 0] = c + u ** 2 * cprime phi[:, 0, 1] = -w * s + u * v * cprime phi[:, 0, 2] = v * s + u * w * cprime phi[:, 1, 0] = w * s + u * v * cprime phi[:, 1, 1] = c + v ** 2 * cprime phi[:, 1, 2] = -u * s + v * w * cprime phi[:, 2, 0] = -v * s + u * w * cprime phi[:, 2, 1] = u * s + v * w * cprime phi[:, 2, 2] = c + w ** 2 * cprime n = np.einsum('kjl, klm -> kj', phi, nprime[..., None]) # Normalize the result ?? n = normalise_vector(n) return normal_image.from_vector(n)
def intrinsic_mean(sd_vectors, mapping_class, max_iters=20): # Compute initial estimate (Euclidean mean of data) mus = normalise_vector(np.mean(sd_vectors, axis=0)) for i in xrange(max_iters): mapping_object = mapping_class(mus) # Iteratively improve estimate of intrinsic mean mean_tangent = np.mean(mapping_object.logmap(sd_vectors), axis=0) mus = np.squeeze(mapping_object.expmap(mean_tangent)) return mus
def photometric_stereo(images, lights): """ Images is a 3 or more channel image representing the same object taken under different lighting conditions. Lights is a n_channels x 3 matrix that represents the direction from which each image is lit. Only the masked pixels are recovered. Parameters ---------- images : (M, N, C) :class:`pybug.image.MaskedNDImage` An image where each channel is an image lit under a unique lighting direction. lights : (C, 3) ndarray A matrix representing the light directions for each of the channels in ``images``. Returns ------- normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage` A 3-channel image representing the components of the recovered normals. albedo_image : (M, N, 1) :class:`pybug.image.MaskedNDImage` A 1-channel image representing the albedo at each pixel. """ # Ensure the light are unit vectors lights = normalise_vector(lights) LL = pinv(lights) # n_masked_pixels x n_channels pixels = images.as_vector(keep_channels=True) n_images = pixels.shape[1] if n_images < 3: raise ValueError('Photometric Stereo is undefined with less than 3 ' 'input images.') if LL.shape[1] != n_images: raise ValueError('You must provide a light direction for each input ' 'channel.') normals = np.dot(pixels, LL.T) magnitudes = np.sqrt((normals * normals).sum(axis=1)) albedo = magnitudes normals[magnitudes != 0.0, :] /= magnitudes[magnitudes != 0.0][..., None] return (images.from_vector(normals, n_channels=3), images.from_vector(albedo, n_channels=1))
def estimate_normals_from_intensity(average_normals, theta_image): theta = theta_image.as_vector() # Represents tan(phi) = sin(partial I/ partial y) / cos(partial I/ partial x) # Where [partial I/ partial y] is the y-direction of the gradient field average_masked_pixels = average_normals.as_vector(keep_channels=True) n = np.sqrt(average_masked_pixels[:, 0] ** 2 + average_masked_pixels[:, 1] ** 2) cosphi = average_masked_pixels[:, 0] / n sinphi = average_masked_pixels[:, 1] / n # Reset any nan-vectors to 0.0 cosphi = np.nan_to_num(cosphi) sinphi = np.nan_to_num(sinphi) nestimates = np.zeros_like(average_masked_pixels) # sin(theta) * cos(phi) nestimates[:, 0] = np.sin(theta) * cosphi # sin(theta) * sin(phi) nestimates[:, 1] = np.sin(theta) * sinphi nestimates[:, 2] = np.cos(theta) # Unit normals nestimates = normalise_vector(nestimates) return average_normals.from_vector(nestimates)
# ## Calculate the Spherical feature space # <codecell> from cosine_normals import Spherical spherical_matrix = Spherical().logmap(normal_matrix) spherical_images = create_feature_space(spherical_matrix, warped_images[0], 'spherical') # <markdowncell> # ## Calculate the AEP feature space # <codecell> from vector_utils import normalise_vector mean_normals = normalise_vector(np.mean(normal_matrix, 0)) # <codecell> from logmap_utils import partial_logmap from aep import AEP aep_matrix = AEP(mean_normals).logmap(normal_matrix) aep_images = create_feature_space(aep_matrix, warped_images[0], 'aep') # <markdowncell> # ## Calculate the PGA feature space # <codecell>
# <codecell> from cosine_normals import Spherical spherical_matrix = Spherical().logmap(normal_matrix) spherical_images = create_feature_space(spherical_matrix, warped_images[0], 'spherical') # <markdowncell> # ## Calculate the AEP feature space # <codecell> from vector_utils import normalise_vector mean_normals = normalise_vector(np.mean(normal_matrix, 0)) # <codecell> from logmap_utils import partial_logmap from aep import AEP aep_matrix = AEP(mean_normals).logmap(normal_matrix) aep_images = create_feature_space(aep_matrix, warped_images[0], 'aep') # <markdowncell> # ## Calculate the PGA feature space # <codecell>
def build_all_models_frgc(images, ref_frame_path, subject_id, out_path='/vol/atlas/homes/pts08/', transform_class=ThinPlateSplines, square_mask=False): print "Beginning model creation for {0}".format(subject_id) # Build reference frame ref_frame = mio.import_image(ref_frame_path) labeller([ref_frame], 'PTS', ibug_68_closed_mouth) ref_frame.crop_to_landmarks(boundary=2, group='ibug_68_closed_mouth', label='all') if not square_mask: ref_frame.constrain_mask_to_landmarks(group='ibug_68_closed_mouth', label='all') reference_shape = ref_frame.landmarks['ibug_68_closed_mouth'].lms # Extract all shapes labeller(images, 'PTS', ibug_68_closed_mouth) shapes = [img.landmarks['ibug_68_closed_mouth'].lms for img in images] # Warp each of the images to the reference image print "Warping all frgc shapes to reference frame of {0}".format(subject_id) tps_transforms = [transform_class(reference_shape, shape) for shape in shapes] warped_images = [img.warp_to(ref_frame.mask, t) for img, t in zip(images, tps_transforms)] # Calculate the normal matrix print 'Extracting all normals' normal_matrix = extract_normals(warped_images) # Save memory by deleting all the images since we don't need them any more. # Keep one around that we can query for it's size etc example_image = deepcopy(warped_images[0]) del warped_images[:] # Normals print 'Computing normal feature space' normal_images = create_feature_space(normal_matrix, example_image, 'normals', subject_id, out_path=out_path) # Spherical print 'Computing spherical feature space' spherical_matrix = Spherical().logmap(normal_matrix) spherical_images = create_feature_space(spherical_matrix, example_image, 'spherical', subject_id, out_path=out_path) # AEP print 'Computing AEP feature space' mean_normals = normalise_vector(np.mean(normal_matrix, 0)) aep_matrix = AEP(mean_normals).logmap(normal_matrix) aep_images = create_feature_space(aep_matrix, example_image, 'aep', subject_id, out_path=out_path) # PGA print 'Computing PGA feature space' mu = intrinsic_mean(normal_matrix, PGA, max_iters=50) pga_matrix = PGA(mu).logmap(normal_matrix) pga_images = create_feature_space(pga_matrix, example_image, 'pga', subject_id, out_path=out_path) # PCA models n_components = 200 print 'Computing PCA models ({} components)'.format(n_components) template = ref_frame normal_model = PCAModel(normal_images, center=True) normal_model.trim_components(200) cosine_model = PCAModel(normal_images, center=False) cosine_model.trim_components(200) spherical_model = PCAModel(spherical_images, center=False) spherical_model.trim_components(200) aep_model = PCAModel(aep_images, center=False) aep_model.trim_components(200) pga_model = PCAModel(pga_images, center=False) pga_model.trim_components(200) mean_normals_image = normal_model.mean mu_image = mean_normals_image.from_vector(mu) # Save out models pickle_model(out_path, subject_id, 'normal', normal_model, template, mean_normals) pickle_model(out_path, subject_id, 'cosine', cosine_model, template, mean_normals) pickle_model(out_path, subject_id, 'spherical', spherical_model, template, mean_normals) pickle_model(out_path, subject_id, 'aep', aep_model, template, mean_normals) pickle_model(out_path, subject_id, 'pga', pga_model, template, mean_normals, intrinsic_means=mu_image)
def geometric_sfs(intensity_image, initial_estimate, normal_model, light_vector, n_iters=100, mapping_object=IdentityMapper()): """ It is assumed that the given intensity image has been pre-aligned so that it is in correspondence with the model. 1. Calculate an initial estimate of the field of surface normals n using (12) 2. Each normal in the estimated field n undergoes an azimuthal equidistant projection (3) to give a vector of transformed coordinates v0. 3. The vector of best fit model parameters is given by ``b = P.T * v0 ``. 4. The vector of transformed coordinates corresponding to the best-fit parameters is given by ``vprime = (P P.T)v0`` 5. Using the inverse azimuthal equidistant projection (4), find the off-cone best fit surface normal nprime from vprime. 6. Find the on-cone surface normal nprimeprime by rotating the off-cone surface normal nprime using ``nprimeprime(i,j) = theta * nprime(i, j)`` 7. Test for convergence. If ``sum over i,j arccos(n(i,j) . nprimeprime(i,j)) < eps``, where eps is a predetermined threshold, then stop and return b as the estimated model parameters and ``nprimeprime`` as rhe recovered needle map. 8. Make ``n(i,j) = nprimeprime(i,j)`` and return to Step 2. Parameters ---------- intensity_image : (M, N, 1) :class:`pybug.image.MaskedNDImage` The 1-channel intensity image of a face to recover normals from. initial_estimate : (M, N, 3) :class:`pybug.image.MaskedNDImage` A 3-channel image representing the initial estimate of the normals for the intensity image. normal_model : :class:`pybug.model.linear.PCAModel` A PCA model representing a subspace of normals. light_vector : (3,) ndarray A single light vector that represent the lighting direction that the image is lit from. n_iters : int, optional The maximum number of iterations to perform. Default: 100 max_error : float, optional The maximum epsilon to test for convergence. Default: 10^-6 mapping_object : MappingClass, optional A class that provides both the logmap and expmap functions. The logmap function is performed on the normals before reconstructing from the PCA model. The expmap function is performed on the subspace after reconstruction from the PCA model Default: Identity mapping (no-op) Returns ------- normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage` A 3-channel image representing the components of the recovered normals. References ---------- [1] Smith, William AP, and Edwin R. Hancock. Facial Shape-from-shading and Recognition Using Principal Geodesic Analysis and Robust Statistics IJCV (2008) [2] Smith, William AP, and Edwin R. Hancock. Recovering facial shape using a statistical model of surface normal direction. T-PAMI (2006) """ # Ensure the light is a unit vector light_vector = normalise_vector(light_vector) # Equation (1): Should never be < 0 if image is properly scaled theta_vec = np.arccos(intensity_image.as_vector()) theta_image = intensity_image.from_vector(theta_vec) n = estimate_normals_from_intensity(initial_estimate, theta_image) for i in xrange(n_iters): v0 = mapping_object.logmap(n) # Vector of best-fit parameters vprime = normal_model.reconstruct(v0) nprime = mapping_object.expmap(vprime) nprime = normalise_image(nprime) # Equivalent to # expmap(theta * logmap(expmap(vprime)) / # row_norm(logmap(expmap(vprime)))) npp = on_cone_rotation(theta_image, nprime, light_vector) n = npp return normalise_image(npp)
from pga import PGA, intrinsic_mean from aep import AEP from cosine_normals import Spherical from vector_utils import normalise_vector from numpy.testing import assert_allclose import numpy as np n_samples = 10 n_vectors = 100 sd_vectors = normalise_vector( np.random.uniform(low=-1.0, high=1.0, size=(n_samples, n_vectors, 3))) base_vectors = normalise_vector( np.random.uniform(low=-1.0, high=1.0, size=(n_vectors, 3))) small_sd_vectors = np.array([[[0.75098051, 0.17029863, 0.6379864], [0.50001385, 0.53529381, -0.68076919]], [[-0.86560911, -0.41208702, -0.28443833], [-0.63544436, 0.60226651, 0.4832034]]]) small_base_vectors = np.array([[0.67114357, -0.01134708, 0.74124055], [-0.76751246, 0.62469913, 0.14379021]]) small_expected_pga_tangent_vectors = np.array([[[0.13036596, 0.18233274], [-0.09662949, 1.71587233]], [[-1.81744851, -1.68283606], [0.32233591, -0.17535739]]]) small_expected_aep_smith_tangent_vectors = np.array( [[[0.18451048, -0.12726506], [-1.26978438, -1.15810307]], [[-1.71331896, 1.78874102], [-0.06747517, 0.36069066]]]) small_expected_spherical_tangent_vectors = np.array( [[[0.97523904, 0.22115337, 0.6379864, 0.77004763], [0.68261463, 0.73077853, -0.68076919, 0.73249799]], [[-0.90290416, -0.42984192, -0.28443833, 0.95869434],
from pga import PGA, intrinsic_mean from aep import AEP from cosine_normals import Spherical from vector_utils import normalise_vector from numpy.testing import assert_allclose import numpy as np n_samples = 10 n_vectors = 100 sd_vectors = normalise_vector( np.random.uniform(low=-1.0, high=1.0, size=(n_samples, n_vectors, 3))) base_vectors = normalise_vector( np.random.uniform(low=-1.0, high=1.0, size=(n_vectors, 3))) small_sd_vectors = np.array([[[ 0.75098051, 0.17029863, 0.6379864 ], [ 0.50001385, 0.53529381, -0.68076919]], [[-0.86560911, -0.41208702, -0.28443833], [-0.63544436, 0.60226651, 0.4832034 ]]]) small_base_vectors = np.array([[ 0.67114357, -0.01134708, 0.74124055], [-0.76751246, 0.62469913, 0.14379021]]) small_expected_pga_tangent_vectors = np.array([[[ 0.13036596, 0.18233274], [-0.09662949, 1.71587233]], [[-1.81744851, -1.68283606], [ 0.32233591, -0.17535739]]]) small_expected_aep_smith_tangent_vectors = np.array( [[[ 0.18451048, -0.12726506], [-1.26978438, -1.15810307]],
def build_all_models_frgc(images, ref_frame_path, subject_id, out_path='/vol/atlas/homes/pts08/', transform_class=ThinPlateSplines, square_mask=False): print "Beginning model creation for {0}".format(subject_id) # Build reference frame ref_frame = mio.import_image(ref_frame_path) labeller([ref_frame], 'PTS', ibug_68_closed_mouth) ref_frame.crop_to_landmarks(boundary=2, group='ibug_68_closed_mouth', label='all') if not square_mask: ref_frame.constrain_mask_to_landmarks(group='ibug_68_closed_mouth', label='all') reference_shape = ref_frame.landmarks['ibug_68_closed_mouth'].lms # Extract all shapes labeller(images, 'PTS', ibug_68_closed_mouth) shapes = [img.landmarks['ibug_68_closed_mouth'].lms for img in images] # Warp each of the images to the reference image print "Warping all frgc shapes to reference frame of {0}".format( subject_id) tps_transforms = [ transform_class(reference_shape, shape) for shape in shapes ] warped_images = [ img.warp_to(ref_frame.mask, t) for img, t in zip(images, tps_transforms) ] # Calculate the normal matrix print 'Extracting all normals' normal_matrix = extract_normals(warped_images) # Save memory by deleting all the images since we don't need them any more. # Keep one around that we can query for it's size etc example_image = deepcopy(warped_images[0]) del warped_images[:] # Normals print 'Computing normal feature space' normal_images = create_feature_space(normal_matrix, example_image, 'normals', subject_id, out_path=out_path) # Spherical print 'Computing spherical feature space' spherical_matrix = Spherical().logmap(normal_matrix) spherical_images = create_feature_space(spherical_matrix, example_image, 'spherical', subject_id, out_path=out_path) # AEP print 'Computing AEP feature space' mean_normals = normalise_vector(np.mean(normal_matrix, 0)) aep_matrix = AEP(mean_normals).logmap(normal_matrix) aep_images = create_feature_space(aep_matrix, example_image, 'aep', subject_id, out_path=out_path) # PGA print 'Computing PGA feature space' mu = intrinsic_mean(normal_matrix, PGA, max_iters=50) pga_matrix = PGA(mu).logmap(normal_matrix) pga_images = create_feature_space(pga_matrix, example_image, 'pga', subject_id, out_path=out_path) # PCA models n_components = 200 print 'Computing PCA models ({} components)'.format(n_components) template = ref_frame normal_model = PCAModel(normal_images, center=True) normal_model.trim_components(200) cosine_model = PCAModel(normal_images, center=False) cosine_model.trim_components(200) spherical_model = PCAModel(spherical_images, center=False) spherical_model.trim_components(200) aep_model = PCAModel(aep_images, center=False) aep_model.trim_components(200) pga_model = PCAModel(pga_images, center=False) pga_model.trim_components(200) mean_normals_image = normal_model.mean mu_image = mean_normals_image.from_vector(mu) # Save out models pickle_model(out_path, subject_id, 'normal', normal_model, template, mean_normals) pickle_model(out_path, subject_id, 'cosine', cosine_model, template, mean_normals) pickle_model(out_path, subject_id, 'spherical', spherical_model, template, mean_normals) pickle_model(out_path, subject_id, 'aep', aep_model, template, mean_normals) pickle_model(out_path, subject_id, 'pga', pga_model, template, mean_normals, intrinsic_means=mu_image)
def geometric_sfs(intensity_image, initial_estimate, normal_model, light_vector, n_iters=100, mapping_object=ImageMapper(IdentityMapper())): """ It is assumed that the given intensity image has been pre-aligned so that it is in correspondence with the model. 1. Calculate an initial estimate of the field of surface normals n using (12) 2. Each normal in the estimated field n undergoes an azimuthal equidistant projection (3) to give a vector of transformed coordinates v0. 3. The vector of best fit model parameters is given by ``b = P.T * v0 ``. 4. The vector of transformed coordinates corresponding to the best-fit parameters is given by ``vprime = (P P.T)v0`` 5. Using the inverse azimuthal equidistant projection (4), find the off-cone best fit surface normal nprime from vprime. 6. Find the on-cone surface normal nprimeprime by rotating the off-cone surface normal nprime using ``nprimeprime(i,j) = theta * nprime(i, j)`` 7. Test for convergence. If ``sum over i,j arccos(n(i,j) . nprimeprime(i,j)) < eps``, where eps is a predetermined threshold, then stop and return b as the estimated model parameters and ``nprimeprime`` as rhe recovered needle map. 8. Make ``n(i,j) = nprimeprime(i,j)`` and return to Step 2. Parameters ---------- intensity_image : (M, N, 1) :class:`pybug.image.MaskedNDImage` The 1-channel intensity image of a face to recover normals from. initial_estimate : (M, N, 3) :class:`pybug.image.MaskedNDImage` A 3-channel image representing the initial estimate of the normals for the intensity image. normal_model : :class:`pybug.model.linear.PCAModel` A PCA model representing a subspace of normals. light_vector : (3,) ndarray A single light vector that represent the lighting direction that the image is lit from. n_iters : int, optional The maximum number of iterations to perform. Default: 100 max_error : float, optional The maximum epsilon to test for convergence. Default: 10^-6 mapping_object : MappingClass, optional A class that provides both the logmap and expmap functions. The logmap function is performed on the normals before reconstructing from the PCA model. The expmap function is performed on the subspace after reconstruction from the PCA model Default: Identity mapping (no-op) Returns ------- normal_image : (M, N, 3) :class:`pybug.image.MaskedNDImage` A 3-channel image representing the components of the recovered normals. References ---------- [1] Smith, William AP, and Edwin R. Hancock. Facial Shape-from-shading and Recognition Using Principal Geodesic Analysis and Robust Statistics IJCV (2008) [2] Smith, William AP, and Edwin R. Hancock. Recovering facial shape using a statistical model of surface normal direction. T-PAMI (2006) """ # Ensure the light is a unit vector light_vector = normalise_vector(light_vector) # Equation (1): Should never be < 0 if image is properly scaled theta_vec = np.arccos(intensity_image.as_vector()) theta_image = intensity_image.from_vector(theta_vec) n = estimate_normals_from_intensity(initial_estimate, theta_image) for i in xrange(n_iters): v0 = mapping_object.logmap(n) # Vector of best-fit parameters vprime = normal_model.reconstruct(v0) nprime = mapping_object.expmap(vprime) nprime = normalise_image(nprime) # Equivalent to # expmap(theta * logmap(expmap(vprime)) / # row_norm(logmap(expmap(vprime)))) npp = on_cone_rotation(theta_image, nprime, light_vector) n = npp return normalise_image(npp)