def gradient(cls, image, forward=None):
r"""
Calculates the gradients of the given method.
If `forward` is provided, then the gradients are warped
(as required in the forward additive algorithm)
Parameters
----------
image : `menpo.image.Image`
The image to calculate the gradients for
forward : `tuple` or ``None``, optional
A `tuple` containing the extra weights required for the function
`warp` (which should be passed as a function handle), i.e.
``(`menpo.image.Image`, `menpo.transform.AlignableTransform>`)``. If
``None``, then the optimization algorithm is assumed to be inverse.
"""
if forward:
# Calculate the gradient over the image
# grad: (dims x ch) x H x W
grad = gradient(image)
# Warp gradient for forward additive using the given transform
# grad: (dims x ch) x h x w
template, transform = forward
grad = grad.warp_to_mask(template.mask, transform,
warp_landmarks=False)
else:
# Calculate the gradient over the image and set one pixels along
# the boundary of the image mask to zero (no reliable gradient
# can be computed there!)
# grad: (dims x ch) x h x w
grad = gradient(image)
grad.set_boundary_pixels()
return grad
def _calculate_gradients(self, image, forward=None):
r"""
Calculates the gradients of the given method.
If `forward` is provided, then the gradients are warped
(as required in the forward additive algorithm)
Parameters
----------
image : :class:`menpo.image.base.Image`
The image to calculate the gradients for
forward : (:map:`Image`, :map:`AlignableTransform>`), optional
A tuple containing the extra weights required for the function
`warp` (which should be passed as a function handle).
Default: `None`
"""
if forward:
# Calculate the gradient over the image
# grad: (dims x ch) x H x W
grad = gradient(image)
# Warp gradient for forward additive using the given transform
# grad: (dims x ch) x h x w
template, transform = forward
grad = grad.warp_to_mask(template.mask, transform,
warp_landmarks=False)
else:
# Calculate the gradient over the image and set one pixels along
# the boundary of the image mask to zero (no reliable gradient
# can be computed there!)
# grad: (dims x ch) x h x w
grad = gradient(image)
grad.set_boundary_pixels()
return grad
def test_gradient_takeo_double():
t = takeo.copy()
t.pixels = t.pixels.astype(np.float64)
grad_image = gradient(t)
np_grad = _np_gradient(t.pixels)
assert_allclose(grad_image.pixels, np_grad)
def test_gradient_takeo_float32():
dtype = np.float32
t = takeo.copy()
t.pixels = t.pixels.astype(dtype)
grad_image = gradient(t)
_check_assertions(grad_image, t.shape, t.n_channels * 2, dtype)
np_grad = _np_gradient(t.pixels)
assert_allclose(grad_image.pixels, np_grad)
def test_gradient_takeo_float32():
dtype = np.float32
t = takeo.copy()
t.pixels = t.pixels.astype(dtype)
grad_image = gradient(t)
_check_assertions(grad_image, t.shape, t.n_channels * 2,
dtype)
np_grad = _np_gradient(t.pixels)
assert_allclose(grad_image.pixels, np_grad)
def test_gradient_double():
dtype = np.float64
p = example_image.astype(dtype)
image = Image(p)
grad_image = gradient(image)
_check_assertions(grad_image, image.shape, image.n_channels * 2,
dtype)
np_grad = np.gradient(p)
assert_allclose(grad_image.pixels[0], np_grad[0])
assert_allclose(grad_image.pixels[1], np_grad[1])
def gradient(self, nullify_values_at_mask_boundaries=False):
r"""
Returns a :map:`MaskedImage` which is the gradient of this one. In the
case of multiple channels, it returns the gradient over each axis over
each channel as a flat list.
Parameters
----------
nullify_values_at_mask_boundaries : bool, optional
If `True` a one pixel boundary is set to 0 around the edge of
the `True` mask region. This is useful in situations where
there is absent data in the image which will cause erroneous
gradient settings.
Default: False
Returns
-------
gradient : :map:`MaskedImage`
The gradient over each axis over each channel. Therefore, the
gradient of a 2D, single channel image, will have length `2`.
The length of a 2D, 3-channel image, will have length `6`.
"""
global binary_erosion, gradient
if gradient is None:
from menpo.feature import gradient # avoid circular reference
# use the feature to take the gradient as normal
grad_image = gradient(self)
if nullify_values_at_mask_boundaries:
if binary_erosion is None:
from scipy.ndimage import binary_erosion # expensive
# Erode the edge of the mask in by one pixel
eroded_mask = binary_erosion(self.mask.mask, iterations=1)
# replace the eroded mask with the diff between the two
# masks. This is only true in the region we want to nullify.
np.logical_and(~eroded_mask, self.mask.mask, out=eroded_mask)
# nullify all the boundary values in the grad image
grad_image.pixels[eroded_mask] = 0.0
return grad_image
def gradient(self, nullify_values_at_mask_boundaries=False):
r"""
Returns a :map:`MaskedImage` which is the gradient of this one. In the
case of multiple channels, it returns the gradient over each axis over
each channel as a flat list.
Parameters
----------
nullify_values_at_mask_boundaries : bool, optional
If `True` a one pixel boundary is set to 0 around the edge of
the `True` mask region. This is useful in situations where
there is absent data in the image which will cause erroneous
gradient settings.
Default: False
Returns
-------
gradient : :map:`MaskedImage`
The gradient over each axis over each channel. Therefore, the
gradient of a 2D, single channel image, will have length `2`.
The length of a 2D, 3-channel image, will have length `6`.
"""
global binary_erosion, gradient
if gradient is None:
from menpo.feature import gradient # avoid circular reference
# use the feature to take the gradient as normal
grad_image = gradient(self)
if nullify_values_at_mask_boundaries:
if binary_erosion is None:
from scipy.ndimage import binary_erosion # expensive
# Erode the edge of the mask in by one pixel
eroded_mask = binary_erosion(self.mask.mask, iterations=1)
# replace the eroded mask with the diff between the two
# masks. This is only true in the region we want to nullify.
np.logical_and(~eroded_mask, self.mask.mask, out=eroded_mask)
# nullify all the boundary values in the grad image
grad_image.pixels[eroded_mask] = 0.0
return grad_image
def fit_from_shape(self,
image,
shape,
n_alphas=100,
n_betas=100,
n_tris=1000,
camera_update=False,
max_iters=100):
# PRE-COMPUTATIONS
# ----------------
# Constants
threshold = 1e-3
# Projection type: 1 is for perspective and 0 for orthographic
projection_type = 1
# store the landmarks
image.landmarks[LM_GROUP] = shape
# Compute the gradient
grad = gradient(image)
# scaling the gradient by image resolution solves the non human-like
# faces problem
if image.shape[1] > image.shape[0]:
scale = image.shape[1] / 2
else:
scale = image.shape[0] / 2
VI_dy = grad.pixels[:3] * scale
VI_dx = grad.pixels[3:] * scale
# Generate instance
instance = self.model.instance(landmark_group=LM_GROUP)
# Get view projection rotation matrices
view_t, proj_t, R = retrieve_view_projection_transforms(image,
instance,
group=LM_GROUP)
# Get camera parameters array
rho = rho_from_view_projection_matrices(proj_t.h_matrix, R.h_matrix)
# Prior probabilities constants
alpha = np.zeros(n_alphas)
beta = np.zeros(n_betas)
da_prior_db = np.zeros(n_betas)
da_prior_da = 2. / (self.model.shape_model.eigenvalues[:n_alphas]**2)
db_prior_da = np.zeros(n_alphas)
db_prior_db = 2. / (self.model.texture_model.eigenvalues[:n_betas]**2)
if camera_update:
prior_drho = np.zeros(len(rho))
SD_alpha_prior = np.concatenate(
(da_prior_da, prior_drho, da_prior_db))
SD_beta_prior = np.concatenate(
(db_prior_da, prior_drho, db_prior_db))
else:
SD_alpha_prior = np.concatenate((da_prior_da, da_prior_db))
SD_beta_prior = np.concatenate((db_prior_da, db_prior_db))
H_alpha_prior = np.diag(SD_alpha_prior)
H_beta_prior = np.diag(SD_beta_prior)
# Initilialize rasterizer
rasterizer = GLRasterizer(height=image.height,
width=image.width,
view_matrix=view_t.h_matrix,
projection_matrix=proj_t.h_matrix)
errors = []
eps = np.inf
k = 0
while k < max_iters and eps > threshold:
# Progress bar
progress = k * 100 / max_iters
sys.stdout.write("\r%d%%" % progress)
sys.stdout.flush()
# Rotation matrices
r_phi, r_theta, r_varphi = compute_rotation_matrices(rho)
# Inverse rendering
tri_index_img, b_coords_img = (
rasterizer.rasterize_barycentric_coordinate_image(instance))
tri_indices = tri_index_img.as_vector()
b_coords = b_coords_img.as_vector(keep_channels=True)
yx = tri_index_img.mask.true_indices()
# Select triangles randomly
rand = np.random.permutation(b_coords.shape[1])
b_coords = b_coords[:, rand[:n_tris]]
yx = yx[rand[:n_tris]]
tri_indices = tri_indices[rand[:n_tris]]
# Build the vertex indices (3 per pixel)
# for the visible triangle
vertex_indices = instance.trilist[tri_indices]
# Warp the shape witht the view matrix
W = view_t.apply(instance.points)
# This solves the perspective projection problems
# It cancels the axes flip done in the view matrix before the
# rasterization
W[:, 1:] *= -1
# Shape and texture principal components are reshaped before
# sampling
shape_pc = self.model.shape_model.components.T
tex_pc = self.model.texture_model.components.T
shape_pc = shape_pc.reshape([instance.n_points, -1])
tex_pc = tex_pc.reshape([instance.n_points, -1])
# Sampling
# norms_uv = sample_object(instance.vertex_normals(),
# vertex_indices, b_coords)
shape_uv = sample_object(instance.points, vertex_indices, b_coords)
tex_uv = sample_object(instance.colours, vertex_indices, b_coords)
warped_uv = sample_object(W, vertex_indices, b_coords)
shape_pc_uv = sample_object(shape_pc, vertex_indices, b_coords)
tex_pc_uv = sample_object(tex_pc, vertex_indices, b_coords)
img_uv = image.sample(yx) # image
VI_dx_uv = Image(VI_dx).sample(yx) # gradient along x
VI_dy_uv = Image(VI_dy).sample(yx) # gradient along y
# Reshape after sampling
new_shape = tex_pc_uv.shape
shape_pc_uv = shape_pc_uv.reshape([new_shape[0], 3, -1])
tex_pc_uv = tex_pc_uv.reshape([new_shape[0], 3, -1])
# DERIVATIVES
dop_dalpha = []
dpp_dalpha = []
dt_dbeta = []
dop_drho = []
dpp_drho = []
if n_alphas > 0:
# Projection derivative wrt shape parameters
dp_dalpha = compute_projection_derivatives_shape_parameters(
shape_pc_uv, rho, warped_uv, R,
self.model.shape_model.eigenvalues, projection_type)
dp_dalpha = dp_dalpha[:, :n_alphas, :]
if n_betas > 0:
# Texture derivative wrt texture parameters
dt_dbeta = compute_texture_derivatives_texture_parameters(
tex_pc_uv, self.model.texture_model.eigenvalues)
dt_dbeta = dt_dbeta[:, :n_betas, :]
if camera_update:
# Projection derivative wrt warp parameters
dp_drho = compute_projection_derivatives_warp_parameters(
shape_uv, warped_uv.T, rho, r_phi, r_theta, r_varphi,
projection_type)
# Compute sd matrix and hessian
if camera_update:
dp_dgamma = np.hstack((dp_dalpha, dp_drho))
else:
dp_dgamma = dp_dalpha
if n_betas > 0 and n_alphas > 0:
dt = -dt_dbeta
SD_gamma = compute_sd(dp_dgamma, VI_dx_uv, VI_dy_uv)
SD_img = np.hstack((SD_gamma, dt))
elif n_alphas > 0 and n_betas == 0:
SD_img = compute_sd(dp_dgamma, VI_dx_uv, VI_dy_uv)
else:
SD_img = -dt_dbeta
# Hessian approximation
H_img = compute_hessian(SD_img)
# Compute error
img_error_uv = img_uv - tex_uv.T
# Compute steepest descent matrix
SD_error_img = compute_sd_error(SD_img, img_error_uv)
# Compute and store the error for future plots
eps = (img_error_uv**2).mean()
errors.append(eps)
# Prior probabilities over shape parameters
if camera_update:
prior_error = np.concatenate((alpha, rho, beta))
else:
prior_error = np.concatenate((alpha, beta))
# Prior probability SD matrices
SD_error_alpha_prior = SD_alpha_prior * prior_error
SD_error_beta_prior = SD_beta_prior * prior_error
# Final hessian and SD error matrix
H = H_img + 1e-2 * H_alpha_prior + H_beta_prior
SD_error = (SD_error_img + 1e-2 * SD_error_alpha_prior +
SD_error_beta_prior)
# Compute increment
delta_sigma = -np.dot(np.linalg.inv(H), SD_error)
# Update parameters
if camera_update:
alpha += delta_sigma[:n_alphas]
rho += delta_sigma[n_alphas:n_alphas + len(rho)]
beta += delta_sigma[n_alphas + len(rho):n_alphas + len(rho) +
n_betas]
else:
alpha += delta_sigma[:n_alphas]
beta += delta_sigma[n_alphas:]
# Generate the updated instance
# The texture is scaled by 255 to cancel the 1./255 scaling in the
# model class
instance = self.model.instance(alpha=alpha, beta=255. * beta)
# Clip to avoid out of range pixels
instance.colours = np.clip(instance.colours, 0, 1)
if camera_update:
# Compute new view matrix
_, R = compute_view_matrix(rho)
view_t.h_matrix[1:3, :3] = -R.h_matrix[1:3, :3]
view_t.h_matrix[0, :3] = R.h_matrix[0, :3]
# Update the rasterizer
rasterizer.set_view_matrix(view_t.h_matrix)
k += 1
# Rasterize the final mesh
rasterized_result = rasterizer.rasterize_mesh(instance)
# Save final values
sys.stdout.write('\rSuccessfully fitted.')
return {
'rasterized_result': rasterized_result,
'result': instance,
'errors': errors
}
def test_gradient_uint8_exception():
image = Image(example_image.astype(np.uint8))
gradient(image)
def _daisy(img, step=4, radius=15, rings=3, histograms=8, orientations=8,
normalization='l1', sigmas=None, ring_radii=None, visualize=False):
'''Extract DAISY feature descriptors densely for the given image.
DAISY is a feature descriptor similar to SIFT formulated in a way that
allows for fast dense extraction. Typically, this is practical for
bag-of-features image representations.
The implementation follows Tola et al. [1]_ but deviate on the following
points:
* Histogram bin contribution are smoothed with a circular Gaussian
window over the tonal range (the angular range).
* The sigma values of the spatial Gaussian smoothing in this code do not
match the sigma values in the original code by Tola et al. [2]_. In
their code, spatial smoothing is applied to both the input image and
the centre histogram. However, this smoothing is not documented in [1]_
and, therefore, it is omitted.
Parameters
----------
img : (M, N) array
Input image (greyscale).
step : int, optional
Distance between descriptor sampling points.
radius : int, optional
Radius (in pixels) of the outermost ring.
rings : int, optional
Number of rings.
histograms : int, optional
Number of histograms sampled per ring.
orientations : int, optional
Number of orientations (bins) per histogram.
normalization : [ 'l1' | 'l2' | 'daisy' | 'off' ], optional
How to normalize the descriptors
* 'l1': L1-normalization of each descriptor.
* 'l2': L2-normalization of each descriptor.
* 'daisy': L2-normalization of individual histograms.
* 'off': Disable normalization.
sigmas : 1D array of float, optional
Standard deviation of spatial Gaussian smoothing for the centre
histogram and for each ring of histograms. The array of sigmas should
be sorted from the centre and out. I.e. the first sigma value defines
the spatial smoothing of the centre histogram and the last sigma value
defines the spatial smoothing of the outermost ring. Specifying sigmas
overrides the following parameter.
``rings = len(sigmas) - 1``
ring_radii : 1D array of int, optional
Radius (in pixels) for each ring. Specifying ring_radii overrides the
following two parameters.
``rings = len(ring_radii)``
``radius = ring_radii[-1]``
If both sigmas and ring_radii are given, they must satisfy the
following predicate since no radius is needed for the centre
histogram.
``len(ring_radii) == len(sigmas) + 1``
visualize : bool, optional
Generate a visualization of the DAISY descriptors
Returns
-------
descs : array
Grid of DAISY descriptors for the given image as an array
dimensionality (P, Q, R) where
``P = ceil((M - radius*2) / step)``
``Q = ceil((N - radius*2) / step)``
``R = (rings * histograms + 1) * orientations``
descs_img : (M, N, 3) array (only if visualize==True)
Visualization of the DAISY descriptors.
References
----------
.. [1] Tola et al. "Daisy: An efficient dense descriptor applied to wide-
baseline stereo." Pattern Analysis and Machine Intelligence, IEEE
Transactions on 32.5 (2010): 815-830.
.. [2] http://cvlab.epfl.ch/alumni/tola/daisy.html
'''
# Compute image derivatives.
# Get number of input image's channels
n_channels = img.shape[-1]
# Compute image gradient
grad = gradient(img)
# For each pixel, select gradient with highest magnitude
tmp_mag = np.zeros(img.shape)
tmp_ori = np.zeros(img.shape)
for c in range(n_channels):
tmp_mag[..., c] = sqrt(grad[..., 2*c] ** 2 + grad[..., 2*c+1] ** 2)
tmp_ori[..., c] = arctan2(grad[..., 2*c+1], grad[..., 2*c])
grad_mag_ind = np.argmax(tmp_mag, axis=2)
# Compute gradient orientation and magnitude and their contribution
# to the histograms.
b = np.concatenate([(grad_mag_ind == i)[..., None]
for i in xrange(n_channels)], axis=-1)
grad_mag = tmp_mag[b].reshape(tmp_mag.shape[:2])
grad_ori = tmp_ori[b].reshape(tmp_ori.shape[:2])
orientation_kappa = orientations / pi
orientation_angles = [2 * o * pi / orientations - pi
for o in range(orientations)]
hist = np.empty((orientations,) + img.shape[:2], dtype=float)
for i, o in enumerate(orientation_angles):
# Weigh bin contribution by the circular normal distribution
hist[i, :, :] = exp(orientation_kappa * cos(grad_ori - o))
# Weigh bin contribution by the gradient magnitude
hist[i, :, :] = np.multiply(hist[i, :, :], grad_mag)
# Smooth orientation histograms for the centre and all rings.
sigmas = [sigmas[0]] + sigmas
hist_smooth = np.empty((rings + 1,) + hist.shape, dtype=float)
for i in range(rings + 1):
for j in range(orientations):
hist_smooth[i, j, :, :] = gaussian_filter(hist[j, :, :],
sigma=sigmas[i])
# Assemble descriptor grid.
theta = [2 * pi * j / histograms for j in range(histograms)]
desc_dims = (rings * histograms + 1) * orientations
descs = np.empty((desc_dims, img.shape[0] - 2 * radius,
img.shape[1] - 2 * radius))
descs[:orientations, :, :] = hist_smooth[0, :, radius:-radius,
radius:-radius]
idx = orientations
for i in range(rings):
for j in range(histograms):
y_min = radius + int(round(ring_radii[i] * sin(theta[j])))
y_max = descs.shape[1] + y_min
x_min = radius + int(round(ring_radii[i] * cos(theta[j])))
x_max = descs.shape[2] + x_min
descs[idx:idx + orientations, :, :] = hist_smooth[i + 1, :,
y_min:y_max,
x_min:x_max]
idx += orientations
descs = descs[:, ::step, ::step]
descs = descs.swapaxes(0, 1).swapaxes(1, 2)
# Normalize descriptors.
if normalization != 'off':
descs += 1e-10
if normalization == 'l1':
descs /= np.sum(descs, axis=2)[:, :, np.newaxis]
elif normalization == 'l2':
descs /= sqrt(np.sum(descs ** 2, axis=2))[:, :, np.newaxis]
elif normalization == 'daisy':
for i in range(0, desc_dims, orientations):
norms = sqrt(np.sum(descs[:, :, i:i + orientations] ** 2,
axis=2))
descs[:, :, i:i + orientations] /= norms[:, :, np.newaxis]
# Change axes so that the channels go to the final axis
descs = np.ascontiguousarray(descs)
return descs
