def _regression_data(self, images, gt_shapes, perturbed_shapes,
verbose=False):
r"""
Method that generates the regression data : features and delta_ps.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images.
gt_shapes : :map:`PointCloud` list
List of the ground truth shapes that correspond to the images.
perturbed_shapes : :map:`PointCloud` list
List of the perturbed shapes in order to regress.
verbose : `boolean`, optional
If ``True``, the progress is printed.
"""
if verbose:
print_dynamic('- Generating regression data')
n_images = len(images)
features = []
delta_ps = []
for j, (i, s, p_shape) in enumerate(zip(images, gt_shapes,
perturbed_shapes)):
if verbose:
print_dynamic('- Generating regression data - {}'.format(
progress_bar_str((j + 1.) / n_images, show_bar=False)))
for ps in p_shape:
features.append(self.features(i, ps))
delta_ps.append(self.delta_ps(s, ps))
return np.asarray(features), np.asarray(delta_ps)
def apply_pyramid_on_images(generators, n_levels, verbose=False):
r"""
Exhausts the pyramid generators verbosely
"""
all_images = []
for j in range(n_levels):
if verbose:
level_str = '- Apply pyramid: '
if n_levels > 1:
level_str = '- Apply pyramid: [Level {} - '.format(j + 1)
level_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
level_images.append(next(g))
all_images.append(level_images)
if verbose:
print_dynamic('- Apply pyramid: Done\n')
return all_images
def apply_pyramid_on_images(generators, n_levels, verbose=False):
r"""
Exhausts the pyramid generators verbosely
"""
all_images = []
for j in range(n_levels):
if verbose:
level_str = '- Apply pyramid: '
if n_levels > 1:
level_str = '- Apply pyramid: [Level {} - '.format(j + 1)
level_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
level_images.append(next(g))
all_images.append(level_images)
if verbose:
print_dynamic('- Apply pyramid: Done\n')
return all_images
def _get_relative_locations(shapes, graph, level_str, verbose):
r"""
returns numpy.array of size 2 x n_images x n_edges
"""
# convert given shapes to point graphs
if isinstance(graph, Tree):
point_graphs = [PointTree(shape.points, graph.adjacency_array,
graph.root_vertex) for shape in shapes]
else:
point_graphs = [PointDirectedGraph(shape.points, graph.adjacency_array)
for shape in shapes]
# initialize an output numpy array
rel_loc_array = np.empty((2, graph.n_edges, len(point_graphs)))
# get relative locations
for c, pt in enumerate(point_graphs):
# print progress
if verbose:
print_dynamic('{}Computing relative locations from '
'shapes - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(point_graphs),
show_bar=False)))
# get relative locations from this shape
rl = pt.relative_locations()
# store
rel_loc_array[..., c] = rl.T
# rollaxis and return
return np.rollaxis(rel_loc_array, 2, 1)
def _import_glob_generator(
pattern,
extension_map,
max_assets=None,
has_landmarks=False,
landmark_resolver=None,
importer_kwargs=None,
verbose=False,
):
filepaths = list(glob_with_suffix(pattern, extension_map))
if max_assets:
filepaths = filepaths[:max_assets]
n_files = len(filepaths)
if n_files == 0:
raise ValueError("The glob {} yields no assets".format(pattern))
for i, asset in enumerate(
_multi_import_generator(
filepaths,
extension_map,
has_landmarks=has_landmarks,
landmark_resolver=landmark_resolver,
importer_kwargs=importer_kwargs,
)
):
if verbose:
print_dynamic(
"- Loading {} assets: {}".format(n_files, progress_bar_str(float(i + 1) / n_files, show_bar=True))
)
yield asset
def _build_appearance_model_sparse(all_patches_array, graph, patch_shape,
n_channels, n_appearance_parameters,
level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'edge'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# initialize block sparse covariance matrix
all_cov = lil_matrix((graph.n_vertices * patch_len,
graph.n_vertices * patch_len))
# compute covariance matrix for each edge
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(graph.adjacency_array[e, :])
v2 = np.max(graph.adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((all_patches_array[v1_from:v1_to, :],
all_patches_array[v2_from:v2_to, :]))
# compute covariance inverse
icov = _covariance_matrix_inverse(np.cov(edge_data),
n_appearance_parameters)
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return app_mean, all_cov.tocsr()
def _build_appearance_model_sparse(all_patches_array, graph, patch_shape,
n_channels, n_appearance_parameters,
level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'edge'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# initialize block sparse covariance matrix
all_cov = lil_matrix(
(graph.n_vertices * patch_len, graph.n_vertices * patch_len))
# compute covariance matrix for each edge
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(graph.adjacency_array[e, :])
v2 = np.max(graph.adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((all_patches_array[v1_from:v1_to, :],
all_patches_array[v2_from:v2_to, :]))
# compute covariance inverse
icov = _covariance_matrix_inverse(np.cov(edge_data),
n_appearance_parameters)
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return app_mean, all_cov.tocsr()
def _build_deformation_model(graph, relative_locations, level_str, verbose):
# build deformation model
if verbose:
print_dynamic('{}Training deformation distribution per '
'graph edge'.format(level_str))
def_len = 2 * graph.n_vertices
def_cov = np.zeros((def_len, def_len))
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training deformation distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# get vertices adjacent to edge
parent = graph.adjacency_array[e, 0]
child = graph.adjacency_array[e, 1]
# compute covariance matrix
edge_cov = np.linalg.inv(np.cov(relative_locations[..., e]))
# store its values
s1 = edge_cov[0, 0]
s2 = edge_cov[1, 1]
s3 = 2 * edge_cov[0, 1]
# Fill the covariance matrix matrix
# get indices
p1 = 2 * parent
p2 = 2 * parent + 1
c1 = 2 * child
c2 = 2 * child + 1
# up-left block
def_cov[p1, p1] += s1
def_cov[p2, p2] += s2
def_cov[p2, p1] += s3
# up-right block
def_cov[p1, c1] = -s1
def_cov[p2, c2] = -s2
def_cov[p1, c2] = -s3 / 2
def_cov[p2, c1] = -s3 / 2
# down-left block
def_cov[c1, p1] = -s1
def_cov[c2, p2] = -s2
def_cov[c1, p2] = -s3 / 2
def_cov[c2, p1] = -s3 / 2
# down-right block
def_cov[c1, c1] += s1
def_cov[c2, c2] += s2
def_cov[c1, c2] += s3
return def_cov
def _build_deformation_model(graph, relative_locations, level_str, verbose):
# build deformation model
if verbose:
print_dynamic('{}Training deformation distribution per '
'graph edge'.format(level_str))
def_len = 2 * graph.n_vertices
def_cov = np.zeros((def_len, def_len))
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training deformation distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# get vertices adjacent to edge
parent = graph.adjacency_array[e, 0]
child = graph.adjacency_array[e, 1]
# compute covariance matrix
edge_cov = np.linalg.inv(np.cov(relative_locations[..., e]))
# store its values
s1 = edge_cov[0, 0]
s2 = edge_cov[1, 1]
s3 = 2 * edge_cov[0, 1]
# Fill the covariance matrix matrix
# get indices
p1 = 2 * parent
p2 = 2 * parent + 1
c1 = 2 * child
c2 = 2 * child + 1
# up-left block
def_cov[p1, p1] += s1
def_cov[p2, p2] += s2
def_cov[p2, p1] += s3
# up-right block
def_cov[p1, c1] = - s1
def_cov[p2, c2] = - s2
def_cov[p1, c2] = - s3 / 2
def_cov[p2, c1] = - s3 / 2
# down-left block
def_cov[c1, p1] = - s1
def_cov[c2, p2] = - s2
def_cov[c1, p2] = - s3 / 2
def_cov[c2, p1] = - s3 / 2
# down-right block
def_cov[c1, c1] += s1
def_cov[c2, c2] += s2
def_cov[c1, c2] += s3
return def_cov
def _create_pyramid(cls, images, n_levels, downscale, pyramid_on_features,
feature_type, verbose=False):
r"""
Function that creates a generator function for Gaussian pyramid. The
pyramid can be created either on the feature space or the original
(intensities) space.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
n_levels: int
The number of multi-resolution pyramidal levels to be used.
downscale: float
The downscale factor that will be used to create the different
pyramidal levels.
pyramid_on_features: boolean
If True, the features are extracted at the highest level and the
pyramid is created on the feature images.
If False, the pyramid is created on the original (intensities)
space.
feature_type: list of size 1 with str or function/closure or None
The feature type to be used in case pyramid_on_features is enabled.
verbose: bool, Optional
Flag that controls information and progress printing.
Default: False
Returns
-------
generator: function
The generator function of the Gaussian pyramid.
"""
if pyramid_on_features:
# compute features at highest level
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Computing feature space: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
feature_images.append(compute_features(i, feature_type[0]))
if verbose:
print_dynamic('- Computing feature space: Done\n')
# create pyramid on feature_images
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in feature_images]
else:
# create pyramid on intensities images
# features will be computed per level
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in images]
return generator
def _scale_images(cls, images, s, level_str, verbose):
scaled_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'- Scaling features: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
scaled_images.append(i.rescale(s))
return scaled_images
def _scale_images(cls, images, s, level_str, verbose):
scaled_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'{}Scaling features: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
scaled_images.append(i.rescale(s))
return scaled_images
def _compute_features(self, images, level_str, verbose):
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'{}Computing feature space: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
if self.features:
i = self.features(i)
feature_images.append(i)
return feature_images
def _compute_features(self, images, level_str, verbose):
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'- Computing feature space: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
if self.features:
i = self.features(i)
feature_images.append(i)
return feature_images
def _normalize_images(self, images, group, label, ref_shape, verbose):
# normalize the scaling of all images wrt the reference_shape size
norm_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
i = i.rescale_to_reference_shape(ref_shape, group=group,
label=label)
if self.sigma:
i.pixels = fsmooth(i.pixels, self.sigma)
norm_images.append(i)
return norm_images
def _normalize_images(self, images, group, label, ref_shape, verbose):
# normalize the scaling of all images wrt the reference_shape size
norm_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
i = rescale_to_reference_shape(i, ref_shape, group=group,
label=label)
if self.sigma:
i.pixels = fsmooth(i.pixels, self.sigma)
norm_images.append(i)
return norm_images
def _compute_minimum_spanning_tree(shapes, root_vertex, level_str, verbose):
# initialize edges and weights matrix
n_vertices = shapes[0].n_points
n_edges = nchoosek(n_vertices, 2)
weights = np.zeros((n_vertices, n_vertices))
edges = np.empty((n_edges, 2), dtype=np.int32)
# fill edges and weights
e = -1
for i in range(n_vertices - 1):
for j in range(i + 1, n_vertices, 1):
# edge counter
e += 1
# print progress
if verbose:
print_dynamic(
'{}Computing complete graph`s weights - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# fill in edges
edges[e, 0] = i
edges[e, 1] = j
# create data matrix of edge
diffs_x = [s.points[i, 0] - s.points[j, 0] for s in shapes]
diffs_y = [s.points[i, 1] - s.points[j, 1] for s in shapes]
coords = np.array([diffs_x, diffs_y])
# compute mean
m = np.mean(coords, axis=1)
# compute covariance
c = np.cov(coords)
# get weight
for im in range(len(shapes)):
weights[i, j] += -np.log(
multivariate_normal.pdf(coords[:, im], mean=m, cov=c))
weights[j, i] = weights[i, j]
# create undirected graph
complete_graph = UndirectedGraph(edges)
if verbose:
print_dynamic('{}Minimum spanning graph computed.\n'.format(level_str))
# compute minimum spanning graph
return complete_graph.minimum_spanning_tree(weights, root_vertex)
def _compute_minimum_spanning_tree(shapes, root_vertex, level_str, verbose):
# initialize edges and weights matrix
n_vertices = shapes[0].n_points
n_edges = nchoosek(n_vertices, 2)
weights = np.zeros((n_vertices, n_vertices))
edges = np.empty((n_edges, 2), dtype=np.int32)
# fill edges and weights
e = -1
for i in range(n_vertices-1):
for j in range(i+1, n_vertices, 1):
# edge counter
e += 1
# print progress
if verbose:
print_dynamic('{}Computing complete graph`s weights - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# fill in edges
edges[e, 0] = i
edges[e, 1] = j
# create data matrix of edge
diffs_x = [s.points[i, 0] - s.points[j, 0] for s in shapes]
diffs_y = [s.points[i, 1] - s.points[j, 1] for s in shapes]
coords = np.array([diffs_x, diffs_y])
# compute mean
m = np.mean(coords, axis=1)
# compute covariance
c = np.cov(coords)
# get weight
for im in range(len(shapes)):
weights[i, j] += -np.log(multivariate_normal.pdf(coords[:, im],
mean=m, cov=c))
weights[j, i] = weights[i, j]
# create undirected graph
complete_graph = UndirectedGraph(edges)
if verbose:
print_dynamic('{}Minimum spanning graph computed.\n'.format(level_str))
# compute minimum spanning graph
return complete_graph.minimum_spanning_tree(weights, root_vertex)
def _normalization_wrt_reference_shape(cls,
images,
group,
label,
reference_shape,
verbose=False):
r"""
Normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the model.
group : `string`
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
reference_shape : :map:`PointCloud`
The reference shape that is used to resize all training images to
a consistent object size.
verbose: bool, optional
Flag that controls information and progress printing.
Returns
-------
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
normalized_images.append(
i.rescale_to_reference_shape(reference_shape,
group=group,
label=label))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return normalized_images
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = Image(i.extract_patches(
s, patch_size=self.parts_shape, as_single_array=True))
parts_images.append(parts_image)
return parts_images
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = build_parts_image(
i, s, self.parts_shape, normalize_parts=self.normalize_parts)
parts_images.append(parts_image)
return parts_images
def __init__(self,
samples,
centre=True,
bias=False,
verbose=False,
n_samples=None):
# get the first element as the template and use it to configure the
# data matrix
if n_samples is None:
# samples is a list
n_samples = len(samples)
template = samples[0]
samples = samples[1:]
else:
# samples is an iterator
template = next(samples)
n_features = template.n_parameters
template_vector = template.as_vector()
data = np.zeros((n_samples, n_features), dtype=template_vector.dtype)
# now we can fill in the first element from the template
data[0] = template_vector
del template_vector
if verbose:
print('Allocated data matrix {:.2f}'
'GB'.format(data.nbytes / 2**30))
# 1-based as we have the template vector set already
for i, sample in enumerate(samples, 1):
if i >= n_samples:
break
if verbose:
print_dynamic(
'Building data matrix from {} samples - {}'.format(
n_samples,
progress_bar_str(float(i + 1) / n_samples,
show_bar=True)))
data[i] = sample.as_vector()
# compute pca
e_vectors, e_values, mean = principal_component_decomposition(
data, whiten=False, centre=centre, bias=bias, inplace=True)
super(PCAModel, self).__init__(e_vectors, mean, template)
self.centred = centre
self.biased = bias
self._eigenvalues = e_values
# start the active components as all the components
self._n_active_components = int(self.n_components)
self._trimmed_eigenvalues = None
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = build_parts_image(
i, s, parts_shape=self.parts_shape,
normalize_parts=self.normalize_parts)
parts_images.append(parts_image)
return parts_images
def compute_sparse_covariance(X, adjacency_array, patch_len, level_str,
verbose):
n_features, n_samples = X.shape
n_edges = adjacency_array.shape[0]
# initialize block sparse covariance matrix
all_cov = np.zeros((n_features, n_features))
# compute covariance matrix for each edge
for e in range(n_edges):
# print progress
if verbose:
print_dynamic('{}Distribution per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(adjacency_array[e, :])
v2 = np.max(adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((X[v1_from:v1_to, :], X[v2_from:v2_to, :]))
# compute covariance inverse
icov = np.linalg.inv(np.cov(edge_data))
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return np.linalg.inv(all_cov)
def _normalization_wrt_reference_shape(cls, images, group, label,
reference_shape, verbose=False):
r"""
Normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the model.
group : `string`
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
reference_shape : :map:`PointCloud`
The reference shape that is used to resize all training images to
a consistent object size.
verbose: bool, optional
Flag that controls information and progress printing.
Returns
-------
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
normalized_images.append(i.rescale_to_reference_shape(
reference_shape, group=group, label=label))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return normalized_images
def _build_appearance_model_block_diagonal(all_patches_array, n_points,
patch_shape, n_channels,
n_appearance_parameters, level_str,
verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'patch'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=-1)
# number of images
n_images = all_patches_array.shape[-1]
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# compute covariance matrix for each patch
all_cov = []
for e in range(n_points):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per patch - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_points,
show_bar=False)))
# select patches and vectorize
patches_vector = all_patches_array[e, ...].reshape(-1, n_images)
# compute covariance
cov_mat = np.cov(patches_vector)
# compute covariance inverse
inv_cov_mat = _covariance_matrix_inverse(cov_mat,
n_appearance_parameters)
# store covariance
all_cov.append(inv_cov_mat)
# create final sparse covariance matrix
return app_mean, block_diag(all_cov).tocsr()
def _warp_images(self, images, shapes, ref_shape, level_str, verbose):
# compute transforms
ref_frame = self._build_reference_frame(ref_shape)
# warp images to reference frame
warped_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
# compute transforms
t = self.transform(ref_frame.landmarks['source'].lms, s)
# warp images
warped_i = i.warp_to_mask(ref_frame.mask, t)
# attach reference frame landmarks to images
warped_i.landmarks['source'] = ref_frame.landmarks['source']
warped_images.append(warped_i)
return warped_images
def __init__(self, samples, centre=True, bias=False, verbose=False,
n_samples=None):
# get the first element as the template and use it to configure the
# data matrix
if n_samples is None:
# samples is a list
n_samples = len(samples)
template = samples[0]
samples = samples[1:]
else:
# samples is an iterator
template = next(samples)
n_features = template.n_parameters
template_vector = template.as_vector()
data = np.zeros((n_samples, n_features), dtype=template_vector.dtype)
# now we can fill in the first element from the template
data[0] = template_vector
del template_vector
if verbose:
print('Allocated data matrix {:.2f}'
'GB'.format(data.nbytes / 2 ** 30))
# 1-based as we have the template vector set already
for i, sample in enumerate(samples, 1):
if i >= n_samples:
break
if verbose:
print_dynamic(
'Building data matrix from {} samples - {}'.format(
n_samples,
progress_bar_str(float(i + 1) / n_samples, show_bar=True)))
data[i] = sample.as_vector()
# compute pca
e_vectors, e_values, mean = principal_component_decomposition(
data, whiten=False, centre=centre, bias=bias, inplace=True)
super(PCAModel, self).__init__(e_vectors, mean, template)
self.centred = centre
self.biased = bias
self._eigenvalues = e_values
# start the active components as all the components
self._n_active_components = int(self.n_components)
self._trimmed_eigenvalues = None
def _warp_images(self, images, shapes, ref_shape, level_str, verbose):
# compute transforms
ref_frame = self._build_reference_frame(ref_shape)
# warp images to reference frame
warped_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
# compute transforms
t = self.transform(ref_frame.landmarks['source'].lms, s)
# warp images
warped_i = i.warp_to_mask(ref_frame.mask, t)
# attach reference frame landmarks to images
warped_i.landmarks['source'] = ref_frame.landmarks['source']
warped_images.append(warped_i)
return warped_images
def _import_glob_generator(pattern, extension_map, max_assets=None,
landmark_resolver=same_name,
landmark_ext_map=None, importer_kwargs=None,
verbose=False):
filepaths = list(glob_with_suffix(pattern, extension_map))
if max_assets:
filepaths = filepaths[:max_assets]
n_files = len(filepaths)
if n_files == 0:
raise ValueError('The glob {} yields no assets'.format(pattern))
for i, asset in enumerate(_multi_import_generator(filepaths, extension_map,
landmark_resolver=landmark_resolver,
landmark_ext_map=landmark_ext_map,
importer_kwargs=importer_kwargs)):
if verbose:
print_dynamic('- Loading {} assets: {}'.format(
n_files, progress_bar_str(float(i + 1) / n_files,
show_bar=True)))
yield asset
def _build_appearance_model_block_diagonal(all_patches_array, n_points,
patch_shape, n_channels,
n_appearance_parameters, level_str,
verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'patch'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# compute covariance matrix for each patch
all_cov = []
for e in range(n_points):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per patch - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_points,
show_bar=False)))
# find indices in target covariance matrix
i_from = e * patch_len
i_to = (e + 1) * patch_len
# compute covariance
cov_mat = np.cov(all_patches_array[i_from:i_to, :])
# compute covariance inverse
inv_cov_mat = _covariance_matrix_inverse(cov_mat,
n_appearance_parameters)
# store covariance
all_cov.append(inv_cov_mat)
# create final sparse covariance matrix
return app_mean, block_diag(all_cov).tocsr()
def create_pyramid(images, n_levels, downscale, features, verbose=False):
r"""
Function that creates a generator function for Gaussian pyramid. The
pyramid can be created either on the feature space or the original
(intensities) space.
Parameters
----------
images: list of :map:`Image`
The set of landmarked images from which to build the AAM.
n_levels: int
The number of multi-resolution pyramidal levels to be used.
downscale: float
The downscale factor that will be used to create the different
pyramidal levels.
features: ``callable`` ``[callable]``
If a single callable, then the feature calculation will happen once
followed by a gaussian pyramid. If a list of callables then a
gaussian pyramid is generated with features extracted at each level
(after downsizing and blurring).
Returns
-------
list of generators :
The generator function of the Gaussian pyramid.
"""
will_take_a_while = is_pyramid_on_features(features)
pyramids = []
for i, img in enumerate(images):
if will_take_a_while and verbose:
print_dynamic(
'Computing top level feature space - {}'.format(
progress_bar_str((i + 1.) / len(images),
show_bar=False)))
pyramids.append(pyramid_of_feature_images(n_levels, downscale,
features, img))
return pyramids
def _build_appearance_model_block_diagonal(all_patches_array, n_points,
patch_shape, n_channels,
n_appearance_parameters, level_str,
verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'patch'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# compute covariance matrix for each patch
all_cov = []
for e in range(n_points):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per patch - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_points,
show_bar=False)))
# find indices in target covariance matrix
i_from = e * patch_len
i_to = (e + 1) * patch_len
# compute covariance
cov_mat = np.cov(all_patches_array[i_from:i_to, :])
# compute covariance inverse
inv_cov_mat = _covariance_matrix_inverse(cov_mat,
n_appearance_parameters)
# store covariance
all_cov.append(inv_cov_mat)
# create final sparse covariance matrix
return app_mean, block_diag(all_cov).tocsr()
def _build_appearance_model_block_diagonal(all_patches_array, n_points,
patch_shape, n_channels,
n_appearance_parameters, level_str,
verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'patch'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=-1)
# number of images
n_images = all_patches_array.shape[-1]
# compute covariance matrix for each patch
all_cov = []
for e in range(n_points):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per patch - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_points,
show_bar=False)))
# select patches and vectorize
patches_vector = all_patches_array[e, ...].reshape(-1, n_images)
# compute covariance
cov_mat = np.cov(patches_vector)
# compute covariance inverse
inv_cov_mat = _covariance_matrix_inverse(cov_mat,
n_appearance_parameters)
# store covariance
all_cov.append(inv_cov_mat)
# create final sparse covariance matrix
return app_mean, block_diag(all_cov).tocsr()
def create_pyramid(images, n_levels, downscale, features, verbose=False):
r"""
Function that creates a generator function for Gaussian pyramid. The
pyramid can be created either on the feature space or the original
(intensities) space.
Parameters
----------
images: list of :map:`Image`
The set of landmarked images from which to build the AAM.
n_levels: int
The number of multi-resolution pyramidal levels to be used.
downscale: float
The downscale factor that will be used to create the different
pyramidal levels.
features: ``callable`` ``[callable]``
If a single callable, then the feature calculation will happen once
followed by a gaussian pyramid. If a list of callables then a
gaussian pyramid is generated with features extracted at each level
(after downsizing and blurring).
Returns
-------
list of generators :
The generator function of the Gaussian pyramid.
"""
will_take_a_while = is_pyramid_on_features(features)
pyramids = []
for i, img in enumerate(images):
if will_take_a_while and verbose:
print_dynamic('Computing top level feature space - {}'.format(
progress_bar_str((i + 1.) / len(images), show_bar=False)))
pyramids.append(
pyramid_of_feature_images(n_levels, downscale, features, img))
return pyramids
def _regression_data(self,
images,
gt_shapes,
perturbed_shapes,
verbose=False):
r"""
Method that generates the regression data : features and delta_ps.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images.
gt_shapes : :map:`PointCloud` list
List of the ground truth shapes that correspond to the images.
perturbed_shapes : :map:`PointCloud` list
List of the perturbed shapes in order to regress.
verbose : `boolean`, optional
If ``True``, the progress is printed.
"""
if verbose:
print_dynamic('- Generating regression data')
n_images = len(images)
features = []
delta_ps = []
for j, (i, s,
p_shape) in enumerate(zip(images, gt_shapes,
perturbed_shapes)):
if verbose:
print_dynamic('- Generating regression data - {}'.format(
progress_bar_str((j + 1.) / n_images, show_bar=False)))
for ps in p_shape:
features.append(self.features(i, ps))
delta_ps.append(self.delta_ps(s, ps))
return np.asarray(features), np.asarray(delta_ps)
def _get_relative_locations(shapes, graph, level_str, verbose):
r"""
returns numpy.array of size 2 x n_images x n_edges
"""
# convert given shapes to point graphs
if isinstance(graph, Tree):
point_graphs = [
PointTree(shape.points, graph.adjacency_array, graph.root_vertex)
for shape in shapes
]
else:
point_graphs = [
PointDirectedGraph(shape.points, graph.adjacency_array)
for shape in shapes
]
# initialize an output numpy array
rel_loc_array = np.empty((2, graph.n_edges, len(point_graphs)))
# get relative locations
for c, pt in enumerate(point_graphs):
# print progress
if verbose:
print_dynamic('{}Computing relative locations from '
'shapes - {}'.format(
level_str,
progress_bar_str(float(c + 1) /
len(point_graphs),
show_bar=False)))
# get relative locations from this shape
rl = pt.relative_locations()
# store
rel_loc_array[..., c] = rl.T
# rollaxis and return
return np.rollaxis(rel_loc_array, 2, 1)
def _warp_images(images, group, label, patch_shape, as_vectors, level_str,
verbose):
r"""
returns numpy.array of size (68*16*16*36) x n_images
"""
# find length of each patch and number of points
n_points = images[0].landmarks[group][label].n_points
patches_len = np.prod(patch_shape) * images[0].n_channels * n_points
n_images = len(images)
# initialize the output
if as_vectors:
all_patches = np.empty((patches_len, n_images))
else:
all_patches = []
# extract parts
for c, i in enumerate(images):
# print progress
if verbose:
print_dynamic('{}Extracting patches from images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images), show_bar=False)))
# extract patches from this image
patches_image = build_patches_image(i,
None,
patch_shape,
group=group,
label=label)
# store
if as_vectors:
all_patches[..., c] = vectorize_patches_image(patches_image)
else:
all_patches.append(patches_image)
return all_patches
def _warp_images(images, group, label, patch_shape, as_vectors, level_str,
verbose):
r"""
returns numpy.array of size (68*16*16*36) x n_images
"""
# find length of each patch and number of points
n_points = images[0].landmarks[group][label].n_points
# TODO: introduce support for offsets
patches_image_shape = (n_points, 1, images[0].n_channels) + patch_shape
n_images = len(images)
# initialize the output
if as_vectors:
all_patches = np.empty(patches_image_shape + (n_images,))
else:
all_patches = []
# extract parts
for c, i in enumerate(images):
# print progress
if verbose:
print_dynamic('{}Extracting patches from images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
# extract patches from this image
patches_image = build_parts_image(
i, i.landmarks[group][label], patch_shape)
# store
if as_vectors:
all_patches[..., c] = patches_image.pixels
else:
all_patches.append(patches_image)
return all_patches
def _warp_images(images, group, label, patch_shape, as_vectors, level_str,
verbose):
r"""
returns numpy.array of size (68*16*16*36) x n_images
"""
# find length of each patch and number of points
n_points = images[0].landmarks[group][label].n_points
# TODO: introduce support for offsets
patches_image_shape = (n_points, 1, images[0].n_channels) + patch_shape
n_images = len(images)
# initialize the output
if as_vectors:
all_patches = np.empty(patches_image_shape + (n_images, ))
else:
all_patches = []
# extract parts
for c, i in enumerate(images):
# print progress
if verbose:
print_dynamic('{}Extracting patches from images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images), show_bar=False)))
# extract patches from this image
patches_image = build_parts_image(i, i.landmarks[group][label],
patch_shape)
# store
if as_vectors:
all_patches[..., c] = patches_image.pixels
else:
all_patches.append(patches_image)
return all_patches
def _warp_images(images, group, label, patch_shape, as_vectors, level_str,
verbose):
r"""
returns numpy.array of size (68*16*16*36) x n_images
"""
# find length of each patch and number of points
n_points = images[0].landmarks[group][label].n_points
patches_len = np.prod(patch_shape) * images[0].n_channels * n_points
n_images = len(images)
# initialize the output
if as_vectors:
all_patches = np.empty((patches_len, n_images))
else:
all_patches = []
# extract parts
for c, i in enumerate(images):
# print progress
if verbose:
print_dynamic('{}Extracting patches from images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
# extract patches from this image
patches_image = build_patches_image(i, None, patch_shape, group=group,
label=label)
# store
if as_vectors:
all_patches[..., c] = vectorize_patches_image(patches_image)
else:
all_patches.append(patches_image)
return all_patches
def _normalization_wrt_reference_shape(cls, images, group, label,
normalization_diagonal,
interpolator, verbose=False):
r"""
Function that normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
The normalization includes:
1) Computation of the reference shape as the mean shape of the images'
landmarks.
2) Scaling of the reference shape using the normalization_diagonal.
3) Rescaling of all the images so that their shape's scale is in
correspondence with the reference shape's scale.
Parameters
----------
images: list of :class:`menpo.image.MaskedImage`
The set of landmarked images from which to build the model.
group : string
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
label: string
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
normalization_diagonal: int
During building an AAM, all images are rescaled to ensure that the
scale of their landmarks matches the scale of the mean shape.
If int, it ensures that the mean shape is scaled so that the
diagonal of the bounding box containing it matches the
normalization_diagonal value.
If None, the mean shape is not rescaled.
Note that, because the reference frame is computed from the mean
landmarks, this kwarg also specifies the diagonal length of the
reference frame (provided that features computation does not change
the image size).
interpolator: string
The interpolator that should be used to perform the warps.
verbose: bool, Optional
Flag that controls information and progress printing.
Default: False
Returns
-------
reference_shape : :map:`PointCloud`
The reference shape that was used to resize all training images to
a consistent object size.
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
# the reference_shape is the mean shape of the images' landmarks
if verbose:
print_dynamic('- Computing reference shape')
shapes = [i.landmarks[group][label] for i in images]
reference_shape = mean_pointcloud(shapes)
# fix the reference_shape's diagonal length if asked
if normalization_diagonal:
x, y = reference_shape.range()
scale = normalization_diagonal / np.sqrt(x**2 + y**2)
Scale(scale, reference_shape.n_dims).apply_inplace(reference_shape)
# normalize the scaling of all images wrt the reference_shape size
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
normalized_images.append(i.rescale_to_reference_shape(
reference_shape, group=group, label=label,
interpolator=interpolator))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return reference_shape, normalized_images
def fit_aps(aps, modelfilename, experiments_path, fast, group,
fitting_images_options, fitting_options, verbose):
# make results filename
filename2 = results_filename(fitting_images_options, fitting_options, group,
fast)
# final save path
filename = modelfilename[:modelfilename.rfind("_")+1] + '_' + filename2
save_path = os.path.join(experiments_path, 'Results', filename)
# fit model
if file_exists(save_path):
if verbose:
print_dynamic('Loading fitting results...')
fitting_results = pickle_load(save_path)
if verbose:
print_dynamic('Fitting results loaded.')
else:
fitter_cls, algorithm_cls = parse_algorithm(
fast, fitting_options['algorithm'])
fitter = fitter_cls(aps, algorithm=algorithm_cls, n_shape=[3, 6],
use_deformation=fitting_options['use_deformation'])
# get fitting images
fitting_images_options['save_path'] = os.path.join(experiments_path,
'Databases')
fitting_images_options['fast'] = fast
fitting_images_options['group'] = group
fitting_images_options['verbose'] = verbose
fitting_images = load_database(**fitting_images_options)
# fit
np.random.seed(seed=1)
fitting_results = []
n_images = len(fitting_images)
if verbose:
perc1 = 0.
perc2 = 0.
perc3 = 0.
for j, i in enumerate(fitting_images):
# fit
if group is not None:
gt_s = i.landmarks[group.__name__].lms
else:
gt_s = i.landmarks['PTS'].lms
s = fitter.perturb_shape(gt_s,
noise_std=fitting_options['noise_std'])
fr = fitter.fit(i, s, gt_shape=gt_s,
max_iters=fitting_options['max_iters'])
fitting_results.append(fr)
# verbose
if verbose:
final_error = fr.final_error(error_type='me_norm')
initial_error = fr.initial_error(error_type='me_norm')
if final_error <= 0.03:
perc1 += 1.
if final_error <= 0.04:
perc2 += 1.
if final_error <= 0.05:
perc3 += 1.
print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%, '
'<=0.05: {3:.1f}%] - Image {4}/{5} (error: '
'{6:.3f} --> {7:.3f})'.format(
progress_bar_str(float(j + 1.) / n_images,
show_bar=False),
perc1 * 100. / n_images, perc2 * 100. / n_images,
perc3 * 100. / n_images, j + 1, n_images,
initial_error, final_error))
if verbose:
print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: '
'{1:.1f}%, <=0.05: {2:.1f}%]\n'.format(
perc1 * 100. / n_images, perc2 * 100. / n_images,
perc3 * 100. / n_images))
errors = []
errors.append([fr.final_error() for fr in fitting_results])
errors.append([fr.initial_error() for fr in fitting_results])
pickle_dump(errors, save_path)
return fitting_results, filename
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the AAM.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
aam : :map:`AAM`
The AAM object. Shape and appearance models are stored from lowest
to highest level
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(images, group, label,
self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.features,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get feature images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = build_shape_model(train_shapes,
self.max_shape_components[rj])
reference_frame = self._build_reference_frame(shape_model.mean())
# add shape model to the list
shape_models.append(shape_model)
# compute transforms
if verbose:
print_dynamic('{}Computing transforms'.format(level_str))
transforms = [
self.transform(reference_frame.landmarks['source'].lms,
i.landmarks[group][label])
for i in feature_images
]
# warp images to reference frame
warped_images = []
for c, (i, t) in enumerate(zip(feature_images, transforms)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(feature_images),
show_bar=False)))
warped_images.append(i.warp_to_mask(reference_frame.mask, t))
# attach reference_frame to images' source shape
for i in warped_images:
i.landmarks['source'] = reference_frame.landmarks['source']
# build appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(level_str))
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components[rj] is not None:
appearance_model.trim_components(
self.max_appearance_components[rj])
# add appearance model to the list
appearance_models.append(appearance_model)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
n_training_images = len(images)
return self._build_aam(shape_models, appearance_models,
n_training_images)
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.features,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = build_shape_model(train_shapes,
self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic('{}Building classifiers - {}'.format(
level_str,
progress_bar_str((k + 1.) / n_points, show_bar=False)))
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0], positive[:,
1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = self.classifier_trainers[rj](X, t)
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
from .base import CLM
return CLM(shape_models, classifiers, n_training_images,
self.patch_shape, self.features, self.reference_shape,
self.downscale, self.scaled_shape_models)
def train(self, images, group=None, label=None, verbose=False, **kwargs):
r"""
Trains a Supervised Descent Regressor given a list of landmarked
images.
Parameters
----------
images: list of :map:`MaskedImage`
The set of landmarked images from which to build the SD.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label: `string`, optional
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose: `boolean`, optional
Flag that controls information and progress printing.
"""
if verbose:
print_dynamic('- Computing reference shape')
self.reference_shape = self._compute_reference_shape(images, group,
label)
# store number of training images
self.n_training_images = len(images)
# normalize the scaling of all images wrt the reference_shape size
self._rescale_reference_shape()
normalized_images = self._normalization_wrt_reference_shape(
images, group, label, self.reference_shape, verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images, self.n_levels,
self.downscale, self.features,
verbose=verbose)
# get feature images of all levels
images = apply_pyramid_on_images(generators, self.n_levels,
verbose=verbose)
# this .reverse sets the lowest resolution as the first level
images.reverse()
# extract the ground truth shapes
gt_shapes = [[i.landmarks[group][label] for i in img]
for img in images]
# build the regressors
if verbose:
if self.n_levels > 1:
print_dynamic('- Building regressors for each of the {} '
'pyramid levels\n'.format(self.n_levels))
else:
print_dynamic('- Building regressors\n')
regressors = []
# for each pyramid level (low --> high)
for j, (level_images, level_gt_shapes) in enumerate(zip(images,
gt_shapes)):
if verbose:
if self.n_levels == 1:
print_dynamic('\n')
elif self.n_levels > 1:
print_dynamic('\nLevel {}:\n'.format(j + 1))
# build regressor
trainer = self._set_regressor_trainer(j)
if j == 0:
regressor = trainer.train(level_images, level_gt_shapes,
verbose=verbose, **kwargs)
else:
regressor = trainer.train(level_images, level_gt_shapes,
level_shapes, verbose=verbose,
**kwargs)
if verbose:
print_dynamic('- Perturbing shapes...')
level_shapes = trainer.perturb_shapes(gt_shapes[0])
regressors.append(regressor)
count = 0
total = len(regressors) * len(images[0]) * len(level_shapes[0])
for k, r in enumerate(regressors):
test_images = images[k]
test_gt_shapes = gt_shapes[k]
fitting_results = []
for (i, gt_s, level_s) in zip(test_images, test_gt_shapes,
level_shapes):
fr_list = []
for ls in level_s:
parameters = r.get_parameters(ls)
fr = r.fit(i, parameters)
fr.gt_shape = gt_s
fr_list.append(fr)
count += 1
fitting_results.append(fr_list)
if verbose:
print_dynamic('- Fitting shapes: {}'.format(
progress_bar_str((count + 1.) / total,
show_bar=False)))
level_shapes = [[Scale(self.downscale,
n_dims=self.reference_shape.n_dims
).apply(fr.final_shape)
for fr in fr_list]
for fr_list in fitting_results]
if verbose:
print_dynamic('- Fitting shapes: computing mean error...')
mean_error = np.mean(np.array([fr.final_error()
for fr_list in fitting_results
for fr in fr_list]))
if verbose:
print_dynamic("- Fitting shapes: mean error "
"is {0:.6f}.\n".format(mean_error))
return self._build_supervised_descent_fitter(regressors)
def build(self, images, group=None, label=None, verbose=False):
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(images, group, label,
self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images, self.n_levels,
self.downscale, self.features,
verbose=verbose)
# if graph_deformation not provided, compute the MST
if self.graph_deformation is None:
shapes = [i.landmarks[group][label] for i in normalized_images]
self.graph_deformation = _compute_minimum_spanning_tree(
shapes, self.root_vertex, '- ', verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
deformation_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
level_str = ' - '
if verbose:
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get feature images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# apply procrustes if asked
if self.use_procrustes:
if verbose:
print_dynamic('{}Procrustes analysis'.format(level_str))
train_shapes = _procrustes_analysis(train_shapes)
# train shape model
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_models.append(_build_shape_model(
train_shapes, self.max_shape_components[rj]))
# compute relative locations from all shapes
relative_locations = _get_relative_locations(
train_shapes, self.graph_deformation, level_str, verbose)
# build and add deformation model to the list
deformation_models.append(_build_deformation_model(
self.graph_deformation, relative_locations, level_str, verbose))
# extract patches from all images
all_patches = _warp_images(feature_images, group, label,
self.patch_shape,
self.gaussian_per_patch, level_str,
verbose)
# build and add appearance model to the list
if self.gaussian_per_patch:
n_channels = feature_images[0].n_channels
if self.graph_appearance is None:
# diagonal block covariance
n_points = images[0].landmarks[group][label].n_points
appearance_models.append(
_build_appearance_model_block_diagonal(
all_patches, n_points, self.patch_shape, n_channels,
self.n_appearance_parameters[rj], level_str,
verbose))
else:
# sparse block covariance
appearance_models.append(_build_appearance_model_sparse(
all_patches, self.graph_appearance, self.patch_shape,
n_channels, self.n_appearance_parameters[rj], level_str,
verbose))
else:
# full covariance
appearance_models.append(_build_appearance_model_full(
all_patches, self.n_appearance_parameters[rj],
level_str, verbose))
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of models so that they are ordered from lower to
# higher resolution
shape_models.reverse()
appearance_models.reverse()
deformation_models.reverse()
n_training_images = len(images)
return self._build_aps(shape_models, deformation_models,
appearance_models, n_training_images)
def as_matrix(vectorizables, length=None, return_template=False, verbose=False):
r"""
Create a matrix from a list/generator of :map:`Vectorizable` objects.
All the objects in the list **must** be the same size when vectorized.
Consider using a generator if the matrix you are creating is large and
passing the length of the generator explicitly.
Parameters
----------
vectorizables : `list` or generator if :map:`Vectorizable` objects
A list or generator of objects that supports the vectorizable interface
length : `int`, optional
Length of the vectorizable list. Useful if you are passing a generator
with a known length.
verbose : `bool`, optional
If ``True``, will print the progress of building the matrix.
return_template : `bool`, optional
If ``True``, will return the first element of the list/generator, which
was used as the template. Useful if you need to map back from the
matrix to a list of vectorizable objects.
Returns
-------
M : (length, n_features) `ndarray`
Every row is an element of the list.
template : :map:`Vectorizable`, optional
If ``return_template == True``, will return the template used to
build the matrix `M`.
"""
# get the first element as the template and use it to configure the
# data matrix
if length is None:
# samples is a list
length = len(vectorizables)
template = vectorizables[0]
vectorizables = vectorizables[1:]
else:
# samples is an iterator
template = next(vectorizables)
n_features = template.n_parameters
template_vector = template.as_vector()
data = np.zeros((length, n_features), dtype=template_vector.dtype)
if verbose:
print('Allocated data matrix {:.2f}'
'GB'.format(data.nbytes / 2 ** 30))
# now we can fill in the first element from the template
data[0] = template_vector
del template_vector
# 1-based as we have the template vector set already
for i, sample in enumerate(vectorizables, 1):
if i >= length:
break
if verbose:
print_dynamic(
'Building data matrix from {} samples - {}'.format(
length,
progress_bar_str(float(i + 1) / length, show_bar=True)))
data[i] = sample.as_vector()
if return_template:
return data, template
else:
return data
def build(self, images, group=None, label=None, verbose=False, **kwargs):
# compute reference shape
reference_shape = self._compute_reference_shape(images, group, label,
verbose)
# normalize images
images = self._normalize_images(images, group, label, reference_shape,
verbose)
# build models at each scale
if verbose:
print_dynamic('- Building models\n')
shape_models = []
appearance_models = []
classifiers = []
# for each pyramid level (high --> low)
for j, s in enumerate(self.scales):
if verbose:
if len(self.scales) > 1:
level_str = ' - Level {}: '.format(j)
else:
level_str = ' - '
# obtain image representation
if j == 0:
# compute features at highest level
feature_images = self._compute_features(images, level_str,
verbose)
level_images = feature_images
elif self.scale_features:
# scale features at other levels
level_images = self._scale_images(feature_images, s,
level_str, verbose)
else:
# scale images and compute features at other levels
scaled_images = self._scale_images(images, s, level_str,
verbose)
level_images = self._compute_features(scaled_images,
level_str, verbose)
# extract potentially rescaled shapes ath highest level
level_shapes = [i.landmarks[group][label]
for i in level_images]
# obtain shape representation
if j == 0 or self.scale_shapes:
# obtain shape model
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = self._build_shape_model(
level_shapes, self.max_shape_components)
# add shape model to the list
shape_models.append(shape_model)
else:
# copy precious shape model and add it to the list
shape_models.append(deepcopy(shape_model))
# obtain warped images
warped_images = self._warp_images(level_images, level_shapes,
shape_model.mean(), level_str,
verbose)
# obtain appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(level_str))
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components is not None:
appearance_model.trim_components(
self.max_appearance_components)
# add appearance model to the list
appearance_models.append(appearance_model)
if isinstance(self, GlobalUnifiedBuilder):
# obtain parts images
parts_images = self._parts_images(level_images, level_shapes,
level_str, verbose)
else:
# parts images are warped images
parts_images = warped_images
# build desired responses
mvn = multivariate_normal(mean=np.zeros(2), cov=self.covariance)
grid = build_sampling_grid(self.parts_shape)
Y = [mvn.pdf(grid + offset) for offset in self.offsets]
# build classifiers
n_landmarks = level_shapes[0].n_points
level_classifiers = []
for l in range(n_landmarks):
if verbose:
print_dynamic('{}Building classifiers - {}'.format(
level_str,
progress_bar_str((l + 1.) / n_landmarks,
show_bar=False)))
X = [i.pixels[l] for i in parts_images]
clf = self.classifier(X, Y, **kwargs)
level_classifiers.append(clf)
# build Multiple classifier
if self.classifier is MCF:
multiple_clf = MultipleMCF(level_classifiers)
elif self.classifier is LinearSVMLR:
multiple_clf = MultipleLinearSVMLR(level_classifiers)
# add appearance model to the list
classifiers.append(multiple_clf)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
classifiers.reverse()
self.scales.reverse()
unified = self._build_unified(shape_models, appearance_models,
classifiers, reference_shape)
return unified
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the AAM.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
aam : :map:`AAM`
The AAM object. Shape and appearance models are stored from lowest
to highest level
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(images, group, label,
self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images, self.n_levels,
self.downscale, self.features,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get feature images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = build_shape_model(
train_shapes, self.max_shape_components[rj])
reference_frame = self._build_reference_frame(shape_model.mean())
# add shape model to the list
shape_models.append(shape_model)
# compute transforms
if verbose:
print_dynamic('{}Computing transforms'.format(level_str))
# Create a dummy initial transform
s_to_t_transform = self.transform(
reference_frame.landmarks['source'].lms,
reference_frame.landmarks['source'].lms)
# warp images to reference frame
warped_images = []
for c, i in enumerate(feature_images):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(feature_images),
show_bar=False)))
# Setting the target can be significantly faster for transforms
# such as CachedPiecewiseAffine
s_to_t_transform.set_target(i.landmarks[group][label])
warped_images.append(i.warp_to_mask(reference_frame.mask,
s_to_t_transform))
# attach reference_frame to images' source shape
for i in warped_images:
i.landmarks['source'] = reference_frame.landmarks['source']
# build appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(level_str))
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components[rj] is not None:
appearance_model.trim_components(
self.max_appearance_components[rj])
# add appearance model to the list
appearance_models.append(appearance_model)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
n_training_images = len(images)
return self._build_aam(shape_models, appearance_models,
n_training_images)
def fit_aps(aps, modelfilename, experiments_path, fast, group,
fitting_images_options, fitting_options, verbose):
# make results filename
filename2 = results_filename(fitting_images_options, fitting_options,
group, fast)
# final save path
filename = modelfilename[:modelfilename.rfind("_") + 1] + '_' + filename2
save_path = os.path.join(experiments_path, 'Results', filename)
# fit model
if file_exists(save_path):
if verbose:
print_dynamic('Loading fitting results...')
fitting_results = pickle_load(save_path)
if verbose:
print_dynamic('Fitting results loaded.')
else:
fitter_cls, algorithm_cls = parse_algorithm(
fast, fitting_options['algorithm'])
fitter = fitter_cls(aps,
algorithm=algorithm_cls,
n_shape=[3, 6],
use_deformation=fitting_options['use_deformation'])
# get fitting images
fitting_images_options['save_path'] = os.path.join(
experiments_path, 'Databases')
fitting_images_options['fast'] = fast
fitting_images_options['group'] = group
fitting_images_options['verbose'] = verbose
fitting_images = load_database(**fitting_images_options)
# fit
np.random.seed(seed=1)
fitting_results = []
n_images = len(fitting_images)
if verbose:
perc1 = 0.
perc2 = 0.
perc3 = 0.
for j, i in enumerate(fitting_images):
# fit
if group is not None:
gt_s = i.landmarks[group.__name__].lms
else:
gt_s = i.landmarks['PTS'].lms
s = fitter.perturb_shape(gt_s,
noise_std=fitting_options['noise_std'])
fr = fitter.fit(i,
s,
gt_shape=gt_s,
max_iters=fitting_options['max_iters'])
fitting_results.append(fr)
# verbose
if verbose:
final_error = fr.final_error(error_type='me_norm')
initial_error = fr.initial_error(error_type='me_norm')
if final_error <= 0.03:
perc1 += 1.
if final_error <= 0.04:
perc2 += 1.
if final_error <= 0.05:
perc3 += 1.
print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%, '
'<=0.05: {3:.1f}%] - Image {4}/{5} (error: '
'{6:.3f} --> {7:.3f})'.format(
progress_bar_str(float(j + 1.) / n_images,
show_bar=False),
perc1 * 100. / n_images,
perc2 * 100. / n_images,
perc3 * 100. / n_images, j + 1, n_images,
initial_error, final_error))
if verbose:
print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: '
'{1:.1f}%, <=0.05: {2:.1f}%]\n'.format(
perc1 * 100. / n_images, perc2 * 100. / n_images,
perc3 * 100. / n_images))
errors = []
errors.append([fr.final_error() for fr in fitting_results])
errors.append([fr.initial_error() for fr in fitting_results])
pickle_dump(errors, save_path)
return fitting_results, filename
def normalization_wrt_reference_shape(images, group, label,
normalization_diagonal, verbose=False):
r"""
Function that normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
The normalization includes:
1) Computation of the reference shape as the mean shape of the images'
landmarks.
2) Scaling of the reference shape using the normalization_diagonal.
3) Rescaling of all the images so that their shape's scale is in
correspondence with the reference shape's scale.
Parameters
----------
images : list of :class:`menpo.image.MaskedImage`
The set of landmarked images to normalize.
group : `str`
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
label : `str`
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
normalization_diagonal: `int`
If int, it ensures that the mean shape is scaled so that the
diagonal of the bounding box containing it matches the
normalization_diagonal value.
If None, the mean shape is not rescaled.
Note that, because the reference frame is computed from the mean
landmarks, this kwarg also specifies the diagonal length of the
reference frame (provided that features computation does not change
the image size).
verbose : `bool`, Optional
Flag that controls information and progress printing.
Returns
-------
reference_shape : :map:`PointCloud`
The reference shape that was used to resize all training images to
a consistent object size.
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
# get shapes
shapes = [i.landmarks[group][label] for i in images]
# compute the reference shape and fix its diagonal length
reference_shape = compute_reference_shape(shapes, normalization_diagonal,
verbose=verbose)
# normalize the scaling of all images wrt the reference_shape size
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
normalized_images.append(i.rescale_to_reference_shape(
reference_shape, group=group, label=label))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return reference_shape, normalized_images
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images, self.n_levels,
self.downscale, self.features,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = build_shape_model(
train_shapes, self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic('{}Building classifiers - {}'.format(
level_str,
progress_bar_str((k + 1.) / n_points,
show_bar=False)))
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0],
positive[:, 1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = self.classifier_trainers[rj](X, t)
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
from .base import CLM
return CLM(shape_models, classifiers, n_training_images,
self.patch_shape, self.features, self.reference_shape,
self.downscale, self.scaled_shape_models)
def build(self, images, group=None, label=None, verbose=False):
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
normalization_wrt_reference_shape(images, group, label,
self.normalization_diagonal,
verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.features,
verbose=verbose)
# if graph_deformation not provided, compute the MST
if self.graph_deformation is None:
shapes = [i.landmarks[group][label] for i in normalized_images]
self.graph_deformation = _compute_minimum_spanning_tree(
shapes, self.root_vertex, '- ', verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
deformation_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
level_str = ' - '
if verbose:
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get feature images of current level
feature_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# apply procrustes if asked
if self.use_procrustes:
if verbose:
print_dynamic('{}Procrustes analysis'.format(level_str))
train_shapes = _procrustes_analysis(train_shapes)
# train shape model
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_models.append(
_build_shape_model(train_shapes, self.graph_shape,
self.max_shape_components[rj]))
# compute relative locations from all shapes
relative_locations = _get_relative_locations(
train_shapes, self.graph_deformation, level_str, verbose)
# build and add deformation model to the list
deformation_models.append(
_build_deformation_model(self.graph_deformation,
relative_locations, level_str,
verbose))
# extract patches from all images
all_patches = _warp_images(feature_images, group, label,
self.patch_shape,
self.gaussian_per_patch, level_str,
verbose)
# build and add appearance model to the list
if self.gaussian_per_patch:
n_channels = feature_images[0].n_channels
if self.graph_appearance is None:
# diagonal block covariance
n_points = images[0].landmarks[group][label].n_points
appearance_models.append(
_build_appearance_model_block_diagonal(
all_patches, n_points, self.patch_shape,
n_channels, self.n_appearance_parameters[rj],
level_str, verbose))
else:
# sparse block covariance
appearance_models.append(
_build_appearance_model_sparse(
all_patches, self.graph_appearance,
self.patch_shape, n_channels,
self.n_appearance_parameters[rj], level_str,
verbose))
else:
if self.graph_appearance is None:
# full covariance
n_points = images[0].landmarks[group][label].n_points
patches_len = np.prod(self.patch_shape) * \
images[0].n_channels * n_points
appearance_models.append(
_build_appearance_model_full(
all_patches, self.n_appearance_parameters[rj],
patches_len, level_str, verbose))
elif self.graph_appearance == 'yorgos':
# full covariance
n_points = images[0].landmarks[group][label].n_points
patches_len = np.prod(self.patch_shape) * \
images[0].n_channels * n_points
appearance_models.append(
_build_appearance_model_full_yorgos(
all_patches, self.n_appearance_parameters[rj],
patches_len, level_str, verbose))
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of models so that they are ordered from lower to
# higher resolution
shape_models.reverse()
appearance_models.reverse()
deformation_models.reverse()
n_training_images = len(images)
return self._build_aps(shape_models, deformation_models,
appearance_models, n_training_images)
def train(self, images, group=None, label=None, verbose=False, **kwargs):
r"""
Trains a Supervised Descent Regressor given a list of landmarked
images.
Parameters
----------
images: list of :map:`MaskedImage`
The set of landmarked images from which to build the SD.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label: `string`, optional
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose: `boolean`, optional
Flag that controls information and progress printing.
"""
if verbose:
print_dynamic('- Computing reference shape')
self.reference_shape = self._compute_reference_shape(
images, group, label)
# store number of training images
self.n_training_images = len(images)
# normalize the scaling of all images wrt the reference_shape size
self._rescale_reference_shape()
normalized_images = self._normalization_wrt_reference_shape(
images, group, label, self.reference_shape, verbose=verbose)
# create pyramid
generators = create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.features,
verbose=verbose)
# get feature images of all levels
images = apply_pyramid_on_images(generators,
self.n_levels,
verbose=verbose)
# this .reverse sets the lowest resolution as the first level
images.reverse()
# extract the ground truth shapes
gt_shapes = [[i.landmarks[group][label] for i in img]
for img in images]
# build the regressors
if verbose:
if self.n_levels > 1:
print_dynamic('- Building regressors for each of the {} '
'pyramid levels\n'.format(self.n_levels))
else:
print_dynamic('- Building regressors\n')
regressors = []
# for each pyramid level (low --> high)
for j, (level_images,
level_gt_shapes) in enumerate(zip(images, gt_shapes)):
if verbose:
if self.n_levels == 1:
print_dynamic('\n')
elif self.n_levels > 1:
print_dynamic('\nLevel {}:\n'.format(j + 1))
# build regressor
trainer = self._set_regressor_trainer(j)
if j == 0:
regressor = trainer.train(level_images,
level_gt_shapes,
verbose=verbose,
**kwargs)
else:
regressor = trainer.train(level_images,
level_gt_shapes,
level_shapes,
verbose=verbose,
**kwargs)
if verbose:
print_dynamic('- Perturbing shapes...')
level_shapes = trainer.perturb_shapes(gt_shapes[0])
regressors.append(regressor)
count = 0
total = len(regressors) * len(images[0]) * len(level_shapes[0])
for k, r in enumerate(regressors):
test_images = images[k]
test_gt_shapes = gt_shapes[k]
fitting_results = []
for (i, gt_s, level_s) in zip(test_images, test_gt_shapes,
level_shapes):
fr_list = []
for ls in level_s:
parameters = r.get_parameters(ls)
fr = r.fit(i, parameters)
fr.gt_shape = gt_s
fr_list.append(fr)
count += 1
fitting_results.append(fr_list)
if verbose:
print_dynamic('- Fitting shapes: {}'.format(
progress_bar_str((count + 1.) / total,
show_bar=False)))
level_shapes = [[
Scale(self.downscale,
n_dims=self.reference_shape.n_dims).apply(
fr.final_shape) for fr in fr_list
] for fr_list in fitting_results]
if verbose:
print_dynamic('- Fitting shapes: computing mean error...')
mean_error = np.mean(
np.array([
fr.final_error() for fr_list in fitting_results
for fr in fr_list
]))
if verbose:
print_dynamic("- Fitting shapes: mean error "
"is {0:.6f}.\n".format(mean_error))
return self._build_supervised_descent_fitter(regressors)
