def __init__(self, reference_shape, regression_type=mlr,
regression_features=sparse_hog, patch_shape=(16, 16),
noise_std=0.04, rotation=False, n_perturbations=10):
super(NonParametricRegressorTrainer, self).__init__(
reference_shape, regression_type=regression_type,
regression_features=regression_features, noise_std=noise_std,
rotation=rotation, n_perturbations=n_perturbations)
self.patch_shape = patch_shape
self.sampling_grid = build_sampling_grid(patch_shape)
def _set_up(self):
# Build the sampling grid associated to the patch shape
self._sampling_grid = build_sampling_grid(self.patch_shape)
# Define the 2-dimensional gaussian distribution
mean = np.zeros(self.transform.n_dims)
covariance = self.scale * self.transform.model.noise_variance
mvn = multivariate_normal(mean=mean, cov=covariance)
# Compute Gaussian-KDE grid
self._kernel_grid = mvn.pdf(self._sampling_grid)
# Jacobian
self._J = self.transform.d_dp([])
# Prior
sim_prior = np.zeros((4,))
pdm_prior = 1 / self.transform.model.eigenvalues
self._J_prior = np.hstack((sim_prior, pdm_prior))
# Inverse Hessian
H = np.einsum('ijk, ilk -> jl', self._J, self._J)
self._inv_H = np.linalg.inv(np.diag(self._J_prior) + H)
def _set_up(self):
# Build the sampling grid associated to the patch shape
self._sampling_grid = build_sampling_grid(self.patch_shape)
# Define the 2-dimensional gaussian distribution
mean = np.zeros(self.transform.n_dims)
covariance = self.scale * self.transform.model.noise_variance
mvn = multivariate_normal(mean=mean, cov=covariance)
# Compute Gaussian-KDE grid
self._kernel_grid = mvn.pdf(self._sampling_grid)
# Jacobian
self._J = self.transform.d_dp([])
# Prior
sim_prior = np.zeros((4, ))
pdm_prior = 1 / self.transform.model.eigenvalues
self._J_prior = np.hstack((sim_prior, pdm_prior))
# Inverse Hessian
H = np.einsum('ijk, ilk -> jl', self._J, self._J)
self._inv_H = np.linalg.inv(np.diag(self._J_prior) + H)
def _set_up(self):
self._sampling_grid = build_sampling_grid(self.patch_shape)
# Gaussian-KDE
mean = np.zeros(self.transform.n_dims)
self._rho = 10 * self.transform.model.noise_variance
mvn = multivariate_normal(mean=mean, cov=self._rho)
self._kernel_grid = mvn.pdf(self._sampling_grid)
# Transform Jacobian
self._J = self.transform.jacobian([])
# Prior
augmented_eigenvalues = np.hstack((np.ones(4),
self.transform.model.eigenvalues))
self._J_regularizer = self._rho / augmented_eigenvalues
# set uninformative prior for similarity weights
self._J_regularizer[:4] = 0
# Inverse Hessian
self._inv_H = np.linalg.inv(np.diag(self._J_regularizer) +
np.dot(self._J.T, self._J))
def _set_up(self):
self._sampling_grid = build_sampling_grid(self.patch_shape)
# Gaussian-KDE
mean = np.zeros(self.transform.n_dims)
self._rho = 10 * self.transform.model.noise_variance
mvn = multivariate_normal(mean=mean, cov=self._rho)
self._kernel_grid = mvn.pdf(self._sampling_grid)
# Transform Jacobian
self._J = self.transform.jacobian([])
# Prior
augmented_eigenvalues = np.hstack(
(np.ones(4), self.transform.model.eigenvalues))
self._J_regularizer = self._rho / augmented_eigenvalues
# set uninformative prior for similarity weights
self._J_regularizer[:4] = 0
# Inverse Hessian
self._inv_H = np.linalg.inv(
np.diag(self._J_regularizer) + np.dot(self._J.T, self._J))
def build(self, images, group=None, label="all", verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal, self.interpolator, verbose=verbose
)
# create pyramid
generators = self._create_pyramid(
normalized_images,
self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type,
verbose=verbose,
)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic("- Building model for each of the {} pyramid " "levels\n".format(self.n_levels))
else:
print_dynamic("- Building model\n")
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = " - "
if self.n_levels > 1:
level_str = " - Level {}: ".format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic(
"{}Rescaling feature space - {}".format(
level_str, progress_bar_str((c + 1.0) / len(generators), show_bar=False)
)
)
feature_images.append(g.next())
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic(
"{}Computing feature space - {}".format(
level_str, progress_bar_str((c + 1.0) / len(generators), show_bar=False)
)
)
feature_images.append(compute_features(g.next(), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic("{}Building shape model".format(level_str))
shape_model = self._build_shape_model(train_shapes, self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic(
"{}Building classifiers - {}".format(
level_str, progress_bar_str((k + 1.0) / n_points, show_bar=False)
)
)
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0], positive[:, 1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples, (-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples, (-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type[rj])
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic("{}Done\n".format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
return CLM(
shape_models,
classifiers,
n_training_images,
self.patch_shape,
self.feature_type,
self.reference_shape,
self.downscale,
self.scaled_shape_models,
self.pyramid_on_features,
self.interpolator,
)
def build(self, images, group=None, label='all'):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
Default: None
label: string, Optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
Default: 'all'
Returns
-------
aam : :class:`menpo.fitmultiple.clm.builder.CLM`
The CLM object
"""
print('- Preprocessing')
self.reference_shape, generator = self._preprocessing(
images, group, label, self.diagonal_range, self.interpolator,
self.scaled_levels, self.n_levels, self.downscale)
print('- Building model pyramids')
shape_models = []
classifiers = []
# for each level
for j in np.arange(self.n_levels):
print(' - Level {}'.format(j))
print(' - Computing feature space')
images = [compute_features(g.next(), self.feature_type)
for g in generator]
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in images]
if j == 0 or self.scaled_levels:
print(' - Building shape model')
shape_model = self._build_shape_model(
shapes, self.max_shape_components)
# add shape model to the list
shape_models.append(shape_model)
print(' - Building classifiers')
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
print(' - {} % '.format(round(100*(k+1)/n_points)), end='\r')
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0],
positive[:, 1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type)
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
return CLM(shape_models, classifiers, self.patch_shape,
self.feature_type, self.reference_shape, self.downscale,
self.scaled_levels, self.interpolator)
def _set_up(self):
# TODO: CLMs should use slices instead of sampling grid, and the
# need of the _set_up method will probably disappear
# set up sampling grid
self.sampling_grid = build_sampling_grid(self.patch_shape)
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
self.interpolator, verbose=verbose)
# create pyramid
generators = self._create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Rescaling feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Computing feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(
compute_features(next(g), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = self._build_shape_model(
train_shapes, self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic('{}Building classifiers - {}'.format(
level_str,
progress_bar_str((k + 1.) / n_points, show_bar=False)))
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0], positive[:,
1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type[rj])
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
return CLM(shape_models, classifiers, n_training_images,
self.patch_shape, self.feature_type, self.reference_shape,
self.downscale, self.scaled_shape_models,
self.pyramid_on_features, self.interpolator)