def _prepare_image(self, image, initial_shape, gt_shape=None):
r"""
The image is first rescaled wrt the reference_landmarks, then
smoothing or gaussian pyramid are computed and, finally, features
are extracted from each pyramidal element.
"""
image.landmarks['initial_shape'] = initial_shape
image = image.rescale_to_reference_shape(
self.reference_shape, group='initial_shape',
interpolator=self.interpolator)
if gt_shape:
image.landmarks['gt_shape'] = initial_shape
if self.n_levels > 1:
if self.scaled_levels:
pyramid = image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
else:
pyramid = image.smoothing_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
images = [compute_features(i, self.feature_type)
for i in pyramid]
images.reverse()
else:
images = [compute_features(image, self.feature_type)]
return images
def _prepare_image(self, image, initial_shape, gt_shape=None):
r"""
The image is first rescaled wrt the reference_landmarks, then
smoothing or gaussian pyramid are computed and, finally, features
are extracted from each pyramidal element.
"""
image.landmarks['initial_shape'] = initial_shape
image = image.rescale_to_reference_shape(
self.reference_shape,
group='initial_shape',
interpolator=self.interpolator)
if gt_shape:
image.landmarks['gt_shape'] = initial_shape
if self.n_levels > 1:
if self.scaled_levels:
pyramid = image.gaussian_pyramid(n_levels=self.n_levels,
downscale=self.downscale)
else:
pyramid = image.smoothing_pyramid(n_levels=self.n_levels,
downscale=self.downscale)
images = [compute_features(i, self.feature_type) for i in pyramid]
images.reverse()
else:
images = [compute_features(image, self.feature_type)]
return images
def _prepare_image(self, image, initial_shape, gt_shape=None):
r"""
The image is first rescaled wrt the reference_landmarks and then the
gaussian pyramid is computed. Depending on the pyramid_on_features
flag, the pyramid is either applied on the feature image or
features are extracted at each pyramidal level.
Parameters
----------
image: :class:`menpo.image.MaskedImage`
The image to be fitted.
initial_shape: class:`menpo.shape.PointCloud`
The initial shape from which the fitting will start.
gt_shape: class:`menpo.shape.PointCloud`, optional
The original ground truth shape associated to the image.
Default: None
Returns
-------
images: list of :class:`menpo.image.masked.MaskedImage`
List of images, each being the result of applying the pyramid.
"""
# rescale image wrt the scale factor between reference_shape and
# initial_shape
image.landmarks['initial_shape'] = initial_shape
image = image.rescale_to_reference_shape(
self.reference_shape, group='initial_shape',
interpolator=self.interpolator)
# attach given ground truth shape
if gt_shape:
image.landmarks['gt_shape'] = gt_shape
# apply pyramid
if self.n_levels > 1:
if self.pyramid_on_features:
# compute features at highest level
feature_image = compute_features(image, self.feature_type[0])
# apply pyramid on feature image
pyramid = feature_image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
# get rescaled feature images
images = list(pyramid)
else:
# create pyramid on intensities image
pyramid = image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
# compute features at each level
images = [compute_features(
i, self.feature_type[self.n_levels - j - 1])
for j, i in enumerate(pyramid)]
images.reverse()
else:
images = [compute_features(image, self.feature_type[0])]
return images
def _prepare_image(self, image, initial_shape, gt_shape=None):
r"""
The image is first rescaled wrt the reference_landmarks and then the
gaussian pyramid is computed. Depending on the pyramid_on_features
flag, the pyramid is either applied on the feature image or
features are extracted at each pyramidal level.
Parameters
----------
image: :class:`menpo.image.MaskedImage`
The image to be fitted.
initial_shape: class:`menpo.shape.PointCloud`
The initial shape from which the fitting will start.
gt_shape: class:`menpo.shape.PointCloud`, optional
The original ground truth shape associated to the image.
Default: None
Returns
-------
images: list of :class:`menpo.image.masked.MaskedImage`
List of images, each being the result of applying the pyramid.
"""
# rescale image wrt the scale factor between reference_shape and
# initial_shape
image.landmarks['initial_shape'] = initial_shape
image = image.rescale_to_reference_shape(
self.reference_shape, group='initial_shape',
interpolator=self.interpolator)
# attach given ground truth shape
if gt_shape:
image.landmarks['gt_shape'] = gt_shape
# apply pyramid
if self.n_levels > 1:
if self.pyramid_on_features:
# compute features at highest level
feature_image = compute_features(image, self.feature_type[0])
# apply pyramid on feature image
pyramid = feature_image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
# get rescaled feature images
images = list(pyramid)
else:
# create pyramid on intensities image
pyramid = image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
# compute features at each level
images = [compute_features(
i, self.feature_type[self.n_levels - j - 1])
for j, i in enumerate(pyramid)]
images.reverse()
else:
images = [compute_features(image, self.feature_type[0])]
return images
def response_image(self, image, group=None, label="all", level=-1):
r"""
Generates a response image result of applying the classifiers of a
particular pyramidal level of the CLM to an image.
Parameters
-----------
image: :map:`Image`
The image.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
level: `int`, optional
The pyramidal level to be used.
Returns
-------
image : :map:`Image`
The response image.
"""
# rescale image
image = image.rescale_to_reference_shape(self.reference_shape, group=group, label=label)
# apply pyramid
if self.n_levels > 1:
if self.pyramid_on_features:
# compute features at highest level
feature_image = compute_features(image, self.feature_type[0])
# apply pyramid on feature image
pyramid = feature_image.gaussian_pyramid(n_levels=self.n_levels, downscale=self.downscale)
# get rescaled feature images
images = list(pyramid)
else:
# create pyramid on intensities image
pyramid = image.gaussian_pyramid(n_levels=self.n_levels, downscale=self.downscale)
# compute features at each level
images = [compute_features(i, self.feature_type[self.n_levels - j - 1]) for j, i in enumerate(pyramid)]
images.reverse()
else:
images = [compute_features(image, self.feature_type[0])]
# initialize responses
image = images[level]
image_pixels = np.reshape(image.pixels, (-1, image.n_channels))
response_data = np.zeros((image.shape[0], image.shape[1], self.n_classifiers_per_level[level]))
# Compute responses
for j, clf in enumerate(self.classifiers[level]):
response_data[:, :, j] = np.reshape(clf(image_pixels), image.shape)
return Image(image_data=response_data)
def features(self, image, shape):
r"""
Method that extracts the features for the regression, which in this
case are patch based.
Parameters
----------
image : :map:`MaskedImage`
The current image.
shape : :map:`PointCloud`
The current shape.
"""
# extract patches
patches = extract_local_patches_fast(image, shape, self.patch_shape)
features = np.zeros((shape.n_points, self._feature_patch_length))
for j, patch in enumerate(patches):
# build patch image
patch_img = Image(patch, copy=False)
# compute features
features[j, ...] = compute_features(
patch_img, self.regression_features).as_vector()
return np.hstack((features.ravel(), 1))
def _create_pyramid(cls, images, n_levels, downscale, pyramid_on_features,
feature_type, verbose=False):
r"""
Function that creates a generator function for Gaussian pyramid. The
pyramid can be created either on the feature space or the original
(intensities) space.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
n_levels: int
The number of multi-resolution pyramidal levels to be used.
downscale: float
The downscale factor that will be used to create the different
pyramidal levels.
pyramid_on_features: boolean
If True, the features are extracted at the highest level and the
pyramid is created on the feature images.
If False, the pyramid is created on the original (intensities)
space.
feature_type: list of size 1 with str or function/closure or None
The feature type to be used in case pyramid_on_features is enabled.
verbose: bool, Optional
Flag that controls information and progress printing.
Default: False
Returns
-------
generator: function
The generator function of the Gaussian pyramid.
"""
if pyramid_on_features:
# compute features at highest level
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Computing feature space: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
feature_images.append(compute_features(i, feature_type[0]))
if verbose:
print_dynamic('- Computing feature space: Done\n')
# create pyramid on feature_images
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in feature_images]
else:
# create pyramid on intensities images
# features will be computed per level
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in images]
return generator
def response_image(self, image, group=None, label='all', level=-1):
r"""
Generates a response image result of applying the classifiers of a
particular pyramidal level of the CLM to an image.
Parameters
-----------
image: :class:`menpo.image.base.Image`
The image.
group : string, Optional
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
Default: None
label: string, Optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
Default: 'all'
level: int, optional
The pyramidal level to be used.
Default: -1
Returns
-------
image: :class:`menpo.image.base.Image`
The response image.
"""
image = image.rescale_to_reference_shape(self.reference_shape,
group=group, label=label)
pyramid = image.gaussian_pyramid(n_levels=self.n_levels,
downscale=self.downscale)
images = [compute_features(i, self.feature_type)
for i in pyramid]
images.reverse()
image = images[level]
image_pixels = np.reshape(image.pixels, (-1, image.n_channels))
response_data = np.zeros((image.shape[0], image.shape[1],
self.n_classifiers_per_level[level]))
# Compute responses
for j, clf in enumerate(self.classifiers[level]):
response_data[:, :, j] = np.reshape(clf(image_pixels),
image.shape)
return Image(image_data=response_data)
def build(self, images, group=None, label='all'):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
Default: None
label: string, Optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
Default: 'all'
Returns
-------
aam : :class:`menpo.fitmultiple.aam.builder.AAM`
The AAM object
"""
print '- Preprocessing'
self.reference_shape, generator = self._preprocessing(
images, group, label, self.diagonal_range, self.interpolator,
self.scaled_levels, self.n_levels, self.downscale)
print '- Building model pyramids'
shape_models = []
appearance_models = []
# for each level
for j in np.arange(self.n_levels):
print ' - Level {}'.format(j)
print ' - Computing feature space'
images = [compute_features(g.next(), self.feature_type)
for g in generator]
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in images]
if j == 0 or self.scaled_levels:
print ' - Building shape model'
if j != 0:
shapes = [Scale(1/self.downscale,
n_dims=shapes[0].n_dims).apply(s)
for s in shapes]
shape_model = self._build_shape_model(
shapes, self.max_shape_components)
print ' - Building reference frame'
reference_frame = self._build_reference_frame(
shape_model.mean)
# add shape model to the list
shape_models.append(shape_model)
print ' - Computing transforms'
transforms = [self.transform(reference_frame.landmarks['source'].lms,
i.landmarks[group][label].lms)
for i in images]
print ' - Warping images'
images = [i.warp_to(reference_frame.mask, t,
interpolator=self.interpolator)
for i, t in zip(images, transforms)]
for i in images:
i.landmarks['source'] = reference_frame.landmarks['source']
self._mask_image(i)
print ' - Building appearance model'
appearance_model = PCAModel(images)
# trim appearance model if required
if self.max_appearance_components is not None:
appearance_model.trim_components(
self.max_appearance_components)
# add appearance model to the list
appearance_models.append(appearance_model)
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
return self._build_aam(shape_models, appearance_models)
def __str__(self):
out = "Supervised Descent Method\n" \
" - Non-Parametric '{}' Regressor\n" \
" - {} training images.\n".format(
self._fitters[0].regressor.__name__, self._n_training_images)
# small strings about number of channels, channels string and downscale
down_str = []
for j in range(self.n_levels):
if j == self.n_levels - 1:
down_str.append('(no downscale)')
else:
down_str.append('(downscale by {})'.format(
self.downscale**(self.n_levels - j - 1)))
temp_img = Image(image_data=np.random.rand(40, 40))
if self.pyramid_on_features:
temp = compute_features(temp_img, self.feature_type[0])
n_channels = [temp.n_channels] * self.n_levels
else:
n_channels = []
for j in range(self.n_levels):
temp = compute_features(temp_img, self.feature_type[j])
n_channels.append(temp.n_channels)
# string about features and channels
if self.pyramid_on_features:
if isinstance(self.feature_type[0], str):
feat_str = "- Feature is {} with ".format(
self.feature_type[0])
elif self.feature_type[0] is None:
feat_str = "- No features extracted. "
else:
feat_str = "- Feature is {} with ".format(
self.feature_type[0].__name__)
if n_channels[0] == 1:
ch_str = ["channel"]
else:
ch_str = ["channels"]
else:
feat_str = []
ch_str = []
for j in range(self.n_levels):
if isinstance(self.feature_type[j], str):
feat_str.append("- Feature is {} with ".format(
self.feature_type[j]))
elif self.feature_type[j] is None:
feat_str.append("- No features extracted. ")
else:
feat_str.append("- Feature is {} with ".format(
self.feature_type[j].__name__))
if n_channels[j] == 1:
ch_str.append("channel")
else:
ch_str.append("channels")
if self.n_levels > 1:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}.\n".format(out, self.n_levels,
self.downscale)
if self.pyramid_on_features:
out = "{} - Pyramid was applied on feature space.\n " \
"{}{} {} per image.\n".format(out, feat_str,
n_channels[0], ch_str[0])
else:
out = "{} - Features were extracted at each pyramid " \
"level.\n".format(out)
for i in range(self.n_levels - 1, -1, -1):
out = "{} - Level {} {}: \n {}{} {} per " \
"image.\n".format(
out, self.n_levels - i, down_str[i], feat_str[i],
n_channels[i], ch_str[i])
else:
if self.pyramid_on_features:
feat_str = [feat_str]
out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format(
out, feat_str[0], n_channels[0], ch_str[0])
return out
def train(self, images, group=None, label="all", **kwargs):
r"""
"""
print("- Computing reference shape")
self.reference_shape = self._compute_reference_shape(images, group, label)
print("- Normalizing object size")
self._rescale_reference_shape()
images = [
i.rescale_to_reference_shape(self.reference_shape, group=group, label=label, interpolator=self.interpolator)
for i in images
]
print("- Generating multilevel scale space")
if self.scaled_levels:
# Gaussian pyramid
generator = [i.gaussian_pyramid(n_levels=self.n_levels, downscale=self.downscale) for i in images]
else:
# Smoothing pyramid
generator = [i.smoothing_pyramid(n_levels=self.n_levels, downscale=self.downscale) for i in images]
print("- Generating multilevel feature space")
images = []
for _ in np.arange(self.n_levels):
images.append([compute_features(g.next(), self.feature_type) for g in generator])
images.reverse()
print("- Extracting ground truth shapes")
gt_shapes = [[i.landmarks[group][label].lms for i in img] for img in images]
print("- Building regressors")
regressors = []
# for each level
for j, (level_images, level_gt_shapes) in enumerate(zip(images, gt_shapes)):
print(" - Level {}".format(j))
trainer = self._set_regressor_trainer(j)
if j == 0:
regressor = trainer.train(level_images, level_gt_shapes, **kwargs)
else:
regressor = trainer.train(level_images, level_gt_shapes, level_shapes, **kwargs)
print(" - Generating next level data")
level_shapes = trainer.perturb_shapes(gt_shapes[0])
regressors.append(regressor)
count = 0
total = len(regressors) * len(images[0]) * len(level_shapes[0])
for k, r in enumerate(regressors):
test_images = images[k]
test_gt_shapes = gt_shapes[k]
fittings = []
for (i, gt_s, level_s) in zip(test_images, test_gt_shapes, level_shapes):
fitting_sublist = []
for ls in level_s:
fitting = r.fit(i, ls)
fitting.gt_shape = gt_s
fitting_sublist.append(fitting)
count += 1
fittings.append(fitting_sublist)
print(" - {} % ".format(round(100 * (count + 1) / total)), end="\r")
if self.scaled_levels:
level_shapes = [
[
Scale(self.downscale, n_dims=self.reference_shape.n_dims).apply(f.final_shape)
for f in fitting_sublist
]
for fitting_sublist in fittings
]
else:
level_shapes = [[f.final_shape for f in fitting_sublist] for fitting_sublist in fittings]
mean_error = np.mean(np.array([f.final_error for fitting_sublist in fittings for f in fitting_sublist]))
print(" - Mean error = {}".format(mean_error))
return self._build_supervised_descent_fitter(regressors)
def build(self, images, group=None, label='all', verbose=False):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the AAM.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
aam : :map:`AAM`
The AAM object. Shape and appearance models are stored from lowest
to highest level
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
self.interpolator, verbose=verbose)
# create pyramid
generators = self._create_pyramid(normalized_images, self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type, verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Rescaling feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(g.next())
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Computing feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(compute_features(
g.next(), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = self._build_shape_model(
train_shapes, self.max_shape_components[rj])
reference_frame = self._build_reference_frame(shape_model.mean)
# add shape model to the list
shape_models.append(shape_model)
# compute transforms
if verbose:
print_dynamic('{}Computing transforms'.format(level_str))
transforms = [self.transform(reference_frame.landmarks['source'].lms,
i.landmarks[group][label].lms)
for i in feature_images]
# warp images to reference frame
warped_images = []
for c, (i, t) in enumerate(zip(feature_images, transforms)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(feature_images),
show_bar=False)))
warped_images.append(i.warp_to(reference_frame.mask, t,
interpolator=self.interpolator))
# attach reference_frame to images' source shape
for i in warped_images:
i.landmarks['source'] = reference_frame.landmarks['source']
# build appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(level_str))
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components[rj] is not None:
appearance_model.trim_components(
self.max_appearance_components[rj])
# add appearance model to the list
appearance_models.append(appearance_model)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
n_training_images = len(images)
return self._build_aam(shape_models, appearance_models,
n_training_images)
def __str__(self):
out = "Supervised Descent Method\n" \
" - Non-Parametric '{}' Regressor\n" \
" - {} training images.\n".format(
self._fitters[0].regressor.__name__, self._n_training_images)
# small strings about number of channels, channels string and downscale
down_str = []
for j in range(self.n_levels):
if j == self.n_levels - 1:
down_str.append('(no downscale)')
else:
down_str.append('(downscale by {})'.format(
self.downscale**(self.n_levels - j - 1)))
temp_img = Image(image_data=np.random.rand(40, 40))
if self.pyramid_on_features:
temp = compute_features(temp_img, self.feature_type[0])
n_channels = [temp.n_channels] * self.n_levels
else:
n_channels = []
for j in range(self.n_levels):
temp = compute_features(temp_img, self.feature_type[j])
n_channels.append(temp.n_channels)
# string about features and channels
if self.pyramid_on_features:
if isinstance(self.feature_type[0], str):
feat_str = "- Feature is {} with ".format(self.feature_type[0])
elif self.feature_type[0] is None:
feat_str = "- No features extracted. "
else:
feat_str = "- Feature is {} with ".format(
self.feature_type[0].__name__)
if n_channels[0] == 1:
ch_str = ["channel"]
else:
ch_str = ["channels"]
else:
feat_str = []
ch_str = []
for j in range(self.n_levels):
if isinstance(self.feature_type[j], str):
feat_str.append("- Feature is {} with ".format(
self.feature_type[j]))
elif self.feature_type[j] is None:
feat_str.append("- No features extracted. ")
else:
feat_str.append("- Feature is {} with ".format(
self.feature_type[j].__name__))
if n_channels[j] == 1:
ch_str.append("channel")
else:
ch_str.append("channels")
if self.n_levels > 1:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}.\n".format(out, self.n_levels,
self.downscale)
if self.pyramid_on_features:
out = "{} - Pyramid was applied on feature space.\n " \
"{}{} {} per image.\n".format(out, feat_str,
n_channels[0], ch_str[0])
else:
out = "{} - Features were extracted at each pyramid " \
"level.\n".format(out)
for i in range(self.n_levels - 1, -1, -1):
out = "{} - Level {} {}: \n {}{} {} per " \
"image.\n".format(
out, self.n_levels - i, down_str[i], feat_str[i],
n_channels[i], ch_str[i])
else:
if self.pyramid_on_features:
feat_str = [feat_str]
out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n".format(
out, feat_str[0], n_channels[0], ch_str[0])
return out
def build(self, images, group=None, label='all'):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
Default: None
label: string, Optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
Default: 'all'
Returns
-------
aam : :class:`menpo.fitmultiple.clm.builder.CLM`
The CLM object
"""
print('- Preprocessing')
self.reference_shape, generator = self._preprocessing(
images, group, label, self.diagonal_range, self.interpolator,
self.scaled_levels, self.n_levels, self.downscale)
print('- Building model pyramids')
shape_models = []
classifiers = []
# for each level
for j in np.arange(self.n_levels):
print(' - Level {}'.format(j))
print(' - Computing feature space')
images = [compute_features(g.next(), self.feature_type)
for g in generator]
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in images]
if j == 0 or self.scaled_levels:
print(' - Building shape model')
shape_model = self._build_shape_model(
shapes, self.max_shape_components)
# add shape model to the list
shape_models.append(shape_model)
print(' - Building classifiers')
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
print(' - {} % '.format(round(100*(k+1)/n_points)), end='\r')
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0],
positive[:, 1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type)
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
return CLM(shape_models, classifiers, self.patch_shape,
self.feature_type, self.reference_shape, self.downscale,
self.scaled_levels, self.interpolator)
def __str__(self):
out = "{0} Fitter\n" \
" - Gradient-Descent {1}\n" \
" - Transform is {2}.\n" \
" - {3} training images.\n".format(
self.clm._str_title, self._fitters[0].algorithm,
self._fitters[0].transform.__class__.__name__,
self.clm.n_training_images)
# small strings about number of channels, channels string and downscale
down_str = []
for j in range(self.n_levels):
if j == self.n_levels - 1:
down_str.append('(no downscale)')
else:
down_str.append('(downscale by {})'.format(
self.downscale**(self.n_levels - j - 1)))
temp_img = Image(image_data=np.random.rand(50, 50))
if self.pyramid_on_features:
temp = compute_features(temp_img, self.feature_type[0])
n_channels = [temp.n_channels] * self.n_levels
else:
n_channels = []
for j in range(self.n_levels):
temp = compute_features(temp_img, self.feature_type[j])
n_channels.append(temp.n_channels)
# string about features and channels
if self.pyramid_on_features:
if isinstance(self.feature_type[0], str):
feat_str = "- Feature is {} with ".format(self.feature_type[0])
elif self.feature_type[0] is None:
feat_str = "- No features extracted. "
else:
feat_str = "- Feature is {} with ".format(
self.feature_type[0].__name__)
if n_channels[0] == 1:
ch_str = ["channel"]
else:
ch_str = ["channels"]
else:
feat_str = []
ch_str = []
for j in range(self.n_levels):
if isinstance(self.feature_type[j], str):
feat_str.append("- Feature is {} with ".format(
self.feature_type[j]))
elif self.feature_type[j] is None:
feat_str.append("- No features extracted. ")
else:
feat_str.append("- Feature is {} with ".format(
self.feature_type[j].__name__))
if n_channels[j] == 1:
ch_str.append("channel")
else:
ch_str.append("channels")
if self.n_levels > 1:
if self.clm.scaled_shape_models:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}.\n - Each level has a scaled shape " \
"model (reference frame).\n - Patch size is {}W x " \
"{}H.\n".format(out, self.n_levels, self.downscale,
self.clm.patch_shape[1],
self.clm.patch_shape[0])
else:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}:\n - Shape models (reference frames) " \
"are not scaled.\n - Patch size is {}W x " \
"{}H.\n".format(out, self.n_levels, self.downscale,
self.clm.patch_shape[1],
self.clm.patch_shape[0])
if self.pyramid_on_features:
out = "{} - Pyramid was applied on feature space.\n " \
"{}{} {} per image.\n".format(out, feat_str,
n_channels[0], ch_str[0])
else:
out = "{} - Features were extracted at each pyramid " \
"level.\n".format(out)
for i in range(self.n_levels - 1, -1, -1):
out = "{} - Level {} {}: \n".format(out, self.n_levels - i,
down_str[i])
if not self.pyramid_on_features:
out = "{} {}{} {} per image.\n".format(
out, feat_str[i], n_channels[i], ch_str[i])
out = "{0} - {1} motion components\n - {2} {3} " \
"classifiers.\n".format(
out, self._fitters[i].transform.n_parameters,
len(self._fitters[i].classifiers),
self._fitters[i].classifiers[0].__name__)
else:
if self.pyramid_on_features:
feat_str = [feat_str]
out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \
" - {4} motion components\n - {5} {6} " \
"classifiers.".format(
out, feat_str[0], n_channels[0], ch_str[0],
out, self._fitters[0].transform.n_parameters,
len(self._fitters[0].classifiers),
self._fitters[0].classifiers[0].__name__)
return out
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the AAM.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
aam : :map:`AAM`
The AAM object. Shape and appearance models are stored from lowest
to highest level
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
self.interpolator, verbose=verbose)
# create pyramid
generators = self._create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
appearance_models = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters in form of list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Rescaling feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Computing feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(
compute_features(next(g), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = self._build_shape_model(
train_shapes, self.max_shape_components[rj])
reference_frame = self._build_reference_frame(shape_model.mean)
# add shape model to the list
shape_models.append(shape_model)
# compute transforms
if verbose:
print_dynamic('{}Computing transforms'.format(level_str))
transforms = [
self.transform(reference_frame.landmarks['source'].lms,
i.landmarks[group][label])
for i in feature_images
]
# warp images to reference frame
warped_images = []
for c, (i, t) in enumerate(zip(feature_images, transforms)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(feature_images),
show_bar=False)))
warped_images.append(
i.warp_to(reference_frame.mask,
t,
interpolator=self.interpolator))
# attach reference_frame to images' source shape
for i in warped_images:
i.landmarks['source'] = reference_frame.landmarks['source']
# build appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(level_str))
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components[rj] is not None:
appearance_model.trim_components(
self.max_appearance_components[rj])
# add appearance model to the list
appearance_models.append(appearance_model)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
n_training_images = len(images)
return self._build_aam(shape_models, appearance_models,
n_training_images)
def build(self, images, group=None, label=None, verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = \
self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal,
self.interpolator, verbose=verbose)
# create pyramid
generators = self._create_pyramid(normalized_images,
self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type,
verbose=verbose)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic('- Building model for each of the {} pyramid '
'levels\n'.format(self.n_levels))
else:
print_dynamic('- Building model\n')
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = ' - '
if self.n_levels > 1:
level_str = ' - Level {}: '.format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Rescaling feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(next(g))
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic('{}Computing feature space - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
feature_images.append(
compute_features(next(g), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label] for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic('{}Building shape model'.format(level_str))
shape_model = self._build_shape_model(
train_shapes, self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic('{}Building classifiers - {}'.format(
level_str,
progress_bar_str((k + 1.) / n_points, show_bar=False)))
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(
patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0], positive[:,
1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples,
(-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples,
(-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type[rj])
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic('{}Done\n'.format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
return CLM(shape_models, classifiers, n_training_images,
self.patch_shape, self.feature_type, self.reference_shape,
self.downscale, self.scaled_shape_models,
self.pyramid_on_features, self.interpolator)
def _set_up(self):
# work out feature length per patch
patch_img = Image.blank(self.patch_shape, fill=0)
self._feature_patch_length = compute_features(
patch_img, self.regression_features).n_parameters
def response_image(self, image, group=None, label=None, level=-1):
r"""
Generates a response image result of applying the classifiers of a
particular pyramidal level of the CLM to an image.
Parameters
-----------
image: :map:`Image`
The image.
group : `string`, optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
level: `int`, optional
The pyramidal level to be used.
Returns
-------
image : :map:`Image`
The response image.
"""
# rescale image
image = image.rescale_to_reference_shape(self.reference_shape,
group=group,
label=label)
# apply pyramid
if self.n_levels > 1:
if self.pyramid_on_features:
# compute features at highest level
feature_image = compute_features(image, self.feature_type[0])
# apply pyramid on feature image
pyramid = feature_image.gaussian_pyramid(
n_levels=self.n_levels, downscale=self.downscale)
# get rescaled feature images
images = list(pyramid)
else:
# create pyramid on intensities image
pyramid = image.gaussian_pyramid(n_levels=self.n_levels,
downscale=self.downscale)
# compute features at each level
images = [
compute_features(i,
self.feature_type[self.n_levels - j - 1])
for j, i in enumerate(pyramid)
]
images.reverse()
else:
images = [compute_features(image, self.feature_type[0])]
# initialize responses
image = images[level]
image_pixels = np.reshape(image.pixels, (-1, image.n_channels))
response_data = np.zeros((image.shape[0], image.shape[1],
self.n_classifiers_per_level[level]))
# Compute responses
for j, clf in enumerate(self.classifiers[level]):
response_data[:, :, j] = np.reshape(clf(image_pixels), image.shape)
return Image(image_data=response_data)
def build(self, images, group=None, label="all", verbose=False):
r"""
Builds a Multilevel Constrained Local Model from a list of
landmarked images.
Parameters
----------
images : list of :map:`Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`, optional
The label of of the landmark manager that you wish to use. If
``None``, the convex hull of all landmarks is used.
verbose : `boolean`, optional
Flag that controls information and progress printing.
Returns
-------
clm : :map:`CLM`
The CLM object
"""
# compute reference_shape and normalize images size
self.reference_shape, normalized_images = self._normalization_wrt_reference_shape(
images, group, label, self.normalization_diagonal, self.interpolator, verbose=verbose
)
# create pyramid
generators = self._create_pyramid(
normalized_images,
self.n_levels,
self.downscale,
self.pyramid_on_features,
self.feature_type,
verbose=verbose,
)
# build the model at each pyramid level
if verbose:
if self.n_levels > 1:
print_dynamic("- Building model for each of the {} pyramid " "levels\n".format(self.n_levels))
else:
print_dynamic("- Building model\n")
shape_models = []
classifiers = []
# for each pyramid level (high --> low)
for j in range(self.n_levels):
# since models are built from highest to lowest level, the
# parameters of type list need to use a reversed index
rj = self.n_levels - j - 1
if verbose:
level_str = " - "
if self.n_levels > 1:
level_str = " - Level {}: ".format(j + 1)
# get images of current level
feature_images = []
if self.pyramid_on_features:
# features are already computed, so just call generator
for c, g in enumerate(generators):
if verbose:
print_dynamic(
"{}Rescaling feature space - {}".format(
level_str, progress_bar_str((c + 1.0) / len(generators), show_bar=False)
)
)
feature_images.append(g.next())
else:
# extract features of images returned from generator
for c, g in enumerate(generators):
if verbose:
print_dynamic(
"{}Computing feature space - {}".format(
level_str, progress_bar_str((c + 1.0) / len(generators), show_bar=False)
)
)
feature_images.append(compute_features(g.next(), self.feature_type[rj]))
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in feature_images]
# define shapes that will be used for training
if j == 0:
original_shapes = shapes
train_shapes = shapes
else:
if self.scaled_shape_models:
train_shapes = shapes
else:
train_shapes = original_shapes
# train shape model and find reference frame
if verbose:
print_dynamic("{}Building shape model".format(level_str))
shape_model = self._build_shape_model(train_shapes, self.max_shape_components[rj])
# add shape model to the list
shape_models.append(shape_model)
# build classifiers
sampling_grid = build_sampling_grid(self.patch_shape)
n_points = shapes[0].n_points
level_classifiers = []
for k in range(n_points):
if verbose:
print_dynamic(
"{}Building classifiers - {}".format(
level_str, progress_bar_str((k + 1.0) / n_points, show_bar=False)
)
)
positive_labels = []
negative_labels = []
positive_samples = []
negative_samples = []
for i, s in zip(feature_images, shapes):
max_x = i.shape[0] - 1
max_y = i.shape[1] - 1
point = (np.round(s.points[k, :])).astype(int)
patch_grid = sampling_grid + point[None, None, ...]
positive, negative = get_pos_neg_grid_positions(patch_grid, positive_grid_size=(1, 1))
x = positive[:, 0]
y = positive[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
positive_sample = i.pixels[positive[:, 0], positive[:, 1], :]
positive_samples.append(positive_sample)
positive_labels.append(np.ones(positive_sample.shape[0]))
x = negative[:, 0]
y = negative[:, 1]
x[x > max_x] = max_x
y[y > max_y] = max_y
x[x < 0] = 0
y[y < 0] = 0
negative_sample = i.pixels[x, y, :]
negative_samples.append(negative_sample)
negative_labels.append(-np.ones(negative_sample.shape[0]))
positive_samples = np.asanyarray(positive_samples)
positive_samples = np.reshape(positive_samples, (-1, positive_samples.shape[-1]))
positive_labels = np.asanyarray(positive_labels).flatten()
negative_samples = np.asanyarray(negative_samples)
negative_samples = np.reshape(negative_samples, (-1, negative_samples.shape[-1]))
negative_labels = np.asanyarray(negative_labels).flatten()
X = np.vstack((positive_samples, negative_samples))
t = np.hstack((positive_labels, negative_labels))
clf = classifier(X, t, self.classifier_type[rj])
level_classifiers.append(clf)
# add level classifiers to the list
classifiers.append(level_classifiers)
if verbose:
print_dynamic("{}Done\n".format(level_str))
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
classifiers.reverse()
n_training_images = len(images)
return CLM(
shape_models,
classifiers,
n_training_images,
self.patch_shape,
self.feature_type,
self.reference_shape,
self.downscale,
self.scaled_shape_models,
self.pyramid_on_features,
self.interpolator,
)
def __str__(self):
out = "{0} Fitter\n" \
" - Gradient-Descent {1}\n" \
" - Transform is {2}.\n" \
" - {3} training images.\n".format(
self.clm._str_title, self._fitters[0].algorithm,
self._fitters[0].transform.__class__.__name__,
self.clm.n_training_images)
# small strings about number of channels, channels string and downscale
down_str = []
for j in range(self.n_levels):
if j == self.n_levels - 1:
down_str.append('(no downscale)')
else:
down_str.append('(downscale by {})'.format(
self.downscale**(self.n_levels - j - 1)))
temp_img = Image(image_data=np.random.rand(50, 50))
if self.pyramid_on_features:
temp = compute_features(temp_img, self.feature_type[0])
n_channels = [temp.n_channels] * self.n_levels
else:
n_channels = []
for j in range(self.n_levels):
temp = compute_features(temp_img, self.feature_type[j])
n_channels.append(temp.n_channels)
# string about features and channels
if self.pyramid_on_features:
if isinstance(self.feature_type[0], str):
feat_str = "- Feature is {} with ".format(
self.feature_type[0])
elif self.feature_type[0] is None:
feat_str = "- No features extracted. "
else:
feat_str = "- Feature is {} with ".format(
self.feature_type[0].__name__)
if n_channels[0] == 1:
ch_str = ["channel"]
else:
ch_str = ["channels"]
else:
feat_str = []
ch_str = []
for j in range(self.n_levels):
if isinstance(self.feature_type[j], str):
feat_str.append("- Feature is {} with ".format(
self.feature_type[j]))
elif self.feature_type[j] is None:
feat_str.append("- No features extracted. ")
else:
feat_str.append("- Feature is {} with ".format(
self.feature_type[j].__name__))
if n_channels[j] == 1:
ch_str.append("channel")
else:
ch_str.append("channels")
if self.n_levels > 1:
if self.clm.scaled_shape_models:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}.\n - Each level has a scaled shape " \
"model (reference frame).\n - Patch size is {}W x " \
"{}H.\n".format(out, self.n_levels, self.downscale,
self.clm.patch_shape[1],
self.clm.patch_shape[0])
else:
out = "{} - Gaussian pyramid with {} levels and downscale " \
"factor of {}:\n - Shape models (reference frames) " \
"are not scaled.\n - Patch size is {}W x " \
"{}H.\n".format(out, self.n_levels, self.downscale,
self.clm.patch_shape[1],
self.clm.patch_shape[0])
if self.pyramid_on_features:
out = "{} - Pyramid was applied on feature space.\n " \
"{}{} {} per image.\n".format(out, feat_str,
n_channels[0], ch_str[0])
else:
out = "{} - Features were extracted at each pyramid " \
"level.\n".format(out)
for i in range(self.n_levels - 1, -1, -1):
out = "{} - Level {} {}: \n".format(out, self.n_levels - i,
down_str[i])
if not self.pyramid_on_features:
out = "{} {}{} {} per image.\n".format(
out, feat_str[i], n_channels[i], ch_str[i])
out = "{0} - {1} motion components\n - {2} {3} " \
"classifiers.\n".format(
out, self._fitters[i].transform.n_parameters,
len(self._fitters[i].classifiers),
self._fitters[i].classifiers[0].__name__)
else:
if self.pyramid_on_features:
feat_str = [feat_str]
out = "{0} - No pyramid used:\n {1}{2} {3} per image.\n" \
" - {4} motion components\n - {5} {6} " \
"classifiers.".format(
out, feat_str[0], n_channels[0], ch_str[0],
out, self._fitters[0].transform.n_parameters,
len(self._fitters[0].classifiers),
self._fitters[0].classifiers[0].__name__)
return out
def __str__(self):
out = "{}\n - {} training images.\n".format(self._str_title, self.n_training_images)
# small strings about number of channels, channels string and downscale
down_str = []
for j in range(self.n_levels):
if j == self.n_levels - 1:
down_str.append("(no downscale)")
else:
down_str.append("(downscale by {})".format(self.downscale ** (self.n_levels - j - 1)))
temp_img = Image(image_data=np.random.rand(50, 50))
if self.pyramid_on_features:
temp = compute_features(temp_img, self.feature_type[0])
n_channels = [temp.n_channels] * self.n_levels
else:
n_channels = []
for j in range(self.n_levels):
temp = compute_features(temp_img, self.feature_type[j])
n_channels.append(temp.n_channels)
# string about features and channels
if self.pyramid_on_features:
if isinstance(self.feature_type[0], str):
feat_str = "- Feature is {} with ".format(self.feature_type[0])
elif self.feature_type[0] is None:
feat_str = "- No features extracted. "
else:
feat_str = "- Feature is {} with ".format(self.feature_type[0].func_name)
if n_channels[0] == 1:
ch_str = ["channel"]
else:
ch_str = ["channels"]
else:
feat_str = []
ch_str = []
for j in range(self.n_levels):
if isinstance(self.feature_type[j], str):
feat_str.append("- Feature is {} with ".format(self.feature_type[j]))
elif self.feature_type[j] is None:
feat_str.append("- No features extracted. ")
else:
feat_str.append("- Feature is {} with ".format(self.feature_type[j].func_name))
if n_channels[j] == 1:
ch_str.append("channel")
else:
ch_str.append("channels")
if self.n_levels > 1:
if self.scaled_shape_models:
out = (
"{} - Gaussian pyramid with {} levels and downscale "
"factor of {}.\n - Each level has a scaled shape "
"model (reference frame).\n - Patch size is {}W x "
"{}H.\n".format(out, self.n_levels, self.downscale, self.patch_shape[1], self.patch_shape[0])
)
else:
out = (
"{} - Gaussian pyramid with {} levels and downscale "
"factor of {}:\n - Shape models (reference frames) "
"are not scaled.\n - Patch size is {}W x "
"{}H.\n".format(out, self.n_levels, self.downscale, self.patch_shape[1], self.patch_shape[0])
)
if self.pyramid_on_features:
out = "{} - Pyramid was applied on feature space.\n " "{}{} {} per image.\n".format(
out, feat_str, n_channels[0], ch_str[0]
)
else:
out = "{} - Features were extracted at each pyramid " "level.\n".format(out)
for i in range(self.n_levels - 1, -1, -1):
out = "{} - Level {} {}: \n".format(out, self.n_levels - i, down_str[i])
if not self.pyramid_on_features:
out = "{} {}{} {} per image.\n".format(out, feat_str[i], n_channels[i], ch_str[i])
out = "{0} - {1} shape components ({2:.2f}% of " "variance)\n - {3} {4} classifiers.\n".format(
out,
self.shape_models[i].n_components,
self.shape_models[i].variance_ratio * 100,
self.n_classifiers_per_level[i],
self.classifiers[i][0].func_name,
)
else:
if self.pyramid_on_features:
feat_str = [feat_str]
out = (
"{0} - No pyramid used:\n {1}{2} {3} per image.\n"
" - {4} shape components ({5:.2f}% of "
"variance)\n - {6} {7} classifiers.".format(
out,
feat_str[0],
n_channels[0],
ch_str[0],
self.shape_models[0].n_components,
self.shape_models[0].variance_ratio * 100,
self.n_classifiers_per_level[0],
self.classifiers[0][0].func_name,
)
)
return out
def features(self, image, shape):
patches = extract_local_patches(image, shape, self.sampling_grid)
features = [compute_features(p, self.regression_features).pixels.ravel()
for p in patches]
return np.hstack((np.asarray(features).ravel(), 1))
def train(self, images, group=None, label='all', **kwargs):
r"""
"""
print('- Computing reference shape')
self.reference_shape = self._compute_reference_shape(
images, group, label)
print('- Normalizing object size')
self._rescale_reference_shape()
images = [
i.rescale_to_reference_shape(self.reference_shape,
group=group,
label=label,
interpolator=self.interpolator)
for i in images
]
print('- Generating multilevel scale space')
if self.scaled_levels:
# Gaussian pyramid
generator = [
i.gaussian_pyramid(n_levels=self.n_levels,
downscale=self.downscale) for i in images
]
else:
# Smoothing pyramid
generator = [
i.smoothing_pyramid(n_levels=self.n_levels,
downscale=self.downscale) for i in images
]
print('- Generating multilevel feature space')
images = []
for _ in np.arange(self.n_levels):
images.append([
compute_features(g.next(), self.feature_type)
for g in generator
])
images.reverse()
print('- Extracting ground truth shapes')
gt_shapes = [[i.landmarks[group][label].lms for i in img]
for img in images]
print('- Building regressors')
regressors = []
# for each level
for j, (level_images,
level_gt_shapes) in enumerate(zip(images, gt_shapes)):
print(' - Level {}'.format(j))
trainer = self._set_regressor_trainer(j)
if j == 0:
regressor = trainer.train(level_images, level_gt_shapes,
**kwargs)
else:
regressor = trainer.train(level_images, level_gt_shapes,
level_shapes, **kwargs)
print(' - Generating next level data')
level_shapes = trainer.perturb_shapes(gt_shapes[0])
regressors.append(regressor)
count = 0
total = len(regressors) * len(images[0]) * len(level_shapes[0])
for k, r in enumerate(regressors):
test_images = images[k]
test_gt_shapes = gt_shapes[k]
fittings = []
for (i, gt_s, level_s) in zip(test_images, test_gt_shapes,
level_shapes):
fitting_sublist = []
for ls in level_s:
fitting = r.fit(i, ls)
fitting.gt_shape = gt_s
fitting_sublist.append(fitting)
count += 1
fittings.append(fitting_sublist)
print(' - {} % '.format(round(100 * (count + 1) / total)),
end='\r')
if self.scaled_levels:
level_shapes = [[
Scale(self.downscale,
n_dims=self.reference_shape.n_dims).apply(
f.final_shape) for f in fitting_sublist
] for fitting_sublist in fittings]
else:
level_shapes = [[f.final_shape for f in fitting_sublist]
for fitting_sublist in fittings]
mean_error = np.mean(
np.array([
f.final_error for fitting_sublist in fittings
for f in fitting_sublist
]))
print(' - Mean error = {}'.format(mean_error))
return self._build_supervised_descent_fitter(regressors)
def build(self, images, group=None, label='all'):
r"""
Builds a Multilevel Active Appearance Model from a list of
landmarked images.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
group : string, Optional
The key of the landmark set that should be used. If None,
and if there is only one set of landmarks, this set will be used.
Default: None
label: string, Optional
The label of of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
Default: 'all'
Returns
-------
aam : :class:`menpo.fitmultiple.aam.builder.AAM`
The AAM object
"""
print '- Preprocessing'
self.reference_shape, generator = self._preprocessing(
images, group, label, self.diagonal_range, self.interpolator,
self.scaled_levels, self.n_levels, self.downscale)
print '- Building model pyramids'
shape_models = []
appearance_models = []
# for each level
for j in np.arange(self.n_levels):
print ' - Level {}'.format(j)
print ' - Computing feature space'
images = [
compute_features(g.next(), self.feature_type)
for g in generator
]
# extract potentially rescaled shapes
shapes = [i.landmarks[group][label].lms for i in images]
if j == 0 or self.scaled_levels:
print ' - Building shape model'
if j != 0:
shapes = [
Scale(1 / self.downscale,
n_dims=shapes[0].n_dims).apply(s) for s in shapes
]
shape_model = self._build_shape_model(
shapes, self.max_shape_components)
print ' - Building reference frame'
reference_frame = self._build_reference_frame(shape_model.mean)
# add shape model to the list
shape_models.append(shape_model)
print ' - Computing transforms'
transforms = [
self.transform(reference_frame.landmarks['source'].lms,
i.landmarks[group][label].lms) for i in images
]
print ' - Warping images'
images = [
i.warp_to(reference_frame.mask,
t,
interpolator=self.interpolator)
for i, t in zip(images, transforms)
]
for i in images:
i.landmarks['source'] = reference_frame.landmarks['source']
self._mask_image(i)
print ' - Building appearance model'
appearance_model = PCAModel(images)
# trim appearance model if required
if self.max_appearance_components is not None:
appearance_model.trim_components(
self.max_appearance_components)
# add appearance model to the list
appearance_models.append(appearance_model)
# reverse the list of shape and appearance models so that they are
# ordered from lower to higher resolution
shape_models.reverse()
appearance_models.reverse()
return self._build_aam(shape_models, appearance_models)