def print_parametric_info(model,
gt_shapes,
n_perturbations,
delta_x,
estimated_delta_x,
level_index,
compute_error_f,
prefix=''):
print_dynamic('{}(Iteration {}) - Calculating errors'.format(
prefix, level_index))
errors = []
for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)):
model._from_vector_inplace(dx)
s1 = model.target
model._from_vector_inplace(edx)
s2 = model.target
gt_s = gt_shapes[np.floor_divide(j, n_perturbations)]
errors.append(compute_error_f(s1, s2, gt_s))
mean = np.mean(errors)
std = np.std(errors)
median = np.median(errors)
print_dynamic('{}(Iteration {}) - Training error -> '
'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'.format(
prefix, level_index, mean, std, median))
def _import_glob_generator(
pattern,
extension_map,
max_assets=None,
has_landmarks=False,
landmark_resolver=None,
importer_kwargs=None,
verbose=False,
):
filepaths = list(glob_with_suffix(pattern, extension_map))
if max_assets:
filepaths = filepaths[:max_assets]
n_files = len(filepaths)
if n_files == 0:
raise ValueError("The glob {} yields no assets".format(pattern))
for i, asset in enumerate(
_multi_import_generator(
filepaths,
extension_map,
has_landmarks=has_landmarks,
landmark_resolver=landmark_resolver,
importer_kwargs=importer_kwargs,
)
):
if verbose:
print_dynamic(
"- Loading {} assets: {}".format(n_files, progress_bar_str(float(i + 1) / n_files, show_bar=True))
)
yield asset
def _build_appearance_model_full_yorgos(all_patches, n_appearance_parameters,
patches_len, level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution'.format(level_str))
# get mean appearance vector
n_images = len(all_patches)
tmp = np.empty((patches_len, n_images))
for c, i in enumerate(all_patches):
tmp[..., c] = vectorize_patches_image(i)
app_mean = np.mean(tmp, axis=1)
# apply pca
appearance_model = PCAModel(all_patches)
# trim components
if n_appearance_parameters is not None:
appearance_model.trim_components(n_appearance_parameters)
# compute covariance matrix
app_cov = np.eye(
appearance_model.n_features,
appearance_model.n_features) - appearance_model.components.T.dot(
appearance_model.components)
return app_mean, app_cov
def compute_reference_shape(shapes, diagonal, verbose=False):
r"""
Function that computes the reference shape as the mean shape of the provided
shapes.
Parameters
----------
shapes : `list` of `menpo.shape.PointCloud`
The set of shapes from which to build the reference shape.
diagonal : `int` or ``None``
If `int`, it ensures that the mean shape is scaled so that the diagonal
of the bounding box containing it matches the provided value.
If ``None``, then the mean shape is not rescaled.
verbose : `bool`, optional
If ``True``, then progress information is printed.
Returns
-------
reference_shape : `menpo.shape.PointCloud`
The reference shape.
"""
# the reference_shape is the mean shape of the images' landmarks
if verbose:
print_dynamic('- Computing reference shape')
reference_shape = mean_pointcloud(shapes)
# fix the reference_shape's diagonal length if asked
if diagonal:
x, y = reference_shape.range()
scale = diagonal / np.sqrt(x**2 + y**2)
reference_shape = Scale(scale,
reference_shape.n_dims).apply(reference_shape)
return reference_shape
def apply_pyramid_on_images(generators, n_levels, verbose=False):
r"""
Exhausts the pyramid generators verbosely
"""
all_images = []
for j in range(n_levels):
if verbose:
level_str = '- Apply pyramid: '
if n_levels > 1:
level_str = '- Apply pyramid: [Level {} - '.format(j + 1)
level_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
level_images.append(next(g))
all_images.append(level_images)
if verbose:
print_dynamic('- Apply pyramid: Done\n')
return all_images
def download(filename, verbose=False):
r"""
Method that downloads the provided filename from SOURCE_URL and
stores it in the data path, if it doesn't already exist.
Parameters
----------
filename : `str`
The filename to download.
verbose : `bool`, optional
If `True`, then the progress will be printed.
Returns
-------
file_path : `pathlib.PosixPath`
The path where the file was stored.
"""
if verbose:
print_dynamic('Downloading {}'.format(filename))
# Path to store data
data_path = src_dir_path() / 'data'
# Check if data path exists, otherwise create it
if not os.path.isdir(str(data_path)):
os.makedirs(str(data_path))
# Check if file exists
file_path = data_path / filename
if not os.path.isfile(str(file_path)):
# It doesn't exist, so download it
urllib.request.urlretrieve(SOURCE_URL + filename,
filename=str(file_path))
# Return the path where the file is stored
return file_path
def test_model(model, test_images, num_init):
face_detector = menpodetect.load_dlib_frontal_face_detector()
test_gt_shapes = util.get_gt_shapes(test_images)
test_boxes = util.get_bounding_boxes(test_images, test_gt_shapes, face_detector)
initial_errors = []
final_errors = []
initial_shapes = []
final_shapes = []
for k, (im, gt_shape, box) in enumerate(zip(test_images, test_gt_shapes, test_boxes)):
init_shapes, fin_shapes = model.apply(im, ([box], num_init, None))
init_shape = util.get_median_shape(init_shapes)
final_shape = fin_shapes[0]
initial_shapes.append(init_shape)
final_shapes.append(final_shape)
initial_errors.append(compute_error(init_shape, gt_shape))
final_errors.append(compute_error(final_shape, gt_shape))
print_dynamic('{}/{}'.format(k + 1, len(test_images)))
return initial_errors, final_errors, initial_shapes, final_shapes
def apply_pyramid_on_images(generators, n_levels, verbose=False):
r"""
Exhausts the pyramid generators verbosely
"""
all_images = []
for j in range(n_levels):
if verbose:
level_str = '- Apply pyramid: '
if n_levels > 1:
level_str = '- Apply pyramid: [Level {} - '.format(j + 1)
level_images = []
for c, g in enumerate(generators):
if verbose:
print_dynamic(
'{}Computing feature space/rescaling - {}'.format(
level_str,
progress_bar_str((c + 1.) / len(generators),
show_bar=False)))
level_images.append(next(g))
all_images.append(level_images)
if verbose:
print_dynamic('- Apply pyramid: Done\n')
return all_images
def build_shape_model(shapes, max_components=None, prefix='', verbose=False):
r"""
Builds a shape model given a set of shapes.
Parameters
----------
shapes: list of :map:`PointCloud`
The set of shapes from which to build the model.
max_components: None or int or float
Specifies the number of components of the trained shape model.
If int, it specifies the exact number of components to be retained.
If float, it specifies the percentage of variance to be retained.
If None, all the available components are kept (100% of variance).
Returns
-------
shape_model: :class:`menpo.model.pca`
The PCA shape model.
"""
if verbose:
print_dynamic('{}Building shape model'.format(prefix))
# compute aligned shapes
aligned_shapes = align_shapes(shapes)
# build shape model
shape_model = PCAModel(aligned_shapes)
if max_components is not None:
# trim shape model if required
shape_model.trim_components(max_components)
return shape_model
def _get_relative_locations(shapes, graph, level_str, verbose):
r"""
returns numpy.array of size 2 x n_images x n_edges
"""
# convert given shapes to point graphs
if isinstance(graph, Tree):
point_graphs = [PointTree(shape.points, graph.adjacency_array,
graph.root_vertex) for shape in shapes]
else:
point_graphs = [PointDirectedGraph(shape.points, graph.adjacency_array)
for shape in shapes]
# initialize an output numpy array
rel_loc_array = np.empty((2, graph.n_edges, len(point_graphs)))
# get relative locations
for c, pt in enumerate(point_graphs):
# print progress
if verbose:
print_dynamic('{}Computing relative locations from '
'shapes - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(point_graphs),
show_bar=False)))
# get relative locations from this shape
rl = pt.relative_locations()
# store
rel_loc_array[..., c] = rl.T
# rollaxis and return
return np.rollaxis(rel_loc_array, 2, 1)
def build(self, images, shapes, targets, extra):
mean_shape, i_stage = extra
n_landmarks = mean_shape.n_points
targets = targets.copy()
MU = self.MU
forests = []
for i in xrange(n_landmarks):
print_dynamic('Building forest for landmark {}.\n'.format(i))
landmark_targets = targets[:, 2 * i:(2 * i + 1) + 1]
feature_extractor_builder = PixelExtractorBuilder(
n_landmarks=1,
n_pixels=self.n_pixels,
kappa=self.kappa,
adaptive=True,
around_landmark=i)
tree_builder = RegressionTreeBuilder(
MU=MU,
depth=self.tree_depth,
n_test_features=self.n_tree_test_features,
exponential_prior=False)
forest_builder = forest.RegressionForestBuilder(
n_trees=self.n_trees_per_landmark,
tree_builder=tree_builder,
feature_extractor_builder=feature_extractor_builder)
f = forest_builder.build(images, landmark_targets,
(shapes, mean_shape, i_stage))
forests.append(f)
return LocalBinaryFeaturesExtractor(forests, mean_shape)
def _build_appearance_model_full(all_patches, n_appearance_parameters,
patches_image_shape, level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution'.format(level_str))
# get mean appearance vector
n_images = len(all_patches)
tmp = np.empty(patches_image_shape + (n_images, ))
for c, i in enumerate(all_patches):
tmp[..., c] = i.pixels
app_mean = np.mean(tmp, axis=-1)
# apply pca
appearance_model = PCAModel(all_patches)
# trim components
if n_appearance_parameters is not None:
appearance_model.trim_components(n_appearance_parameters)
# compute covariance matrix
app_cov = appearance_model.components.T.dot(
np.diag(1 / appearance_model.eigenvalues)).dot(
appearance_model.components)
return app_mean, app_cov
def _regression_data(self, images, gt_shapes, perturbed_shapes,
verbose=False):
r"""
Method that generates the regression data : features and delta_ps.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images.
gt_shapes : :map:`PointCloud` list
List of the ground truth shapes that correspond to the images.
perturbed_shapes : :map:`PointCloud` list
List of the perturbed shapes in order to regress.
verbose : `boolean`, optional
If ``True``, the progress is printed.
"""
if verbose:
print_dynamic('- Generating regression data')
n_images = len(images)
features = []
delta_ps = []
for j, (i, s, p_shape) in enumerate(zip(images, gt_shapes,
perturbed_shapes)):
if verbose:
print_dynamic('- Generating regression data - {}'.format(
progress_bar_str((j + 1.) / n_images, show_bar=False)))
for ps in p_shape:
features.append(self.features(i, ps))
delta_ps.append(self.delta_ps(s, ps))
return np.asarray(features), np.asarray(delta_ps)
def build(self, images, targets, extra):
shapes, mean_shape, i_stage = extra
n_landmarks = mean_shape.n_points
feature_extractor = self.feature_extractor_builder.build(
images, shapes, targets, (mean_shape, i_stage))
print("Extracting local binary features for each image.\n")
features = [
list(feature_extractor.apply(images[i], shapes[i]))
for i in xrange(len(images))
]
print("Features extracted.\n")
w = np.zeros(shape=(2 * n_landmarks, len(features[0])))
for lmark in xrange(2 * n_landmarks):
print_dynamic(
"Learning linear regression coefficients for landmark coordinate {}/{}.\n"
.format(lmark, 2 * n_landmarks))
linreg = liblinearutil.train(
list(targets[:, lmark]), features,
"-s 12 -p 0 -c {}".format(1 / float(len(features))))
w_list = linreg.get_decfun()[0]
w[lmark][0:len(w_list)] = w_list
return GlobalRegression(feature_extractor, w, mean_shape)
def build(self, images, targets, extra):
shapes, mean_shape, i_stage = extra
self.mean_shape = mean_shape
assert (len(images) == len(shapes))
feature_extractor = self.feature_extractor_builder.build(
images, shapes, targets, (mean_shape, i_stage))
# Extract shape-indexed pixels from images.
pixel_vectors = np.array([
feature_extractor.extract_features(
img, shape,
self.to_mean(shape).pseudoinverse())
for (img, shape) in zip(images, shapes)
])
data = self.precompute(pixel_vectors, feature_extractor, mean_shape)
primitive_regressors = []
for i in range(self.n_primitive_regressors):
print_dynamic("Building primitive regressor {}".format(i))
primitive_regressor = self.primitive_builder.build(
pixel_vectors, targets, data)
# Update targets.
targets -= [
primitive_regressor.apply(pixel_vector, data)
for pixel_vector in pixel_vectors
]
primitive_regressors.append(primitive_regressor)
return InnerCascade(
(feature_extractor, primitive_regressors, mean_shape,
self.post_process(primitive_regressors)))
def compute_reference_shape(shapes, diagonal, verbose=False):
r"""
Function that computes the reference shape as the mean shape of the provided
shapes.
Parameters
----------
shapes : `list` of `menpo.shape.PointCloud`
The set of shapes from which to build the reference shape.
diagonal : `int` or ``None``
If `int`, it ensures that the mean shape is scaled so that the diagonal
of the bounding box containing it matches the provided value.
If ``None``, then the mean shape is not rescaled.
verbose : `bool`, optional
If ``True``, then progress information is printed.
Returns
-------
reference_shape : `menpo.shape.PointCloud`
The reference shape.
"""
# the reference_shape is the mean shape of the images' landmarks
if verbose:
print_dynamic('- Computing reference shape')
reference_shape = mean_pointcloud(shapes)
# fix the reference_shape's diagonal length if asked
if diagonal:
x, y = reference_shape.range()
scale = diagonal / np.sqrt(x ** 2 + y ** 2)
reference_shape = Scale(scale, reference_shape.n_dims).apply(
reference_shape)
return reference_shape
def train_aps(experiments_path, fast, group, training_images_options,
training_options, save_model, verbose):
# update training_images_options
training_images_options['save_path'] = os.path.join(experiments_path,
'Databases')
training_images_options['fast'] = fast
training_images_options['group'] = group
training_images_options['verbose'] = verbose
# parse training options
adj, rv = parse_deformation_graph(training_options['graph_deformation'])
training_options['adjacency_array_deformation'] = adj
training_options['root_vertex_deformation'] = rv
adj, fl = parse_appearance_graph(training_options['graph_appearance'])
training_options['adjacency_array_appearance'] = adj
training_options['gaussian_per_patch'] = fl
training_options['features'] = parse_features(training_options['features'],
fast)
graph_deformation_str = training_options['graph_deformation']
graph_appearance_str = training_options['graph_appearance']
del training_options['graph_deformation']
del training_options['graph_appearance']
# Load training images
training_images = load_database(**training_images_options)
# make model filename
filename = model_filename(training_images_options, training_options, group,
fast, graph_deformation_str, graph_appearance_str)
save_path = os.path.join(experiments_path, 'Models', filename)
# train model
if file_exists(save_path):
if verbose:
print_dynamic('Loading model...')
aps = pickle_load(save_path)
if verbose:
print_dynamic('Model loaded.')
else:
training_options['max_shape_components'] = None
# Train model
if fast:
from antonakoscvpr2015.menpofast.builder import APSBuilder
else:
from antonakoscvpr2015.menpo.builder import APSBuilder
if group is not None:
aps = APSBuilder(**training_options).build(training_images,
group=group.__name__,
verbose=verbose)
else:
aps = APSBuilder(**training_options).build(training_images,
verbose=verbose)
# save model
if save_model:
pickle_dump(aps, save_path)
return aps, filename, training_images
def _build_appearance_model_sparse(all_patches_array, graph, patch_shape,
n_channels, n_appearance_parameters,
level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'edge'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# initialize block sparse covariance matrix
all_cov = lil_matrix(
(graph.n_vertices * patch_len, graph.n_vertices * patch_len))
# compute covariance matrix for each edge
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(graph.adjacency_array[e, :])
v2 = np.max(graph.adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((all_patches_array[v1_from:v1_to, :],
all_patches_array[v2_from:v2_to, :]))
# compute covariance inverse
icov = _covariance_matrix_inverse(np.cov(edge_data),
n_appearance_parameters)
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return app_mean, all_cov.tocsr()
def _train(self,
images,
gt_shapes,
current_shapes,
increment=False,
prefix='',
verbose=False):
if not increment:
# Reset the regressors
self.regressors = []
elif increment and not (hasattr(self, 'regressors')
and self.regressors):
raise ValueError('Algorithm must be trained before it can be '
'incremented.')
n_perturbations = len(current_shapes[0])
template_shape = gt_shapes[0]
# obtain delta_x and gt_x
delta_x, gt_x = self._compute_delta_x(gt_shapes, current_shapes)
# Cascaded Regression loop
for k in range(self.n_iterations):
# generate regression data
features_prefix = '{}(Iteration {}) - '.format(prefix, k)
features = self._compute_training_features(images,
gt_shapes,
current_shapes,
prefix=features_prefix,
verbose=verbose)
if verbose:
print_dynamic(
'{}(Iteration {}) - Performing regression'.format(
prefix, k))
if not increment:
r = self._regressor_cls()
r.train(features, delta_x)
self.regressors.append(r)
else:
self.regressors[k].increment(features, delta_x)
# Estimate delta_points
estimated_delta_x = self.regressors[k].predict(features)
if verbose:
self._print_regression_info(template_shape,
gt_shapes,
n_perturbations,
delta_x,
estimated_delta_x,
k,
prefix=prefix)
self._update_estimates(estimated_delta_x, delta_x, gt_x,
current_shapes)
return current_shapes
def _build_appearance_model_sparse(all_patches_array, graph, patch_shape,
n_channels, n_appearance_parameters,
level_str, verbose):
# build appearance model
if verbose:
print_dynamic('{}Training appearance distribution per '
'edge'.format(level_str))
# compute mean appearance vector
app_mean = np.mean(all_patches_array, axis=1)
# appearance vector and patch vector lengths
patch_len = np.prod(patch_shape) * n_channels
# initialize block sparse covariance matrix
all_cov = lil_matrix((graph.n_vertices * patch_len,
graph.n_vertices * patch_len))
# compute covariance matrix for each edge
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training appearance distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(graph.adjacency_array[e, :])
v2 = np.max(graph.adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((all_patches_array[v1_from:v1_to, :],
all_patches_array[v2_from:v2_to, :]))
# compute covariance inverse
icov = _covariance_matrix_inverse(np.cov(edge_data),
n_appearance_parameters)
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return app_mean, all_cov.tocsr()
def _train(self, images, gt_shapes, current_shapes, increment=False,
prefix='', verbose=False):
if not increment:
# Reset the regressors
self.regressors = []
n_perturbations = len(current_shapes[0])
template_shape = gt_shapes[0]
# obtain delta_x and gt_x (parameters rather than shapes)
delta_x, gt_x = obtain_parametric_delta_x(gt_shapes, current_shapes,
self.transform)
# Cascaded Regression loop
for k in range(self.n_iterations):
# generate regression data
features = self._generate_features(
images, current_shapes,
prefix='{}(Iteration {}) - '.format(prefix, k),
verbose=verbose)
if verbose:
print_dynamic('{}(Iteration {}) - Performing regression'.format(
prefix, k))
if not increment:
r = self._regressor_cls()
r.train(features, delta_x)
self.regressors.append(r)
else:
self.regressors[k].increment(features, delta_x)
# Estimate delta_points
estimated_delta_x = self.regressors[k].predict(features)
if verbose:
self._print_regression_info(template_shape, gt_shapes,
n_perturbations, delta_x,
estimated_delta_x, k,
prefix=prefix)
j = 0
for shapes in current_shapes:
for s in shapes:
# Estimate parameters
edx = estimated_delta_x[j]
# Current parameters
cx = _weights_for_target(self.transform, s) + edx
# Uses less memory to find updated target shape
self.transform.from_vector_inplace(cx)
# Update current shape inplace
s.from_vector_inplace(self.transform.target.as_vector())
delta_x[j] = gt_x[j] - cx
j += 1
return current_shapes
def _build_deformation_model(graph, relative_locations, level_str, verbose):
# build deformation model
if verbose:
print_dynamic('{}Training deformation distribution per '
'graph edge'.format(level_str))
def_len = 2 * graph.n_vertices
def_cov = np.zeros((def_len, def_len))
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training deformation distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# get vertices adjacent to edge
parent = graph.adjacency_array[e, 0]
child = graph.adjacency_array[e, 1]
# compute covariance matrix
edge_cov = np.linalg.inv(np.cov(relative_locations[..., e]))
# store its values
s1 = edge_cov[0, 0]
s2 = edge_cov[1, 1]
s3 = 2 * edge_cov[0, 1]
# Fill the covariance matrix matrix
# get indices
p1 = 2 * parent
p2 = 2 * parent + 1
c1 = 2 * child
c2 = 2 * child + 1
# up-left block
def_cov[p1, p1] += s1
def_cov[p2, p2] += s2
def_cov[p2, p1] += s3
# up-right block
def_cov[p1, c1] = - s1
def_cov[p2, c2] = - s2
def_cov[p1, c2] = - s3 / 2
def_cov[p2, c1] = - s3 / 2
# down-left block
def_cov[c1, p1] = - s1
def_cov[c2, p2] = - s2
def_cov[c1, p2] = - s3 / 2
def_cov[c2, p1] = - s3 / 2
# down-right block
def_cov[c1, c1] += s1
def_cov[c2, c2] += s2
def_cov[c1, c2] += s3
return def_cov
def _build_deformation_model(graph, relative_locations, level_str, verbose):
# build deformation model
if verbose:
print_dynamic('{}Training deformation distribution per '
'graph edge'.format(level_str))
def_len = 2 * graph.n_vertices
def_cov = np.zeros((def_len, def_len))
for e in range(graph.n_edges):
# print progress
if verbose:
print_dynamic('{}Training deformation distribution '
'per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / graph.n_edges,
show_bar=False)))
# get vertices adjacent to edge
parent = graph.adjacency_array[e, 0]
child = graph.adjacency_array[e, 1]
# compute covariance matrix
edge_cov = np.linalg.inv(np.cov(relative_locations[..., e]))
# store its values
s1 = edge_cov[0, 0]
s2 = edge_cov[1, 1]
s3 = 2 * edge_cov[0, 1]
# Fill the covariance matrix matrix
# get indices
p1 = 2 * parent
p2 = 2 * parent + 1
c1 = 2 * child
c2 = 2 * child + 1
# up-left block
def_cov[p1, p1] += s1
def_cov[p2, p2] += s2
def_cov[p2, p1] += s3
# up-right block
def_cov[p1, c1] = -s1
def_cov[p2, c2] = -s2
def_cov[p1, c2] = -s3 / 2
def_cov[p2, c1] = -s3 / 2
# down-left block
def_cov[c1, p1] = -s1
def_cov[c2, p2] = -s2
def_cov[c1, p2] = -s3 / 2
def_cov[c2, p1] = -s3 / 2
# down-right block
def_cov[c1, c1] += s1
def_cov[c2, c2] += s2
def_cov[c1, c2] += s3
return def_cov
def _create_pyramid(cls, images, n_levels, downscale, pyramid_on_features,
feature_type, verbose=False):
r"""
Function that creates a generator function for Gaussian pyramid. The
pyramid can be created either on the feature space or the original
(intensities) space.
Parameters
----------
images: list of :class:`menpo.image.Image`
The set of landmarked images from which to build the AAM.
n_levels: int
The number of multi-resolution pyramidal levels to be used.
downscale: float
The downscale factor that will be used to create the different
pyramidal levels.
pyramid_on_features: boolean
If True, the features are extracted at the highest level and the
pyramid is created on the feature images.
If False, the pyramid is created on the original (intensities)
space.
feature_type: list of size 1 with str or function/closure or None
The feature type to be used in case pyramid_on_features is enabled.
verbose: bool, Optional
Flag that controls information and progress printing.
Default: False
Returns
-------
generator: function
The generator function of the Gaussian pyramid.
"""
if pyramid_on_features:
# compute features at highest level
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Computing feature space: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
feature_images.append(compute_features(i, feature_type[0]))
if verbose:
print_dynamic('- Computing feature space: Done\n')
# create pyramid on feature_images
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in feature_images]
else:
# create pyramid on intensities images
# features will be computed per level
generator = [i.gaussian_pyramid(n_levels=n_levels,
downscale=downscale)
for i in images]
return generator
def post_process(self, ferns):
basis = None
if self.compress:
print("\nPerforming fern compression.\n")
# Create a new basis by randomly sampling from all fern outputs.
basis = self._random_basis(ferns, self.basis_size)
for i, fern in enumerate(ferns):
print_dynamic("Compressing fern {}/{}.".format(i, len(ferns)))
fern.compress(basis, self.compression_maxnonzero)
return basis
def _scale_images(cls, images, s, level_str, verbose):
scaled_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'- Scaling features: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
scaled_images.append(i.rescale(s))
return scaled_images
def _scale_images(cls, images, s, level_str, verbose):
scaled_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'{}Scaling features: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
scaled_images.append(i.rescale(s))
return scaled_images
def post_process(self, ferns):
basis = None
if self.compress:
print("\nPerforming fern compression.\n")
# Create a new basis by randomly sampling from all fern outputs.
basis = self._random_basis(ferns, self.basis_size)
for i, fern in enumerate(ferns):
print_dynamic("Compressing fern {}/{}.".format(i, len(ferns)))
fern.compress(basis, self.compression_maxnonzero)
return basis
def _create_dense_diagonal_precision(
X,
graph,
n_features,
n_features_per_vertex,
dtype=np.float32,
n_components=None,
bias=0,
return_covariances=False,
verbose=False,
):
# Initialize precision
precision = np.zeros((n_features, n_features), dtype=dtype)
if return_covariances:
all_covariances = np.zeros(
(graph.n_vertices, n_features_per_vertex, n_features_per_vertex),
dtype=dtype,
)
if verbose:
print_dynamic("Allocated precision matrix of size {}".format(
bytes_str(precision.nbytes)))
# Print information if asked
if verbose:
vertices = print_progress(
range(graph.n_vertices),
n_items=graph.n_vertices,
prefix="Precision per vertex",
end_with_newline=False,
)
else:
vertices = range(graph.n_vertices)
# Compute covariance matrix for each patch
for v in vertices:
# find indices in target precision matrix
i_from = v * n_features_per_vertex
i_to = (v + 1) * n_features_per_vertex
# compute covariance
covmat = np.cov(X[:, i_from:i_to], rowvar=0, bias=bias)
if return_covariances:
all_covariances[v] = covmat
# invert it
covmat = _covariance_matrix_inverse(covmat, n_components)
# insert to precision matrix
precision[i_from:i_to, i_from:i_to] = covmat
# return covariances
if return_covariances:
return precision, all_covariances
else:
return precision
def _compute_reference_shape(self, images, group, label, verbose):
# the reference_shape is the mean shape of the images' landmarks
if verbose:
print_dynamic('- Computing reference shape')
shapes = [i.landmarks[group][label] for i in images]
ref_shape = mean_pointcloud(shapes)
# fix the reference_shape's diagonal length if specified
if self.diagonal:
x, y = ref_shape.range()
scale = self.diagonal / np.sqrt(x**2 + y**2)
Scale(scale, ref_shape.n_dims).apply_inplace(ref_shape)
return ref_shape
def _compute_reference_shape(self, images, group, label, verbose):
# the reference_shape is the mean shape of the images' landmarks
if verbose:
print_dynamic('- Computing reference shape')
shapes = [i.landmarks[group][label] for i in images]
ref_shape = mean_pointcloud(shapes)
# fix the reference_shape's diagonal length if specified
if self.diagonal:
x, y = ref_shape.range()
scale = self.diagonal / np.sqrt(x**2 + y**2)
ref_shape = Scale(scale, ref_shape.n_dims).apply(ref_shape)
return ref_shape
def _train(self, images, gt_shapes, current_shapes, increment=False,
prefix='', verbose=False):
if not increment:
# Reset the regressors
self.regressors = []
n_perturbations = len(current_shapes[0])
template_shape = gt_shapes[0]
# obtain delta_x and gt_x
delta_x, gt_x = obtain_delta_x(gt_shapes, current_shapes)
# Cascaded Regression loop
for k in range(self.n_iterations):
# generate regression data
features = features_per_image(
images, current_shapes, self.patch_shape, self.patch_features,
prefix='{}(Iteration {}) - '.format(prefix, k),
verbose=verbose)
if verbose:
print_dynamic('{}(Iteration {}) - Performing regression'.format(
prefix, k))
if not increment:
r = self._regressor_cls()
r.train(features, delta_x)
self.regressors.append(r)
else:
self.regressors[k].increment(features, delta_x)
# Estimate delta_points
estimated_delta_x = self.regressors[k].predict(features)
if verbose:
self._print_regression_info(template_shape, gt_shapes,
n_perturbations, delta_x,
estimated_delta_x, k,
prefix=prefix)
j = 0
for shapes in current_shapes:
for s in shapes:
# update current x
current_x = s.as_vector() + estimated_delta_x[j]
# update current shape inplace
s.from_vector_inplace(current_x)
# update delta_x
delta_x[j] = gt_x[j] - current_x
# increase index
j += 1
return current_shapes
def _normalize_images(self, images, group, label, ref_shape, verbose):
# normalize the scaling of all images wrt the reference_shape size
norm_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
i = i.rescale_to_reference_shape(ref_shape, group=group,
label=label)
if self.sigma:
i.pixels = fsmooth(i.pixels, self.sigma)
norm_images.append(i)
return norm_images
def _compute_features(self, images, level_str, verbose):
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'- Computing feature space: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
if self.features:
i = self.features(i)
feature_images.append(i)
return feature_images
def _compute_features(self, images, level_str, verbose):
feature_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic(
'{}Computing feature space: {}'.format(
level_str, progress_bar_str((c + 1.) / len(images),
show_bar=False)))
if self.features:
i = self.features(i)
feature_images.append(i)
return feature_images
def _normalize_images(self, images, group, label, ref_shape, verbose):
# normalize the scaling of all images wrt the reference_shape size
norm_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
i = rescale_to_reference_shape(i, ref_shape, group=group,
label=label)
if self.sigma:
i.pixels = fsmooth(i.pixels, self.sigma)
norm_images.append(i)
return norm_images
def increment_shape_model(shape_model, shapes, forgetting_factor=None,
max_components=None, prefix='', verbose=False):
r"""
"""
if verbose:
print_dynamic('{}Incrementing shape model'.format(prefix))
# compute aligned shapes
aligned_shapes = align_shapes(shapes)
# increment shape model
shape_model.increment(aligned_shapes, forgetting_factor=forgetting_factor)
if max_components is not None:
shape_model.trim_components(max_components)
return shape_model
def _compute_minimum_spanning_tree(shapes, root_vertex, level_str, verbose):
# initialize edges and weights matrix
n_vertices = shapes[0].n_points
n_edges = nchoosek(n_vertices, 2)
weights = np.zeros((n_vertices, n_vertices))
edges = np.empty((n_edges, 2), dtype=np.int32)
# fill edges and weights
e = -1
for i in range(n_vertices - 1):
for j in range(i + 1, n_vertices, 1):
# edge counter
e += 1
# print progress
if verbose:
print_dynamic(
'{}Computing complete graph`s weights - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# fill in edges
edges[e, 0] = i
edges[e, 1] = j
# create data matrix of edge
diffs_x = [s.points[i, 0] - s.points[j, 0] for s in shapes]
diffs_y = [s.points[i, 1] - s.points[j, 1] for s in shapes]
coords = np.array([diffs_x, diffs_y])
# compute mean
m = np.mean(coords, axis=1)
# compute covariance
c = np.cov(coords)
# get weight
for im in range(len(shapes)):
weights[i, j] += -np.log(
multivariate_normal.pdf(coords[:, im], mean=m, cov=c))
weights[j, i] = weights[i, j]
# create undirected graph
complete_graph = UndirectedGraph(edges)
if verbose:
print_dynamic('{}Minimum spanning graph computed.\n'.format(level_str))
# compute minimum spanning graph
return complete_graph.minimum_spanning_tree(weights, root_vertex)
def _compute_minimum_spanning_tree(shapes, root_vertex, level_str, verbose):
# initialize edges and weights matrix
n_vertices = shapes[0].n_points
n_edges = nchoosek(n_vertices, 2)
weights = np.zeros((n_vertices, n_vertices))
edges = np.empty((n_edges, 2), dtype=np.int32)
# fill edges and weights
e = -1
for i in range(n_vertices-1):
for j in range(i+1, n_vertices, 1):
# edge counter
e += 1
# print progress
if verbose:
print_dynamic('{}Computing complete graph`s weights - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# fill in edges
edges[e, 0] = i
edges[e, 1] = j
# create data matrix of edge
diffs_x = [s.points[i, 0] - s.points[j, 0] for s in shapes]
diffs_y = [s.points[i, 1] - s.points[j, 1] for s in shapes]
coords = np.array([diffs_x, diffs_y])
# compute mean
m = np.mean(coords, axis=1)
# compute covariance
c = np.cov(coords)
# get weight
for im in range(len(shapes)):
weights[i, j] += -np.log(multivariate_normal.pdf(coords[:, im],
mean=m, cov=c))
weights[j, i] = weights[i, j]
# create undirected graph
complete_graph = UndirectedGraph(edges)
if verbose:
print_dynamic('{}Minimum spanning graph computed.\n'.format(level_str))
# compute minimum spanning graph
return complete_graph.minimum_spanning_tree(weights, root_vertex)
def _increment_dense_diagonal_precision(
X,
mean_vector,
covariances,
n,
graph,
n_features,
n_features_per_vertex,
dtype=np.float32,
n_components=None,
bias=0,
verbose=False,
):
# Initialize precision
precision = np.zeros((n_features, n_features), dtype=dtype)
# Print information if asked
if verbose:
print_dynamic("Allocated precision matrix of size {}".format(
bytes_str(precision.nbytes)))
vertices = print_progress(
range(graph.n_vertices),
n_items=graph.n_vertices,
prefix="Precision per vertex",
end_with_newline=False,
)
else:
vertices = range(graph.n_vertices)
# Compute covariance matrix for each patch
for v in vertices:
# find indices in target precision matrix
i_from = v * n_features_per_vertex
i_to = (v + 1) * n_features_per_vertex
# get data
edge_data = X[:, i_from:i_to]
m = mean_vector[i_from:i_to]
# increment
_, covariances[v] = _increment_multivariate_gaussian_cov(
edge_data, m, covariances[v], n, bias=bias)
# invert it
precision[i_from:i_to, i_from:i_to] = _covariance_matrix_inverse(
covariances[v], n_components)
# return covariances
return precision, covariances
def _normalization_wrt_reference_shape(cls,
images,
group,
label,
reference_shape,
verbose=False):
r"""
Normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the model.
group : `string`
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
reference_shape : :map:`PointCloud`
The reference shape that is used to resize all training images to
a consistent object size.
verbose: bool, optional
Flag that controls information and progress printing.
Returns
-------
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images), show_bar=False)))
normalized_images.append(
i.rescale_to_reference_shape(reference_shape,
group=group,
label=label))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return normalized_images
def serialize_data(images, labels, filename, verbose=False):
r"""
Method that saves the provided images and labels to tfrecords file using the
tensorflow record writer.
Parameters
----------
images : `list` or `array`
The images to serialize.
labels : `array`
The corresponding labels.
filename : `str`
The filename to use. Note that the data will be saved in the 'data'
folder.
verbose : `bool`, optional
If `True`, then the progress will be printed.
"""
# If images is list, convert it to numpy array
images = convert_images_to_array(images)
# Get number of images, height, width and number of channels
num_examples = labels.shape[0]
if images.shape[0] != num_examples:
raise ValueError(
"Images size %d does not match labels size %d.".format(
images.shape[0], num_examples))
height = images.shape[1]
width = images.shape[2]
n_channels = images.shape[3]
# Save data
filename = str(src_dir_path() / 'data' / (filename + '.tfrecords'))
if verbose:
print_dynamic('Writing {}'.format(filename))
writer = tf.python_io.TFRecordWriter(filename)
for index in range(num_examples):
image_raw = images[index].tostring()
example = tf.train.Example(features=tf.train.Features(
feature={
'height': _int64_feature(height),
'width': _int64_feature(width),
'depth': _int64_feature(n_channels),
'label': _int64_feature(int(labels[index])),
'image_raw': _bytes_feature(image_raw)
}))
writer.write(example.SerializeToString())
writer.close()
if verbose:
print_dynamic('Completed successfully!')
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = build_parts_image(
i, s, self.parts_shape, normalize_parts=self.normalize_parts)
parts_images.append(parts_image)
return parts_images
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = Image(i.extract_patches(
s, patch_size=self.parts_shape, as_single_array=True))
parts_images.append(parts_image)
return parts_images
def build_appearance_model(images, gt_shapes, patch_shape, patch_features,
appearance_model_cls, verbose=False, prefix=''):
wrap = partial(print_progress,
prefix='{}Extracting ground truth patches'.format(prefix),
end_with_newline=not prefix, verbose=verbose)
n_images = len(images)
# Extract patches from ground truth
gt_patches = [features_per_patch(im, gt_s, patch_shape,
patch_features)
for gt_s, im in wrap(zip(gt_shapes, images))]
# Calculate appearance model from extracted gt patches
gt_patches = np.array(gt_patches).reshape([n_images, -1])
if verbose:
print_dynamic('{}Building Appearance Model'.format(prefix))
return appearance_model_cls(gt_patches)
def _train(self, images, gt_shapes, current_shapes, increment=False,
prefix='', verbose=False):
if not increment:
# Reset the regressors
self.regressors = []
elif increment and not (hasattr(self, 'regressors') and self.regressors):
raise ValueError('Algorithm must be trained before it can be '
'incremented.')
n_perturbations = len(current_shapes[0])
template_shape = gt_shapes[0]
# obtain delta_x and gt_x
delta_x, gt_x = self._compute_delta_x(gt_shapes, current_shapes)
# Cascaded Regression loop
for k in range(self.n_iterations):
# generate regression data
features_prefix = '{}(Iteration {}) - '.format(prefix, k)
features = self._compute_training_features(images, gt_shapes,
current_shapes,
prefix=features_prefix,
verbose=verbose)
if verbose:
print_dynamic('{}(Iteration {}) - Performing regression'.format(
prefix, k))
if not increment:
r = self._regressor_cls()
r.train(features, delta_x)
self.regressors.append(r)
else:
self.regressors[k].increment(features, delta_x)
# Estimate delta_points
estimated_delta_x = self.regressors[k].predict(features)
if verbose:
self._print_regression_info(template_shape, gt_shapes,
n_perturbations, delta_x,
estimated_delta_x, k,
prefix=prefix)
self._update_estimates(estimated_delta_x, delta_x, gt_x,
current_shapes)
return current_shapes
def __init__(self,
samples,
centre=True,
bias=False,
verbose=False,
n_samples=None):
# get the first element as the template and use it to configure the
# data matrix
if n_samples is None:
# samples is a list
n_samples = len(samples)
template = samples[0]
samples = samples[1:]
else:
# samples is an iterator
template = next(samples)
n_features = template.n_parameters
template_vector = template.as_vector()
data = np.zeros((n_samples, n_features), dtype=template_vector.dtype)
# now we can fill in the first element from the template
data[0] = template_vector
del template_vector
if verbose:
print('Allocated data matrix {:.2f}'
'GB'.format(data.nbytes / 2**30))
# 1-based as we have the template vector set already
for i, sample in enumerate(samples, 1):
if i >= n_samples:
break
if verbose:
print_dynamic(
'Building data matrix from {} samples - {}'.format(
n_samples,
progress_bar_str(float(i + 1) / n_samples,
show_bar=True)))
data[i] = sample.as_vector()
# compute pca
e_vectors, e_values, mean = principal_component_decomposition(
data, whiten=False, centre=centre, bias=bias, inplace=True)
super(PCAModel, self).__init__(e_vectors, mean, template)
self.centred = centre
self.biased = bias
self._eigenvalues = e_values
# start the active components as all the components
self._n_active_components = int(self.n_components)
self._trimmed_eigenvalues = None
def build(self, images, gt_shapes, boxes):
self.mean_shape = tools.centered_mean_shape(gt_shapes)
self.n_landmarks = self.mean_shape.n_points
# Generate initial shapes with perturbations.
print_dynamic('Generating initial shapes')
shapes = np.array(
[tools.fit_shape_to_box(self.mean_shape, box) for box in boxes])
print_dynamic('Perturbing initial estimates')
if self.n_perturbations > 1:
images, shapes, gt_shapes, boxes = tools.perturb_shapes(
images,
shapes,
gt_shapes,
boxes,
self.n_perturbations,
mode='mean_shape')
assert (len(boxes) == len(images))
assert (len(shapes) == len(images))
assert (len(gt_shapes) == len(images))
print('\nSize of augmented dataset: {} images.\n'.format(len(images)))
weak_regressors = []
for j in range(self.n_stages):
# Calculate normalized targets.
deltas = [
gt_shapes[i].points - shapes[i].points
for i in range(len(images))
]
targets = np.array([
tools.transform_to_mean_shape(
shapes[i], self.mean_shape).apply(deltas[i]).reshape(
(2 * self.n_landmarks, )) for i in range(len(images))
])
weak_regressor = self.weak_builder.build(
images, targets, (shapes, self.mean_shape, j))
# Update current estimates of shapes.
for i in range(len(images)):
offset = weak_regressor.apply(images[i], shapes[i])
shapes[i].points += offset.points
weak_regressors.append(weak_regressor)
print("\nBuilt outer regressor {}\n".format(j))
return CascadedShapeRegressor(self.n_landmarks, weak_regressors,
self.mean_shape)
def _warp_images(self, images, shapes, _, level_str, verbose):
# extract parts
parts_images = []
for c, (i, s) in enumerate(zip(images, shapes)):
if verbose:
print_dynamic('{}Warping images - {}'.format(
level_str,
progress_bar_str(float(c + 1) / len(images),
show_bar=False)))
parts_image = build_parts_image(
i, s, parts_shape=self.parts_shape,
normalize_parts=self.normalize_parts)
parts_images.append(parts_image)
return parts_images
def build(self, images, targets, extra):
shapes, mean_shape, i_stage = extra
n_landmarks = mean_shape.n_points
feature_extractor = self.feature_extractor_builder.build(images, shapes, targets, (mean_shape, i_stage))
print("Extracting local binary features for each image.\n")
features = [ list(feature_extractor.apply(images[i], shapes[i])) for i in xrange(len(images)) ]
print("Features extracted.\n")
w = np.zeros(shape=(2*n_landmarks, len(features[0])))
for lmark in xrange(2*n_landmarks):
print_dynamic("Learning linear regression coefficients for landmark coordinate {}/{}.\n".format(lmark, 2*n_landmarks))
linreg = liblinearutil.train(list(targets[:, lmark]), features, "-s 12 -p 0 -c {}".format(1/float(len(features))))
w_list = linreg.get_decfun()[0]
w[lmark][0:len(w_list)] = w_list
return GlobalRegression(feature_extractor, w, mean_shape)
def print_non_parametric_info(template_shape, gt_shapes, n_perturbations,
delta_x, estimated_delta_x, level_index,
compute_error_f, prefix=''):
print_dynamic('{}(Iteration {}) - Calculating errors'.format(
prefix, level_index))
errors = []
for j, (dx, edx) in enumerate(zip(delta_x, estimated_delta_x)):
s1 = template_shape.from_vector(dx)
s2 = template_shape.from_vector(edx)
gt_s = gt_shapes[np.floor_divide(j, n_perturbations)]
errors.append(compute_error_f(s1, s2, gt_s))
mean = np.mean(errors)
std = np.std(errors)
median = np.median(errors)
print_dynamic('{}(Iteration {}) - Training error -> '
'mean: {:.4f}, std: {:.4f}, median: {:.4f}.\n'.
format(prefix, level_index, mean, std, median))
def get_bounding_boxes(images, gt_shapes, face_detector):
ret = []
for i, (img, gt_shape) in enumerate(zip(images, gt_shapes)):
print_dynamic("Detecting face {}/{}".format(i, len(images)))
boxes = face_detector(img)
# Some images contain multiple faces, but are annotated only once.
# We only remember the box that contains the gt_shape.
found = False
for box in boxes:
if is_point_within(gt_shape.centre(), box.bounds()):
ret.append(box)
found = True
break
if not found:
ret.append(gt_shape.bounding_box())
return np.array(ret)
def compute_sparse_covariance(X, adjacency_array, patch_len, level_str,
verbose):
n_features, n_samples = X.shape
n_edges = adjacency_array.shape[0]
# initialize block sparse covariance matrix
all_cov = np.zeros((n_features, n_features))
# compute covariance matrix for each edge
for e in range(n_edges):
# print progress
if verbose:
print_dynamic('{}Distribution per edge - {}'.format(
level_str,
progress_bar_str(float(e + 1) / n_edges,
show_bar=False)))
# edge vertices
v1 = np.min(adjacency_array[e, :])
v2 = np.max(adjacency_array[e, :])
# find indices in target covariance matrix
v1_from = v1 * patch_len
v1_to = (v1 + 1) * patch_len
v2_from = v2 * patch_len
v2_to = (v2 + 1) * patch_len
# extract data
edge_data = np.concatenate((X[v1_from:v1_to, :], X[v2_from:v2_to, :]))
# compute covariance inverse
icov = np.linalg.inv(np.cov(edge_data))
# v1, v2
all_cov[v1_from:v1_to, v2_from:v2_to] += icov[:patch_len, patch_len::]
# v2, v1
all_cov[v2_from:v2_to, v1_from:v1_to] += icov[patch_len::, :patch_len]
# v1, v1
all_cov[v1_from:v1_to, v1_from:v1_to] += icov[:patch_len, :patch_len]
# v2, v2
all_cov[v2_from:v2_to, v2_from:v2_to] += icov[patch_len::, patch_len::]
return np.linalg.inv(all_cov)
def load_basel_from_mat(recreate_meshes=False,
output_base_path='/vol/atlas/homes/pts08/',
input_base_path='/vol/atlas/pts08/basel/',
max_images=None):
previously_pickled_path = os.path.join(output_base_path,
'basel_python_68.pkl')
mat_file_path = os.path.join(input_base_path, 'basel_68.mat')
if not recreate_meshes and os.path.exists(previously_pickled_path):
with open(previously_pickled_path) as f:
images = cPickle.load(f)
else:
from scipy.io import loadmat
from menpo.image import Image
from menpo.shape import PointCloud
basel_dataset = loadmat(mat_file_path)
textures = basel_dataset['textures']
shape = basel_dataset['shapes']
landmarks = np.swapaxes(basel_dataset['landmarks'], 0, 1)
N = max_images if max_images else landmarks.shape[2]
all_images = []
for i in xrange(N):
# Change to the correct handedness
shape[..., 1:3, i] *= -1
shape_image = ShapeImage(shape[..., i],
texture=Image(textures[..., i]))
shape_image.landmarks['PTS'] = PointCloud(landmarks[..., i])
shape_image.mesh.texture.landmarks['PTS'] = shape_image.landmarks['PTS']
shape_image.constrain_mask_to_landmarks()
shape_image.rebuild_mesh()
all_images.append(shape_image)
print_dynamic('Image {0} of {1}'.format(i + 1, N))
print('\n')
images = [im for im in all_images if im.n_landmark_groups == 1]
print('{0}% of the images had landmarks'.format(
(float(len(images)) / len(all_images)) * 100))
with open(previously_pickled_path, 'wb') as f:
cPickle.dump(images, f, protocol=2)
return images
def load_database(path_to_images,
save_path,
db_name,
crop_percentage,
fast,
group,
verbose=False):
# create filename
if group is not None:
filename = (db_name + '_' + group.__name__ + '_crop' +
str(int(crop_percentage * 100)))
else:
filename = db_name + 'PTS' + '_crop' + str(int(crop_percentage * 100))
if fast:
filename += '_menpofast.pickle'
else:
filename += '_menpo.pickle'
save_path = os.path.join(save_path, filename)
# check if file exists
if file_exists(save_path):
if verbose:
print_dynamic('Loading images...')
images = pickle_load(save_path)
if verbose:
print_dynamic('Images Loaded.')
else:
# load images
images = []
for i in mio.import_images(path_to_images, verbose=verbose):
if fast:
i = convert_from_menpo(i)
i.crop_to_landmarks_proportion_inplace(crop_percentage,
group='PTS')
if group is not None:
labeller(i, 'PTS', group)
if i.n_channels == 3:
i = i.as_greyscale(mode='average')
images.append(i)
# save images
pickle_dump(images, save_path)
# return images
return images
def _create_dense_diagonal_precision(X, graph, n_features,
n_features_per_vertex,
dtype=np.float32, n_components=None,
bias=0, return_covariances=False,
verbose=False):
# Initialize precision
precision = np.zeros((n_features, n_features), dtype=dtype)
if return_covariances:
all_covariances = np.zeros(
(graph.n_vertices, n_features_per_vertex, n_features_per_vertex),
dtype=dtype)
if verbose:
print_dynamic('Allocated precision matrix of size {}'.format(
bytes_str(precision.nbytes)))
# Print information if asked
if verbose:
vertices = print_progress(
range(graph.n_vertices), n_items=graph.n_vertices,
prefix='Precision per vertex', end_with_newline=False)
else:
vertices = range(graph.n_vertices)
# Compute covariance matrix for each patch
for v in vertices:
# find indices in target precision matrix
i_from = v * n_features_per_vertex
i_to = (v + 1) * n_features_per_vertex
# compute covariance
covmat = np.cov(X[:, i_from:i_to], rowvar=0, bias=bias)
if return_covariances:
all_covariances[v] = covmat
# invert it
covmat = _covariance_matrix_inverse(covmat, n_components)
# insert to precision matrix
precision[i_from:i_to, i_from:i_to] = covmat
# return covariances
if return_covariances:
return precision, all_covariances
else:
return precision
def get_bounding_boxes(images, gt_shapes, face_detector):
ret = []
for i, (img, gt_shape) in enumerate(zip(images, gt_shapes)):
print_dynamic("Detecting face {}/{}".format(i, len(images)))
boxes = face_detector(img)
# Some images contain multiple faces, but are annotated only once.
# We only remember the box that contains the gt_shape.
found = False
for box in boxes:
if is_point_within(gt_shape.centre(), box.bounds()):
ret.append(box)
found = True
break
if not found:
ret.append(gt_shape.bounding_box())
return np.array(ret)
def _normalization_wrt_reference_shape(cls, images, group, label,
reference_shape, verbose=False):
r"""
Normalizes the images sizes with respect to the reference
shape (mean shape) scaling. This step is essential before building a
deformable model.
Parameters
----------
images : list of :map:`MaskedImage`
The set of landmarked images from which to build the model.
group : `string`
The key of the landmark set that should be used. If ``None``,
and if there is only one set of landmarks, this set will be used.
label : `string`
The label of the landmark manager that you wish to use. If no
label is passed, the convex hull of all landmarks is used.
reference_shape : :map:`PointCloud`
The reference shape that is used to resize all training images to
a consistent object size.
verbose: bool, optional
Flag that controls information and progress printing.
Returns
-------
normalized_images : :map:`MaskedImage` list
A list with the normalized images.
"""
normalized_images = []
for c, i in enumerate(images):
if verbose:
print_dynamic('- Normalizing images size: {}'.format(
progress_bar_str((c + 1.) / len(images),
show_bar=False)))
normalized_images.append(i.rescale_to_reference_shape(
reference_shape, group=group, label=label))
if verbose:
print_dynamic('- Normalizing images size: Done\n')
return normalized_images
