def fit_parametric_shape(image, initial_shape, parametric_algorithm,
gt_shape=None, return_costs=False):
r"""
Method that fits a parametric cascaded regression algorithm to an image.
Parameters
----------
image : `menpo.image.Image`
The input image.
initial_shape : `menpo.shape.PointCloud`
The initial estimation of the shape.
parametric_algorithm : `class`
A cascaded regression algorithm that employs a parametric shape model.
Please refer to `menpofit.sdm.algorithm`.
gt_shape : `menpo.shape.PointCloud`
The ground truth shape that corresponds to the image.
return_costs : `bool`, optional
If ``True``, then the cost function values will be computed during
the fitting procedure. Then these cost values will be assigned to the
returned `fitting_result`. *Note that this argument currently has no
effect and will raise a warning if set to ``True``. This is because
it is not possible to evaluate the cost function of this algorithm.*
Returns
-------
fitting_result : :map:`ParametricIterativeResult`
The final fitting result.
"""
# costs warning
if return_costs:
raise_costs_warning(parametric_algorithm)
# set current shape and initialize list of shapes
parametric_algorithm.shape_model.set_target(initial_shape)
current_shape = initial_shape.from_vector(
parametric_algorithm.shape_model.target.as_vector().copy())
shapes = []
shape_parameters = [parametric_algorithm.shape_model.as_vector()]
# Cascaded Regression loop
for r in parametric_algorithm.regressors:
# compute regression features
features = parametric_algorithm._compute_test_features(image,
current_shape)
# solve for increments on the shape vector
dx = r.predict(features).ravel()
# update current shape
p = parametric_algorithm.shape_model.as_vector() + dx
parametric_algorithm.shape_model._from_vector_inplace(p)
current_shape = current_shape.from_vector(
parametric_algorithm.shape_model.target.as_vector().copy())
shapes.append(current_shape)
shape_parameters.append(p)
# return algorithm result
return ParametricIterativeResult(
shapes=shapes, shape_parameters=shape_parameters,
initial_shape=initial_shape, image=image, gt_shape=gt_shape)
def algorithm_result(self,
image,
shapes,
shape_parameters,
initial_shape=None,
costs=None,
gt_shape=None):
r"""
Returns an ATM iterative optimization result object.
Parameters
----------
image : `menpo.image.Image` or subclass
The image on which the optimization is applied.
shapes : `list` of `menpo.shape.PointCloud`
The `list` of sparse shapes per iteration.
shape_parameters : `list` of `ndarray`
The `list` of shape parameters per iteration.
initial_shape : `menpo.shape.PointCloud` or ``None``, optional
The initial shape from which the fitting process started. If
``None``, then no initial shape is assigned.
gt_shape : `menpo.shape.PointCloud` or ``None``, optional
The ground truth shape that corresponds to the test image.
costs : `list` of `float` or ``None``, optional
The `list` of costs per iteration. If ``None``, then it is
assumed that the cost computation for that particular algorithm
is not well defined.
Returns
-------
result : :map:`ParametricIterativeResult`
The optimization result object.
"""
# TODO: Separate result for linear ATM that stores both the sparse
# and dense shapes per iteration (@patricksnape will fix this)
# This means that the linear ATM will only store the sparse shapes
shapes = [
self.transform.from_vector(p).sparse_target
for p in shape_parameters
]
return ParametricIterativeResult(shapes=shapes,
shape_parameters=shape_parameters,
initial_shape=initial_shape,
image=image,
gt_shape=gt_shape,
costs=costs)
def algorithm_result(self,
image,
shapes,
shape_parameters,
initial_shape=None,
gt_shape=None,
costs=None):
r"""
Returns an ATM iterative optimization result object.
Parameters
----------
image : `menpo.image.Image` or subclass
The image on which the optimization is applied.
shapes : `list` of `menpo.shape.PointCloud`
The `list` of shapes per iteration.
shape_parameters : `list` of `ndarray`
The `list` of shape parameters per iteration.
initial_shape : `menpo.shape.PointCloud` or ``None``, optional
The initial shape from which the fitting process started. If
``None``, then no initial shape is assigned.
gt_shape : `menpo.shape.PointCloud` or ``None``, optional
The ground truth shape that corresponds to the test image.
costs : `list` of `float` or ``None``, optional
The `list` of costs per iteration. If ``None``, then it is
assumed that the cost computation for that particular algorithm
is not well defined.
Returns
-------
result : :map:`ParametricIterativeResult`
The optimization result object.
"""
return ParametricIterativeResult(shapes=shapes,
shape_parameters=shape_parameters,
initial_shape=initial_shape,
image=image,
gt_shape=gt_shape,
costs=costs)
def run(self,
image,
initial_shape,
gt_shape=None,
max_iters=20,
return_costs=False,
map_inference=True):
r"""
Execute the optimization algorithm.
Parameters
----------
image : `menpo.image.Image`
The input test image.
initial_shape : `menpo.shape.PointCloud`
The initial shape from which the optimization will start.
gt_shape : `menpo.shape.PointCloud` or ``None``, optional
The ground truth shape of the image. It is only needed in order
to get passed in the optimization result object, which has the
ability to compute the fitting error.
max_iters : `int`, optional
The maximum number of iterations. Note that the algorithm may
converge, and thus stop, earlier.
return_costs : `bool`, optional
If ``True``, then the cost function values will be computed
during the fitting procedure. Then these cost values will be
assigned to the returned `fitting_result`. *Note that this
argument currently has no effect and will raise a warning if set
to ``True``. This is because it is not possible to evaluate the
cost function of this algorithm.*
map_inference : `bool`, optional
If ``True``, then the solution will be given after performing MAP
inference.
Returns
-------
fitting_result : :map:`ParametricIterativeResult`
The parametric iterative fitting result.
"""
# costs warning
if return_costs:
raise_costs_warning(self)
patch_size = self.opt['ratio1']
patch_size2 = self.opt['ratio2']
search_ratio = [patch_size] * 3 + [patch_size2
] * (self.opt['numIter'] - 3)
numLandmarks = image.rspmap_data.shape[1]
timeFitStart = time()
# pad responsemaps with zeros
padH = int(image.shape[0] / 2)
padW = int(image.shape[1] / 2)
# image.rspmap_data = np.pad(image.rspmap_data, ((0, 0), (0, 0), (padH, padH), (padW, padW)), 'constant')
self.rps_zeros[0, :, padH:padH + image.shape[0],
padW:padW + image.shape[1]] = image.rspmap_data
image.rspmap_data = self.rps_zeros
# cost = time() - timeFitStart
timeFitStart = time()
try:
weighted_project = self.opt['ablation'][0]
weighted_meanshift = self.opt['ablation'][1]
except:
weighted_project = True
weighted_meanshift = True
if weighted_project:
# Correction step
search_grid = build_grid(
np.array(search_ratio[0] * np.array(image.shape), int))
# Compute patch responses
patch_responses = self.expert_ensemble.predict_probability(
image, initial_shape, search_grid.shape)
project_weight = rspimage.calculate_evidence(
patch_responses,
rate=self.confidence_gama,
offset=self.kernel_idealmap).reshape((1, -1))
# write project_weight
inirho = self.opt['pdm_rho']
# inirho = 0
ini_rho2_inv_prior = np.hstack((np.zeros(
(4, )), inirho / self.transform.model.eigenvalues))
initial_shape_mean = initial_shape.points.ravel(
) - self.transform.model._mean
iniJe = -self.J.T.dot(initial_shape_mean * project_weight[0])
iniJWJ = self.J.T.dot(np.diag(project_weight[0]).dot(self.J))
inv_JJ = np.linalg.inv(iniJWJ + np.diag(ini_rho2_inv_prior))
initial_p = -inv_JJ.dot(iniJe) # self.inv_JJ_prior
# Update pdm
self.transform._from_vector_inplace(initial_p)
else:
self.transform.set_target(initial_shape)
# Initialize transform
p_list = [self.transform.as_vector()]
# check boundary
if np.any(self.transform.target.points < 0) or np.any(
self.transform.target.points >= image.shape[0]):
self.transform.target.points[self.transform.target.points < 0] = 1
self.transform.target.points[self.transform.target.points >=
image.shape[0]] = image.shape[0] - 2
self.transform.set_target(self.transform.target)
self.transform.target.points[self.transform.target.points < 0] = 1
self.transform.target.points[self.transform.target.points >=
image.shape[0]] = image.shape[0] - 2
# print image.path.name
# print 'out of bound'
shapes = [self.transform.target]
# tuning step
# Initialize iteration counter and epsilon
k = 0
eps = np.Inf
errSum = [np.Inf]
epsList = [np.Inf]
searchR = []
try:
max_iters = self.opt['numIter']
except:
max_iters = 10
rho2 = self.opt['rho2']
# timeFitStart = time()
while k < max_iters and eps > self.eps:
#target = self.transform.target
target = shapes[k]
search_grid = build_grid(
np.array(search_ratio[k] * np.array(image.shape), int))
# search_grid = self.search_grid *np.array(image.shape, dtype=np.float32)/np.array(image.rspmap_data.shape)[-2:-1]
candidate_landmarks = (target.points[:, None, None, None, :] +
search_grid)
# Compute patch responses
patch_responses = self.expert_ensemble.predict_probability(
image, target, search_grid.shape)
if weighted_meanshift:
kernel_covariance = rspimage.calculate_evidence(
patch_responses,
rate=self.confidence_gama,
offset=self.kernel_idealmap)
else:
kernel_covariance = np.ones(2 * patch_responses.shape[0])
try:
smooth_rspmap = self.opt['smooth']
except:
smooth_rspmap = True
if smooth_rspmap:
try:
patch_responses /= np.sum(patch_responses,
axis=(-2, -1))[..., None, None]
except:
print image.path.name
print('Normalize fail.')
mvnList = [
multivariate_normal(mean=np.zeros(2),
cov=2.0 * rho2 /
(kernel_covariance[2 * i] +
kernel_covariance[2 * i + 1]))
for i in range(patch_responses.shape[0])
]
kernel_grid = np.array([
mvnList[i].pdf(search_grid)[None]
for i in range(patch_responses.shape[0])
])
patch_kernels = patch_responses * kernel_grid
else:
patch_kernels = patch_responses
# Normalise smoothed responses
patch_kernels /= np.sum(patch_kernels, axis=(-2, -1))[..., None,
None]
# Compute mean shift target
mean_shift_target = np.sum(patch_kernels[..., None] *
candidate_landmarks,
axis=(-3, -2))
# Compute shape error term
error = mean_shift_target.ravel() - target.as_vector()
errSum.append(np.sum(np.abs(error)))
# error = error
# Solve for increments on the shape parameters
if map_inference:
'''
Je = (self.rho2_inv_L * self.transform.as_vector() -
self.J.T.dot(error))
# inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L))
dp = -self.inv_JJ_prior.dot(Je) #self.inv_JJ_prior
'''
Je = (self.rho2_inv_L * self.transform.as_vector() -
self.J.T.dot(error * kernel_covariance))
JWJ = self.J.T.dot(np.diag(kernel_covariance).dot(self.J))
inv_JJ_prior = np.linalg.inv(JWJ + np.diag(self.rho2_inv_L))
dp = -inv_JJ_prior.dot(Je) # self.inv_JJ_prior
'''
sim_prior = np.zeros((4,))
pdm_prior = rho2 / self.transform.model.eigenvalues
rho2_inv_L = np.hstack((sim_prior, pdm_prior))
Je = (rho2_inv_L * self.transform.as_vector() -
self.J.T.dot(error*kernel_covariance))
JWJ = self.J.T.dot(np.diag(kernel_covariance).dot(self.J))
inv_JJ_prior = np.linalg.inv(JWJ + np.diag(rho2_inv_L))
dp = -inv_JJ_prior.dot(Je) # self.inv_JJ_prior
'''
else:
J = np.rollaxis(self.transform.d_dp(target.points), -1, 1)
J = J.reshape((-1, J.shape[-1]))
JJ = J.T.dot(np.diag(kernel_covariance**2).dot(J))
# Compute Jacobian pseudo-inverse
pinv_J = np.linalg.solve(JJ, J.T.dot(
np.diag(kernel_covariance))) #
dp = pinv_J.dot(error)
print('notMAP')
# Update pdm
s_k = self.transform.target.points
self.transform._from_vector_inplace(self.transform.as_vector() +
dp)
if np.any(self.transform.target.points < 0) or np.any(
self.transform.target.points >= image.shape[0]):
self.transform.target.points[
self.transform.target.points < 0] = 1
self.transform.target.points[
self.transform.target.points >=
image.shape[0]] = image.shape[0] - 2
self.transform.set_target(self.transform.target)
self.transform.target.points[
self.transform.target.points < 0] = 1
self.transform.target.points[
self.transform.target.points >=
image.shape[0]] = image.shape[0] - 2
p_list.append(self.transform.as_vector())
shapes.append(self.transform.target)
# Test convergence
eps = np.abs(np.linalg.norm(s_k - self.transform.target.points))
epsList.append(eps)
# Increase iteration counter
k += 1
cost = time() - timeFitStart
cost = [cost]
visualize = self.opt['verbose']
if visualize:
# wrFilebase = self.opt['imgDir']
# wrFilebase = '../'
image.landmarks['initial'] = shapes[0]
image.landmarks['final'] = shapes[-1]
plt.clf()
image._view_landmarks_2d(group='final',
marker_size=5,
marker_edge_colour='k',
marker_face_colour='g')
plt.show(block=False)
plt.pause(0.05)
# plt.savefig(wrFilebase + '/output/' + image.path.stem + '.png')
return ParametricIterativeResult(
shapes=shapes,
shape_parameters=p_list,
initial_shape=
initial_shape, #image.landmarks['__initial_shape'] initial_shape
image=image,
gt_shape=gt_shape,
costs=cost)
def run(self,
image,
initial_shape,
gt_shape=None,
max_iters=20,
return_costs=False,
map_inference=False):
r"""
Execute the optimization algorithm.
Parameters
----------
image : `menpo.image.Image`
The input test image.
initial_shape : `menpo.shape.PointCloud`
The initial shape from which the optimization will start.
gt_shape : `menpo.shape.PointCloud` or ``None``, optional
The ground truth shape of the image. It is only needed in order
to get passed in the optimization result object, which has the
ability to compute the fitting error.
max_iters : `int`, optional
The maximum number of iterations. Note that the algorithm may
converge, and thus stop, earlier.
return_costs : `bool`, optional
If ``True``, then the cost function values will be computed
during the fitting procedure. Then these cost values will be
assigned to the returned `fitting_result`. *Note that this
argument currently has no effect and will raise a warning if set
to ``True``. This is because it is not possible to evaluate the
cost function of this algorithm.*
map_inference : `bool`, optional
If ``True``, then the solution will be given after performing MAP
inference.
Returns
-------
fitting_result : :map:`ParametricIterativeResult`
The parametric iterative fitting result.
"""
# costs warning
if return_costs:
raise_costs_warning(self)
# Initialize transform
self.transform.set_target(initial_shape)
p_list = [self.transform.as_vector()]
shapes = [self.transform.target]
# Initialize iteration counter and epsilon
k = 0
eps = np.Inf
# Expectation-Maximisation loop
while k < max_iters and eps > self.eps:
target = self.transform.target
# Obtain all landmark positions l_i = (x_i, y_i) being considered
# ie all pixel positions in each landmark's search space
candidate_landmarks = (target.points[:, None, None, None, :] +
self.search_grid)
# Compute responses
responses = self.expert_ensemble.predict_probability(image, target)
# Approximate responses using isotropic Gaussian
max_indices = np.argmax(responses.reshape(responses.shape[:2] +
(-1, )),
axis=-1)
max_indices = np.unravel_index(max_indices, responses.shape)[-2:]
max_indices = np.hstack((max_indices[0], max_indices[1]))
max_indices = max_indices[:, None, None, None, ...]
max_indices -= self.half_search_shape
gaussian_responses = self.mvn.pdf(max_indices + self.search_grid)
# Normalise smoothed responses
gaussian_responses /= np.sum(gaussian_responses,
axis=(-2, -1))[..., None, None]
# Compute new target
new_target = np.sum(gaussian_responses[:, None, ..., None] *
candidate_landmarks,
axis=(-3, -2))
# Compute shape error term
error = target.as_vector() - new_target.ravel()
# Solve for increments on the shape parameters
if map_inference:
Je = (self.rho2_inv_L * self.transform.as_vector() -
self.J.T.dot(error))
dp = -self.inv_JJ_prior.dot(Je)
else:
dp = self.pinv_J.dot(error)
# Update pdm
s_k = self.transform.target.points
self.transform._from_vector_inplace(self.transform.as_vector() +
dp)
p_list.append(self.transform.as_vector())
shapes.append(self.transform.target)
# Test convergence
eps = np.abs(np.linalg.norm(s_k - self.transform.target.points))
# Increase iteration counter
k += 1
# Return algorithm result
return ParametricIterativeResult(shapes=shapes,
shape_parameters=p_list,
initial_shape=initial_shape,
image=image,
gt_shape=gt_shape)