def fit_parametric_shape(image, initial_shape, parametric_algorithm, gt_shape=None, return_costs=False): r""" Method that fits a parametric cascaded regression algorithm to an image. Parameters ---------- image : `menpo.image.Image` The input image. initial_shape : `menpo.shape.PointCloud` The initial estimation of the shape. parametric_algorithm : `class` A cascaded regression algorithm that employs a parametric shape model. Please refer to `menpofit.sdm.algorithm`. gt_shape : `menpo.shape.PointCloud` The ground truth shape that corresponds to the image. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* Returns ------- fitting_result : :map:`ParametricIterativeResult` The final fitting result. """ # costs warning if return_costs: raise_costs_warning(parametric_algorithm) # set current shape and initialize list of shapes parametric_algorithm.shape_model.set_target(initial_shape) current_shape = initial_shape.from_vector( parametric_algorithm.shape_model.target.as_vector().copy()) shapes = [] shape_parameters = [parametric_algorithm.shape_model.as_vector()] # Cascaded Regression loop for r in parametric_algorithm.regressors: # compute regression features features = parametric_algorithm._compute_test_features(image, current_shape) # solve for increments on the shape vector dx = r.predict(features).ravel() # update current shape p = parametric_algorithm.shape_model.as_vector() + dx parametric_algorithm.shape_model._from_vector_inplace(p) current_shape = current_shape.from_vector( parametric_algorithm.shape_model.target.as_vector().copy()) shapes.append(current_shape) shape_parameters.append(p) # return algorithm result return ParametricIterativeResult( shapes=shapes, shape_parameters=shape_parameters, initial_shape=initial_shape, image=image, gt_shape=gt_shape)
def fit_parametric_shape(image, initial_shape, parametric_algorithm, gt_shape=None, return_costs=False): r""" Method that fits a parametric cascaded regression algorithm to an image. Parameters ---------- image : `menpo.image.Image` The input image. initial_shape : `menpo.shape.PointCloud` The initial estimation of the shape. parametric_algorithm : `class` A cascaded regression algorithm that employs a parametric shape model. Please refer to `menpofit.sdm.algorithm`. gt_shape : `menpo.shape.PointCloud` The ground truth shape that corresponds to the image. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* Returns ------- fitting_result : :map:`ParametricIterativeResult` The final fitting result. """ # costs warning if return_costs: raise_costs_warning(parametric_algorithm) # set current shape and initialize list of shapes parametric_algorithm.shape_model.set_target(initial_shape) current_shape = initial_shape.from_vector(parametric_algorithm.shape_model.target.as_vector().copy()) shapes = [] shape_parameters = [parametric_algorithm.shape_model.as_vector()] # Cascaded Regression loop for r in parametric_algorithm.regressors: # compute regression features features = parametric_algorithm._compute_test_features(image, current_shape) # solve for increments on the shape vector dx = r.predict(features).ravel() # update current shape p = parametric_algorithm.shape_model.as_vector() + dx parametric_algorithm.shape_model._from_vector_inplace(p) current_shape = current_shape.from_vector(parametric_algorithm.shape_model.target.as_vector().copy()) shapes.append(current_shape) shape_parameters.append(p) # return algorithm result return ParametricIterativeResult( shapes=shapes, shape_parameters=shape_parameters, initial_shape=initial_shape, image=image, gt_shape=gt_shape )
def run(self, image, initial_shape, gt_shape=None, return_costs=False, **kwargs): r""" Run the algorithm to an image given an initial shape. Parameters ---------- image : `menpo.image.Image` or subclass The image to be fitted. initial_shape : `menpo.shape.PointCloud` The initial shape from which the fitting procedure will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape associated to the image. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* Returns ------- fitting_result : :map:`AAMAlgorithmResult` The parametric iterative fitting result. """ # costs warning if return_costs: raise_costs_warning(self) # initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] shapes = [self.transform.target] # Cascaded Regression loop for r in self.regressors: # Assumes that the transform is correctly set features = self._compute_test_features(image, self.transform.target) # solve for increments on the shape parameters dx = r.predict(features) # We need to update the transform to set the state for the warping # of the image above. new_x = p_list[-1] + dx self.transform._from_vector_inplace(new_x) p_list.append(new_x) shapes.append(self.transform.target) # return algorithm result return self.interface.algorithm_result( image=image, shapes=shapes, shape_parameters=p_list, initial_shape=initial_shape, gt_shape=gt_shape)
def run(self, image, bounding_box, gt_shape=None, return_costs=False, **kwargs): r""" Run the predictor to an image given an initial bounding box. Parameters ---------- image : `menpo.image.Image` or subclass The image to be fitted. bounding_box : `menpo.shape.PointDirectedGraph` The initial bounding box from which the fitting procedure will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape associated to the image. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* Returns ------- fitting_result: `menpofit.result.NonParametricIterativeResult` The result of the fitting procedure. """ # costs warning if return_costs: raise_costs_warning(self) # Perform prediction pix = image_to_dlib_pixels(image) rect = pointcloud_to_dlib_rect(bounding_box) pred = dlib_full_object_detection_to_pointcloud( self.dlib_model(pix, rect)) return NonParametricIterativeResult(shapes=[pred], initial_shape=None, image=image, gt_shape=gt_shape)
def run(self, image, bounding_box, gt_shape=None, return_costs=False, **kwargs): r""" Run the predictor to an image given an initial bounding box. Parameters ---------- image : `menpo.image.Image` or subclass The image to be fitted. bounding_box : `menpo.shape.PointDirectedGraph` The initial bounding box from which the fitting procedure will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape associated to the image. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* Returns ------- fitting_result: `menpofit.result.NonParametricIterativeResult` The result of the fitting procedure. """ # costs warning if return_costs: raise_costs_warning(self) # Perform prediction pix = image_to_dlib_pixels(image) rect = pointcloud_to_dlib_rect(bounding_box) pred = dlib_full_object_detection_to_pointcloud( self.dlib_model(pix, rect)) return NonParametricIterativeResult( shapes=[pred], initial_shape=None, image=image, gt_shape=gt_shape)
def run(self, image, initial_shape, gt_shape=None, max_iters=20, return_costs=False, map_inference=True): r""" Execute the optimization algorithm. Parameters ---------- image : `menpo.image.Image` The input test image. initial_shape : `menpo.shape.PointCloud` The initial shape from which the optimization will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape of the image. It is only needed in order to get passed in the optimization result object, which has the ability to compute the fitting error. max_iters : `int`, optional The maximum number of iterations. Note that the algorithm may converge, and thus stop, earlier. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* map_inference : `bool`, optional If ``True``, then the solution will be given after performing MAP inference. Returns ------- fitting_result : :map:`ParametricIterativeResult` The parametric iterative fitting result. """ # costs warning if return_costs: raise_costs_warning(self) patch_size = self.opt['ratio1'] patch_size2 = self.opt['ratio2'] search_ratio = [patch_size] * 3 + [patch_size2 ] * (self.opt['numIter'] - 3) numLandmarks = image.rspmap_data.shape[1] timeFitStart = time() # pad responsemaps with zeros padH = int(image.shape[0] / 2) padW = int(image.shape[1] / 2) # image.rspmap_data = np.pad(image.rspmap_data, ((0, 0), (0, 0), (padH, padH), (padW, padW)), 'constant') self.rps_zeros[0, :, padH:padH + image.shape[0], padW:padW + image.shape[1]] = image.rspmap_data image.rspmap_data = self.rps_zeros # cost = time() - timeFitStart timeFitStart = time() try: weighted_project = self.opt['ablation'][0] weighted_meanshift = self.opt['ablation'][1] except: weighted_project = True weighted_meanshift = True if weighted_project: # Correction step search_grid = build_grid( np.array(search_ratio[0] * np.array(image.shape), int)) # Compute patch responses patch_responses = self.expert_ensemble.predict_probability( image, initial_shape, search_grid.shape) project_weight = rspimage.calculate_evidence( patch_responses, rate=self.confidence_gama, offset=self.kernel_idealmap).reshape((1, -1)) # write project_weight inirho = self.opt['pdm_rho'] # inirho = 0 ini_rho2_inv_prior = np.hstack((np.zeros( (4, )), inirho / self.transform.model.eigenvalues)) initial_shape_mean = initial_shape.points.ravel( ) - self.transform.model._mean iniJe = -self.J.T.dot(initial_shape_mean * project_weight[0]) iniJWJ = self.J.T.dot(np.diag(project_weight[0]).dot(self.J)) inv_JJ = np.linalg.inv(iniJWJ + np.diag(ini_rho2_inv_prior)) initial_p = -inv_JJ.dot(iniJe) # self.inv_JJ_prior # Update pdm self.transform._from_vector_inplace(initial_p) else: self.transform.set_target(initial_shape) # Initialize transform p_list = [self.transform.as_vector()] # check boundary if np.any(self.transform.target.points < 0) or np.any( self.transform.target.points >= image.shape[0]): self.transform.target.points[self.transform.target.points < 0] = 1 self.transform.target.points[self.transform.target.points >= image.shape[0]] = image.shape[0] - 2 self.transform.set_target(self.transform.target) self.transform.target.points[self.transform.target.points < 0] = 1 self.transform.target.points[self.transform.target.points >= image.shape[0]] = image.shape[0] - 2 # print image.path.name # print 'out of bound' shapes = [self.transform.target] # tuning step # Initialize iteration counter and epsilon k = 0 eps = np.Inf errSum = [np.Inf] epsList = [np.Inf] searchR = [] try: max_iters = self.opt['numIter'] except: max_iters = 10 rho2 = self.opt['rho2'] # timeFitStart = time() while k < max_iters and eps > self.eps: #target = self.transform.target target = shapes[k] search_grid = build_grid( np.array(search_ratio[k] * np.array(image.shape), int)) # search_grid = self.search_grid *np.array(image.shape, dtype=np.float32)/np.array(image.rspmap_data.shape)[-2:-1] candidate_landmarks = (target.points[:, None, None, None, :] + search_grid) # Compute patch responses patch_responses = self.expert_ensemble.predict_probability( image, target, search_grid.shape) if weighted_meanshift: kernel_covariance = rspimage.calculate_evidence( patch_responses, rate=self.confidence_gama, offset=self.kernel_idealmap) else: kernel_covariance = np.ones(2 * patch_responses.shape[0]) try: smooth_rspmap = self.opt['smooth'] except: smooth_rspmap = True if smooth_rspmap: try: patch_responses /= np.sum(patch_responses, axis=(-2, -1))[..., None, None] except: print image.path.name print('Normalize fail.') mvnList = [ multivariate_normal(mean=np.zeros(2), cov=2.0 * rho2 / (kernel_covariance[2 * i] + kernel_covariance[2 * i + 1])) for i in range(patch_responses.shape[0]) ] kernel_grid = np.array([ mvnList[i].pdf(search_grid)[None] for i in range(patch_responses.shape[0]) ]) patch_kernels = patch_responses * kernel_grid else: patch_kernels = patch_responses # Normalise smoothed responses patch_kernels /= np.sum(patch_kernels, axis=(-2, -1))[..., None, None] # Compute mean shift target mean_shift_target = np.sum(patch_kernels[..., None] * candidate_landmarks, axis=(-3, -2)) # Compute shape error term error = mean_shift_target.ravel() - target.as_vector() errSum.append(np.sum(np.abs(error))) # error = error # Solve for increments on the shape parameters if map_inference: ''' Je = (self.rho2_inv_L * self.transform.as_vector() - self.J.T.dot(error)) # inv_JJ_prior = np.linalg.inv(self.JJ + np.diag(self.rho2_inv_L)) dp = -self.inv_JJ_prior.dot(Je) #self.inv_JJ_prior ''' Je = (self.rho2_inv_L * self.transform.as_vector() - self.J.T.dot(error * kernel_covariance)) JWJ = self.J.T.dot(np.diag(kernel_covariance).dot(self.J)) inv_JJ_prior = np.linalg.inv(JWJ + np.diag(self.rho2_inv_L)) dp = -inv_JJ_prior.dot(Je) # self.inv_JJ_prior ''' sim_prior = np.zeros((4,)) pdm_prior = rho2 / self.transform.model.eigenvalues rho2_inv_L = np.hstack((sim_prior, pdm_prior)) Je = (rho2_inv_L * self.transform.as_vector() - self.J.T.dot(error*kernel_covariance)) JWJ = self.J.T.dot(np.diag(kernel_covariance).dot(self.J)) inv_JJ_prior = np.linalg.inv(JWJ + np.diag(rho2_inv_L)) dp = -inv_JJ_prior.dot(Je) # self.inv_JJ_prior ''' else: J = np.rollaxis(self.transform.d_dp(target.points), -1, 1) J = J.reshape((-1, J.shape[-1])) JJ = J.T.dot(np.diag(kernel_covariance**2).dot(J)) # Compute Jacobian pseudo-inverse pinv_J = np.linalg.solve(JJ, J.T.dot( np.diag(kernel_covariance))) # dp = pinv_J.dot(error) print('notMAP') # Update pdm s_k = self.transform.target.points self.transform._from_vector_inplace(self.transform.as_vector() + dp) if np.any(self.transform.target.points < 0) or np.any( self.transform.target.points >= image.shape[0]): self.transform.target.points[ self.transform.target.points < 0] = 1 self.transform.target.points[ self.transform.target.points >= image.shape[0]] = image.shape[0] - 2 self.transform.set_target(self.transform.target) self.transform.target.points[ self.transform.target.points < 0] = 1 self.transform.target.points[ self.transform.target.points >= image.shape[0]] = image.shape[0] - 2 p_list.append(self.transform.as_vector()) shapes.append(self.transform.target) # Test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) epsList.append(eps) # Increase iteration counter k += 1 cost = time() - timeFitStart cost = [cost] visualize = self.opt['verbose'] if visualize: # wrFilebase = self.opt['imgDir'] # wrFilebase = '../' image.landmarks['initial'] = shapes[0] image.landmarks['final'] = shapes[-1] plt.clf() image._view_landmarks_2d(group='final', marker_size=5, marker_edge_colour='k', marker_face_colour='g') plt.show(block=False) plt.pause(0.05) # plt.savefig(wrFilebase + '/output/' + image.path.stem + '.png') return ParametricIterativeResult( shapes=shapes, shape_parameters=p_list, initial_shape= initial_shape, #image.landmarks['__initial_shape'] initial_shape image=image, gt_shape=gt_shape, costs=cost)
def run(self, image, initial_shape, gt_shape=None, max_iters=20, return_costs=False, map_inference=False): r""" Execute the optimization algorithm. Parameters ---------- image : `menpo.image.Image` The input test image. initial_shape : `menpo.shape.PointCloud` The initial shape from which the optimization will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape of the image. It is only needed in order to get passed in the optimization result object, which has the ability to compute the fitting error. max_iters : `int`, optional The maximum number of iterations. Note that the algorithm may converge, and thus stop, earlier. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* map_inference : `bool`, optional If ``True``, then the solution will be given after performing MAP inference. Returns ------- fitting_result : :map:`ParametricIterativeResult` The parametric iterative fitting result. """ # costs warning if return_costs: raise_costs_warning(self) # Initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] shapes = [self.transform.target] # Initialize iteration counter and epsilon k = 0 eps = np.Inf # Expectation-Maximisation loop while k < max_iters and eps > self.eps: target = self.transform.target # Obtain all landmark positions l_i = (x_i, y_i) being considered # ie all pixel positions in each landmark's search space candidate_landmarks = (target.points[:, None, None, None, :] + self.search_grid) # Compute responses responses = self.expert_ensemble.predict_probability(image, target) # Approximate responses using isotropic Gaussian max_indices = np.argmax(responses.reshape(responses.shape[:2] + (-1, )), axis=-1) max_indices = np.unravel_index(max_indices, responses.shape)[-2:] max_indices = np.hstack((max_indices[0], max_indices[1])) max_indices = max_indices[:, None, None, None, ...] max_indices -= self.half_search_shape gaussian_responses = self.mvn.pdf(max_indices + self.search_grid) # Normalise smoothed responses gaussian_responses /= np.sum(gaussian_responses, axis=(-2, -1))[..., None, None] # Compute new target new_target = np.sum(gaussian_responses[:, None, ..., None] * candidate_landmarks, axis=(-3, -2)) # Compute shape error term error = target.as_vector() - new_target.ravel() # Solve for increments on the shape parameters if map_inference: Je = (self.rho2_inv_L * self.transform.as_vector() - self.J.T.dot(error)) dp = -self.inv_JJ_prior.dot(Je) else: dp = self.pinv_J.dot(error) # Update pdm s_k = self.transform.target.points self.transform._from_vector_inplace(self.transform.as_vector() + dp) p_list.append(self.transform.as_vector()) shapes.append(self.transform.target) # Test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) # Increase iteration counter k += 1 # Return algorithm result return ParametricIterativeResult(shapes=shapes, shape_parameters=p_list, initial_shape=initial_shape, image=image, gt_shape=gt_shape)
def run(self, image, initial_shape, gt_shape=None, max_iters=20, return_costs=False, map_inference=False): r""" Execute the optimization algorithm. Parameters ---------- image : `menpo.image.Image` The input test image. initial_shape : `menpo.shape.PointCloud` The initial shape from which the optimization will start. gt_shape : `menpo.shape.PointCloud` or ``None``, optional The ground truth shape of the image. It is only needed in order to get passed in the optimization result object, which has the ability to compute the fitting error. max_iters : `int`, optional The maximum number of iterations. Note that the algorithm may converge, and thus stop, earlier. return_costs : `bool`, optional If ``True``, then the cost function values will be computed during the fitting procedure. Then these cost values will be assigned to the returned `fitting_result`. *Note that this argument currently has no effect and will raise a warning if set to ``True``. This is because it is not possible to evaluate the cost function of this algorithm.* map_inference : `bool`, optional If ``True``, then the solution will be given after performing MAP inference. Returns ------- fitting_result : :map:`ParametricIterativeResult` The parametric iterative fitting result. """ # costs warning if return_costs: raise_costs_warning(self) # Initialize transform self.transform.set_target(initial_shape) p_list = [self.transform.as_vector()] shapes = [self.transform.target] # Initialize iteration counter and epsilon k = 0 eps = np.Inf # Expectation-Maximisation loop while k < max_iters and eps > self.eps: target = self.transform.target # Obtain all landmark positions l_i = (x_i, y_i) being considered # ie all pixel positions in each landmark's search space candidate_landmarks = (target.points[:, None, None, None, :] + self.search_grid) # Compute patch responses patch_responses = self.expert_ensemble.predict_probability(image, target) # Smooth responses using the Gaussian-KDE grid patch_kernels = patch_responses * self.kernel_grid # Normalise smoothed responses patch_kernels /= np.sum(patch_kernels, axis=(-2, -1))[..., None, None] # Compute mean shift target mean_shift_target = np.sum(patch_kernels[..., None] * candidate_landmarks, axis=(-3, -2)) # Compute shape error term error = mean_shift_target.ravel() - target.as_vector() # Solve for increments on the shape parameters if map_inference: Je = (self.rho2_inv_L * self.transform.as_vector() - self.J.T.dot(error)) dp = -self.inv_JJ_prior.dot(Je) else: dp = self.pinv_J.dot(error) # Update pdm s_k = self.transform.target.points self.transform._from_vector_inplace(self.transform.as_vector() + dp) p_list.append(self.transform.as_vector()) shapes.append(self.transform.target) # Test convergence eps = np.abs(np.linalg.norm(s_k - self.transform.target.points)) # Increase iteration counter k += 1 # Return algorithm result return ParametricIterativeResult(shapes=shapes, shape_parameters=p_list, initial_shape=initial_shape, image=image, gt_shape=gt_shape)