def test_str_mock(mock_stdout): print(aam) fitter = LucasKanadeAAMFitter(aam, algorithm=AlternatingInverseCompositional) print(fitter) print(aam2) fitter = LucasKanadeAAMFitter(aam2, algorithm=ProbabilisticForwardAdditive) print(fitter)
def fit(self, images, landmarks): num_samples = len(landmarks[0]) self.num_landmarks_by_shape = [] for i in range(len(landmarks)): self.num_landmarks_by_shape.append(len(landmarks[i][0])) aam_images = self.prepare_data_for_aam(images, landmarks) aam = PatchAAM(aam_images, group=None, patch_shape=[(self.scales[0], self.scales[0]), (self.scales[1], self.scales[1])], diagonal=150, scales=(0.5, 1.0), holistic_features=fast_dsift, max_shape_components=20, max_appearance_components=150, verbose=True) self.fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[5, 20], n_appearance=[30, 150]) pc = [] for img in aam_images: pc.append(img.landmarks[None].lms) self.mean_shape = mean_pointcloud(pc) self.image_shape = images[0].shape fitting_results = [] for img in aam_images[:10]: fr = self.fitter.fit_from_shape(img, self.mean_shape, gt_shape=img.landmarks[None].lms) fitting_results.append(fr)
def train(self, train_img_path): train_imgs = self.load_database(train_img_path) train_imgs = self.equalize_hist(train_imgs) if self.algo == 'aam': trainer = HolisticAAM(train_imgs, group='PTS', verbose=True, diagonal=120, scales=(0.5, 1.0)) self.fitter = LucasKanadeAAMFitter(trainer, n_shape=[6, 12], n_appearance=0.5) mio.export_pickle(self.fitter, self.model_filename) print('aam model trained and exported!') elif self.algo == 'asm': trainer = CLM(train_imgs, group='PTS', verbose=True, diagonal=120, scales=(0.5, 1.0)) self.fitter = GradientDescentCLMFitter(trainer, n_shape=[6, 12]) mio.export_pickle(self.fitter, self.model_filename) print('asm model trained and exported!') else: ValueError('algorithm must be aam or asm!')
def train(): path_to_images = 'lfpw/ceph_trainset/' training_images = [] for img in mio.import_images(path_to_images, verbose=True): # if img.n_channels == 3: # img = img.as_greyscale() img = img.crop_to_landmarks_proportion(0.2) d = img.diagonal() if d > 400: img = img.rescale(400.0 / d) training_images.append(img) # patch_aam = PatchAAM(training_images, group='PTS', patch_shape=[(15, 15), (23, 23)], # diagonal=200, scales=(0.5, 1.0), holistic_features=fast_dsift, # max_shape_components=60, max_appearance_components=200, # verbose=True) patch_aam = PatchAAM(training_images, group='PTS', patch_shape=[(16, 19), (19, 16)], diagonal=200, scales=(0.5, 1), holistic_features=fast_dsift, max_shape_components=74, max_appearance_components=175, verbose=True) fitter = LucasKanadeAAMFitter(patch_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 30], n_appearance=[40, 160]) mio.export_pickle(fitter, '26_img_35_pnt.pkl', overwrite=True) return fitter
def lucasKanadeAAMFitter(self, aam): patch_aam = aam fitter = LucasKanadeAAMFitter(patch_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[5, 20], n_appearance=[30, 150]) return fitter
def fitter_AAM(aam): from menpofit.aam import LucasKanadeAAMFitter, WibergInverseCompositional fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[5, 20], n_appearance=[30, 150]) return fitter
def test_pertrurb_shape(): fitter = LucasKanadeAAMFitter(aam) s = fitter.perturb_shape(training_images[0].landmarks[None].lms, noise_std=0.08, rotation=False) assert (s.n_dims == 2) assert (s.n_landmark_groups == 0) assert (s.n_points == 68)
def test_obtain_shape_from_bb(): fitter = LucasKanadeAAMFitter(aam) s = fitter.obtain_shape_from_bb( np.array([[53.916, 1.853], [408.469, 339.471]])) assert ((np.around(s.points) == np.around(initial_shape[0].points)).all()) assert (s.n_dims == 2) assert (s.n_landmark_groups == 0) assert (s.n_points == 68)
def _load_landmark_fitter(self): if self._fitter_type == 'aam': self._aam_fitter = mio.import_pickle(self._aam_fitter_file) fitter = LucasKanadeAAMFitter(self._aam_fitter, lk_algorithm_cls=WibergInverseCompositional, n_shape=self._shape_components, n_appearance=self._appearance_components) elif self._fitter_type == 'ert': fitter = DlibWrapper( path.join(current_path, '../pretrained/shape_predictor_68_face_landmarks.dat')) else: raise Exception('unknown fitter, did you mean aam/ert?') self._landmark_fitter = fitter
def train(img_generator): # clean up the images with the standard menpo pre-processing images = [menpo_img_process(img) for img in img_generator] # build the AAM # TODO implement settings for strongest AAM we can for menpofit aam = AAMBuilder(features=fast_dsift, normalization_diagonal=120).build(images, verbose=True, group='gt') fitter = LucasKanadeAAMFitter(aam, n_shape=[3, 6, 12], n_appearance=0.9) # return a callable that wraps the menpo fitter in order to integrate with # menpobench return MenpoFitWrapper(fitter)
def aam_helper(aam, algorithm, im_number, max_iters, initial_error, final_error, error_type): fitter = LucasKanadeAAMFitter(aam, algorithm=algorithm) fitting_result = fitter.fit( training_images[im_number], initial_shape[im_number], gt_shape=training_images[im_number].landmarks[None].lms, max_iters=max_iters) assert_allclose( np.around(fitting_result.initial_error(error_type=error_type), 5), initial_error) assert_allclose( np.around(fitting_result.final_error(error_type=error_type), 5), final_error)
def train_aam(path_to_images): training_images = mio.import_images(path_to_images, verbose=True) training_images = training_images.map(process) aam = HolisticAAM(training_images, group='PTS', diagonal=150, scales=(0.5, 1.0), verbose=True, #holistic_features=double_igo, max_shape_components=40, max_appearance_components=300 ) fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=AlternatingInverseCompositional, n_shape=[10, 40], n_appearance=[60, 300] ) return aam, fitter
def _load_landmark_fitter(self): if self._fitter_type == 'aam': self._aam_fitter = mio.import_pickle(self._aam_fitter_file) fitter = LucasKanadeAAMFitter( self._aam_fitter, lk_algorithm_cls=WibergInverseCompositional, n_shape=self._shape_components, n_appearance=self._appearance_components) elif self._fitter_type == 'ert': _, _, fitter68 = maybe_download_models() fitter = DlibWrapper(fitter68) else: raise Exception('unknown fitter, did you mean aam/ert?') self._landmark_fitter = fitter
def eval_aams(aam_file, images: List[Image]): aam = joblib.load(aam_file) img_size = int(aam_file.split("resolution_")[1].split("_", 1)[0]) img_size = (img_size, img_size // 4) fitter = LucasKanadeAAMFitter(aam) for i in tqdm(images): i = i.resize(img_size) fitting_result = fitter.fit_from_bb( i, i.landmarks[i.landmarks.group_labels[-1]].bounding_box(), max_iters=50, gt_shape=i.landmarks[i.landmarks.group_labels[-1]], ) print("")
def Train(self, i_diag=150, i_scale=[0.5, 1.0], i_max_greyscale_dims=200, i_max_shape_dims=20): # laterals tuned for performance gain - Sacrifice mouth modes self.model = HolisticAAM( self.LoadDataset(), group='PTS', verbose=True, holistic_features=float32_fast_dsift, diagonal=i_diag, scales=i_scale, max_appearance_components=i_max_greyscale_dims, max_shape_components=i_max_shape_dims) self.fitter = LucasKanadeAAMFitter(self.model, n_shape=[5, 15], n_appearance=[50, 150])
def process_data(dir): dirlist = os.listdir(dir) dirlist = [d for d in dirlist if not os.path.isfile(os.path.join(dir, d))] dirlist = sorted(dirlist) # load the aam model aam = mio.import_pickle("aam.pkl") # create fiiter fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=16, n_appearance=104) # Load detector detector = load_dlib_frontal_face_detector() #load the sentences for j, subdir in enumerate(dirlist): ids = os.listdir(os.path.join(dir, subdir, "video/")) ids = [i for i in ids if "head" not in i] ids = sorted(ids) for k, id in enumerate(ids): t = time.time() video = os.path.join(dir, subdir, "video", id + '/') process_one_sentence(video, fitter, detector) print(j, k, time.time() - t)
from menpofit.aam import HolisticAAM aam = HolisticAAM(training_images, group='PTS', verbose=True, holistic_features=custom_double_igo, diagonal=120, scales=(0.5, 1.0)) print(aam) aam.view_aam_widget() from menpofit.aam import LucasKanadeAAMFitter fitter = LucasKanadeAAMFitter(aam, n_shape=[6, 12], n_appearance=0.5) from menpofit.fitter import noisy_shape_from_bounding_box fitting_results = [] for i in test_images: # obtain original landmarks gt_s = i.landmarks['PTS'].lms # generate perturbed landmarks s = noisy_shape_from_bounding_box(gt_s, gt_s.bounding_box()) # fit image fr = fitter.fit_from_shape(i, s, gt_shape=gt_s) fitting_results.append(fr)
def fit(path_to_images, path_to_test, c, r, w): training_images = [] for img in print_progress(mio.import_images(path_to_images, verbose=True)): # convert to greyscale if img.n_channels == 3: img = img.as_greyscale() # crop to landmarks bounding box with an extra 20% padding img = img.crop_to_landmarks_proportion(0.2) # rescale image if its diagonal is bigger than 400 pixels d = img.diagonal() if d > 1000: img = img.rescale(1000.0 / d) # define a TriMesh which will be useful for Piecewise Affine Warp of HolisticAAM # labeller(img, 'PTS', face_ibug_68_to_face_ibug_68_trimesh) # append to list training_images.append(img) # ## Training ribcage - Patch # from menpofit.aam import PatchAAM # from menpo.feature import fast_dsift # # patch_aam = PatchAAM(training_images, group='PTS', patch_shape=[(15, 15), (23, 23)], # diagonal=500, scales=(0.5, 1.0), holistic_features=fast_dsift, # max_shape_components=20, max_appearance_components=150, # verbose=True) ## Training ribcage - Holistic patch_aam = HolisticAAM(training_images, group='PTS', diagonal=500, scales=(0.5, 1.0), holistic_features=fast_dsift, verbose=True, max_shape_components=20, max_appearance_components=150) ## Prediction fitter = LucasKanadeAAMFitter(patch_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[5, 20], n_appearance=[30, 150]) image = mio.import_image(path_to_test) #initialize box adjacency_matrix = np.array([ [0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [1, 0, 0, 0], ]) # points = np.array([[0,0], [0,2020], [2020, 2020], [2020, 0]]) points = np.array([[r - w / 2, c - w / 2], [r - w / 2, c + w / 2], [r + w / 2, c + w / 2], [r + w / 2, c - w / 2]]) graph = PointDirectedGraph(points, adjacency_matrix) box = graph.bounding_box() # initial bbox initial_bbox = box # fit image result = fitter.fit_from_bb(image, initial_bbox, max_iters=[15, 5]) pts = result.final_shape.points return pts
def test_max_iters_exception(): fitter = LucasKanadeAAMFitter(aam, algorithm=AlternatingInverseCompositional) fitter.fit(training_images[0], initial_shape[0], max_iters=[10, 20, 30, 40])
if d > 400: img = img.rescale(400.0 / d) # append to list training_images.append(img) patch_aam = PatchAAM(training_images, group='PTS', patch_shape=[(16, 19), (19, 16)], diagonal=200, scales=(0.5, 1.0), holistic_features=fast_dsift, max_shape_components=60, max_appearance_components=200, verbose=True) fitter = LucasKanadeAAMFitter(patch_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 30], n_appearance=[40, 160]) adjacency_matrix = np.array([[0, 1, 0, 0], [0, 0, 1, 0], [0, 0, 0, 1], [1, 0, 0, 0]]) def chen_get_bbx(txt_path): coordinate = np.loadtxt(txt_path, comments='\n', delimiter=',') y, x = coordinate.T max_x = max(x) min_x = min(x) max_y = max(y) min_y = min(y) - 18 points = np.array([[min_x, min_y], [min_x, max_y], [max_x, max_y],
def auto_construct(pdm, images, trilist=None, fit_group='init', train_group='final', models=[], errors=[], costs=[], isplot=False, feature=[igo] * 10, diagonal=200, scales=(0.5, 1.0), n_shape=[2, 4], n_appearance=[20, 30], max_iters=10, generative_iter=30, discriminative_iter = 10, n_processes=24, inc_appearance=0, model_class=HolisticAAM, increament=False, update_shape=False, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, export_path=None ): # initialisation DB_size = len(images) / 2 DB1 = images[:DB_size] DB2 = images[DB_size:] init_shape = pdm.shape_models[-1].model.mean() n_iteration = 0 if trilist is None: trilist = TriMesh(init_shape.points).trilist for j in xrange(discriminative_iter): i_appearance = np.array(n_appearance) + np.array(inc_appearance) if (i_appearance > 1).any(): i_appearance = i_appearance.astype(int).tolist() else: i_appearance = i_appearance.tolist() # ------------ generative iterations ------------- for i in xrange(generative_iter): print 'Discriminative Iter: {}, Generative Iter: {}'.format(j, i) aam_fitter = LucasKanadeAAMFitter(pdm, n_shape=n_shape, n_appearance=i_appearance) pdm, error = generative_construct( DB1, aam_fitter, trilist, fit_group=fit_group, train_group=train_group, label='iteration_{:03d}'.format(n_iteration), feature=feature[j], diagonal=diagonal, scales=scales, original_shape_model=None if update_shape else pdm.shape_models, n_processes=n_processes, model_class=model_class, increament_model=pdm if increament else None, shape_forgetting_factor=shape_forgetting_factor, appearance_forgetting_factor=appearance_forgetting_factor, max_iters=max_iters ) n_iteration += 1 models.append(pdm) errors.append(error) if export_path: mio.export_pickle([images, models, errors], export_path, overwrite=True) if isplot: plot(errors) # ----------- discriminative iterations------------ aam_fitter = LucasKanadeAAMFitter(pdm, n_shape=n_shape, n_appearance=i_appearance) frs = mp_fit(DB2, aam_fitter, group=fit_group, n_processes=n_processes, max_iters=max_iters) for img, fr in zip(DB2, frs): img.landmarks[train_group] = fr.final_shape sdm = RegularizedSDM( DB2, diagonal=diagonal, alpha=100, group=train_group, n_iterations=4, scales=(0.5,0.5,1.0,1.0), patch_features=patch_features, n_perturbations=30, patch_shape=[(25, 25), (15, 15), (15,15), (9,9)], verbose=True ) pdm, error = generative_construct( DB1, sdm, trilist, fit_group=fit_group, train_group=train_group, label='discriminative_{:02d}'.format(j), original_shape_model=None if update_shape else pdm.shape_models, feature=feature[j], diagonal=diagonal, scales=scales, n_processes=n_processes, model_class=model_class, increament_model=pdm if increament else None, shape_forgetting_factor=shape_forgetting_factor, appearance_forgetting_factor=appearance_forgetting_factor, max_iters=max_iters ) models.append(pdm) errors.append(error) if export_path: mio.export_pickle([images, models, errors], export_path, overwrite=True) if isplot: plot(errors) return models[-2]
def test_n_appearance_exception(): fitter = LucasKanadeAAMFitter(aam, n_appearance=[10, 20])
def test_n_shape_exception(): fitter = LucasKanadeAAMFitter(aam, n_shape=[3, 6, 'a'])
def __init__(self, extract_opts=None, process_opts=None, output_dir=None): r""" Parameters ---------- extract_opts : `dict` holding the configuration for feature extraction For complete description of some parameters, please refer upstream to their documentation in the menpofit project Must specify the following options: ``warp`` : `holistic` or `patch`; chooses between menpofit.aam.HolisticAAM and menpofit.aam.PatchAAM ``resolution_scales`` : `tuple` of `floats` between 0.0 and 1.0 A pyramid of AAMs will be created, one for each element in the tuple A value of 1.0 corresponds to the full resolution images, 0.5 to a half and so on ``patch_shape`` : `tuple` of `tuple` of two `ints` Parameter required when ``warp`` is `patch` One tuple per resolution scale The patch shape is specified as a window of MxN pixels around each landmark ``max_shape_components`` : `int` or `list` of `ints` maximum number of eigenvectors (per resolution scale) kept from shape PCA True value can be less that max, depending on the variance in the training images ``max_appearance_components: `int` or `list` of `ints` maximum number of eigenvectors (per resolution scale) kept from texture PCA True value can be less that max, depending on the variance in the training images ``diagonal`` : `int` serving as the diagonal size of the rescaled training images ``features`` : `no_op`, `hog`, `dsift`, `fast_dsift` `no_op` uses the image pixels for the texture model `hog, dsift, fast_dsift` extract popular image descriptors instead ``landmark_dir`` : `str`, directory containing the facial landmarks for the training images ``landmark_group`` : `pts_face`, `pts_chin`, `pts_lips` `pts_face` constructs a full facial model using all the 68 landmark points `pts_chin` uses landmarks [2:15) plus [48:68) to model the chin and lips region `pts_lips` uses only [48:68) to model the lip region ``confidence_thresh`` : `float` in range [0:1] Makes use of the OpenFace average confidence score, keeping only the frames above this threshold ``kept_frames`` : `float` in range [0:1] Samples the remaining video frames (above the confidence threshold) to keep only a small proportion This avoids training the AAM with a large number of consecutive video frames Before sampling, the frames from each video are sorted by the amount of lip opening. Then sampling is done at evenly spaced intervals ``greyscale`` : `boolean`; if ``True``, converts the frames to a single channel of grey / luminance levels if ``False``, the model is built on the original RGB channels ``model_name`` : `str`; name of the AAM pickle object to be stored offline process_opts: `dict` holding the configuration for feature processing Must specif y the following options: ``face_detector`` : `dlib` or `opencv` or `dpm` Selects the implementation that detects a face in an image `dlib` is the fastest, `dpm` may be more accurate (check G.Chrysos, Feb 2017) ``landmark_fitter : `aam` or `ert` Selects the algorithm that fits the landmarks on a detected face `ert` uses a model pre-trained on challenging datasets `aam` may use your own model ``aam_fitter`` : `str`, full file name storing an AAM pickle to be used for landmark fitting Mandatory if ``landmark_fitter`` is AAM ``parameters_from`` : `fitting`, `projection` If `fitting`, the shape and appearance parameters optimized by the Lukas-Kanade fitting algorithm are returned. If `projection`, only the final shape of the fitting process will be used, initializing another fitter based on a new AAM specified below `` projection_aam`` : `str`, full file name storing an AAM pickle to be used in the process described above ``shape`` : `face`, `chin` or `lips` Chooses an AAM that may describe an entire face, or sub-parts of it If `chin` or `lips`, the associated landmarks will be selected from the face fitting process, then a few more iterations of a fitting algorithm will be run using the part AAM specified below ``part_AAM`` : `None` or a `str` representing the file storing a part AAM pickle (chin or lips) Must be different from `None` if `shape` is `chin` or `lips` Such part_AAM can be obtained by choosing the ``landmark_group`` parameter accordingly in the extraction process ``confidence_thresh`` : `float`, DEPRECATED It was used to filter out the frames having a confidence threshold for the landmarks lower than this value. Their corresponding features were simply arrays of zeros. Now we consider every frame where a face is detected. ``shape_components`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of the kept shape eigenvectors for the projection and fitter AAMs The shape feature size will be up to this value ``appearance_components`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of the kept texture eigenvectors for the projection and fitter AAMs The appearance feature size will be up to this value ``max_iters`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of iterations (per resolution scale) of the optimisation algorithm Only used for the fitter AAM, since 0 iterations are used with the projection AAM ``landmark_dir`` : `str`, directory containing the ground-truth facial landmarks for every frame of each video. Used only to compute an error between prediction and ground-truth. Can be `None` if the error log is not necessary ``log_errors`` : `boolean` If ``True``, generates a log file per video, stating the models used and the prediction error for each frame ``log_dir`` : `str`, directory to store the error logs above output_dir : `str`, absolute path where the features are to be stored """ self._outDir = output_dir if extract_opts is not None: self._extractOpts = extract_opts self._warpType = extract_opts['warp'] self._landmarkDir = extract_opts['landmark_dir'] self._landmarkGroup = extract_opts['landmark_group'] self._max_shape_components = extract_opts['max_shape_components'] self._max_appearance_components = extract_opts['max_appearance_components'] self._diagonal = extract_opts['diagonal'] self._scales = extract_opts['resolution_scales'] self._confidence_thresh = extract_opts['confidence_thresh'] self._kept_frames = extract_opts['kept_frames'] if extract_opts['features'] == 'fast_dsift': self._features = fast_dsift elif extract_opts['features'] == 'dsift': self._features = dsift elif extract_opts['features'] == 'hog': self._features = hog elif extract_opts['features'] == 'no_op': self._features = no_op else: raise Exception('Unknown feature type to extract, did you mean fast_dsift ?') if 'greyscale' in extract_opts.keys(): self._greyscale = extract_opts['greyscale'] else: self._greyscale = False self._outModelName = extract_opts['model_name'] if process_opts is not None: # Face detection self._face_detect_method = process_opts['face_detector'] if self._face_detect_method == 'dlib': from menpodetect import load_dlib_frontal_face_detector detector = load_dlib_frontal_face_detector() elif self._face_detect_method == 'opencv': from menpodetect import load_opencv_frontal_face_detector detector = load_opencv_frontal_face_detector() elif self._face_detect_method == 'dpm': from menpodetect.ffld2 import load_ffld2_frontal_face_detector detector = load_ffld2_frontal_face_detector() else: raise Exception('unknown detector, did you mean dlib/opencv/dpm?') self._face_detect = detector self._shape_components = process_opts['shape_components'] self._appearance_components = process_opts['appearance_components'] self._max_iters = process_opts['max_iters'] self._fitter_type = process_opts['landmark_fitter'] # Landmark fitter (pretrained ERT or AAM), actually loaded later to avoid pickling with Pool if self._fitter_type == 'aam': self._aam_fitter_file = process_opts['aam_fitter'] # Parameters source # If fitting, self._parameters = process_opts['parameters_from'] if self._parameters == 'aam_projection': self._projection_aam_file = process_opts['projection_aam'] self._projection_aam = mio.import_pickle(self._projection_aam_file) self._projection_fitter = LucasKanadeAAMFitter( aam=self._projection_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=self._shape_components, n_appearance=self._appearance_components) else: pass self._confidence_thresh = process_opts['confidence_thresh'] self._landmarkDir = process_opts['landmark_dir'] self._shape = process_opts['shape'] self._part_aam = process_opts['part_aam'] self._log_errors = process_opts['log_errors'] if self._log_errors is False: self._myresolver = None self._log_dir = process_opts['log_dir']
def process_clip(clip_name, paths, training_images, img_type, loop, svm_params, mi=110, d_aam=130, n_s=None, n_a=None): """ Processes a clip. Accepts a clip (along with its params and paths), trains a person-specific part based AAM (pbaam) and then fits it to all the frames. :param clip_name: str: Name of the clip. :param paths: dict: Required paths for training/fitting/exporting data. :param training_images: list: List of menpo images (generic images) appended to the person specific ones. :param img_type: str: Suffix (extension) of the frames, e.g. '.png'. :param loop: bool: Declares whether this is a 2nd fit for AAM (loop). :param svm_params: dict: Required params for SVM classification. If 'apply' is False, the rest are not used. Otherwise, requires reference frame and classifier loaded. :param mi: int: (optional) Max images of the clip loaded for the pbaam. :param d_aam: int: (optional) Diagonal of AAM (param in building it). :param n_s: int/list/None: (optional) Number of shapes for AAM (as expected in menpofit). :param n_a: int/list/None: (optional) Number of appearances for AAM (as expected in menpofit). :return: """ global fitter # paths and list of frames frames_path = paths['clips'] + frames + clip_name + sep if not check_path_and_landmarks(frames_path, clip_name, paths['in_lns'] + clip_name): return False list_frames = sorted(os.listdir(frames_path)) pts_p = mkdir_p(paths['out_lns'] + clip_name + sep) svm_p = mkdir_p(paths['out_svm'] + clip_name + sep) # svm path # loading images from the clip training_detector = load_images(list(list_frames), frames_path, paths['in_lns'], clip_name, training_images=list(training_images), max_images=mi) print('\nBuilding Part based AAM for the clip {}.'.format(clip_name)) aam = PatchAAM(training_detector, verbose=True, holistic_features=features, patch_shape=patch_shape, diagonal=d_aam, scales=(.5, 1)) del training_detector fitter = LucasKanadeAAMFitter(aam, n_shape=n_s, n_appearance=n_a) # save the AAM model (requires plenty of disk space for each model). aam.features = None export_pickle(aam, paths['out_model'] + clip_name + '.pkl', overwrite=True) del aam clip = Clip(clip_name, paths['clips'], frames, [paths['in_lns'], paths['in_fit_lns']], [pts_p, svm_p]) # [process_frame(frame_name, clip, img_type, svm_params,loop) for frame_name in list_frames]; Parallel(n_jobs=-1, verbose=4)( delayed(process_frame)(frame_name, clip, img_type, svm_params, loop) for frame_name in list_frames) fitter = [] # reset fitter return True
def get_feature(self, file, process_opts=None): r""" Computes the AAM features, according to the `process_opts` Parameters ---------- file process_opts Returns ------- A dictionary of five elements, each representing a variation of the computed features (shape and appearance alone or concatenated, with or without derivatives) """ self._maybe_start_logging(file) self._load_landmark_fitter() frames = mio.import_video(file, landmark_resolver=self._myresolver, normalize=True, exact_frame_count=True) feat_shape = [] feat_app = [] feat_shape_app = [] for frameIdx, frame in enumerate(frames): bounding_boxes = self._face_detect(frame) if len(bounding_boxes) > 0: initial_bbox = bounding_boxes[0] if self._log_errors is True: gt_shape = frame.landmarks['pts_face'] else: gt_shape = None if isinstance(self._landmark_fitter, LucasKanadeAAMFitter): result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, max_iters=self._max_iters, gt_shape=gt_shape) elif isinstance(self._landmark_fitter, DlibWrapper): # DLIB fitter, doesn't have max_iters result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, gt_shape=gt_shape) else: raise Exception('incompatible landmark fitter') self._maybe_append_to_log(file, frameIdx, result) if self._shape == 'face': if self._parameters == 'lk_fitting': # skip the first 4 similarity params, probably not useful for classification shape_param_frame = result.shape_parameters[-1][4:] app_param_frame = result.appearance_parameters[-1] elif self._parameters == 'aam_projection': result_aam = self._projection_fitter.fit_from_shape( frame, result.final_shape, max_iters=[0, 0, 0]) # TODO: analyse the case when aam true components are less than max components shape_param_frame = result_aam.shape_parameters[-1][4:] app_param_frame = result_aam.appearance_parameters[-1] feat_shape.append(shape_param_frame) feat_app.append(app_param_frame) feat_shape_app.append(np.concatenate((shape_param_frame, app_param_frame))) elif self._shape == 'lips': # extract lips landmarks from the final face fitting to initialize the part model fitting aam_lips = mio.import_pickle(self._part_aam) fitter_lips = LucasKanadeAAMFitter(aam_lips, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20], n_appearance=[20, 150]) result_lips = fitter_lips.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'lips'), max_iters=[5, 5]) shape_param_frame_lips = result_lips.shape_parameters[-1][4:] app_param_frame_lips = result_lips.appearance_parameters[-1] feat_shape.append(shape_param_frame_lips) feat_app.append(app_param_frame_lips) feat_shape_app.append(np.concatenate((shape_param_frame_lips, app_param_frame_lips))) elif self._shape == 'chin': # extract chin and lips landmarks from the final face fitting to initialize the part model fitting aam_chin = mio.import_pickle(self._part_aam) fitter_chin = LucasKanadeAAMFitter(aam_chin, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20, 25], n_appearance=[20, 50, 150]) result_chin = fitter_chin.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'chin'), max_iters=[10, 10, 5]) shape_param_frame_mchin = result_chin.shape_parameters[-1][4:] app_param_frame_mchin = result_chin.appearance_parameters[-1] feat_shape.append(shape_param_frame_mchin) feat_app.append(app_param_frame_mchin) feat_shape_app.append(np.concatenate((shape_param_frame_mchin, app_param_frame_mchin))) else: raise Exception('Unknown shape model, currently supported are: face, lips, chin') else: # we did not detect any face zero_feat_shape = np.zeros(process_opts['shape_components'][-1]) zero_feat_app = np.zeros(process_opts['appearance_components'][-1]) zero_feat_shape_app = np.zeros( process_opts['shape_components'][-1] + process_opts['appearance_components'][-1]) feat_shape.append(zero_feat_shape) feat_app.append(zero_feat_app) feat_shape_app.append(zero_feat_shape_app) npfeat_shape = np.array(feat_shape) npfeat_app = np.array(feat_app) npfeat_shape_app = np.array(feat_shape_app) npfeat_app_delta = vsrmath.accurate_derivative(npfeat_app, 'delta') npfeat_shape_app_delta = vsrmath.accurate_derivative(npfeat_shape_app, 'delta') return {'shape': npfeat_shape, 'app': npfeat_app, 'shape_app': npfeat_shape_app, 'app_delta': npfeat_app_delta, 'shape_app_delta': npfeat_shape_app_delta}
def getAAMFitter(self, aam): aam_fitter = LucasKanadeAAMFitter(aam, n_shape=[5, 15], n_appearance=[50, 150]) return aam_fitter
return image #训练集 path_to_images = 'D:/电信研究院/人脸矫正/labelme-master/examples/transfer' training_images = [] for img in print_progress(mio.import_images(path_to_images, verbose=True)): training_images.append(process(img)) # %matplotlib inline,这步只能在jupyter里可视化 # from menpowidgets import visualize_images # visualize_images(training_images) aam = HolisticAAM(training_images, reference_shape=None, diagonal=150, scales=(0.5, 1.0), holistic_features=igo, verbose=True) fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[3, 20], n_appearance=[30, 150]) # % matplotlib inline # method to load a database def load_database(path_to_images, crop_percentage, max_images=None): images = [] # load landmarked images for i in mio.import_images(path_to_images, max_images=max_images, verbose=True): # crop image i = i.crop_to_landmarks_proportion(crop_percentage) # convert it to grayscale if needed if i.n_channels == 3: i = i.as_greyscale(mode='luminosity')
def aam_fit_benchmark(fitting_images, aam, fitting_options=None, perturb_options=None, verbose=False): r""" Fits a trained AAM model to a database. Parameters ---------- fitting_images: list of :class:MaskedImage objects A list of the fitting images. aam: :class:menpo.fitmultilevel.aam.AAM object The trained AAM object. It can be generated from the aam_build_benchmark() method. fitting_options: dictionary, optional A dictionary with the parameters that will be passed in the LucasKanadeAAMFitter (:class:menpo.fitmultilevel.aam.base). If None, the default options will be used. This is an example of the dictionary with the default options: fitting_options = {'algorithm': AlternatingInverseCompositional, 'md_transform': OrthoMDTransform, 'global_transform': AlignmentSimilarity, 'n_shape': None, 'n_appearance': None, 'max_iters': 50, 'error_type': 'me_norm' } For an explanation of the options, please refer to the LucasKanadeAAMFitter documentation. Default: None bounding_boxes: list of (2, 2) ndarray, optional If provided, fits will be initialized from a bounding box. If None, perturbation of ground truth will be used instead. can be provided). Interpreted as [[min_x, min_y], [max_x, max_y]]. perturb_options: dictionary, optional A dictionary with parameters that control the perturbation on the ground truth shape with noise of specified std. Note that if bounding_box is provided perturb_options is ignored and not used. If None, the default options will be used. This is an example of the dictionary with the default options: initialization_options = {'noise_std': 0.04, 'rotation': False } For an explanation of the options, please refer to the perturb_shape() method documentation of :map:`MultilevelFitter`. verbose: bool, optional If True, it prints information regarding the AAM fitting including progress bar, current image error and percentage of images with errors less or equal than a value. Default: False Returns ------- fitting_results: :map:`FittingResultList` A list with the :map:`FittingResult` object per image. """ if verbose: print('AAM Fitting:') perc1 = 0. perc2 = 0. # parse options if fitting_options is None: fitting_options = {} if perturb_options is None: perturb_options = {} # extract some options group = fitting_options.pop('gt_group', 'PTS') max_iters = fitting_options.pop('max_iters', 50) error_type = fitting_options.pop('error_type', 'me_norm') # create fitter fitter = LucasKanadeAAMFitter(aam, **fitting_options) # fit images n_images = len(fitting_images) fitting_results = [] for j, i in enumerate(fitting_images): # perturb shape gt_s = i.landmarks[group].lms if 'bbox' in i.landmarks: # shape from bounding box s = fitter.obtain_shape_from_bb(i.landmarks['bbox'].lms.points) else: # shape from perturbation s = fitter.perturb_shape(gt_s, **perturb_options) # fit fr = fitter.fit(i, s, gt_shape=gt_s, max_iters=max_iters) fitting_results.append(fr) # print final_error = fr.final_error(error_type=error_type) initial_error = fr.initial_error(error_type=error_type) if verbose: if error_type == 'me_norm': if final_error <= 0.03: perc1 += 1. if final_error <= 0.04: perc2 += 1. elif error_type == 'rmse': if final_error <= 0.05: perc1 += 1. if final_error <= 0.06: perc2 += 1. print_dynamic('- {0} - [<=0.03: {1:.1f}%, <=0.04: {2:.1f}%] - ' 'Image {3}/{4} (error: {5:.3f} --> {6:.3f})'.format( progress_bar_str(float(j + 1) / n_images, show_bar=False), perc1 * 100. / n_images, perc2 * 100. / n_images, j + 1, n_images, initial_error, final_error)) if verbose: print_dynamic('- Fitting completed: [<=0.03: {0:.1f}%, <=0.04: ' '{1:.1f}%]\n'.format(perc1 * 100. / n_images, perc2 * 100. / n_images)) return fitting_results