class AAM: ''' Initialise members and train AAM \param[in] i_dataset Full filepath to training and test dataset \param[in] i_debug True to display debug info ''' def __init__(self, i_dataset, i_debug=False): self.debug = i_debug self.dataset = i_dataset if not os.path.exists(self.dataset): raise RuntimeError('Database dir does not exist in ' + self.dataset) self.Train() self.viola_face_detector = FaceDetectViola(False) self.menpo_face_detector = FaceDetectMenpo() if self.debug: self.PrintDebug() ''' Load training images and annotated landmarks from a training set in the file system ''' def LoadDataset(self): trainset = os.path.join(self.dataset, 'trainset', '*') training_images = [ self.LoadImage(img) for img in menpoio.import_images(trainset, verbose=True) ] return training_images ''' Crops image landmarks (0.1 referenced from AAMs Basics) and convert to greyscale \param[in] i_img Menpo image to process \return processed menpo image ''' def LoadImage(self, i_img, i_landmark_crop=0.5): img = i_img.crop_to_landmarks_proportion(i_landmark_crop) img = GreyscaleConversionMenpo(img) return img ''' Train an Active Appearance Model and compute the \param[in] i_diag Search gradient along model landmark \param[in] i_scale Scale applied to search direction (search) || (initial, search) \param[in] i_max_greyscale_dims Dimensionality limit for PCA appearance model \param[in] i_max_shape_dims Dimensionality limit for PCA keypoint components ''' def Train(self, i_diag=150, i_scale=[0.5, 1.0], i_max_greyscale_dims=200, i_max_shape_dims=20): # laterals tuned for performance gain - Sacrifice mouth modes self.model = HolisticAAM( self.LoadDataset(), group='PTS', verbose=True, holistic_features=float32_fast_dsift, diagonal=i_diag, scales=i_scale, max_appearance_components=i_max_greyscale_dims, max_shape_components=i_max_shape_dims) self.fitter = LucasKanadeAAMFitter(self.model, n_shape=[5, 15], n_appearance=[50, 150]) ''' Fit an appearance model to an image with annotated landmarks \return Converged candidate fit ''' def FitAnnotatedImage(self, i_img): gt = i_img.landmarks['PTS'].lms initial_shape = self.fitter.perturb_from_bb(gt, gt.bounding_box()) return self.fitter.fit_from_shape(i_img, initial_shape, gt_shape=gt) ''' Fit an appearance model to an image without annotations using Menpo Face Detection \return Converged landmarks ''' def FitWildImageMenpo(self, i_img, i_initial_guess=None, i_max_iters=10): # Convert menpo image to expected openCV format i_img = GreyscaleConversionMenpo(i_img) ret = None if i_initial_guess is not None: pts = menpo.shape.PointCloud(i_initial_guess, False) ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points else: bb = self.menpo_face_detector.Detect(i_img) if bb is not None: ret = self.fitter.fit_from_bb(i_img, bb, i_max_iters).final_shape.points return ret ''' Fit an appearance model to an image without annotations using Viola Face Detection \return Converged landmarks ''' def FitWildImageViola(self, i_img, i_initial_guess=None, i_max_iters=10): # Convert menpo image to expected openCV format i_img = GreyscaleConversionMenpo(i_img) img = i_img.pixels[0] * 255 img = numpy.array(img, dtype=numpy.uint8) # Detect face with experiment tuning according to lfpw testset ret = None if i_initial_guess is None: faces = self.viola_face_detector.Detect(img, 3, 1.1, 0.125, 1.0) # Fit candidate model if len(faces) > 1: faces = [GetLargestROI(faces)] faces = ConvertRectToMenpoBoundingBox(faces) fit = self.fitter.fit_from_bb(i_img, faces[0], i_max_iters) ret = fit.final_shape.points elif i_initial_guess is not None: pts = menpo.shape.PointCloud(i_initial_guess, False) ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points return ret ''' Print debug information for the AAM class ''' def PrintDebug(self): print('Dataset', self.dataset) print self.model
class ActiveAppearanceModel(SegmentationModel): def __init__(self, join_masks=True, scales=[15, 23]): self.mean_shape = None self.image_shape = None self.fitter = None self.num_landmarks_by_shape = None self.join_masks = join_masks self.scales = scales def normalize(self, image): mean = np.mean(image) std = np.std(image) image = (image - mean)/std # FIX: min_ = np.min(image) max_ = np.max(image) image = (image - min_)/(max_ - min_ + 1e-7) return image def prepare_data_for_aam(self, images, landmarks): images_aam = [] for i in range(len(images)): images[i] = self.normalize(images[i]) images_aam.append(Image(images[i])) lmx = landmarks[0][i].T[0] lmy = landmarks[0][i].T[1] num_points = len(landmarks[0]) rmx = landmarks[1][i].T[0] rmy = landmarks[1][i].T[1] pc = PointCloud(points=np.vstack((np.array([lmy, lmx]).T, np.array([rmy, rmx]).T))) lg = LandmarkGroup.init_from_indices_mapping(pc, OrderedDict({'left':range(len(landmarks[0][i])), 'right':range(len(landmarks[0][i]), len(landmarks[0][i]) + len(landmarks[1][i]))})) lm = LandmarkManager() lm.setdefault('left', lg) images_aam[-1].landmarks = lm return images_aam def fit(self, images, landmarks): num_samples = len(landmarks[0]) self.num_landmarks_by_shape = [] for i in range(len(landmarks)): self.num_landmarks_by_shape.append(len(landmarks[i][0])) aam_images = self.prepare_data_for_aam(images, landmarks) aam = PatchAAM(aam_images, group=None, patch_shape=[(self.scales[0], self.scales[0]), (self.scales[1], self.scales[1])], diagonal=150, scales=(0.5, 1.0), holistic_features=fast_dsift, max_shape_components=20, max_appearance_components=150, verbose=True) self.fitter = LucasKanadeAAMFitter(aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=[5, 20], n_appearance=[30, 150]) pc = [] for img in aam_images: pc.append(img.landmarks[None].lms) self.mean_shape = mean_pointcloud(pc) self.image_shape = images[0].shape fitting_results = [] for img in aam_images[:10]: fr = self.fitter.fit_from_shape(img, self.mean_shape, gt_shape=img.landmarks[None].lms) fitting_results.append(fr) def transform_one_image(self, image): print 'init ...', mask = np.zeros(shape=image.shape) image = Image(self.normalize(image)) fr = self.fitter.fit_from_shape(image, self.mean_shape) pred_landmarks = fr.final_shape.points begin = 0 masks = [] for i in range(0, len(self.num_landmarks_by_shape)): masks.append(self.create_mask_from_landmarks(pred_landmarks[begin:begin + self.num_landmarks_by_shape[i]])) mask = np.logical_or(mask, masks[-1]) begin += self.num_landmarks_by_shape[i] print 'done' if self.join_masks: return mask else: return masks
# append it to the list images.append(i) return images # 测试集 image_path_test = "D:/电信研究院/人脸矫正/labelme-master/examples/transfer/test" test_images = load_database(image_path_test, 0.5, max_images=5) fitting_results = [] for i in test_images: # obtain original landmarks gt_s = i.landmarks['PTS'].lms # generate perturbed landmarks s = noisy_shape_from_bounding_box(gt_s, gt_s.bounding_box()) # fit image fr = fitter.fit_from_shape(i, s, gt_shape=gt_s) fitting_results.append(fr) print('fr',type(fr),fr.final_shape.points,fr) # from menpowidgets import visualize_fitting_result # #预测 # image_path_pred = "D:/电信研究院/人脸矫正/labelme-master/examples/transfer/pred" # # pred_images = [] # # load landmarked images # for i in mio.import_images(image_path_pred, max_images=None, verbose=True): # # convert it to grayscale if needed # if i.n_channels == 3: # i = i.as_greyscale(mode='luminosity') # # # append it to the list # pred_images.append(i)
class AAM: ''' Initialise members and train AAM \param[in] i_dataset Full filepath to training and test dataset \param[in] i_debug True to display debug info ''' def __init__(self, i_dataset, i_debug = False): self.debug = i_debug self.dataset = i_dataset if not os.path.exists(self.dataset): raise RuntimeError('Database dir does not exist in ' + self.dataset) self.Train() self.viola_face_detector = FaceDetectViola(False) self.menpo_face_detector = FaceDetectMenpo() if self.debug: self.PrintDebug() ''' Load training images and annotated landmarks from a training set in the file system ''' def LoadDataset(self): trainset = os.path.join(self.dataset,'trainset','*') training_images = [self.LoadImage(img) for img in menpoio.import_images(trainset, verbose = True) ] return training_images ''' Crops image landmarks (0.1 referenced from AAMs Basics) and convert to greyscale \param[in] i_img Menpo image to process \return processed menpo image ''' def LoadImage(self, i_img, i_landmark_crop = 0.5): img = i_img.crop_to_landmarks_proportion( i_landmark_crop ) img = GreyscaleConversionMenpo(img) return img ''' Train an Active Appearance Model and compute the \param[in] i_diag Search gradient along model landmark \param[in] i_scale Scale applied to search direction (search) || (initial, search) \param[in] i_max_greyscale_dims Dimensionality limit for PCA appearance model \param[in] i_max_shape_dims Dimensionality limit for PCA keypoint components ''' def Train(self, i_diag = 150, i_scale = [0.5, 1.0], i_max_greyscale_dims = 200, i_max_shape_dims = 20): # laterals tuned for performance gain - Sacrifice mouth modes self.model = HolisticAAM( self.LoadDataset(), group='PTS', verbose=True, holistic_features=float32_fast_dsift, diagonal=i_diag, scales=i_scale, max_appearance_components = i_max_greyscale_dims, max_shape_components = i_max_shape_dims) self.fitter = LucasKanadeAAMFitter( self.model, n_shape = [5, 15], n_appearance= [50, 150]); ''' Fit an appearance model to an image with annotated landmarks \return Converged candidate fit ''' def FitAnnotatedImage(self, i_img): gt = i_img.landmarks['PTS'].lms initial_shape = self.fitter.perturb_from_bb( gt, gt.bounding_box() ) return self.fitter.fit_from_shape(i_img, initial_shape, gt_shape=gt) ''' Fit an appearance model to an image without annotations using Menpo Face Detection \return Converged landmarks ''' def FitWildImageMenpo(self, i_img, i_initial_guess = None, i_max_iters = 10): # Convert menpo image to expected openCV format i_img = GreyscaleConversionMenpo(i_img) ret = None if i_initial_guess is not None: pts = menpo.shape.PointCloud(i_initial_guess, False) ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points else: bb = self.menpo_face_detector.Detect(i_img) if bb is not None: ret = self.fitter.fit_from_bb(i_img, bb, i_max_iters).final_shape.points return ret ''' Fit an appearance model to an image without annotations using Viola Face Detection \return Converged landmarks ''' def FitWildImageViola(self, i_img, i_initial_guess = None, i_max_iters = 10): # Convert menpo image to expected openCV format i_img = GreyscaleConversionMenpo(i_img) img = i_img.pixels[0] * 255 img = numpy.array(img, dtype=numpy.uint8) # Detect face with experiment tuning according to lfpw testset ret = None if i_initial_guess is None: faces = self.viola_face_detector.Detect(img, 3, 1.1, 0.125, 1.0) # Fit candidate model if len(faces) > 1: faces = [GetLargestROI(faces)] faces = ConvertRectToMenpoBoundingBox(faces) fit = self.fitter.fit_from_bb(i_img, faces[0], i_max_iters) ret = fit.final_shape.points elif i_initial_guess is not None: pts = menpo.shape.PointCloud(i_initial_guess, False) ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points return ret ''' Print debug information for the AAM class ''' def PrintDebug(self): print('Dataset', self.dataset) print self.model
def get_feature(self, file, process_opts=None): r""" Computes the AAM features, according to the `process_opts` Parameters ---------- file process_opts Returns ------- A dictionary of five elements, each representing a variation of the computed features (shape and appearance alone or concatenated, with or without derivatives) """ self._maybe_start_logging(file) self._load_landmark_fitter() frames = mio.import_video(file, landmark_resolver=self._myresolver, normalize=True, exact_frame_count=True) feat_shape = [] feat_app = [] feat_shape_app = [] for frameIdx, frame in enumerate(frames): bounding_boxes = self._face_detect(frame) if len(bounding_boxes) > 0: initial_bbox = bounding_boxes[0] if self._log_errors is True: gt_shape = frame.landmarks['pts_face'] else: gt_shape = None if isinstance(self._landmark_fitter, LucasKanadeAAMFitter): result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, max_iters=self._max_iters, gt_shape=gt_shape) elif isinstance(self._landmark_fitter, DlibWrapper): # DLIB fitter, doesn't have max_iters result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, gt_shape=gt_shape) else: raise Exception('incompatible landmark fitter') self._maybe_append_to_log(file, frameIdx, result) if self._shape == 'face': if self._parameters == 'lk_fitting': # skip the first 4 similarity params, probably not useful for classification shape_param_frame = result.shape_parameters[-1][4:] app_param_frame = result.appearance_parameters[-1] elif self._parameters == 'aam_projection': result_aam = self._projection_fitter.fit_from_shape( frame, result.final_shape, max_iters=[0, 0, 0]) # TODO: analyse the case when aam true components are less than max components shape_param_frame = result_aam.shape_parameters[-1][4:] app_param_frame = result_aam.appearance_parameters[-1] feat_shape.append(shape_param_frame) feat_app.append(app_param_frame) feat_shape_app.append(np.concatenate((shape_param_frame, app_param_frame))) elif self._shape == 'lips': # extract lips landmarks from the final face fitting to initialize the part model fitting aam_lips = mio.import_pickle(self._part_aam) fitter_lips = LucasKanadeAAMFitter(aam_lips, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20], n_appearance=[20, 150]) result_lips = fitter_lips.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'lips'), max_iters=[5, 5]) shape_param_frame_lips = result_lips.shape_parameters[-1][4:] app_param_frame_lips = result_lips.appearance_parameters[-1] feat_shape.append(shape_param_frame_lips) feat_app.append(app_param_frame_lips) feat_shape_app.append(np.concatenate((shape_param_frame_lips, app_param_frame_lips))) elif self._shape == 'chin': # extract chin and lips landmarks from the final face fitting to initialize the part model fitting aam_chin = mio.import_pickle(self._part_aam) fitter_chin = LucasKanadeAAMFitter(aam_chin, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20, 25], n_appearance=[20, 50, 150]) result_chin = fitter_chin.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'chin'), max_iters=[10, 10, 5]) shape_param_frame_mchin = result_chin.shape_parameters[-1][4:] app_param_frame_mchin = result_chin.appearance_parameters[-1] feat_shape.append(shape_param_frame_mchin) feat_app.append(app_param_frame_mchin) feat_shape_app.append(np.concatenate((shape_param_frame_mchin, app_param_frame_mchin))) else: raise Exception('Unknown shape model, currently supported are: face, lips, chin') else: # we did not detect any face zero_feat_shape = np.zeros(process_opts['shape_components'][-1]) zero_feat_app = np.zeros(process_opts['appearance_components'][-1]) zero_feat_shape_app = np.zeros( process_opts['shape_components'][-1] + process_opts['appearance_components'][-1]) feat_shape.append(zero_feat_shape) feat_app.append(zero_feat_app) feat_shape_app.append(zero_feat_shape_app) npfeat_shape = np.array(feat_shape) npfeat_app = np.array(feat_app) npfeat_shape_app = np.array(feat_shape_app) npfeat_app_delta = vsrmath.accurate_derivative(npfeat_app, 'delta') npfeat_shape_app_delta = vsrmath.accurate_derivative(npfeat_shape_app, 'delta') return {'shape': npfeat_shape, 'app': npfeat_app, 'shape_app': npfeat_shape_app, 'app_delta': npfeat_app_delta, 'shape_app_delta': npfeat_shape_app_delta}
class AAMFeature(Feature): r""" Active Appearance Model (AAM) feature extraction """ def __init__(self, extract_opts=None, process_opts=None, output_dir=None): r""" Parameters ---------- extract_opts : `dict` holding the configuration for feature extraction For complete description of some parameters, please refer upstream to their documentation in the menpofit project Must specify the following options: ``warp`` : `holistic` or `patch`; chooses between menpofit.aam.HolisticAAM and menpofit.aam.PatchAAM ``resolution_scales`` : `tuple` of `floats` between 0.0 and 1.0 A pyramid of AAMs will be created, one for each element in the tuple A value of 1.0 corresponds to the full resolution images, 0.5 to a half and so on ``patch_shape`` : `tuple` of `tuple` of two `ints` Parameter required when ``warp`` is `patch` One tuple per resolution scale The patch shape is specified as a window of MxN pixels around each landmark ``max_shape_components`` : `int` or `list` of `ints` maximum number of eigenvectors (per resolution scale) kept from shape PCA True value can be less that max, depending on the variance in the training images ``max_appearance_components: `int` or `list` of `ints` maximum number of eigenvectors (per resolution scale) kept from texture PCA True value can be less that max, depending on the variance in the training images ``diagonal`` : `int` serving as the diagonal size of the rescaled training images ``features`` : `no_op`, `hog`, `dsift`, `fast_dsift` `no_op` uses the image pixels for the texture model `hog, dsift, fast_dsift` extract popular image descriptors instead ``landmark_dir`` : `str`, directory containing the facial landmarks for the training images ``landmark_group`` : `pts_face`, `pts_chin`, `pts_lips` `pts_face` constructs a full facial model using all the 68 landmark points `pts_chin` uses landmarks [2:15) plus [48:68) to model the chin and lips region `pts_lips` uses only [48:68) to model the lip region ``confidence_thresh`` : `float` in range [0:1] Makes use of the OpenFace average confidence score, keeping only the frames above this threshold ``kept_frames`` : `float` in range [0:1] Samples the remaining video frames (above the confidence threshold) to keep only a small proportion This avoids training the AAM with a large number of consecutive video frames Before sampling, the frames from each video are sorted by the amount of lip opening. Then sampling is done at evenly spaced intervals ``greyscale`` : `boolean`; if ``True``, converts the frames to a single channel of grey / luminance levels if ``False``, the model is built on the original RGB channels ``model_name`` : `str`; name of the AAM pickle object to be stored offline process_opts: `dict` holding the configuration for feature processing Must specif y the following options: ``face_detector`` : `dlib` or `opencv` or `dpm` Selects the implementation that detects a face in an image `dlib` is the fastest, `dpm` may be more accurate (check G.Chrysos, Feb 2017) ``landmark_fitter : `aam` or `ert` Selects the algorithm that fits the landmarks on a detected face `ert` uses a model pre-trained on challenging datasets `aam` may use your own model ``aam_fitter`` : `str`, full file name storing an AAM pickle to be used for landmark fitting Mandatory if ``landmark_fitter`` is AAM ``parameters_from`` : `fitting`, `projection` If `fitting`, the shape and appearance parameters optimized by the Lukas-Kanade fitting algorithm are returned. If `projection`, only the final shape of the fitting process will be used, initializing another fitter based on a new AAM specified below `` projection_aam`` : `str`, full file name storing an AAM pickle to be used in the process described above ``shape`` : `face`, `chin` or `lips` Chooses an AAM that may describe an entire face, or sub-parts of it If `chin` or `lips`, the associated landmarks will be selected from the face fitting process, then a few more iterations of a fitting algorithm will be run using the part AAM specified below ``part_AAM`` : `None` or a `str` representing the file storing a part AAM pickle (chin or lips) Must be different from `None` if `shape` is `chin` or `lips` Such part_AAM can be obtained by choosing the ``landmark_group`` parameter accordingly in the extraction process ``confidence_thresh`` : `float`, DEPRECATED It was used to filter out the frames having a confidence threshold for the landmarks lower than this value. Their corresponding features were simply arrays of zeros. Now we consider every frame where a face is detected. ``shape_components`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of the kept shape eigenvectors for the projection and fitter AAMs The shape feature size will be up to this value ``appearance_components`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of the kept texture eigenvectors for the projection and fitter AAMs The appearance feature size will be up to this value ``max_iters`` : `int` or `list` of `ints` (one per resolution scale) Selects the number of iterations (per resolution scale) of the optimisation algorithm Only used for the fitter AAM, since 0 iterations are used with the projection AAM ``landmark_dir`` : `str`, directory containing the ground-truth facial landmarks for every frame of each video. Used only to compute an error between prediction and ground-truth. Can be `None` if the error log is not necessary ``log_errors`` : `boolean` If ``True``, generates a log file per video, stating the models used and the prediction error for each frame ``log_dir`` : `str`, directory to store the error logs above output_dir : `str`, absolute path where the features are to be stored """ self._outDir = output_dir if extract_opts is not None: self._extractOpts = extract_opts self._warpType = extract_opts['warp'] self._landmarkDir = extract_opts['landmark_dir'] self._landmarkGroup = extract_opts['landmark_group'] self._max_shape_components = extract_opts['max_shape_components'] self._max_appearance_components = extract_opts['max_appearance_components'] self._diagonal = extract_opts['diagonal'] self._scales = extract_opts['resolution_scales'] self._confidence_thresh = extract_opts['confidence_thresh'] self._kept_frames = extract_opts['kept_frames'] if extract_opts['features'] == 'fast_dsift': self._features = fast_dsift elif extract_opts['features'] == 'dsift': self._features = dsift elif extract_opts['features'] == 'hog': self._features = hog elif extract_opts['features'] == 'no_op': self._features = no_op else: raise Exception('Unknown feature type to extract, did you mean fast_dsift ?') if 'greyscale' in extract_opts.keys(): self._greyscale = extract_opts['greyscale'] else: self._greyscale = False self._outModelName = extract_opts['model_name'] if process_opts is not None: # Face detection self._face_detect_method = process_opts['face_detector'] if self._face_detect_method == 'dlib': from menpodetect import load_dlib_frontal_face_detector detector = load_dlib_frontal_face_detector() elif self._face_detect_method == 'opencv': from menpodetect import load_opencv_frontal_face_detector detector = load_opencv_frontal_face_detector() elif self._face_detect_method == 'dpm': from menpodetect.ffld2 import load_ffld2_frontal_face_detector detector = load_ffld2_frontal_face_detector() else: raise Exception('unknown detector, did you mean dlib/opencv/dpm?') self._face_detect = detector self._shape_components = process_opts['shape_components'] self._appearance_components = process_opts['appearance_components'] self._max_iters = process_opts['max_iters'] self._fitter_type = process_opts['landmark_fitter'] # Landmark fitter (pretrained ERT or AAM), actually loaded later to avoid pickling with Pool if self._fitter_type == 'aam': self._aam_fitter_file = process_opts['aam_fitter'] # Parameters source # If fitting, self._parameters = process_opts['parameters_from'] if self._parameters == 'aam_projection': self._projection_aam_file = process_opts['projection_aam'] self._projection_aam = mio.import_pickle(self._projection_aam_file) self._projection_fitter = LucasKanadeAAMFitter( aam=self._projection_aam, lk_algorithm_cls=WibergInverseCompositional, n_shape=self._shape_components, n_appearance=self._appearance_components) else: pass self._confidence_thresh = process_opts['confidence_thresh'] self._landmarkDir = process_opts['landmark_dir'] self._shape = process_opts['shape'] self._part_aam = process_opts['part_aam'] self._log_errors = process_opts['log_errors'] if self._log_errors is False: self._myresolver = None self._log_dir = process_opts['log_dir'] def extract_save_features(self, files): r""" Uses the input files as train AAMs and store the resulting pickle on the disk Parameters ---------- files Returns ------- """ # 1. fetch all video frames, attach landmarks frames = mio.import_video(files[0], landmark_resolver=self._myresolver, normalize=True, exact_frame_count=True) # frames = frames.map(AAMFeature._preprocess) idx_above_thresh, idx_lip_opening = landmark_filter( files[0], self._landmarkDir, threshold=self._confidence_thresh, keep=self._kept_frames) frames = frames[idx_above_thresh] frames = frames[idx_lip_opening] frames = frames.map(attach_semantic_landmarks) if self._greyscale is True: frames = frames.map(convert_to_grayscale) # initial AAM training if self._warpType == 'holistic': aam = HolisticAAM(frames, group=self._landmarkGroup, holistic_features=self._features, reference_shape=None, diagonal=self._diagonal, scales=self._scales, max_shape_components=self._max_shape_components, max_appearance_components=self._max_appearance_components, verbose=False) elif self._warpType == 'patch': aam = PatchAAM(frames, group=self._landmarkGroup, holistic_features=self._features, diagonal=self._diagonal, scales=self._scales, max_shape_components=self._max_shape_components, max_appearance_components=self._max_appearance_components, patch_shape=self._extractOpts['patch_shape'], verbose=False) else: raise Exception('Unknown warp type. Did you mean holistic/patch ?') frame_buffer = LazyList.init_from_iterable([]) buffer_len = 256 for idx, file in enumerate(files[1:]): # useful to check progress with open('./run/log_' + self._outModelName + '.txt', 'w') as log: log.write(str(idx) + ' ' + file + '\n') frames = mio.import_video(file, landmark_resolver=self._myresolver, normalize=True, exact_frame_count=True) idx_above_thresh, idx_lip_opening = landmark_filter( file, landmark_dir=self._landmarkDir, threshold=self._confidence_thresh, keep=self._kept_frames) frames = frames[idx_above_thresh] frames = frames[idx_lip_opening] frames = frames.map(attach_semantic_landmarks) if self._greyscale is True: frames = frames.map(convert_to_grayscale) frame_buffer += frames if len(frame_buffer) > buffer_len: # 2. retrain AAM aam.increment(frame_buffer, group=self._landmarkGroup, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, verbose=False, batch_size=None) del frame_buffer frame_buffer = LazyList.init_from_iterable([]) else: pass if len(frame_buffer) != 0: # # deplete remaining frames aam.increment(frame_buffer, group=self._landmarkGroup, shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0, verbose=False, batch_size=None) del frame_buffer mio.export_pickle(obj=aam, fp=self._outDir + self._outModelName, overwrite=True, protocol=4) def get_feature(self, file, process_opts=None): r""" Computes the AAM features, according to the `process_opts` Parameters ---------- file process_opts Returns ------- A dictionary of five elements, each representing a variation of the computed features (shape and appearance alone or concatenated, with or without derivatives) """ self._maybe_start_logging(file) self._load_landmark_fitter() frames = mio.import_video(file, landmark_resolver=self._myresolver, normalize=True, exact_frame_count=True) feat_shape = [] feat_app = [] feat_shape_app = [] for frameIdx, frame in enumerate(frames): bounding_boxes = self._face_detect(frame) if len(bounding_boxes) > 0: initial_bbox = bounding_boxes[0] if self._log_errors is True: gt_shape = frame.landmarks['pts_face'] else: gt_shape = None if isinstance(self._landmark_fitter, LucasKanadeAAMFitter): result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, max_iters=self._max_iters, gt_shape=gt_shape) elif isinstance(self._landmark_fitter, DlibWrapper): # DLIB fitter, doesn't have max_iters result = self._landmark_fitter.fit_from_bb( frame, initial_bbox, gt_shape=gt_shape) else: raise Exception('incompatible landmark fitter') self._maybe_append_to_log(file, frameIdx, result) if self._shape == 'face': if self._parameters == 'lk_fitting': # skip the first 4 similarity params, probably not useful for classification shape_param_frame = result.shape_parameters[-1][4:] app_param_frame = result.appearance_parameters[-1] elif self._parameters == 'aam_projection': result_aam = self._projection_fitter.fit_from_shape( frame, result.final_shape, max_iters=[0, 0, 0]) # TODO: analyse the case when aam true components are less than max components shape_param_frame = result_aam.shape_parameters[-1][4:] app_param_frame = result_aam.appearance_parameters[-1] feat_shape.append(shape_param_frame) feat_app.append(app_param_frame) feat_shape_app.append(np.concatenate((shape_param_frame, app_param_frame))) elif self._shape == 'lips': # extract lips landmarks from the final face fitting to initialize the part model fitting aam_lips = mio.import_pickle(self._part_aam) fitter_lips = LucasKanadeAAMFitter(aam_lips, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20], n_appearance=[20, 150]) result_lips = fitter_lips.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'lips'), max_iters=[5, 5]) shape_param_frame_lips = result_lips.shape_parameters[-1][4:] app_param_frame_lips = result_lips.appearance_parameters[-1] feat_shape.append(shape_param_frame_lips) feat_app.append(app_param_frame_lips) feat_shape_app.append(np.concatenate((shape_param_frame_lips, app_param_frame_lips))) elif self._shape == 'chin': # extract chin and lips landmarks from the final face fitting to initialize the part model fitting aam_chin = mio.import_pickle(self._part_aam) fitter_chin = LucasKanadeAAMFitter(aam_chin, lk_algorithm_cls=WibergInverseCompositional, n_shape=[10, 20, 25], n_appearance=[20, 50, 150]) result_chin = fitter_chin.fit_from_shape( image=frame, initial_shape=_pointcloud_subset(result.final_shape, 'chin'), max_iters=[10, 10, 5]) shape_param_frame_mchin = result_chin.shape_parameters[-1][4:] app_param_frame_mchin = result_chin.appearance_parameters[-1] feat_shape.append(shape_param_frame_mchin) feat_app.append(app_param_frame_mchin) feat_shape_app.append(np.concatenate((shape_param_frame_mchin, app_param_frame_mchin))) else: raise Exception('Unknown shape model, currently supported are: face, lips, chin') else: # we did not detect any face zero_feat_shape = np.zeros(process_opts['shape_components'][-1]) zero_feat_app = np.zeros(process_opts['appearance_components'][-1]) zero_feat_shape_app = np.zeros( process_opts['shape_components'][-1] + process_opts['appearance_components'][-1]) feat_shape.append(zero_feat_shape) feat_app.append(zero_feat_app) feat_shape_app.append(zero_feat_shape_app) npfeat_shape = np.array(feat_shape) npfeat_app = np.array(feat_app) npfeat_shape_app = np.array(feat_shape_app) npfeat_app_delta = vsrmath.accurate_derivative(npfeat_app, 'delta') npfeat_shape_app_delta = vsrmath.accurate_derivative(npfeat_shape_app, 'delta') return {'shape': npfeat_shape, 'app': npfeat_app, 'shape_app': npfeat_shape_app, 'app_delta': npfeat_app_delta, 'shape_app_delta': npfeat_shape_app_delta} def _myresolver(self, file, frame): frames_dir = file_to_feature(str(file), extension='') return {'pts_face': self._landmarkDir + frames_dir + '/frame_' + str(frame + 1) + '.pts'} def _maybe_start_logging(self, file): if self._log_errors is True: from os import makedirs makedirs('./run/logs/' + self._log_dir, exist_ok=True) cf = file_to_feature(file, extension='') with open('./run/logs/' + self._log_dir + '/log_' + cf + '.txt', 'w') as log: log.write('{} \n'.format(file)) log.write('Face detector: {}\n'.format(self._face_detect_method)) if self._fitter_type == 'aam': log.write('AAM Landmark fitter: {}\n'.format(self._aam_fitter_file)) elif self._fitter_type == 'ert': log.write('Pretrained ERT Landmark fitter\n') if self._parameters == 'projection': log.write('AAM Projector: {}\n'.format(self._projection_aam_file)) def _maybe_append_to_log(self, file, frame_idx, result): if self._log_errors is True: cf = file_to_feature(file, extension='') error = result.final_error() with open('./run/logs/' + self._log_dir + '/log_' + cf + '.txt', 'a') as log: log.write('frame {}. error: {} \n'.format(str(frame_idx), str(error))) def _load_landmark_fitter(self): if self._fitter_type == 'aam': self._aam_fitter = mio.import_pickle(self._aam_fitter_file) fitter = LucasKanadeAAMFitter(self._aam_fitter, lk_algorithm_cls=WibergInverseCompositional, n_shape=self._shape_components, n_appearance=self._appearance_components) elif self._fitter_type == 'ert': fitter = DlibWrapper( path.join(current_path, '../pretrained/shape_predictor_68_face_landmarks.dat')) else: raise Exception('unknown fitter, did you mean aam/ert?') self._landmark_fitter = fitter