class AAM:
'''
Initialise members and train AAM
\param[in] i_dataset Full filepath to training and test dataset
\param[in] i_debug True to display debug info
'''
def __init__(self, i_dataset, i_debug=False):
self.debug = i_debug
self.dataset = i_dataset
if not os.path.exists(self.dataset):
raise RuntimeError('Database dir does not exist in ' +
self.dataset)
self.Train()
self.viola_face_detector = FaceDetectViola(False)
self.menpo_face_detector = FaceDetectMenpo()
if self.debug:
self.PrintDebug()
'''
Load training images and annotated landmarks from a training set in the file system
'''
def LoadDataset(self):
trainset = os.path.join(self.dataset, 'trainset', '*')
training_images = [
self.LoadImage(img)
for img in menpoio.import_images(trainset, verbose=True)
]
return training_images
'''
Crops image landmarks (0.1 referenced from AAMs Basics) and convert to greyscale
\param[in] i_img Menpo image to process
\return processed menpo image
'''
def LoadImage(self, i_img, i_landmark_crop=0.5):
img = i_img.crop_to_landmarks_proportion(i_landmark_crop)
img = GreyscaleConversionMenpo(img)
return img
'''
Train an Active Appearance Model and compute the
\param[in] i_diag Search gradient along model landmark
\param[in] i_scale Scale applied to search direction (search) || (initial, search)
\param[in] i_max_greyscale_dims Dimensionality limit for PCA appearance model
\param[in] i_max_shape_dims Dimensionality limit for PCA keypoint components
'''
def Train(self,
i_diag=150,
i_scale=[0.5, 1.0],
i_max_greyscale_dims=200,
i_max_shape_dims=20):
# laterals tuned for performance gain - Sacrifice mouth modes
self.model = HolisticAAM(
self.LoadDataset(),
group='PTS',
verbose=True,
holistic_features=float32_fast_dsift,
diagonal=i_diag,
scales=i_scale,
max_appearance_components=i_max_greyscale_dims,
max_shape_components=i_max_shape_dims)
self.fitter = LucasKanadeAAMFitter(self.model,
n_shape=[5, 15],
n_appearance=[50, 150])
'''
Fit an appearance model to an image with annotated landmarks
\return Converged candidate fit
'''
def FitAnnotatedImage(self, i_img):
gt = i_img.landmarks['PTS'].lms
initial_shape = self.fitter.perturb_from_bb(gt, gt.bounding_box())
return self.fitter.fit_from_shape(i_img, initial_shape, gt_shape=gt)
'''
Fit an appearance model to an image without annotations using Menpo Face Detection
\return Converged landmarks
'''
def FitWildImageMenpo(self, i_img, i_initial_guess=None, i_max_iters=10):
# Convert menpo image to expected openCV format
i_img = GreyscaleConversionMenpo(i_img)
ret = None
if i_initial_guess is not None:
pts = menpo.shape.PointCloud(i_initial_guess, False)
ret = self.fitter.fit_from_shape(i_img, pts,
i_max_iters).final_shape.points
else:
bb = self.menpo_face_detector.Detect(i_img)
if bb is not None:
ret = self.fitter.fit_from_bb(i_img, bb,
i_max_iters).final_shape.points
return ret
'''
Fit an appearance model to an image without annotations using Viola Face Detection
\return Converged landmarks
'''
def FitWildImageViola(self, i_img, i_initial_guess=None, i_max_iters=10):
# Convert menpo image to expected openCV format
i_img = GreyscaleConversionMenpo(i_img)
img = i_img.pixels[0] * 255
img = numpy.array(img, dtype=numpy.uint8)
# Detect face with experiment tuning according to lfpw testset
ret = None
if i_initial_guess is None:
faces = self.viola_face_detector.Detect(img, 3, 1.1, 0.125, 1.0)
# Fit candidate model
if len(faces) > 1:
faces = [GetLargestROI(faces)]
faces = ConvertRectToMenpoBoundingBox(faces)
fit = self.fitter.fit_from_bb(i_img, faces[0], i_max_iters)
ret = fit.final_shape.points
elif i_initial_guess is not None:
pts = menpo.shape.PointCloud(i_initial_guess, False)
ret = self.fitter.fit_from_shape(i_img, pts,
i_max_iters).final_shape.points
return ret
'''
Print debug information for the AAM class
'''
def PrintDebug(self):
print('Dataset', self.dataset)
print self.model
class ActiveAppearanceModel(SegmentationModel):
def __init__(self, join_masks=True, scales=[15, 23]):
self.mean_shape = None
self.image_shape = None
self.fitter = None
self.num_landmarks_by_shape = None
self.join_masks = join_masks
self.scales = scales
def normalize(self, image):
mean = np.mean(image)
std = np.std(image)
image = (image - mean)/std
# FIX:
min_ = np.min(image)
max_ = np.max(image)
image = (image - min_)/(max_ - min_ + 1e-7)
return image
def prepare_data_for_aam(self, images, landmarks):
images_aam = []
for i in range(len(images)):
images[i] = self.normalize(images[i])
images_aam.append(Image(images[i]))
lmx = landmarks[0][i].T[0]
lmy = landmarks[0][i].T[1]
num_points = len(landmarks[0])
rmx = landmarks[1][i].T[0]
rmy = landmarks[1][i].T[1]
pc = PointCloud(points=np.vstack((np.array([lmy, lmx]).T, np.array([rmy, rmx]).T)))
lg = LandmarkGroup.init_from_indices_mapping(pc,
OrderedDict({'left':range(len(landmarks[0][i])), 'right':range(len(landmarks[0][i]), len(landmarks[0][i]) + len(landmarks[1][i]))}))
lm = LandmarkManager()
lm.setdefault('left', lg)
images_aam[-1].landmarks = lm
return images_aam
def fit(self, images, landmarks):
num_samples = len(landmarks[0])
self.num_landmarks_by_shape = []
for i in range(len(landmarks)):
self.num_landmarks_by_shape.append(len(landmarks[i][0]))
aam_images = self.prepare_data_for_aam(images, landmarks)
aam = PatchAAM(aam_images, group=None, patch_shape=[(self.scales[0], self.scales[0]), (self.scales[1], self.scales[1])],
diagonal=150, scales=(0.5, 1.0), holistic_features=fast_dsift,
max_shape_components=20, max_appearance_components=150,
verbose=True)
self.fitter = LucasKanadeAAMFitter(aam,
lk_algorithm_cls=WibergInverseCompositional,
n_shape=[5, 20], n_appearance=[30, 150])
pc = []
for img in aam_images:
pc.append(img.landmarks[None].lms)
self.mean_shape = mean_pointcloud(pc)
self.image_shape = images[0].shape
fitting_results = []
for img in aam_images[:10]:
fr = self.fitter.fit_from_shape(img, self.mean_shape, gt_shape=img.landmarks[None].lms)
fitting_results.append(fr)
def transform_one_image(self, image):
print 'init ...',
mask = np.zeros(shape=image.shape)
image = Image(self.normalize(image))
fr = self.fitter.fit_from_shape(image, self.mean_shape)
pred_landmarks = fr.final_shape.points
begin = 0
masks = []
for i in range(0, len(self.num_landmarks_by_shape)):
masks.append(self.create_mask_from_landmarks(pred_landmarks[begin:begin + self.num_landmarks_by_shape[i]]))
mask = np.logical_or(mask, masks[-1])
begin += self.num_landmarks_by_shape[i]
print 'done'
if self.join_masks:
return mask
else:
return masks
# append it to the list
images.append(i)
return images
# 测试集
image_path_test = "D:/电信研究院/人脸矫正/labelme-master/examples/transfer/test"
test_images = load_database(image_path_test, 0.5, max_images=5)
fitting_results = []
for i in test_images:
# obtain original landmarks
gt_s = i.landmarks['PTS'].lms
# generate perturbed landmarks
s = noisy_shape_from_bounding_box(gt_s, gt_s.bounding_box())
# fit image
fr = fitter.fit_from_shape(i, s, gt_shape=gt_s)
fitting_results.append(fr)
print('fr',type(fr),fr.final_shape.points,fr)
# from menpowidgets import visualize_fitting_result
# #预测
# image_path_pred = "D:/电信研究院/人脸矫正/labelme-master/examples/transfer/pred"
#
# pred_images = []
# # load landmarked images
# for i in mio.import_images(image_path_pred, max_images=None, verbose=True):
# # convert it to grayscale if needed
# if i.n_channels == 3:
# i = i.as_greyscale(mode='luminosity')
#
# # append it to the list
# pred_images.append(i)
class AAM:
'''
Initialise members and train AAM
\param[in] i_dataset Full filepath to training and test dataset
\param[in] i_debug True to display debug info
'''
def __init__(self, i_dataset, i_debug = False):
self.debug = i_debug
self.dataset = i_dataset
if not os.path.exists(self.dataset):
raise RuntimeError('Database dir does not exist in ' + self.dataset)
self.Train()
self.viola_face_detector = FaceDetectViola(False)
self.menpo_face_detector = FaceDetectMenpo()
if self.debug:
self.PrintDebug()
'''
Load training images and annotated landmarks from a training set in the file system
'''
def LoadDataset(self):
trainset = os.path.join(self.dataset,'trainset','*')
training_images = [self.LoadImage(img) for img in menpoio.import_images(trainset, verbose = True) ]
return training_images
'''
Crops image landmarks (0.1 referenced from AAMs Basics) and convert to greyscale
\param[in] i_img Menpo image to process
\return processed menpo image
'''
def LoadImage(self, i_img, i_landmark_crop = 0.5):
img = i_img.crop_to_landmarks_proportion( i_landmark_crop )
img = GreyscaleConversionMenpo(img)
return img
'''
Train an Active Appearance Model and compute the
\param[in] i_diag Search gradient along model landmark
\param[in] i_scale Scale applied to search direction (search) || (initial, search)
\param[in] i_max_greyscale_dims Dimensionality limit for PCA appearance model
\param[in] i_max_shape_dims Dimensionality limit for PCA keypoint components
'''
def Train(self, i_diag = 150, i_scale = [0.5, 1.0], i_max_greyscale_dims = 200, i_max_shape_dims = 20):
# laterals tuned for performance gain - Sacrifice mouth modes
self.model = HolisticAAM(
self.LoadDataset(),
group='PTS',
verbose=True,
holistic_features=float32_fast_dsift,
diagonal=i_diag,
scales=i_scale,
max_appearance_components = i_max_greyscale_dims,
max_shape_components = i_max_shape_dims)
self.fitter = LucasKanadeAAMFitter(
self.model,
n_shape = [5, 15],
n_appearance= [50, 150]);
'''
Fit an appearance model to an image with annotated landmarks
\return Converged candidate fit
'''
def FitAnnotatedImage(self, i_img):
gt = i_img.landmarks['PTS'].lms
initial_shape = self.fitter.perturb_from_bb( gt, gt.bounding_box() )
return self.fitter.fit_from_shape(i_img, initial_shape, gt_shape=gt)
'''
Fit an appearance model to an image without annotations using Menpo Face Detection
\return Converged landmarks
'''
def FitWildImageMenpo(self, i_img, i_initial_guess = None, i_max_iters = 10):
# Convert menpo image to expected openCV format
i_img = GreyscaleConversionMenpo(i_img)
ret = None
if i_initial_guess is not None:
pts = menpo.shape.PointCloud(i_initial_guess, False)
ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points
else:
bb = self.menpo_face_detector.Detect(i_img)
if bb is not None:
ret = self.fitter.fit_from_bb(i_img, bb, i_max_iters).final_shape.points
return ret
'''
Fit an appearance model to an image without annotations using Viola Face Detection
\return Converged landmarks
'''
def FitWildImageViola(self, i_img, i_initial_guess = None, i_max_iters = 10):
# Convert menpo image to expected openCV format
i_img = GreyscaleConversionMenpo(i_img)
img = i_img.pixels[0] * 255
img = numpy.array(img, dtype=numpy.uint8)
# Detect face with experiment tuning according to lfpw testset
ret = None
if i_initial_guess is None:
faces = self.viola_face_detector.Detect(img, 3, 1.1, 0.125, 1.0)
# Fit candidate model
if len(faces) > 1:
faces = [GetLargestROI(faces)]
faces = ConvertRectToMenpoBoundingBox(faces)
fit = self.fitter.fit_from_bb(i_img, faces[0], i_max_iters)
ret = fit.final_shape.points
elif i_initial_guess is not None:
pts = menpo.shape.PointCloud(i_initial_guess, False)
ret = self.fitter.fit_from_shape(i_img, pts, i_max_iters).final_shape.points
return ret
'''
Print debug information for the AAM class
'''
def PrintDebug(self):
print('Dataset', self.dataset)
print self.model
def get_feature(self, file, process_opts=None):
r"""
Computes the AAM features, according to the `process_opts`
Parameters
----------
file
process_opts
Returns
-------
A dictionary of five elements, each representing a variation of the computed features
(shape and appearance alone or concatenated, with or without derivatives)
"""
self._maybe_start_logging(file)
self._load_landmark_fitter()
frames = mio.import_video(file,
landmark_resolver=self._myresolver,
normalize=True,
exact_frame_count=True)
feat_shape = []
feat_app = []
feat_shape_app = []
for frameIdx, frame in enumerate(frames):
bounding_boxes = self._face_detect(frame)
if len(bounding_boxes) > 0:
initial_bbox = bounding_boxes[0]
if self._log_errors is True:
gt_shape = frame.landmarks['pts_face']
else:
gt_shape = None
if isinstance(self._landmark_fitter, LucasKanadeAAMFitter):
result = self._landmark_fitter.fit_from_bb(
frame, initial_bbox,
max_iters=self._max_iters,
gt_shape=gt_shape)
elif isinstance(self._landmark_fitter, DlibWrapper): # DLIB fitter, doesn't have max_iters
result = self._landmark_fitter.fit_from_bb(
frame,
initial_bbox,
gt_shape=gt_shape)
else:
raise Exception('incompatible landmark fitter')
self._maybe_append_to_log(file, frameIdx, result)
if self._shape == 'face':
if self._parameters == 'lk_fitting':
# skip the first 4 similarity params, probably not useful for classification
shape_param_frame = result.shape_parameters[-1][4:]
app_param_frame = result.appearance_parameters[-1]
elif self._parameters == 'aam_projection':
result_aam = self._projection_fitter.fit_from_shape(
frame,
result.final_shape,
max_iters=[0, 0, 0])
# TODO: analyse the case when aam true components are less than max components
shape_param_frame = result_aam.shape_parameters[-1][4:]
app_param_frame = result_aam.appearance_parameters[-1]
feat_shape.append(shape_param_frame)
feat_app.append(app_param_frame)
feat_shape_app.append(np.concatenate((shape_param_frame, app_param_frame)))
elif self._shape == 'lips':
# extract lips landmarks from the final face fitting to initialize the part model fitting
aam_lips = mio.import_pickle(self._part_aam)
fitter_lips = LucasKanadeAAMFitter(aam_lips, lk_algorithm_cls=WibergInverseCompositional,
n_shape=[10, 20], n_appearance=[20, 150])
result_lips = fitter_lips.fit_from_shape(
image=frame,
initial_shape=_pointcloud_subset(result.final_shape, 'lips'),
max_iters=[5, 5])
shape_param_frame_lips = result_lips.shape_parameters[-1][4:]
app_param_frame_lips = result_lips.appearance_parameters[-1]
feat_shape.append(shape_param_frame_lips)
feat_app.append(app_param_frame_lips)
feat_shape_app.append(np.concatenate((shape_param_frame_lips, app_param_frame_lips)))
elif self._shape == 'chin':
# extract chin and lips landmarks from the final face fitting to initialize the part model fitting
aam_chin = mio.import_pickle(self._part_aam)
fitter_chin = LucasKanadeAAMFitter(aam_chin, lk_algorithm_cls=WibergInverseCompositional,
n_shape=[10, 20, 25], n_appearance=[20, 50, 150])
result_chin = fitter_chin.fit_from_shape(
image=frame,
initial_shape=_pointcloud_subset(result.final_shape, 'chin'),
max_iters=[10, 10, 5])
shape_param_frame_mchin = result_chin.shape_parameters[-1][4:]
app_param_frame_mchin = result_chin.appearance_parameters[-1]
feat_shape.append(shape_param_frame_mchin)
feat_app.append(app_param_frame_mchin)
feat_shape_app.append(np.concatenate((shape_param_frame_mchin, app_param_frame_mchin)))
else:
raise Exception('Unknown shape model, currently supported are: face, lips, chin')
else: # we did not detect any face
zero_feat_shape = np.zeros(process_opts['shape_components'][-1])
zero_feat_app = np.zeros(process_opts['appearance_components'][-1])
zero_feat_shape_app = np.zeros(
process_opts['shape_components'][-1] + process_opts['appearance_components'][-1])
feat_shape.append(zero_feat_shape)
feat_app.append(zero_feat_app)
feat_shape_app.append(zero_feat_shape_app)
npfeat_shape = np.array(feat_shape)
npfeat_app = np.array(feat_app)
npfeat_shape_app = np.array(feat_shape_app)
npfeat_app_delta = vsrmath.accurate_derivative(npfeat_app, 'delta')
npfeat_shape_app_delta = vsrmath.accurate_derivative(npfeat_shape_app, 'delta')
return {'shape': npfeat_shape,
'app': npfeat_app,
'shape_app': npfeat_shape_app,
'app_delta': npfeat_app_delta,
'shape_app_delta': npfeat_shape_app_delta}
class AAMFeature(Feature):
r"""
Active Appearance Model (AAM) feature extraction
"""
def __init__(self, extract_opts=None, process_opts=None, output_dir=None):
r"""
Parameters
----------
extract_opts : `dict` holding the configuration for feature extraction
For complete description of some parameters, please refer upstream
to their documentation in the menpofit project
Must specify the following options:
``warp`` : `holistic` or `patch`;
chooses between menpofit.aam.HolisticAAM and menpofit.aam.PatchAAM
``resolution_scales`` : `tuple` of `floats` between 0.0 and 1.0
A pyramid of AAMs will be created, one for each element in the tuple
A value of 1.0 corresponds to the full resolution images, 0.5 to a half and so on
``patch_shape`` : `tuple` of `tuple` of two `ints`
Parameter required when ``warp`` is `patch`
One tuple per resolution scale
The patch shape is specified as a window of MxN pixels around each landmark
``max_shape_components`` : `int` or `list` of `ints`
maximum number of eigenvectors (per resolution scale) kept from shape PCA
True value can be less that max, depending on the variance in the training images
``max_appearance_components: `int` or `list` of `ints`
maximum number of eigenvectors (per resolution scale) kept from texture PCA
True value can be less that max, depending on the variance in the training images
``diagonal`` : `int` serving as the diagonal size of the rescaled training images
``features`` : `no_op`, `hog`, `dsift`, `fast_dsift`
`no_op` uses the image pixels for the texture model
`hog, dsift, fast_dsift` extract popular image descriptors instead
``landmark_dir`` : `str`, directory containing the facial landmarks for the training images
``landmark_group`` : `pts_face`, `pts_chin`, `pts_lips`
`pts_face` constructs a full facial model using all the 68 landmark points
`pts_chin` uses landmarks [2:15) plus [48:68) to model the chin and lips region
`pts_lips` uses only [48:68) to model the lip region
``confidence_thresh`` : `float` in range [0:1]
Makes use of the OpenFace average confidence score, keeping only the frames above this threshold
``kept_frames`` : `float` in range [0:1]
Samples the remaining video frames (above the confidence threshold) to keep only a small proportion
This avoids training the AAM with a large number of consecutive video frames
Before sampling, the frames from each video are sorted by the amount of lip opening.
Then sampling is done at evenly spaced intervals
``greyscale`` : `boolean`; if ``True``, converts the frames to a single channel of grey / luminance levels
if ``False``, the model is built on the original RGB channels
``model_name`` : `str`; name of the AAM pickle object to be stored offline
process_opts: `dict` holding the configuration for feature processing
Must specif y the following options:
``face_detector`` : `dlib` or `opencv` or `dpm`
Selects the implementation that detects a face in an image
`dlib` is the fastest, `dpm` may be more accurate (check G.Chrysos, Feb 2017)
``landmark_fitter : `aam` or `ert`
Selects the algorithm that fits the landmarks on a detected face
`ert` uses a model pre-trained on challenging datasets
`aam` may use your own model
``aam_fitter`` : `str`, full file name storing an AAM pickle to be used for landmark fitting
Mandatory if ``landmark_fitter`` is AAM
``parameters_from`` : `fitting`, `projection`
If `fitting`, the shape and appearance parameters optimized by the Lukas-Kanade fitting algorithm
are returned. If `projection`, only the final shape of the fitting process will be used, initializing
another fitter based on a new AAM specified below
`` projection_aam`` : `str`, full file name storing an AAM pickle to be used in the process described above
``shape`` : `face`, `chin` or `lips`
Chooses an AAM that may describe an entire face, or sub-parts of it
If `chin` or `lips`, the associated landmarks will be selected from the face fitting process,
then a few more iterations of a fitting algorithm will be run using the part AAM specified below
``part_AAM`` : `None` or a `str` representing the file storing a part AAM pickle (chin or lips)
Must be different from `None` if `shape` is `chin` or `lips`
Such part_AAM can be obtained by choosing the ``landmark_group`` parameter accordingly in the
extraction process
``confidence_thresh`` : `float`, DEPRECATED
It was used to filter out the frames having a confidence threshold for the landmarks lower than
this value. Their corresponding features were simply arrays of zeros. Now we consider every frame
where a face is detected.
``shape_components`` : `int` or `list` of `ints` (one per resolution scale)
Selects the number of the kept shape eigenvectors for the projection and fitter AAMs
The shape feature size will be up to this value
``appearance_components`` : `int` or `list` of `ints` (one per resolution scale)
Selects the number of the kept texture eigenvectors for the projection and fitter AAMs
The appearance feature size will be up to this value
``max_iters`` : `int` or `list` of `ints` (one per resolution scale)
Selects the number of iterations (per resolution scale) of the optimisation algorithm
Only used for the fitter AAM, since 0 iterations are used with the projection AAM
``landmark_dir`` : `str`, directory containing the ground-truth facial landmarks
for every frame of each video. Used only to compute an error between prediction and ground-truth.
Can be `None` if the error log is not necessary
``log_errors`` : `boolean`
If ``True``, generates a log file per video, stating the models used
and the prediction error for each frame
``log_dir`` : `str`, directory to store the error logs above
output_dir : `str`, absolute path where the features are to be stored
"""
self._outDir = output_dir
if extract_opts is not None:
self._extractOpts = extract_opts
self._warpType = extract_opts['warp']
self._landmarkDir = extract_opts['landmark_dir']
self._landmarkGroup = extract_opts['landmark_group']
self._max_shape_components = extract_opts['max_shape_components']
self._max_appearance_components = extract_opts['max_appearance_components']
self._diagonal = extract_opts['diagonal']
self._scales = extract_opts['resolution_scales']
self._confidence_thresh = extract_opts['confidence_thresh']
self._kept_frames = extract_opts['kept_frames']
if extract_opts['features'] == 'fast_dsift':
self._features = fast_dsift
elif extract_opts['features'] == 'dsift':
self._features = dsift
elif extract_opts['features'] == 'hog':
self._features = hog
elif extract_opts['features'] == 'no_op':
self._features = no_op
else:
raise Exception('Unknown feature type to extract, did you mean fast_dsift ?')
if 'greyscale' in extract_opts.keys():
self._greyscale = extract_opts['greyscale']
else:
self._greyscale = False
self._outModelName = extract_opts['model_name']
if process_opts is not None:
# Face detection
self._face_detect_method = process_opts['face_detector']
if self._face_detect_method == 'dlib':
from menpodetect import load_dlib_frontal_face_detector
detector = load_dlib_frontal_face_detector()
elif self._face_detect_method == 'opencv':
from menpodetect import load_opencv_frontal_face_detector
detector = load_opencv_frontal_face_detector()
elif self._face_detect_method == 'dpm':
from menpodetect.ffld2 import load_ffld2_frontal_face_detector
detector = load_ffld2_frontal_face_detector()
else:
raise Exception('unknown detector, did you mean dlib/opencv/dpm?')
self._face_detect = detector
self._shape_components = process_opts['shape_components']
self._appearance_components = process_opts['appearance_components']
self._max_iters = process_opts['max_iters']
self._fitter_type = process_opts['landmark_fitter']
# Landmark fitter (pretrained ERT or AAM), actually loaded later to avoid pickling with Pool
if self._fitter_type == 'aam':
self._aam_fitter_file = process_opts['aam_fitter']
# Parameters source
# If fitting,
self._parameters = process_opts['parameters_from']
if self._parameters == 'aam_projection':
self._projection_aam_file = process_opts['projection_aam']
self._projection_aam = mio.import_pickle(self._projection_aam_file)
self._projection_fitter = LucasKanadeAAMFitter(
aam=self._projection_aam,
lk_algorithm_cls=WibergInverseCompositional,
n_shape=self._shape_components,
n_appearance=self._appearance_components)
else:
pass
self._confidence_thresh = process_opts['confidence_thresh']
self._landmarkDir = process_opts['landmark_dir']
self._shape = process_opts['shape']
self._part_aam = process_opts['part_aam']
self._log_errors = process_opts['log_errors']
if self._log_errors is False:
self._myresolver = None
self._log_dir = process_opts['log_dir']
def extract_save_features(self, files):
r"""
Uses the input files as train AAMs and store the resulting pickle on the disk
Parameters
----------
files
Returns
-------
"""
# 1. fetch all video frames, attach landmarks
frames = mio.import_video(files[0],
landmark_resolver=self._myresolver,
normalize=True,
exact_frame_count=True)
# frames = frames.map(AAMFeature._preprocess)
idx_above_thresh, idx_lip_opening = landmark_filter(
files[0],
self._landmarkDir,
threshold=self._confidence_thresh,
keep=self._kept_frames)
frames = frames[idx_above_thresh]
frames = frames[idx_lip_opening]
frames = frames.map(attach_semantic_landmarks)
if self._greyscale is True:
frames = frames.map(convert_to_grayscale)
# initial AAM training
if self._warpType == 'holistic':
aam = HolisticAAM(frames,
group=self._landmarkGroup,
holistic_features=self._features,
reference_shape=None,
diagonal=self._diagonal,
scales=self._scales,
max_shape_components=self._max_shape_components,
max_appearance_components=self._max_appearance_components,
verbose=False)
elif self._warpType == 'patch':
aam = PatchAAM(frames,
group=self._landmarkGroup,
holistic_features=self._features,
diagonal=self._diagonal,
scales=self._scales,
max_shape_components=self._max_shape_components,
max_appearance_components=self._max_appearance_components,
patch_shape=self._extractOpts['patch_shape'],
verbose=False)
else:
raise Exception('Unknown warp type. Did you mean holistic/patch ?')
frame_buffer = LazyList.init_from_iterable([])
buffer_len = 256
for idx, file in enumerate(files[1:]):
# useful to check progress
with open('./run/log_' + self._outModelName + '.txt', 'w') as log:
log.write(str(idx) + ' ' + file + '\n')
frames = mio.import_video(file,
landmark_resolver=self._myresolver,
normalize=True,
exact_frame_count=True)
idx_above_thresh, idx_lip_opening = landmark_filter(
file,
landmark_dir=self._landmarkDir,
threshold=self._confidence_thresh,
keep=self._kept_frames)
frames = frames[idx_above_thresh]
frames = frames[idx_lip_opening]
frames = frames.map(attach_semantic_landmarks)
if self._greyscale is True:
frames = frames.map(convert_to_grayscale)
frame_buffer += frames
if len(frame_buffer) > buffer_len:
# 2. retrain AAM
aam.increment(frame_buffer,
group=self._landmarkGroup,
shape_forgetting_factor=1.0,
appearance_forgetting_factor=1.0,
verbose=False,
batch_size=None)
del frame_buffer
frame_buffer = LazyList.init_from_iterable([])
else:
pass
if len(frame_buffer) != 0: #
# deplete remaining frames
aam.increment(frame_buffer,
group=self._landmarkGroup,
shape_forgetting_factor=1.0,
appearance_forgetting_factor=1.0,
verbose=False,
batch_size=None)
del frame_buffer
mio.export_pickle(obj=aam, fp=self._outDir + self._outModelName, overwrite=True, protocol=4)
def get_feature(self, file, process_opts=None):
r"""
Computes the AAM features, according to the `process_opts`
Parameters
----------
file
process_opts
Returns
-------
A dictionary of five elements, each representing a variation of the computed features
(shape and appearance alone or concatenated, with or without derivatives)
"""
self._maybe_start_logging(file)
self._load_landmark_fitter()
frames = mio.import_video(file,
landmark_resolver=self._myresolver,
normalize=True,
exact_frame_count=True)
feat_shape = []
feat_app = []
feat_shape_app = []
for frameIdx, frame in enumerate(frames):
bounding_boxes = self._face_detect(frame)
if len(bounding_boxes) > 0:
initial_bbox = bounding_boxes[0]
if self._log_errors is True:
gt_shape = frame.landmarks['pts_face']
else:
gt_shape = None
if isinstance(self._landmark_fitter, LucasKanadeAAMFitter):
result = self._landmark_fitter.fit_from_bb(
frame, initial_bbox,
max_iters=self._max_iters,
gt_shape=gt_shape)
elif isinstance(self._landmark_fitter, DlibWrapper): # DLIB fitter, doesn't have max_iters
result = self._landmark_fitter.fit_from_bb(
frame,
initial_bbox,
gt_shape=gt_shape)
else:
raise Exception('incompatible landmark fitter')
self._maybe_append_to_log(file, frameIdx, result)
if self._shape == 'face':
if self._parameters == 'lk_fitting':
# skip the first 4 similarity params, probably not useful for classification
shape_param_frame = result.shape_parameters[-1][4:]
app_param_frame = result.appearance_parameters[-1]
elif self._parameters == 'aam_projection':
result_aam = self._projection_fitter.fit_from_shape(
frame,
result.final_shape,
max_iters=[0, 0, 0])
# TODO: analyse the case when aam true components are less than max components
shape_param_frame = result_aam.shape_parameters[-1][4:]
app_param_frame = result_aam.appearance_parameters[-1]
feat_shape.append(shape_param_frame)
feat_app.append(app_param_frame)
feat_shape_app.append(np.concatenate((shape_param_frame, app_param_frame)))
elif self._shape == 'lips':
# extract lips landmarks from the final face fitting to initialize the part model fitting
aam_lips = mio.import_pickle(self._part_aam)
fitter_lips = LucasKanadeAAMFitter(aam_lips, lk_algorithm_cls=WibergInverseCompositional,
n_shape=[10, 20], n_appearance=[20, 150])
result_lips = fitter_lips.fit_from_shape(
image=frame,
initial_shape=_pointcloud_subset(result.final_shape, 'lips'),
max_iters=[5, 5])
shape_param_frame_lips = result_lips.shape_parameters[-1][4:]
app_param_frame_lips = result_lips.appearance_parameters[-1]
feat_shape.append(shape_param_frame_lips)
feat_app.append(app_param_frame_lips)
feat_shape_app.append(np.concatenate((shape_param_frame_lips, app_param_frame_lips)))
elif self._shape == 'chin':
# extract chin and lips landmarks from the final face fitting to initialize the part model fitting
aam_chin = mio.import_pickle(self._part_aam)
fitter_chin = LucasKanadeAAMFitter(aam_chin, lk_algorithm_cls=WibergInverseCompositional,
n_shape=[10, 20, 25], n_appearance=[20, 50, 150])
result_chin = fitter_chin.fit_from_shape(
image=frame,
initial_shape=_pointcloud_subset(result.final_shape, 'chin'),
max_iters=[10, 10, 5])
shape_param_frame_mchin = result_chin.shape_parameters[-1][4:]
app_param_frame_mchin = result_chin.appearance_parameters[-1]
feat_shape.append(shape_param_frame_mchin)
feat_app.append(app_param_frame_mchin)
feat_shape_app.append(np.concatenate((shape_param_frame_mchin, app_param_frame_mchin)))
else:
raise Exception('Unknown shape model, currently supported are: face, lips, chin')
else: # we did not detect any face
zero_feat_shape = np.zeros(process_opts['shape_components'][-1])
zero_feat_app = np.zeros(process_opts['appearance_components'][-1])
zero_feat_shape_app = np.zeros(
process_opts['shape_components'][-1] + process_opts['appearance_components'][-1])
feat_shape.append(zero_feat_shape)
feat_app.append(zero_feat_app)
feat_shape_app.append(zero_feat_shape_app)
npfeat_shape = np.array(feat_shape)
npfeat_app = np.array(feat_app)
npfeat_shape_app = np.array(feat_shape_app)
npfeat_app_delta = vsrmath.accurate_derivative(npfeat_app, 'delta')
npfeat_shape_app_delta = vsrmath.accurate_derivative(npfeat_shape_app, 'delta')
return {'shape': npfeat_shape,
'app': npfeat_app,
'shape_app': npfeat_shape_app,
'app_delta': npfeat_app_delta,
'shape_app_delta': npfeat_shape_app_delta}
def _myresolver(self, file, frame):
frames_dir = file_to_feature(str(file), extension='')
return {'pts_face': self._landmarkDir + frames_dir + '/frame_' + str(frame + 1) + '.pts'}
def _maybe_start_logging(self, file):
if self._log_errors is True:
from os import makedirs
makedirs('./run/logs/' + self._log_dir, exist_ok=True)
cf = file_to_feature(file, extension='')
with open('./run/logs/' + self._log_dir + '/log_' + cf + '.txt', 'w') as log:
log.write('{} \n'.format(file))
log.write('Face detector: {}\n'.format(self._face_detect_method))
if self._fitter_type == 'aam':
log.write('AAM Landmark fitter: {}\n'.format(self._aam_fitter_file))
elif self._fitter_type == 'ert':
log.write('Pretrained ERT Landmark fitter\n')
if self._parameters == 'projection':
log.write('AAM Projector: {}\n'.format(self._projection_aam_file))
def _maybe_append_to_log(self, file, frame_idx, result):
if self._log_errors is True:
cf = file_to_feature(file, extension='')
error = result.final_error()
with open('./run/logs/' + self._log_dir + '/log_' + cf + '.txt', 'a') as log:
log.write('frame {}. error: {} \n'.format(str(frame_idx), str(error)))
def _load_landmark_fitter(self):
if self._fitter_type == 'aam':
self._aam_fitter = mio.import_pickle(self._aam_fitter_file)
fitter = LucasKanadeAAMFitter(self._aam_fitter, lk_algorithm_cls=WibergInverseCompositional,
n_shape=self._shape_components, n_appearance=self._appearance_components)
elif self._fitter_type == 'ert':
fitter = DlibWrapper(
path.join(current_path, '../pretrained/shape_predictor_68_face_landmarks.dat'))
else:
raise Exception('unknown fitter, did you mean aam/ert?')
self._landmark_fitter = fitter
