def test_utils():
sample = read_openface(join(get_test_data_path(), "OpenFace_Test.csv"))
lm_cols = ["x_" + str(i) for i in range(0, 68)] + [
"y_" + str(i) for i in range(0, 68)
]
sample_face = np.array([sample[lm_cols].values[0]])
registered_lm = registration(sample_face)
assert registered_lm.shape == (1, 136)
with pytest.raises(ValueError):
registration(sample_face, method="badmethod")
with pytest.raises(TypeError):
registration(sample_face, method=np.array([1, 2, 3, 4]))
with pytest.raises(AssertionError):
registration([sample_face[0]])
with pytest.raises(AssertionError):
registration(sample_face[0])
with pytest.raises(AssertionError):
registration(sample_face[:, :-1])
# Test softmax
assert softmax(0) == 0.5
# Test badfile.
with pytest.raises(Exception):
load_h5("badfile.h5")
# Test loading of pickled model
out = load_pickled_model()
with pytest.raises(Exception):
load_pickled_model("badfile.pkl")
def predict(au, model=None):
"""Helper function to predict landmarks from au given a sklearn model
Args:
au: vector of action unit intensities
model: sklearn pls object (uses pretrained model by default)
Returns:
landmarks: Array of landmarks (2,68)
"""
if model is None:
model = load_h5()
elif not isinstance(model, PLSRegression):
raise ValueError("make sure that model is a PLSRegression instance")
if len(au) != len(model.x_mean_):
print(au)
print(len(model.x_mean_))
raise ValueError("au vector must be len(", len(model.x_mean_), ").")
if len(au.shape) == 1:
au = np.reshape(au, (1, -1))
landmarks = np.reshape(model.predict(au), (2, 68))
landmarks[1, :] = -1 * landmarks[
1, :] # this might not generalize to other models
return landmarks
def plot_face(model=None, au=None, vectorfield=None, muscles = None, ax=None, color='k', linewidth=1,
linestyle='-', gaze = None, *args, **kwargs):
''' Function to plot facesself
Args:
model: sklearn PLSRegression instance
au: vector of action units (same length as model.x_mean_)
vectorfield: (dict) {'target':target_array,'reference':reference_array}
muscles: (dict) {'muscle': color}
ax: matplotlib axis handle
color: matplotlib color
linewidth: matplotlib linewidth
linestyle: matplotlib linestyle
gaze: array of gaze vectors (len(4))
Returns:
'''
if model is None:
model = load_h5()
else:
if not isinstance(model, PLSRegression):
raise ValueError('make sure that model is a PLSRegression instance')
if au is None:
au = np.zeros(len(model.x_mean_))
warnings.warn("Don't forget to pass an 'au' vector of len(20), "
"using neutral as default")
landmarks = predict(au, model)
currx, curry = ([landmarks[x,:] for x in range(2)])
if ax is None:
ax = _create_empty_figure()
if muscles is not None:
if not isinstance(muscles, dict):
raise ValueError('muscles must be a dictionary ')
draw_muscles(currx, curry, ax=ax, au=au, **muscles)
if gaze is not None and len((gaze)) != 4:
warnings.warn("Don't forget to pass a 'gaze' vector of len(4), "
"using neutral as default")
gaze = None
draw_lineface(currx, curry, color=color, linewidth=linewidth,
linestyle=linestyle, ax=ax, gaze = gaze, *args, **kwargs)
if vectorfield is not None:
if not isinstance(vectorfield, dict):
raise ValueError('vectorfield must be a dictionary ')
if 'reference' not in vectorfield:
raise ValueError("vectorfield must contain 'reference' key")
if 'target' not in vectorfield.keys():
vectorfield['target'] = landmarks
draw_vectorfield(ax=ax, **vectorfield)
ax.set_xlim([25,172])
ax.set_ylim((-240,-50))
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
return ax
def predict(au, model=None, feature_range=None):
"""Helper function to predict landmarks from au given a sklearn model
Args:
au: vector of action unit intensities
model: sklearn pls object (uses pretrained model by default)
feature_range (tuple, default: None): If a tuple with (min, max), scale input AU intensities to (min, max) before prediction.
Returns:
landmarks: Array of landmarks (2,68)
"""
if model is None:
model = load_h5()
elif not isinstance(model, PLSRegression):
raise ValueError("make sure that model is a PLSRegression instance")
if len(au) != model.n_components:
print(au)
print(model.n_components)
raise ValueError("au vector must be len(", model.n_components, ").")
if len(au.shape) == 1:
au = np.reshape(au, (1, -1))
if feature_range:
au = minmax_scale(au, feature_range=feature_range, axis=1)
landmarks = np.reshape(model.predict(au), (2, 68))
# landmarks[1, :] = -1 * landmarks[1, :] # this might not generalize to other models
return landmarks
def plot(self,
row_n,
model=None,
vectorfield=None,
muscles=None,
ax=None,
color='k',
linewidth=1,
linestyle='-',
gaze=None,
*args,
**kwargs):
""" Plot facial representation of data
Args:
row_n: the row of data to use
model: sklearn PLSRegression instance
vectorfield: (dict) {'target':target_array,'reference':reference_array}
muscles: (dict) {'muscle': color}
ax: matplotlib axis handle
color: matplotlib color
linewidth: matplotlib linewidth
linestyle: matplotlib linestyle
gaze: array of gaze vectors (len(5))
"""
feats = [
'AU1', 'AU2', 'AU4', 'AU5', 'AU6', 'AU7', 'AU9', 'AU10', 'AU12',
'AU14', 'AU15', 'AU17', 'AU18', 'AU20', 'AU23', 'AU24', 'AU25',
'AU26', 'AU28', 'AU43', 'Pitch', 'Roll', 'Yaw'
]
if (row_n > len(self)):
raise ValueError("Row number out of range.")
try:
au = []
for feat in feats:
aun = self[feat]
au.append(aun.copy()[row_n])
au = np.array(au)
if model is None:
model = load_h5('facet.h5')
if muscles is not None:
muscles['facet'] = 1
ax = plot_face(model=model,
au=au,
vectorfield=vectorfield,
muscles=muscles,
ax=ax,
color=color,
linewidth=linewidth,
linestyle=linestyle,
gaze=gaze,
*args,
**kwargs)
return ax
except Exception as e:
print('Unable to plot data:', e)
# Activate AU1: Inner brow raiser
au = [3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
# Plot face
fig, axes = plt.subplots(1,2)
plot_face(model=None, ax = axes[0], au = np.zeros(20), color='k', linewidth=1, linestyle='-')
plot_face(model=None, ax = axes[1], au = np.array(au), color='k', linewidth=1, linestyle='-')
## Add a vectorfield with arrows from the changed face back to neutral and vice versa
from feat.plotting import plot_face, predict
from feat.utils import load_h5
import numpy as np
import matplotlib.pyplot as plt
model = load_h5('pyfeat_aus_to_landmarks.h5')
# Add data activate AU1, and AU12
au = np.array([2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ])
# Get neutral landmarks
neutral = predict(np.zeros(len(au)))
# Provide target landmarks and other vector specifications
vectors = {'target': predict(au),
'reference': neutral, 'color': 'blue'}
fig, axes = plt.subplots(1,2)
# Plot face where vectorfield goes from neutral to target, with target as final face
plot_face(model = model, ax = axes[0], au = np.array(au),
vectorfield = vectors, color='k', linewidth=1, linestyle='-')
# save out trained model
# import h5py
# hf = h5py.File('../feat/resources/pyfeat_aus_to_landmarks.h5', 'w')
# hf.create_dataset('coef', data=clf.coef_)
# hf.create_dataset('x_mean', data=clf._x_mean)
# hf.create_dataset('x_std', data=clf._x_std)
# hf.create_dataset('y_mean', data=clf._y_mean)
# hf.close()
# Load h5 model
# In[4]:
from feat.utils import load_h5
clf = load_h5('../../feat/resources/pyfeat_aus_to_landmarks.h5')
# ## Preprocessing datasets
# Here we provide sample code for how you might preprocess the datasets to be used in this tutorial.
#
#
# In[ ]:
from PIL import Image, ImageOps
import math, cv2, csv
from scipy.spatial import ConvexHull
from skimage.morphology.convex_hull import grid_points_in_poly
from feat import Detector
import os, glob, pandas as pd, numpy as np
import matplotlib.pyplot as plt
def plot_face(model=None,
au=None,
vectorfield=None,
muscles=None,
ax=None,
feature_range=False,
color="k",
linewidth=1,
linestyle="-",
gaze=None,
*args,
**kwargs):
"""Function to plot facesself
Args:
model: sklearn PLSRegression instance
au: vector of action units (same length as model.n_components)
vectorfield: (dict) {'target':target_array,'reference':reference_array}
muscles: (dict) {'muscle': color}
ax: matplotlib axis handle
feature_range (tuple, default: None): If a tuple with (min, max), scale input AU intensities to (min, max) before prediction.
color: matplotlib color
linewidth: matplotlib linewidth
linestyle: matplotlib linestyle
gaze: array of gaze vectors (len(4))
Returns:
ax: plot handle
"""
if model is None:
model = load_h5()
else:
if not isinstance(model, PLSRegression):
raise ValueError(
"make sure that model is a PLSRegression instance")
if au is None:
au = np.zeros(model.n_components)
warnings.warn("Don't forget to pass an 'au' vector of len(20), "
"using neutral as default")
landmarks = predict(au, model, feature_range=feature_range)
currx, curry = [landmarks[x, :] for x in range(2)]
if ax is None:
ax = _create_empty_figure()
if muscles is not None:
# Muscles are always scaled 0 - 100 b/c color palette is 0-100
au = minmax_scale(au, feature_range=(0, 100))
if not isinstance(muscles, dict):
raise ValueError("muscles must be a dictionary ")
draw_muscles(currx, curry, ax=ax, au=au, **muscles)
if gaze is not None and len((gaze)) != 4:
warnings.warn("Don't forget to pass a 'gaze' vector of len(4), "
"using neutral as default")
gaze = None
draw_lineface(currx,
curry,
color=color,
linewidth=linewidth,
linestyle=linestyle,
ax=ax,
gaze=gaze,
*args,
**kwargs)
if vectorfield is not None:
if not isinstance(vectorfield, dict):
raise ValueError("vectorfield must be a dictionary ")
if "reference" not in vectorfield:
raise ValueError("vectorfield must contain 'reference' key")
if "target" not in vectorfield.keys():
vectorfield["target"] = landmarks
draw_vectorfield(ax=ax, **vectorfield)
ax.set_xlim([25, 172])
ax.set_ylim((240, 50))
ax.axes.get_xaxis().set_visible(False)
ax.axes.get_yaxis().set_visible(False)
return ax
