def __init__(self) -> None:
self.pca_model = load_classifier(
os.path.join(get_resource_path(), "hog_pca_all_emotio.joblib"))
self.classifier = load_classifier(
os.path.join(get_resource_path(), "Logistic_520.joblib"))
self.scaler = load_classifier(
os.path.join(get_resource_path(), "hog_scalar_aus.joblib"))
def __init__(self) -> None:
self.pca_model = load_classifier(os.path.join(
get_resource_path(), "emo_hog_pca.joblib"))
self.classifier = load_classifier(
os.path.join(get_resource_path(), "emoSVM38.joblib"))
self.scaler = load_classifier(os.path.join(
get_resource_path(), "emo_hog_scalar.joblib"))
def __init__(self, pretrained=True):
super().__init__()
self.conv1 = nn.Conv2d(3, 32, kernel_size=3)
self.prelu1 = nn.PReLU(32)
self.pool1 = nn.MaxPool2d(3, 2, ceil_mode=True)
self.conv2 = nn.Conv2d(32, 64, kernel_size=3)
self.prelu2 = nn.PReLU(64)
self.pool2 = nn.MaxPool2d(3, 2, ceil_mode=True)
self.conv3 = nn.Conv2d(64, 64, kernel_size=3)
self.prelu3 = nn.PReLU(64)
self.pool3 = nn.MaxPool2d(2, 2, ceil_mode=True)
self.conv4 = nn.Conv2d(64, 128, kernel_size=2)
self.prelu4 = nn.PReLU(128)
self.dense5 = nn.Linear(1152, 256)
self.prelu5 = nn.PReLU(256)
self.dense6_1 = nn.Linear(256, 2)
self.softmax6_1 = nn.Softmax(dim=1)
self.dense6_2 = nn.Linear(256, 4)
self.dense6_3 = nn.Linear(256, 10)
self.training = False
if pretrained:
state_dict_path = os.path.join(get_resource_path(), "onet.pt")
state_dict = torch.load(state_dict_path)
self.load_state_dict(state_dict)
def __init__(self):
super(RNet, self).__init__()
self.features = nn.Sequential(
OrderedDict([
("conv1", nn.Conv2d(3, 28, 3, 1)),
("prelu1", nn.PReLU(28)),
("pool1", nn.MaxPool2d(3, 2, ceil_mode=True)),
("conv2", nn.Conv2d(28, 48, 3, 1)),
("prelu2", nn.PReLU(48)),
("pool2", nn.MaxPool2d(3, 2, ceil_mode=True)),
("conv3", nn.Conv2d(48, 64, 2, 1)),
("prelu3", nn.PReLU(64)),
("flatten", Flatten()),
("conv4", nn.Linear(576, 128)),
("prelu4", nn.PReLU(128)),
]))
self.conv5_1 = nn.Linear(128, 2)
self.conv5_2 = nn.Linear(128, 4)
weights = np.load(os.path.join(get_resource_path(), "rnet.npy"),
allow_pickle=True)[()]
for n, p in self.named_parameters():
p.data = torch.FloatTensor(weights[n])
def __init__(self, cpu_mode, constrained=True):
""" Creates an img2pose model. Constrained model is optimized for face detection/ pose estimation for
front-facing faces ( [-90, 90] degree range) only. Unconstrained model can detect faces and poses at any angle,
but shows slightly dampened performance on face pose estimation.
Args:
cpu_mode (bool): whether or not to use CPU (True) or GPU (False)
Returns:
Img2Pose object
"""
pose_mean = np.load(POSE_MEAN, allow_pickle=True)
pose_stddev = np.load(POSE_STDDEV, allow_pickle=True)
threed_points = np.load(THREED_FACE_MODEL, allow_pickle=True)
self.model = img2poseModel(
DEPTH, MIN_SIZE, MAX_SIZE,
pose_mean=pose_mean, pose_stddev=pose_stddev,
threed_68_points=threed_points
)
self.transform = transforms.Compose([transforms.ToTensor()])
# Load the constrained model
model_file = "img2pose_v1_ft_300w_lp.pth" if constrained else "img2pose_v1.pth"
self.load_model(
os.path.join(get_resource_path(), model_file),
cpu_mode=cpu_mode
)
self.model.evaluate()
# Set threshold score for bounding box detection
self.detection_threshold = vis_thres
def __init__(self):
super(PNet, self).__init__()
# suppose we have input with size HxW, then
# after first layer: H - 2,
# after pool: ceil((H - 2)/2),
# after second conv: ceil((H - 2)/2) - 2,
# after last conv: ceil((H - 2)/2) - 4,
# and the same for W
self.features = nn.Sequential(
OrderedDict([
("conv1", nn.Conv2d(3, 10, 3, 1)),
("prelu1", nn.PReLU(10)),
("pool1", nn.MaxPool2d(2, 2, ceil_mode=True)),
("conv2", nn.Conv2d(10, 16, 3, 1)),
("prelu2", nn.PReLU(16)),
("conv3", nn.Conv2d(16, 32, 3, 1)),
("prelu3", nn.PReLU(32)),
]))
self.conv4_1 = nn.Conv2d(32, 2, 1, 1)
self.conv4_2 = nn.Conv2d(32, 4, 1, 1)
weights = np.load(os.path.join(get_resource_path(), "pnet.npy"),
allow_pickle=True)[()]
for n, p in self.named_parameters():
p.data = torch.FloatTensor(weights[n])
def __init__(self):
super(ONet, self).__init__()
self.features = nn.Sequential(
OrderedDict([
("conv1", nn.Conv2d(3, 32, 3, 1)),
("prelu1", nn.PReLU(32)),
("pool1", nn.MaxPool2d(3, 2, ceil_mode=True)),
("conv2", nn.Conv2d(32, 64, 3, 1)),
("prelu2", nn.PReLU(64)),
("pool2", nn.MaxPool2d(3, 2, ceil_mode=True)),
("conv3", nn.Conv2d(64, 64, 3, 1)),
("prelu3", nn.PReLU(64)),
("pool3", nn.MaxPool2d(2, 2, ceil_mode=True)),
("conv4", nn.Conv2d(64, 128, 2, 1)),
("prelu4", nn.PReLU(128)),
("flatten", Flatten()),
("conv5", nn.Linear(1152, 256)),
("drop5", nn.Dropout(0.25)),
("prelu5", nn.PReLU(256)),
]))
self.conv6_1 = nn.Linear(256, 2)
self.conv6_2 = nn.Linear(256, 4)
self.conv6_3 = nn.Linear(256, 10)
weights = np.load(os.path.join(get_resource_path(), "onet.npy"),
allow_pickle=True)[()]
for n, p in self.named_parameters():
p.data = torch.FloatTensor(weights[n])
def __init__(self) -> None:
"""
Initialize model. Loads model weights
"""
super(ferNetModule, self).__init__()
self.pretrained_path = os.path.join(
get_resource_path(), 'best_ferModel.pth')
self.net0 = fer_net(in_chs=3, num_classes=7, img_size=200)
self.use_gpu = torch.cuda.is_available()
if self.use_gpu:
self.net0 = self.net0.cuda()
if self.use_gpu:
self.net0.load_state_dict(torch.load(self.pretrained_path))
else:
self.net0.load_state_dict(torch.load(
self.pretrained_path, map_location={'cuda:0': 'cpu'}))
def __init__(self, pretrained=True):
super().__init__()
self.conv1 = nn.Conv2d(3, 10, kernel_size=3)
self.prelu1 = nn.PReLU(10)
self.pool1 = nn.MaxPool2d(2, 2, ceil_mode=True)
self.conv2 = nn.Conv2d(10, 16, kernel_size=3)
self.prelu2 = nn.PReLU(16)
self.conv3 = nn.Conv2d(16, 32, kernel_size=3)
self.prelu3 = nn.PReLU(32)
self.conv4_1 = nn.Conv2d(32, 2, kernel_size=1)
self.softmax4_1 = nn.Softmax(dim=1)
self.conv4_2 = nn.Conv2d(32, 4, kernel_size=1)
self.training = False
if pretrained:
state_dict_path = os.path.join(get_resource_path(), "pnet.pt")
state_dict = torch.load(state_dict_path)
self.load_state_dict(state_dict)
def __init__(self) -> None:
"""
Initialize.
"""
super(DRMLNet,self).__init__()
self.params = {
"config_au_num": 12,
"config_write_path_prefix": os.path.join(
get_resource_path(), "DRMLNetParams.pth"
),
}
self.use_gpu = torch.cuda.is_available()
self.drml_net = DRML_net(AU_num=self.params["config_au_num"])
if self.use_gpu:
self.drml_net.load_state_dict(torch.load(self.params["config_write_path_prefix"]))
self.drml_net = self.drml_net.cuda()
else:
self.drml_net.load_state_dict(
torch.load(self.params["config_write_path_prefix"], map_location={"cuda:0": "cpu"})
)
self.drml_net.eval()
def __init__(self, pretrained=True):
super().__init__()
self.conv1 = nn.Conv2d(3, 28, kernel_size=3)
self.prelu1 = nn.PReLU(28)
self.pool1 = nn.MaxPool2d(3, 2, ceil_mode=True)
self.conv2 = nn.Conv2d(28, 48, kernel_size=3)
self.prelu2 = nn.PReLU(48)
self.pool2 = nn.MaxPool2d(3, 2, ceil_mode=True)
self.conv3 = nn.Conv2d(48, 64, kernel_size=2)
self.prelu3 = nn.PReLU(64)
self.dense4 = nn.Linear(576, 128)
self.prelu4 = nn.PReLU(128)
self.dense5_1 = nn.Linear(128, 2)
self.softmax5_1 = nn.Softmax(dim=1)
self.dense5_2 = nn.Linear(128, 4)
self.training = False
if pretrained:
state_dict_path = os.path.join(get_resource_path(), "rnet.pt")
state_dict = torch.load(state_dict_path)
self.load_state_dict(state_dict)
def __init__(
self,
face_model="retinaface",
landmark_model="mobilenet",
au_model="rf",
emotion_model="resmasknet",
n_jobs=1,
):
"""Detector class to detect FEX from images or videos.
Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos.
Args:
n_jobs (int, default=1): Number of processes to use for extraction.
Attributes:
info (dict):
n_jobs (int): Number of jobs to be used in parallel.
face_model (str, default=retinaface): Name of face detection model
landmark_model (str, default=mobilenet): Nam eof landmark model
au_model (str, default=rf): Name of Action Unit detection model
emotion_model (str, default=resmasknet): Path to emotion detection model.
face_detection_columns (list): Column names for face detection ouput (x, y, w, h)
face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...)
emotion_model_columns (list): Column names for emotion model output
mapper (dict): Class names for emotion model output by index.
input_shape (dict)
face_detector: face detector object
face_landmark: face_landmark object
emotion_model: emotion_model object
Examples:
>> detector = Detector(n_jobs=1)
>> detector.detect_image("input.jpg")
>> detector.detect_video("input.mp4")
"""
self.info = {}
self.info["n_jobs"] = n_jobs
if torch.cuda.is_available():
self.map_location = lambda storage, loc: storage.cuda()
else:
self.map_location = "cpu"
""" LOAD UP THE MODELS """
print("Loading Face Detection model: ", face_model)
# Check if model files have been downloaded. Otherwise download model.
# get model url.
with open(os.path.join(get_resource_path(), "model_list.json"),
"r") as f:
model_urls = json.load(f)
if face_model:
for url in model_urls["face_detectors"][
face_model.lower()]["urls"]:
download_url(url, get_resource_path())
if landmark_model:
for url in model_urls["landmark_detectors"][
landmark_model.lower()]["urls"]:
download_url(url, get_resource_path())
if au_model:
for url in model_urls["au_detectors"][au_model.lower()]["urls"]:
download_url(url, get_resource_path())
if ".zip" in url:
import zipfile
with zipfile.ZipFile(
os.path.join(get_resource_path(),
"JAANetparams.zip"), 'r') as zip_ref:
zip_ref.extractall(os.path.join(get_resource_path()))
if au_model.lower() in ['logistic', 'svm', 'rf']:
download_url(
model_urls["au_detectors"]['hog-pca']['urls'][0],
get_resource_path())
download_url(
model_urls["au_detectors"]['au_scalar']['urls'][0],
get_resource_path())
if emotion_model:
for url in model_urls["emotion_detectors"][
emotion_model.lower()]["urls"]:
download_url(url, get_resource_path())
if emotion_model.lower() in ['svm', 'rf']:
download_url(
model_urls["emotion_detectors"]['emo_pca']['urls'][0],
get_resource_path())
download_url(
model_urls["emotion_detectors"]['emo_scalar']['urls']
[0], get_resource_path())
if face_model:
if face_model.lower() == "faceboxes":
self.face_detector = FaceBoxes()
elif face_model.lower() == "retinaface":
self.face_detector = Retinaface_test.Retinaface()
elif face_model.lower() == "mtcnn":
self.face_detector = MTCNN()
self.info["face_model"] = face_model
facebox_columns = FEAT_FACEBOX_COLUMNS
self.info["face_detection_columns"] = facebox_columns
predictions = np.empty((1, len(facebox_columns)))
predictions[:] = np.nan
empty_facebox = pd.DataFrame(predictions, columns=facebox_columns)
self._empty_facebox = empty_facebox
print("Loading Face Landmark model: ", landmark_model)
if landmark_model:
if landmark_model.lower() == "mobilenet":
self.landmark_detector = MobileNet_GDConv(136)
self.landmark_detector = torch.nn.DataParallel(
self.landmark_detector)
checkpoint = torch.load(
os.path.join(
get_resource_path(),
"mobilenet_224_model_best_gdconv_external.pth.tar",
),
map_location=self.map_location,
)
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
elif landmark_model.lower() == "pfld":
self.landmark_detector = PFLDInference()
checkpoint = torch.load(
os.path.join(get_resource_path(),
"pfld_model_best.pth.tar"),
map_location=self.map_location,
)
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
elif landmark_model.lower() == "mobilefacenet":
self.landmark_detector = MobileFaceNet([112, 112], 136)
checkpoint = torch.load(
os.path.join(get_resource_path(),
"mobilefacenet_model_best.pth.tar"),
map_location=self.map_location,
)
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
self.info["landmark_model"] = landmark_model
self.info["mapper"] = openface_2d_landmark_columns
landmark_columns = openface_2d_landmark_columns
self.info["face_landmark_columns"] = landmark_columns
predictions = np.empty((1, len(openface_2d_landmark_columns)))
predictions[:] = np.nan
empty_landmarks = pd.DataFrame(predictions, columns=landmark_columns)
self._empty_landmark = empty_landmarks
print("Loading au model: ", au_model)
self.info["au_model"] = au_model
if au_model:
if au_model.lower() == "jaanet":
self.au_model = JAANet()
elif au_model.lower() == "drml":
self.au_model = DRMLNet()
elif au_model.lower() == "logistic":
self.au_model = LogisticClassifier()
elif au_model.lower() == "svm":
self.au_model = SVMClassifier()
elif au_model.lower() == 'rf':
self.au_model = RandomForestClassifier()
if (au_model is None) or (au_model.lower() in ['jaanet', 'drml']):
auoccur_columns = jaanet_AU_presence
else:
auoccur_columns = RF_AU_presence
self.info["au_presence_columns"] = auoccur_columns
predictions = np.empty((1, len(auoccur_columns)))
predictions[:] = np.nan
empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns)
self._empty_auoccurence = empty_au_occurs
print("Loading emotion model: ", emotion_model)
self.info["emotion_model"] = emotion_model
if emotion_model:
if emotion_model.lower() == "fer":
self.emotion_model = ferNetModule()
elif emotion_model.lower() == "resmasknet":
self.emotion_model = ResMaskNet()
elif emotion_model.lower() == 'svm':
self.emotion_model = EmoSVMClassifier()
elif emotion_model.lower() == 'rf':
self.emotion_model = EmoRandomForestClassifier()
self.info["emotion_model_columns"] = FEAT_EMOTION_COLUMNS
predictions = np.empty((1, len(FEAT_EMOTION_COLUMNS)))
predictions[:] = np.nan
empty_emotion = pd.DataFrame(predictions, columns=FEAT_EMOTION_COLUMNS)
self._empty_emotion = empty_emotion
predictions = np.empty((1, len(auoccur_columns)))
predictions[:] = np.nan
empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns)
self._empty_auoccurence = empty_au_occurs
self.info["output_columns"] = (FEAT_TIME_COLUMNS + facebox_columns +
landmark_columns + auoccur_columns +
FEAT_EMOTION_COLUMNS + ["input"])
Timer,
load_model,
nms,
)
from feat.utils import get_resource_path
# some global configs
confidence_threshold = 0.05
top_k = 5000
keep_top_k = 750
nms_threshold = 0.3
vis_thres = 0.5
resize = 1
scale_flag = True
HEIGHT, WIDTH = 720, 1080
pretrained_path = os.path.join(get_resource_path(), "FaceBoxesProd.pth")
cfg = {
"name": "FaceBoxes",
"min_sizes": [[32, 64, 128], [256], [512]],
"steps": [32, 64, 128],
"variance": [0.1, 0.2],
"clip": False,
}
def viz_bbox(img, dets, wfp="out.jpg"):
# show
for b in dets:
if b[4] < vis_thres:
continue
text = "{:.4f}".format(b[4])
from torchvision import transforms
from .img2pose_model import img2poseModel
from feat.utils import get_resource_path
from feat.face_detectors.Retinaface.Retinaface_utils import py_cpu_nms
from ..utils import convert_to_euler
BORDER_SIZE = 100
DEPTH = 18
MAX_SIZE = 1400
MIN_SIZE = 400
nms_inclusion_threshold = 0.05 # face score below which face box is excluded from nms
nms_threshold = 0.6 # default from img2pose paper
top_k = 5000
keep_top_k = 750
vis_thres = 0.5
POSE_MEAN = os.path.join(get_resource_path(), "WIDER_train_pose_mean_v1.npy")
POSE_STDDEV = os.path.join(get_resource_path(), "WIDER_train_pose_stddev_v1.npy")
THREED_FACE_MODEL = os.path.join(get_resource_path(), "reference_3d_68_points_trans.npy")
class Img2Pose:
def __init__(self, cpu_mode, constrained=True):
""" Creates an img2pose model. Constrained model is optimized for face detection/ pose estimation for
front-facing faces ( [-90, 90] degree range) only. Unconstrained model can detect faces and poses at any angle,
but shows slightly dampened performance on face pose estimation.
Args:
cpu_mode (bool): whether or not to use CPU (True) or GPU (False)
Returns:
Img2Pose object
def __init__(
self,
face_model="retinaface",
landmark_model="MobileNet",
au_model="jaanet",
emotion_model="fer",
n_jobs=1,
):
"""Detector class to detect FEX from images or videos.
Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos.
Args:
n_jobs (int, default=1): Number of processes to use for extraction.
Attributes:
info (dict):
n_jobs (int): Number of jobs to be used in parallel.
face_detection_model (str, default=haarcascade_frontalface_alt.xml): Path to face detection model.
face_detection_columns (list): Column names for face detection ouput (x, y, w, h)
face_landmark_model (str, default=lbfmodel.yaml): Path to landmark model.
face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...)
emotion_model (str, default=fer_aug_model.h5): Path to emotion detection model.
emotion_model_columns (list): Column names for emotion model output
mapper (dict): Class names for emotion model output by index.
input_shape (dict)
face_detector: face detector object
face_landmark: face_landmark object
emotion_model: emotion_model object
Examples:
>> detector = Detector(n_jobs=1)
>> detector.detect_image("input.jpg")
>> detector.detect_video("input.mp4")
"""
self.info = {}
self.info["n_jobs"] = n_jobs
if torch.cuda.is_available():
self.map_location = lambda storage, loc: storage.cuda()
else:
self.map_location = "cpu"
""" LOAD UP THE MODELS """
print("Loading Face Detection model: ", face_model)
if face_model:
if face_model.lower() == "faceboxes":
self.face_detector = FaceBoxes()
elif face_model.lower() == "retinaface":
self.face_detector = Retinaface_test.Retinaface()
elif face_model.lower() == "mtcnn":
self.face_detector = MTCNN()
self.info["face_model"] = face_model
# self.info["mapper"] = FEAT_FACEBOX_COLUMNS
facebox_columns = FEAT_FACEBOX_COLUMNS
self.info["face_detection_columns"] = facebox_columns
predictions = np.empty((1, len(facebox_columns)))
predictions[:] = np.nan
empty_facebox = pd.DataFrame(predictions, columns=facebox_columns)
self._empty_facebox = empty_facebox
print("Loading Face Landmark model: ", landmark_model)
# self.info['Landmark_Model'] = landmark_model
if landmark_model:
if landmark_model.lower() == "mobilenet":
self.landmark_detector = MobileNet_GDConv(136)
self.landmark_detector = torch.nn.DataParallel(
self.landmark_detector)
# or download model from https://drive.google.com/file/d/1Le5UdpMkKOTRr1sTp4lwkw8263sbgdSe/view?usp=sharing
checkpoint = torch.load(
os.path.join(
get_resource_path(),
"mobilenet_224_model_best_gdconv_external.pth.tar",
),
map_location=self.map_location,
)
# print("Use MobileNet as backbone")
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
elif landmark_model.lower() == "pfld":
self.landmark_detector = PFLDInference()
# or download from https://drive.google.com/file/d/1gjgtm6qaBQJ_EY7lQfQj3EuMJCVg9lVu/view?usp=sharing
checkpoint = torch.load(
os.path.join(get_resource_path(),
"pfld_model_best.pth.tar"),
map_location=self.map_location,
)
# print("Use PFLD as backbone")
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
# or download from https://drive.google.com/file/d/1T8J73UTcB25BEJ_ObAJczCkyGKW5VaeY/view?usp=sharing
elif landmark_model.lower() == "mobilefacenet":
self.landmark_detector = MobileFaceNet([112, 112], 136)
checkpoint = torch.load(
os.path.join(get_resource_path(),
"mobilefacenet_model_best.pth.tar"),
map_location=self.map_location,
)
# print("Use MobileFaceNet as backbone")
self.landmark_detector.load_state_dict(
checkpoint["state_dict"])
self.info["landmark_model"] = landmark_model
self.info["mapper"] = openface_2d_landmark_columns
landmark_columns = openface_2d_landmark_columns
self.info["face_landmark_columns"] = landmark_columns
predictions = np.empty((1, len(openface_2d_landmark_columns)))
predictions[:] = np.nan
empty_landmarks = pd.DataFrame(predictions, columns=landmark_columns)
self._empty_landmark = empty_landmarks
print("Loading au occurence model: ", au_model)
self.info["au_model"] = au_model
if au_model:
if au_model.lower() == "jaanet":
self.au_model = JAANet()
elif au_model.lower() == "drml":
self.au_model = DRMLNet()
# self.info["mapper"] = jaanet_AU_presence
auoccur_columns = jaanet_AU_presence
self.info["au_presence_columns"] = auoccur_columns
predictions = np.empty((1, len(auoccur_columns)))
predictions[:] = np.nan
empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns)
self._empty_auoccurence = empty_au_occurs
print("Loading emotion model: ", emotion_model)
self.info["emotion_model"] = emotion_model
if emotion_model:
if emotion_model.lower() == "fer":
self.emotion_model = ferNetModule()
elif emotion_model.lower() == "resmasknet":
self.emotion_model = ResMaskNet()
self.info["emotion_model_columns"] = FEAT_EMOTION_COLUMNS
predictions = np.empty((1, len(FEAT_EMOTION_COLUMNS)))
predictions[:] = np.nan
empty_emotion = pd.DataFrame(predictions, columns=FEAT_EMOTION_COLUMNS)
self._empty_emotion = empty_emotion
# self.info['auoccur_model'] = au_model
# self.info["mapper"] = jaanet_AU_presence
auoccur_columns = jaanet_AU_presence
self.info["au_presence_columns"] = auoccur_columns
predictions = np.empty((1, len(auoccur_columns)))
predictions[:] = np.nan
empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns)
self._empty_auoccurence = empty_au_occurs
self.info["output_columns"] = (FEAT_TIME_COLUMNS + facebox_columns +
landmark_columns + auoccur_columns +
FEAT_EMOTION_COLUMNS + ["input"])
from feat.utils import get_resource_path
def face_rect_to_coords(rectangle):
"""
Takes in a (x, y, w, h) array and transforms it into (x, y, x2, y2)
"""
return [
rectangle[0], rectangle[1], rectangle[0] + rectangle[2],
rectangle[1] + rectangle[3]
]
# load pre trained emotion model
print("Loading FEX DCNN model.")
dcnn_model_path = os.path.join(get_resource_path(), 'fer_aug_model.h5')
if not os.path.exists(dcnn_model_path):
print("Emotion prediction model not found. Please run download_models.py.")
model = models.load_model(dcnn_model_path) # Load model to use.
(_, img_w, img_h, img_c) = model.layers[0].input_shape # model input shape.
mapper = {
0: 'anger',
1: 'disgust',
2: 'fear',
3: 'happiness',
4: 'sadness',
5: 'surprise',
6: 'neutral'
}
emotion_columns = [key for key in mapper.values()]
def __init__(self):
"""Initialize ResMaskNet
@misc{luanresmaskingnet2020,
Author = {Luan Pham & Tuan Anh Tran},
Title = {Facial Expression Recognition using Residual Masking Network},
url = {https://github.com/phamquiluan/ResidualMaskingNetwork},
Year = {2020}
}
"""
self.transform = transforms.Compose(
[transforms.ToPILImage(),
transforms.ToTensor()])
self.FER_2013_EMO_DICT = {
0: "angry",
1: "disgust",
2: "fear",
3: "happy",
4: "sad",
5: "surprise",
6: "neutral",
}
# load configs and set random seed
configs = json.load(
open(
os.path.join(get_resource_path(),
"ResMaskNet_fer2013_config.json")))
self.image_size = (configs["image_size"], configs["image_size"])
self.use_gpu = torch.cuda.is_available()
# if self.use_gpu:
# self.state = torch.load(
# os.path.join(
# get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth"
# )
# )
# else:
# self.state = torch.load(
# os.path.join(
# get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth"
# ),
# map_location={"cuda:0": "cpu"},
# )
self.model = resmasking_dropout1(in_channels=3, num_classes=7)
if self.use_gpu:
self.model.load_state_dict(
torch.load(
os.path.join(
get_resource_path(),
"ResMaskNet_Z_resmasking_dropout1_rot30.pth"))['net'])
self.model.cuda()
else:
self.model.load_state_dict(
torch.load(
os.path.join(get_resource_path(),
"ResMaskNet_Z_resmasking_dropout1_rot30.pth"),
map_location={"cuda:0": "cpu"},
)['net'])
self.model.eval()
def __init__(self) -> None:
"""
Initialize.
Args:
img_data: numpy array image data files of shape (N,3,W,H)
land_data: numpy array landmark data of shape (N, 49*2)
"""
# self.imgs = img_data
# self.land_data = land_data
super(JAANet, self).__init__()
self.params = {
"config_unit_dim": 8,
"config_crop_size": 176,
"config_map_size": 44,
"config_au_num": 12,
"config_land_num": 49,
"config_fill_coeff": 0.56,
"config_write_path_prefix": get_resource_path(),
}
config_unit_dim = self.params["config_unit_dim"]
config_crop_size = self.params["config_crop_size"]
config_map_size = self.params["config_map_size"]
config_au_num = self.params["config_au_num"]
config_land_num = self.params["config_land_num"]
config_fill_coeff = self.params["config_fill_coeff"]
config_write_path_prefix = self.params["config_write_path_prefix"]
self.region_learning = network.network_dict["HMRegionLearning"](
input_dim=3, unit_dim=config_unit_dim)
self.align_net = network.network_dict["AlignNet"](
crop_size=config_crop_size,
map_size=config_map_size,
au_num=config_au_num,
land_num=config_land_num,
input_dim=config_unit_dim * 8,
fill_coeff=config_fill_coeff,
)
self.local_attention_refine = network.network_dict[
"LocalAttentionRefine"](au_num=config_au_num,
unit_dim=config_unit_dim)
self.local_au_net = network.network_dict["LocalAUNetv2"](
au_num=config_au_num,
input_dim=config_unit_dim * 8,
unit_dim=config_unit_dim,
)
self.global_au_feat = network.network_dict["HLFeatExtractor"](
input_dim=config_unit_dim * 8, unit_dim=config_unit_dim)
self.au_net = network.network_dict["AUNet"](au_num=config_au_num,
input_dim=12000,
unit_dim=config_unit_dim)
self.use_gpu = torch.cuda.is_available()
if self.use_gpu:
self.region_learning = self.region_learning.cuda()
self.align_net = self.align_net.cuda()
self.local_attention_refine = self.local_attention_refine.cuda()
self.local_au_net = self.local_au_net.cuda()
self.global_au_feat = self.global_au_feat.cuda()
self.au_net = self.au_net.cuda()
# Load parameters
# load_map = 'cpu' if True else 'false'
# au_occur_model_path = os.path.join(
# config_write_path_prefix , '/region_learning' , '.pth')
# print("should load data at ",os.path.join(config_write_path_prefix , 'region_learning.pth'))
# print("Directory Files:")
# print(os.listdir(config_write_path_prefix))
self.region_learning.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"region_learning.pth")))
self.align_net.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix, "align_net.pth")))
self.local_attention_refine.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"local_attention_refine.pth")))
self.local_au_net.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"local_au_net.pth")))
self.global_au_feat.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"global_au_feat.pth")))
self.au_net.load_state_dict(
torch.load(os.path.join(config_write_path_prefix,
"au_net.pth")))
else:
self.region_learning.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"region_learning.pth"),
map_location={"cuda:0": "cpu"},
))
self.align_net.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix, "align_net.pth"),
map_location={"cuda:0": "cpu"},
))
self.local_attention_refine.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"local_attention_refine.pth"),
map_location={"cuda:0": "cpu"},
))
self.local_au_net.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix, "local_au_net.pth"),
map_location={"cuda:0": "cpu"},
))
self.global_au_feat.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix,
"global_au_feat.pth"),
map_location={"cuda:0": "cpu"},
))
self.au_net.load_state_dict(
torch.load(
os.path.join(config_write_path_prefix, "au_net.pth"),
map_location={"cuda:0": "cpu"},
))
self.region_learning.eval()
self.align_net.eval()
self.local_attention_refine.eval()
self.local_au_net.eval()
self.global_au_feat.eval()
self.au_net.eval()
def __init__(self, n_jobs=1):
"""Detector class to detect FEX from images or videos.
Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos.
Args:
n_jobs (int, default=1): Number of processes to use for extraction.
Attributes:
info (dict):
n_jobs (int): Number of jobs to be used in parallel.
face_detection_model (str, default=haarcascade_frontalface_alt.xml): Path to face detection model.
face_detection_columns (list): Column names for face detection ouput (x, y, w, h)
face_landmark_model (str, default=lbfmodel.yaml): Path to landmark model.
face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...)
emotion_model (str, default=fer_aug_model.h5): Path to emotion detection model.
emotion_model_columns (list): Column names for emotion model output
mapper (dict): Class names for emotion model output by index.
input_shape (dict)
face_detector: face detector object
face_landmark: face_landmark object
emotion_model: emotion_model object
Examples:
>> detector = Detector(n_jobs=1)
>> detector.detect_image("input.jpg")
>> detector.detect_video("input.mp4")
"""
self.info = {}
self.info['n_jobs'] = n_jobs
""" LOAD UP THE MODELS """
print("Loading Face Detection model.")
face_detection_model_path = cv.data.haarcascades + "haarcascade_frontalface_alt.xml"
if not os.path.exists(face_detection_model_path):
print("Face detection model not found. Check haarcascade_frontalface_alt.xml exists in your opencv installation (cv.data).")
face_cascade = cv.CascadeClassifier(face_detection_model_path)
face_detection_columns = FEAT_FACEBOX_COLUMNS
facebox_empty = np.empty((1,4))
facebox_empty[:] = np.nan
empty_facebox = pd.DataFrame(facebox_empty, columns = face_detection_columns)
self.info["face_detection_model"] = face_detection_model_path
self.info["face_detection_columns"] = face_detection_columns
self.face_detector = face_cascade
self._empty_facebox = empty_facebox
print("Loading Face Landmark model.")
face_landmark = cv.face.createFacemarkLBF()
face_landmark_model_path = os.path.join(get_resource_path(), 'lbfmodel.yaml')
if not os.path.exists(face_landmark_model_path):
print("Face landmark model not found. Please run download_models.py.")
face_landmark.loadModel(face_landmark_model_path)
face_landmark_columns = np.array([(f'x_{i}',f'y_{i}') for i in range(68)]).reshape(1,136)[0].tolist()
landmark_empty = np.empty((1,136))
landmark_empty[:] = np.nan
empty_landmark = pd.DataFrame(landmark_empty, columns = face_landmark_columns)
self.info["face_landmark_model"] = face_landmark_model_path
self.info['face_landmark_columns'] = face_landmark_columns
self.face_landmark = face_landmark
self._empty_landmark = empty_landmark
print("Loading FEX DCNN emotion model.")
emotion_model = 'fer_aug_model.h5'
emotion_model_path = os.path.join(get_resource_path(),'fer_aug_model.h5')
if not os.path.exists(emotion_model_path):
print("Emotion prediction model not found. Please run download_models.py.")
model = models.load_model(emotion_model_path) # Load model to use.
(_, img_w, img_h, img_c) = model.layers[0].input_shape # model input shape.
self.info["input_shape"] = {"img_w": img_w, "img_h": img_h, "img_c": img_c}
self.info["emotion_model"] = emotion_model_path
self.info["mapper"] = FEAT_EMOTION_MAPPER
self.emotion_model = model
emotion_columns = [key for key in self.info["mapper"].values()]
self.info['emotion_model_columns'] = emotion_columns
# create empty df for predictions
predictions = np.empty((1, len(self.info["mapper"])))
predictions[:] = np.nan
empty_emotion = pd.DataFrame(predictions, columns = self.info["mapper"].values())
self._empty_emotion = empty_emotion
frame_columns = ["frame"]
self.info["output_columns"] = frame_columns + emotion_columns + face_detection_columns + face_landmark_columns
import os
import cv2
import numpy as np
from feat.utils import get_resource_path
from feat.facepose_detectors.utils import convert_to_euler
THREED_FACE_MODEL = os.path.join(get_resource_path(),
"reference_3d_68_points_trans.npy")
class PerspectiveNPointModel:
""" Class that leverages 68 2D facial landmark points to estimate head pose using the Perspective-n-Point
algorithm.
Code adapted from https://github.com/yinguobing/head-pose-estimation/ and
https://github.com/lincolnhard/head-pose-estimation/. Each code base licensed under MIT Licenses, which can be
found here: https://github.com/yinguobing/head-pose-estimation/blob/master/LICENSE and here:
https://github.com/lincolnhard/head-pose-estimation/blob/master/LICENSE
"""
def __init__(self):
""" Initializes the model, with a reference 3D model (xyz coordinates) of a standard face"""
# self.model_points = get_full_model_points(os.path.join(get_resource_path(), "3d_face_model.txt"))
self.model_points = np.load(THREED_FACE_MODEL, allow_pickle=True)
def predict(self, img, landmarks):
""" Determines headpose using passed 68 2D landmarks
Args:
img (np.ndarray) : The cv2 image from which the landmarks were produced
landmarks (np.ndarray) : The landmarks to use to produce the headpose estimate
Returns:
#!/usr/bin/env python3
"""
This script will download the necessary models to the feat path.
Usage
python3 download_models.py
"""
from feat.utils import get_resource_path
import wget
import os
print("Downloading FEX emotion model.")
fex_emotion_model = "https://github.com/cosanlab/feat/releases/download/v0.1/fer_aug_model.h5"
wget.download(fex_emotion_model, get_resource_path())
if os.path.exists(os.path.join(get_resource_path(), "fer_aug_model.h5")):
print("\nFEX emotion model downloaded successfully.\n")
else:
print("Something went wrong. Model not found in directory.")
print("Downloading landmark detection model.")
lbfmodel = "https://github.com/cosanlab/feat/releases/download/v0.1/lbfmodel.yaml"
wget.download(lbfmodel, get_resource_path())
if os.path.exists(os.path.join(get_resource_path(), "lbfmodel.yaml")):
print("\nLandmark detection model downloaded successfully.\n")
else:
print("Something went wrong. Model not found in directory.")
from __future__ import print_function
import os
import torch
import numpy as np
import time
import feat
from feat.face_detectors.Retinaface.Retinaface_model import PriorBox, RetinaFace
from feat.face_detectors.Retinaface.Retinaface_utils import (
py_cpu_nms,
decode,
decode_landm,
)
from feat.utils import get_resource_path
# some global configs
trained_model = os.path.join(get_resource_path(), "mobilenet0.25_Final.pth")
network = "mobile0.25"
confidence_threshold = 0.05
top_k = 5000
keep_top_k = 750
nms_threshold = 0.3
vis_thres = 0.5
resize = 1
cpu = True
cfg_mnet = {
"name": "mobilenet0.25",
"min_sizes": [[16, 32], [64, 128], [256, 512]],
"steps": [8, 16, 32],
"variance": [0.1, 0.2],
"clip": False,
"loc_weight": 2.0,
#!/usr/bin/env python3
"""
This script will download the necessary models to the feat path.
Usage
python3 download_models.py
"""
from feat.utils import get_resource_path
import wget
import os
import zipfile
if os.path.exists(os.path.join(get_resource_path(), "best_ferModel.pth")):
print("Fex model already exists; skipping download.")
else:
print("Downloading FEX emotion model.")
try:
fex_emotion_model = "https://github.com/cosanlab/feat/releases/download/v0.1/best_ferModel.pth"
wget.download(fex_emotion_model, get_resource_path(), bar=None)
# except:
# try:
# fex_emotion_model = "https://www.dropbox.com/s/d3yhtsqggqcrjl2/fer_emotion_model.h5?dl=1"
# wget.download(fex_emotion_model, get_resource_path(), bar=None)
except:
print("FeX emotion model failed to download")
if os.path.exists(os.path.join(get_resource_path(), "best_ferModel.pth")):
print("\nFEX emotion model downloaded successfully.\n")
else:
print("Something went wrong. Model not found in directory.")
