def __init__(self) -> None: self.pca_model = load_classifier( os.path.join(get_resource_path(), "hog_pca_all_emotio.joblib")) self.classifier = load_classifier( os.path.join(get_resource_path(), "Logistic_520.joblib")) self.scaler = load_classifier( os.path.join(get_resource_path(), "hog_scalar_aus.joblib"))
def __init__(self) -> None: self.pca_model = load_classifier(os.path.join( get_resource_path(), "emo_hog_pca.joblib")) self.classifier = load_classifier( os.path.join(get_resource_path(), "emoSVM38.joblib")) self.scaler = load_classifier(os.path.join( get_resource_path(), "emo_hog_scalar.joblib"))
def __init__(self, pretrained=True): super().__init__() self.conv1 = nn.Conv2d(3, 32, kernel_size=3) self.prelu1 = nn.PReLU(32) self.pool1 = nn.MaxPool2d(3, 2, ceil_mode=True) self.conv2 = nn.Conv2d(32, 64, kernel_size=3) self.prelu2 = nn.PReLU(64) self.pool2 = nn.MaxPool2d(3, 2, ceil_mode=True) self.conv3 = nn.Conv2d(64, 64, kernel_size=3) self.prelu3 = nn.PReLU(64) self.pool3 = nn.MaxPool2d(2, 2, ceil_mode=True) self.conv4 = nn.Conv2d(64, 128, kernel_size=2) self.prelu4 = nn.PReLU(128) self.dense5 = nn.Linear(1152, 256) self.prelu5 = nn.PReLU(256) self.dense6_1 = nn.Linear(256, 2) self.softmax6_1 = nn.Softmax(dim=1) self.dense6_2 = nn.Linear(256, 4) self.dense6_3 = nn.Linear(256, 10) self.training = False if pretrained: state_dict_path = os.path.join(get_resource_path(), "onet.pt") state_dict = torch.load(state_dict_path) self.load_state_dict(state_dict)
def __init__(self): super(RNet, self).__init__() self.features = nn.Sequential( OrderedDict([ ("conv1", nn.Conv2d(3, 28, 3, 1)), ("prelu1", nn.PReLU(28)), ("pool1", nn.MaxPool2d(3, 2, ceil_mode=True)), ("conv2", nn.Conv2d(28, 48, 3, 1)), ("prelu2", nn.PReLU(48)), ("pool2", nn.MaxPool2d(3, 2, ceil_mode=True)), ("conv3", nn.Conv2d(48, 64, 2, 1)), ("prelu3", nn.PReLU(64)), ("flatten", Flatten()), ("conv4", nn.Linear(576, 128)), ("prelu4", nn.PReLU(128)), ])) self.conv5_1 = nn.Linear(128, 2) self.conv5_2 = nn.Linear(128, 4) weights = np.load(os.path.join(get_resource_path(), "rnet.npy"), allow_pickle=True)[()] for n, p in self.named_parameters(): p.data = torch.FloatTensor(weights[n])
def __init__(self, cpu_mode, constrained=True): """ Creates an img2pose model. Constrained model is optimized for face detection/ pose estimation for front-facing faces ( [-90, 90] degree range) only. Unconstrained model can detect faces and poses at any angle, but shows slightly dampened performance on face pose estimation. Args: cpu_mode (bool): whether or not to use CPU (True) or GPU (False) Returns: Img2Pose object """ pose_mean = np.load(POSE_MEAN, allow_pickle=True) pose_stddev = np.load(POSE_STDDEV, allow_pickle=True) threed_points = np.load(THREED_FACE_MODEL, allow_pickle=True) self.model = img2poseModel( DEPTH, MIN_SIZE, MAX_SIZE, pose_mean=pose_mean, pose_stddev=pose_stddev, threed_68_points=threed_points ) self.transform = transforms.Compose([transforms.ToTensor()]) # Load the constrained model model_file = "img2pose_v1_ft_300w_lp.pth" if constrained else "img2pose_v1.pth" self.load_model( os.path.join(get_resource_path(), model_file), cpu_mode=cpu_mode ) self.model.evaluate() # Set threshold score for bounding box detection self.detection_threshold = vis_thres
def __init__(self): super(PNet, self).__init__() # suppose we have input with size HxW, then # after first layer: H - 2, # after pool: ceil((H - 2)/2), # after second conv: ceil((H - 2)/2) - 2, # after last conv: ceil((H - 2)/2) - 4, # and the same for W self.features = nn.Sequential( OrderedDict([ ("conv1", nn.Conv2d(3, 10, 3, 1)), ("prelu1", nn.PReLU(10)), ("pool1", nn.MaxPool2d(2, 2, ceil_mode=True)), ("conv2", nn.Conv2d(10, 16, 3, 1)), ("prelu2", nn.PReLU(16)), ("conv3", nn.Conv2d(16, 32, 3, 1)), ("prelu3", nn.PReLU(32)), ])) self.conv4_1 = nn.Conv2d(32, 2, 1, 1) self.conv4_2 = nn.Conv2d(32, 4, 1, 1) weights = np.load(os.path.join(get_resource_path(), "pnet.npy"), allow_pickle=True)[()] for n, p in self.named_parameters(): p.data = torch.FloatTensor(weights[n])
def __init__(self): super(ONet, self).__init__() self.features = nn.Sequential( OrderedDict([ ("conv1", nn.Conv2d(3, 32, 3, 1)), ("prelu1", nn.PReLU(32)), ("pool1", nn.MaxPool2d(3, 2, ceil_mode=True)), ("conv2", nn.Conv2d(32, 64, 3, 1)), ("prelu2", nn.PReLU(64)), ("pool2", nn.MaxPool2d(3, 2, ceil_mode=True)), ("conv3", nn.Conv2d(64, 64, 3, 1)), ("prelu3", nn.PReLU(64)), ("pool3", nn.MaxPool2d(2, 2, ceil_mode=True)), ("conv4", nn.Conv2d(64, 128, 2, 1)), ("prelu4", nn.PReLU(128)), ("flatten", Flatten()), ("conv5", nn.Linear(1152, 256)), ("drop5", nn.Dropout(0.25)), ("prelu5", nn.PReLU(256)), ])) self.conv6_1 = nn.Linear(256, 2) self.conv6_2 = nn.Linear(256, 4) self.conv6_3 = nn.Linear(256, 10) weights = np.load(os.path.join(get_resource_path(), "onet.npy"), allow_pickle=True)[()] for n, p in self.named_parameters(): p.data = torch.FloatTensor(weights[n])
def __init__(self) -> None: """ Initialize model. Loads model weights """ super(ferNetModule, self).__init__() self.pretrained_path = os.path.join( get_resource_path(), 'best_ferModel.pth') self.net0 = fer_net(in_chs=3, num_classes=7, img_size=200) self.use_gpu = torch.cuda.is_available() if self.use_gpu: self.net0 = self.net0.cuda() if self.use_gpu: self.net0.load_state_dict(torch.load(self.pretrained_path)) else: self.net0.load_state_dict(torch.load( self.pretrained_path, map_location={'cuda:0': 'cpu'}))
def __init__(self, pretrained=True): super().__init__() self.conv1 = nn.Conv2d(3, 10, kernel_size=3) self.prelu1 = nn.PReLU(10) self.pool1 = nn.MaxPool2d(2, 2, ceil_mode=True) self.conv2 = nn.Conv2d(10, 16, kernel_size=3) self.prelu2 = nn.PReLU(16) self.conv3 = nn.Conv2d(16, 32, kernel_size=3) self.prelu3 = nn.PReLU(32) self.conv4_1 = nn.Conv2d(32, 2, kernel_size=1) self.softmax4_1 = nn.Softmax(dim=1) self.conv4_2 = nn.Conv2d(32, 4, kernel_size=1) self.training = False if pretrained: state_dict_path = os.path.join(get_resource_path(), "pnet.pt") state_dict = torch.load(state_dict_path) self.load_state_dict(state_dict)
def __init__(self) -> None: """ Initialize. """ super(DRMLNet,self).__init__() self.params = { "config_au_num": 12, "config_write_path_prefix": os.path.join( get_resource_path(), "DRMLNetParams.pth" ), } self.use_gpu = torch.cuda.is_available() self.drml_net = DRML_net(AU_num=self.params["config_au_num"]) if self.use_gpu: self.drml_net.load_state_dict(torch.load(self.params["config_write_path_prefix"])) self.drml_net = self.drml_net.cuda() else: self.drml_net.load_state_dict( torch.load(self.params["config_write_path_prefix"], map_location={"cuda:0": "cpu"}) ) self.drml_net.eval()
def __init__(self, pretrained=True): super().__init__() self.conv1 = nn.Conv2d(3, 28, kernel_size=3) self.prelu1 = nn.PReLU(28) self.pool1 = nn.MaxPool2d(3, 2, ceil_mode=True) self.conv2 = nn.Conv2d(28, 48, kernel_size=3) self.prelu2 = nn.PReLU(48) self.pool2 = nn.MaxPool2d(3, 2, ceil_mode=True) self.conv3 = nn.Conv2d(48, 64, kernel_size=2) self.prelu3 = nn.PReLU(64) self.dense4 = nn.Linear(576, 128) self.prelu4 = nn.PReLU(128) self.dense5_1 = nn.Linear(128, 2) self.softmax5_1 = nn.Softmax(dim=1) self.dense5_2 = nn.Linear(128, 4) self.training = False if pretrained: state_dict_path = os.path.join(get_resource_path(), "rnet.pt") state_dict = torch.load(state_dict_path) self.load_state_dict(state_dict)
def __init__( self, face_model="retinaface", landmark_model="mobilenet", au_model="rf", emotion_model="resmasknet", n_jobs=1, ): """Detector class to detect FEX from images or videos. Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos. Args: n_jobs (int, default=1): Number of processes to use for extraction. Attributes: info (dict): n_jobs (int): Number of jobs to be used in parallel. face_model (str, default=retinaface): Name of face detection model landmark_model (str, default=mobilenet): Nam eof landmark model au_model (str, default=rf): Name of Action Unit detection model emotion_model (str, default=resmasknet): Path to emotion detection model. face_detection_columns (list): Column names for face detection ouput (x, y, w, h) face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...) emotion_model_columns (list): Column names for emotion model output mapper (dict): Class names for emotion model output by index. input_shape (dict) face_detector: face detector object face_landmark: face_landmark object emotion_model: emotion_model object Examples: >> detector = Detector(n_jobs=1) >> detector.detect_image("input.jpg") >> detector.detect_video("input.mp4") """ self.info = {} self.info["n_jobs"] = n_jobs if torch.cuda.is_available(): self.map_location = lambda storage, loc: storage.cuda() else: self.map_location = "cpu" """ LOAD UP THE MODELS """ print("Loading Face Detection model: ", face_model) # Check if model files have been downloaded. Otherwise download model. # get model url. with open(os.path.join(get_resource_path(), "model_list.json"), "r") as f: model_urls = json.load(f) if face_model: for url in model_urls["face_detectors"][ face_model.lower()]["urls"]: download_url(url, get_resource_path()) if landmark_model: for url in model_urls["landmark_detectors"][ landmark_model.lower()]["urls"]: download_url(url, get_resource_path()) if au_model: for url in model_urls["au_detectors"][au_model.lower()]["urls"]: download_url(url, get_resource_path()) if ".zip" in url: import zipfile with zipfile.ZipFile( os.path.join(get_resource_path(), "JAANetparams.zip"), 'r') as zip_ref: zip_ref.extractall(os.path.join(get_resource_path())) if au_model.lower() in ['logistic', 'svm', 'rf']: download_url( model_urls["au_detectors"]['hog-pca']['urls'][0], get_resource_path()) download_url( model_urls["au_detectors"]['au_scalar']['urls'][0], get_resource_path()) if emotion_model: for url in model_urls["emotion_detectors"][ emotion_model.lower()]["urls"]: download_url(url, get_resource_path()) if emotion_model.lower() in ['svm', 'rf']: download_url( model_urls["emotion_detectors"]['emo_pca']['urls'][0], get_resource_path()) download_url( model_urls["emotion_detectors"]['emo_scalar']['urls'] [0], get_resource_path()) if face_model: if face_model.lower() == "faceboxes": self.face_detector = FaceBoxes() elif face_model.lower() == "retinaface": self.face_detector = Retinaface_test.Retinaface() elif face_model.lower() == "mtcnn": self.face_detector = MTCNN() self.info["face_model"] = face_model facebox_columns = FEAT_FACEBOX_COLUMNS self.info["face_detection_columns"] = facebox_columns predictions = np.empty((1, len(facebox_columns))) predictions[:] = np.nan empty_facebox = pd.DataFrame(predictions, columns=facebox_columns) self._empty_facebox = empty_facebox print("Loading Face Landmark model: ", landmark_model) if landmark_model: if landmark_model.lower() == "mobilenet": self.landmark_detector = MobileNet_GDConv(136) self.landmark_detector = torch.nn.DataParallel( self.landmark_detector) checkpoint = torch.load( os.path.join( get_resource_path(), "mobilenet_224_model_best_gdconv_external.pth.tar", ), map_location=self.map_location, ) self.landmark_detector.load_state_dict( checkpoint["state_dict"]) elif landmark_model.lower() == "pfld": self.landmark_detector = PFLDInference() checkpoint = torch.load( os.path.join(get_resource_path(), "pfld_model_best.pth.tar"), map_location=self.map_location, ) self.landmark_detector.load_state_dict( checkpoint["state_dict"]) elif landmark_model.lower() == "mobilefacenet": self.landmark_detector = MobileFaceNet([112, 112], 136) checkpoint = torch.load( os.path.join(get_resource_path(), "mobilefacenet_model_best.pth.tar"), map_location=self.map_location, ) self.landmark_detector.load_state_dict( checkpoint["state_dict"]) self.info["landmark_model"] = landmark_model self.info["mapper"] = openface_2d_landmark_columns landmark_columns = openface_2d_landmark_columns self.info["face_landmark_columns"] = landmark_columns predictions = np.empty((1, len(openface_2d_landmark_columns))) predictions[:] = np.nan empty_landmarks = pd.DataFrame(predictions, columns=landmark_columns) self._empty_landmark = empty_landmarks print("Loading au model: ", au_model) self.info["au_model"] = au_model if au_model: if au_model.lower() == "jaanet": self.au_model = JAANet() elif au_model.lower() == "drml": self.au_model = DRMLNet() elif au_model.lower() == "logistic": self.au_model = LogisticClassifier() elif au_model.lower() == "svm": self.au_model = SVMClassifier() elif au_model.lower() == 'rf': self.au_model = RandomForestClassifier() if (au_model is None) or (au_model.lower() in ['jaanet', 'drml']): auoccur_columns = jaanet_AU_presence else: auoccur_columns = RF_AU_presence self.info["au_presence_columns"] = auoccur_columns predictions = np.empty((1, len(auoccur_columns))) predictions[:] = np.nan empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns) self._empty_auoccurence = empty_au_occurs print("Loading emotion model: ", emotion_model) self.info["emotion_model"] = emotion_model if emotion_model: if emotion_model.lower() == "fer": self.emotion_model = ferNetModule() elif emotion_model.lower() == "resmasknet": self.emotion_model = ResMaskNet() elif emotion_model.lower() == 'svm': self.emotion_model = EmoSVMClassifier() elif emotion_model.lower() == 'rf': self.emotion_model = EmoRandomForestClassifier() self.info["emotion_model_columns"] = FEAT_EMOTION_COLUMNS predictions = np.empty((1, len(FEAT_EMOTION_COLUMNS))) predictions[:] = np.nan empty_emotion = pd.DataFrame(predictions, columns=FEAT_EMOTION_COLUMNS) self._empty_emotion = empty_emotion predictions = np.empty((1, len(auoccur_columns))) predictions[:] = np.nan empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns) self._empty_auoccurence = empty_au_occurs self.info["output_columns"] = (FEAT_TIME_COLUMNS + facebox_columns + landmark_columns + auoccur_columns + FEAT_EMOTION_COLUMNS + ["input"])
Timer, load_model, nms, ) from feat.utils import get_resource_path # some global configs confidence_threshold = 0.05 top_k = 5000 keep_top_k = 750 nms_threshold = 0.3 vis_thres = 0.5 resize = 1 scale_flag = True HEIGHT, WIDTH = 720, 1080 pretrained_path = os.path.join(get_resource_path(), "FaceBoxesProd.pth") cfg = { "name": "FaceBoxes", "min_sizes": [[32, 64, 128], [256], [512]], "steps": [32, 64, 128], "variance": [0.1, 0.2], "clip": False, } def viz_bbox(img, dets, wfp="out.jpg"): # show for b in dets: if b[4] < vis_thres: continue text = "{:.4f}".format(b[4])
from torchvision import transforms from .img2pose_model import img2poseModel from feat.utils import get_resource_path from feat.face_detectors.Retinaface.Retinaface_utils import py_cpu_nms from ..utils import convert_to_euler BORDER_SIZE = 100 DEPTH = 18 MAX_SIZE = 1400 MIN_SIZE = 400 nms_inclusion_threshold = 0.05 # face score below which face box is excluded from nms nms_threshold = 0.6 # default from img2pose paper top_k = 5000 keep_top_k = 750 vis_thres = 0.5 POSE_MEAN = os.path.join(get_resource_path(), "WIDER_train_pose_mean_v1.npy") POSE_STDDEV = os.path.join(get_resource_path(), "WIDER_train_pose_stddev_v1.npy") THREED_FACE_MODEL = os.path.join(get_resource_path(), "reference_3d_68_points_trans.npy") class Img2Pose: def __init__(self, cpu_mode, constrained=True): """ Creates an img2pose model. Constrained model is optimized for face detection/ pose estimation for front-facing faces ( [-90, 90] degree range) only. Unconstrained model can detect faces and poses at any angle, but shows slightly dampened performance on face pose estimation. Args: cpu_mode (bool): whether or not to use CPU (True) or GPU (False) Returns: Img2Pose object
def __init__( self, face_model="retinaface", landmark_model="MobileNet", au_model="jaanet", emotion_model="fer", n_jobs=1, ): """Detector class to detect FEX from images or videos. Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos. Args: n_jobs (int, default=1): Number of processes to use for extraction. Attributes: info (dict): n_jobs (int): Number of jobs to be used in parallel. face_detection_model (str, default=haarcascade_frontalface_alt.xml): Path to face detection model. face_detection_columns (list): Column names for face detection ouput (x, y, w, h) face_landmark_model (str, default=lbfmodel.yaml): Path to landmark model. face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...) emotion_model (str, default=fer_aug_model.h5): Path to emotion detection model. emotion_model_columns (list): Column names for emotion model output mapper (dict): Class names for emotion model output by index. input_shape (dict) face_detector: face detector object face_landmark: face_landmark object emotion_model: emotion_model object Examples: >> detector = Detector(n_jobs=1) >> detector.detect_image("input.jpg") >> detector.detect_video("input.mp4") """ self.info = {} self.info["n_jobs"] = n_jobs if torch.cuda.is_available(): self.map_location = lambda storage, loc: storage.cuda() else: self.map_location = "cpu" """ LOAD UP THE MODELS """ print("Loading Face Detection model: ", face_model) if face_model: if face_model.lower() == "faceboxes": self.face_detector = FaceBoxes() elif face_model.lower() == "retinaface": self.face_detector = Retinaface_test.Retinaface() elif face_model.lower() == "mtcnn": self.face_detector = MTCNN() self.info["face_model"] = face_model # self.info["mapper"] = FEAT_FACEBOX_COLUMNS facebox_columns = FEAT_FACEBOX_COLUMNS self.info["face_detection_columns"] = facebox_columns predictions = np.empty((1, len(facebox_columns))) predictions[:] = np.nan empty_facebox = pd.DataFrame(predictions, columns=facebox_columns) self._empty_facebox = empty_facebox print("Loading Face Landmark model: ", landmark_model) # self.info['Landmark_Model'] = landmark_model if landmark_model: if landmark_model.lower() == "mobilenet": self.landmark_detector = MobileNet_GDConv(136) self.landmark_detector = torch.nn.DataParallel( self.landmark_detector) # or download model from https://drive.google.com/file/d/1Le5UdpMkKOTRr1sTp4lwkw8263sbgdSe/view?usp=sharing checkpoint = torch.load( os.path.join( get_resource_path(), "mobilenet_224_model_best_gdconv_external.pth.tar", ), map_location=self.map_location, ) # print("Use MobileNet as backbone") self.landmark_detector.load_state_dict( checkpoint["state_dict"]) elif landmark_model.lower() == "pfld": self.landmark_detector = PFLDInference() # or download from https://drive.google.com/file/d/1gjgtm6qaBQJ_EY7lQfQj3EuMJCVg9lVu/view?usp=sharing checkpoint = torch.load( os.path.join(get_resource_path(), "pfld_model_best.pth.tar"), map_location=self.map_location, ) # print("Use PFLD as backbone") self.landmark_detector.load_state_dict( checkpoint["state_dict"]) # or download from https://drive.google.com/file/d/1T8J73UTcB25BEJ_ObAJczCkyGKW5VaeY/view?usp=sharing elif landmark_model.lower() == "mobilefacenet": self.landmark_detector = MobileFaceNet([112, 112], 136) checkpoint = torch.load( os.path.join(get_resource_path(), "mobilefacenet_model_best.pth.tar"), map_location=self.map_location, ) # print("Use MobileFaceNet as backbone") self.landmark_detector.load_state_dict( checkpoint["state_dict"]) self.info["landmark_model"] = landmark_model self.info["mapper"] = openface_2d_landmark_columns landmark_columns = openface_2d_landmark_columns self.info["face_landmark_columns"] = landmark_columns predictions = np.empty((1, len(openface_2d_landmark_columns))) predictions[:] = np.nan empty_landmarks = pd.DataFrame(predictions, columns=landmark_columns) self._empty_landmark = empty_landmarks print("Loading au occurence model: ", au_model) self.info["au_model"] = au_model if au_model: if au_model.lower() == "jaanet": self.au_model = JAANet() elif au_model.lower() == "drml": self.au_model = DRMLNet() # self.info["mapper"] = jaanet_AU_presence auoccur_columns = jaanet_AU_presence self.info["au_presence_columns"] = auoccur_columns predictions = np.empty((1, len(auoccur_columns))) predictions[:] = np.nan empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns) self._empty_auoccurence = empty_au_occurs print("Loading emotion model: ", emotion_model) self.info["emotion_model"] = emotion_model if emotion_model: if emotion_model.lower() == "fer": self.emotion_model = ferNetModule() elif emotion_model.lower() == "resmasknet": self.emotion_model = ResMaskNet() self.info["emotion_model_columns"] = FEAT_EMOTION_COLUMNS predictions = np.empty((1, len(FEAT_EMOTION_COLUMNS))) predictions[:] = np.nan empty_emotion = pd.DataFrame(predictions, columns=FEAT_EMOTION_COLUMNS) self._empty_emotion = empty_emotion # self.info['auoccur_model'] = au_model # self.info["mapper"] = jaanet_AU_presence auoccur_columns = jaanet_AU_presence self.info["au_presence_columns"] = auoccur_columns predictions = np.empty((1, len(auoccur_columns))) predictions[:] = np.nan empty_au_occurs = pd.DataFrame(predictions, columns=auoccur_columns) self._empty_auoccurence = empty_au_occurs self.info["output_columns"] = (FEAT_TIME_COLUMNS + facebox_columns + landmark_columns + auoccur_columns + FEAT_EMOTION_COLUMNS + ["input"])
from feat.utils import get_resource_path def face_rect_to_coords(rectangle): """ Takes in a (x, y, w, h) array and transforms it into (x, y, x2, y2) """ return [ rectangle[0], rectangle[1], rectangle[0] + rectangle[2], rectangle[1] + rectangle[3] ] # load pre trained emotion model print("Loading FEX DCNN model.") dcnn_model_path = os.path.join(get_resource_path(), 'fer_aug_model.h5') if not os.path.exists(dcnn_model_path): print("Emotion prediction model not found. Please run download_models.py.") model = models.load_model(dcnn_model_path) # Load model to use. (_, img_w, img_h, img_c) = model.layers[0].input_shape # model input shape. mapper = { 0: 'anger', 1: 'disgust', 2: 'fear', 3: 'happiness', 4: 'sadness', 5: 'surprise', 6: 'neutral' } emotion_columns = [key for key in mapper.values()]
def __init__(self): """Initialize ResMaskNet @misc{luanresmaskingnet2020, Author = {Luan Pham & Tuan Anh Tran}, Title = {Facial Expression Recognition using Residual Masking Network}, url = {https://github.com/phamquiluan/ResidualMaskingNetwork}, Year = {2020} } """ self.transform = transforms.Compose( [transforms.ToPILImage(), transforms.ToTensor()]) self.FER_2013_EMO_DICT = { 0: "angry", 1: "disgust", 2: "fear", 3: "happy", 4: "sad", 5: "surprise", 6: "neutral", } # load configs and set random seed configs = json.load( open( os.path.join(get_resource_path(), "ResMaskNet_fer2013_config.json"))) self.image_size = (configs["image_size"], configs["image_size"]) self.use_gpu = torch.cuda.is_available() # if self.use_gpu: # self.state = torch.load( # os.path.join( # get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth" # ) # ) # else: # self.state = torch.load( # os.path.join( # get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth" # ), # map_location={"cuda:0": "cpu"}, # ) self.model = resmasking_dropout1(in_channels=3, num_classes=7) if self.use_gpu: self.model.load_state_dict( torch.load( os.path.join( get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth"))['net']) self.model.cuda() else: self.model.load_state_dict( torch.load( os.path.join(get_resource_path(), "ResMaskNet_Z_resmasking_dropout1_rot30.pth"), map_location={"cuda:0": "cpu"}, )['net']) self.model.eval()
def __init__(self) -> None: """ Initialize. Args: img_data: numpy array image data files of shape (N,3,W,H) land_data: numpy array landmark data of shape (N, 49*2) """ # self.imgs = img_data # self.land_data = land_data super(JAANet, self).__init__() self.params = { "config_unit_dim": 8, "config_crop_size": 176, "config_map_size": 44, "config_au_num": 12, "config_land_num": 49, "config_fill_coeff": 0.56, "config_write_path_prefix": get_resource_path(), } config_unit_dim = self.params["config_unit_dim"] config_crop_size = self.params["config_crop_size"] config_map_size = self.params["config_map_size"] config_au_num = self.params["config_au_num"] config_land_num = self.params["config_land_num"] config_fill_coeff = self.params["config_fill_coeff"] config_write_path_prefix = self.params["config_write_path_prefix"] self.region_learning = network.network_dict["HMRegionLearning"]( input_dim=3, unit_dim=config_unit_dim) self.align_net = network.network_dict["AlignNet"]( crop_size=config_crop_size, map_size=config_map_size, au_num=config_au_num, land_num=config_land_num, input_dim=config_unit_dim * 8, fill_coeff=config_fill_coeff, ) self.local_attention_refine = network.network_dict[ "LocalAttentionRefine"](au_num=config_au_num, unit_dim=config_unit_dim) self.local_au_net = network.network_dict["LocalAUNetv2"]( au_num=config_au_num, input_dim=config_unit_dim * 8, unit_dim=config_unit_dim, ) self.global_au_feat = network.network_dict["HLFeatExtractor"]( input_dim=config_unit_dim * 8, unit_dim=config_unit_dim) self.au_net = network.network_dict["AUNet"](au_num=config_au_num, input_dim=12000, unit_dim=config_unit_dim) self.use_gpu = torch.cuda.is_available() if self.use_gpu: self.region_learning = self.region_learning.cuda() self.align_net = self.align_net.cuda() self.local_attention_refine = self.local_attention_refine.cuda() self.local_au_net = self.local_au_net.cuda() self.global_au_feat = self.global_au_feat.cuda() self.au_net = self.au_net.cuda() # Load parameters # load_map = 'cpu' if True else 'false' # au_occur_model_path = os.path.join( # config_write_path_prefix , '/region_learning' , '.pth') # print("should load data at ",os.path.join(config_write_path_prefix , 'region_learning.pth')) # print("Directory Files:") # print(os.listdir(config_write_path_prefix)) self.region_learning.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "region_learning.pth"))) self.align_net.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "align_net.pth"))) self.local_attention_refine.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "local_attention_refine.pth"))) self.local_au_net.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "local_au_net.pth"))) self.global_au_feat.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "global_au_feat.pth"))) self.au_net.load_state_dict( torch.load(os.path.join(config_write_path_prefix, "au_net.pth"))) else: self.region_learning.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "region_learning.pth"), map_location={"cuda:0": "cpu"}, )) self.align_net.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "align_net.pth"), map_location={"cuda:0": "cpu"}, )) self.local_attention_refine.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "local_attention_refine.pth"), map_location={"cuda:0": "cpu"}, )) self.local_au_net.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "local_au_net.pth"), map_location={"cuda:0": "cpu"}, )) self.global_au_feat.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "global_au_feat.pth"), map_location={"cuda:0": "cpu"}, )) self.au_net.load_state_dict( torch.load( os.path.join(config_write_path_prefix, "au_net.pth"), map_location={"cuda:0": "cpu"}, )) self.region_learning.eval() self.align_net.eval() self.local_attention_refine.eval() self.local_au_net.eval() self.global_au_feat.eval() self.au_net.eval()
def __init__(self, n_jobs=1): """Detector class to detect FEX from images or videos. Detector is a class used to detect faces, facial landmarks, emotions, and action units from images and videos. Args: n_jobs (int, default=1): Number of processes to use for extraction. Attributes: info (dict): n_jobs (int): Number of jobs to be used in parallel. face_detection_model (str, default=haarcascade_frontalface_alt.xml): Path to face detection model. face_detection_columns (list): Column names for face detection ouput (x, y, w, h) face_landmark_model (str, default=lbfmodel.yaml): Path to landmark model. face_landmark_columns (list): Column names for face landmark output (x0, y0, x1, y1, ...) emotion_model (str, default=fer_aug_model.h5): Path to emotion detection model. emotion_model_columns (list): Column names for emotion model output mapper (dict): Class names for emotion model output by index. input_shape (dict) face_detector: face detector object face_landmark: face_landmark object emotion_model: emotion_model object Examples: >> detector = Detector(n_jobs=1) >> detector.detect_image("input.jpg") >> detector.detect_video("input.mp4") """ self.info = {} self.info['n_jobs'] = n_jobs """ LOAD UP THE MODELS """ print("Loading Face Detection model.") face_detection_model_path = cv.data.haarcascades + "haarcascade_frontalface_alt.xml" if not os.path.exists(face_detection_model_path): print("Face detection model not found. Check haarcascade_frontalface_alt.xml exists in your opencv installation (cv.data).") face_cascade = cv.CascadeClassifier(face_detection_model_path) face_detection_columns = FEAT_FACEBOX_COLUMNS facebox_empty = np.empty((1,4)) facebox_empty[:] = np.nan empty_facebox = pd.DataFrame(facebox_empty, columns = face_detection_columns) self.info["face_detection_model"] = face_detection_model_path self.info["face_detection_columns"] = face_detection_columns self.face_detector = face_cascade self._empty_facebox = empty_facebox print("Loading Face Landmark model.") face_landmark = cv.face.createFacemarkLBF() face_landmark_model_path = os.path.join(get_resource_path(), 'lbfmodel.yaml') if not os.path.exists(face_landmark_model_path): print("Face landmark model not found. Please run download_models.py.") face_landmark.loadModel(face_landmark_model_path) face_landmark_columns = np.array([(f'x_{i}',f'y_{i}') for i in range(68)]).reshape(1,136)[0].tolist() landmark_empty = np.empty((1,136)) landmark_empty[:] = np.nan empty_landmark = pd.DataFrame(landmark_empty, columns = face_landmark_columns) self.info["face_landmark_model"] = face_landmark_model_path self.info['face_landmark_columns'] = face_landmark_columns self.face_landmark = face_landmark self._empty_landmark = empty_landmark print("Loading FEX DCNN emotion model.") emotion_model = 'fer_aug_model.h5' emotion_model_path = os.path.join(get_resource_path(),'fer_aug_model.h5') if not os.path.exists(emotion_model_path): print("Emotion prediction model not found. Please run download_models.py.") model = models.load_model(emotion_model_path) # Load model to use. (_, img_w, img_h, img_c) = model.layers[0].input_shape # model input shape. self.info["input_shape"] = {"img_w": img_w, "img_h": img_h, "img_c": img_c} self.info["emotion_model"] = emotion_model_path self.info["mapper"] = FEAT_EMOTION_MAPPER self.emotion_model = model emotion_columns = [key for key in self.info["mapper"].values()] self.info['emotion_model_columns'] = emotion_columns # create empty df for predictions predictions = np.empty((1, len(self.info["mapper"]))) predictions[:] = np.nan empty_emotion = pd.DataFrame(predictions, columns = self.info["mapper"].values()) self._empty_emotion = empty_emotion frame_columns = ["frame"] self.info["output_columns"] = frame_columns + emotion_columns + face_detection_columns + face_landmark_columns
import os import cv2 import numpy as np from feat.utils import get_resource_path from feat.facepose_detectors.utils import convert_to_euler THREED_FACE_MODEL = os.path.join(get_resource_path(), "reference_3d_68_points_trans.npy") class PerspectiveNPointModel: """ Class that leverages 68 2D facial landmark points to estimate head pose using the Perspective-n-Point algorithm. Code adapted from https://github.com/yinguobing/head-pose-estimation/ and https://github.com/lincolnhard/head-pose-estimation/. Each code base licensed under MIT Licenses, which can be found here: https://github.com/yinguobing/head-pose-estimation/blob/master/LICENSE and here: https://github.com/lincolnhard/head-pose-estimation/blob/master/LICENSE """ def __init__(self): """ Initializes the model, with a reference 3D model (xyz coordinates) of a standard face""" # self.model_points = get_full_model_points(os.path.join(get_resource_path(), "3d_face_model.txt")) self.model_points = np.load(THREED_FACE_MODEL, allow_pickle=True) def predict(self, img, landmarks): """ Determines headpose using passed 68 2D landmarks Args: img (np.ndarray) : The cv2 image from which the landmarks were produced landmarks (np.ndarray) : The landmarks to use to produce the headpose estimate Returns:
#!/usr/bin/env python3 """ This script will download the necessary models to the feat path. Usage python3 download_models.py """ from feat.utils import get_resource_path import wget import os print("Downloading FEX emotion model.") fex_emotion_model = "https://github.com/cosanlab/feat/releases/download/v0.1/fer_aug_model.h5" wget.download(fex_emotion_model, get_resource_path()) if os.path.exists(os.path.join(get_resource_path(), "fer_aug_model.h5")): print("\nFEX emotion model downloaded successfully.\n") else: print("Something went wrong. Model not found in directory.") print("Downloading landmark detection model.") lbfmodel = "https://github.com/cosanlab/feat/releases/download/v0.1/lbfmodel.yaml" wget.download(lbfmodel, get_resource_path()) if os.path.exists(os.path.join(get_resource_path(), "lbfmodel.yaml")): print("\nLandmark detection model downloaded successfully.\n") else: print("Something went wrong. Model not found in directory.")
from __future__ import print_function import os import torch import numpy as np import time import feat from feat.face_detectors.Retinaface.Retinaface_model import PriorBox, RetinaFace from feat.face_detectors.Retinaface.Retinaface_utils import ( py_cpu_nms, decode, decode_landm, ) from feat.utils import get_resource_path # some global configs trained_model = os.path.join(get_resource_path(), "mobilenet0.25_Final.pth") network = "mobile0.25" confidence_threshold = 0.05 top_k = 5000 keep_top_k = 750 nms_threshold = 0.3 vis_thres = 0.5 resize = 1 cpu = True cfg_mnet = { "name": "mobilenet0.25", "min_sizes": [[16, 32], [64, 128], [256, 512]], "steps": [8, 16, 32], "variance": [0.1, 0.2], "clip": False, "loc_weight": 2.0,
#!/usr/bin/env python3 """ This script will download the necessary models to the feat path. Usage python3 download_models.py """ from feat.utils import get_resource_path import wget import os import zipfile if os.path.exists(os.path.join(get_resource_path(), "best_ferModel.pth")): print("Fex model already exists; skipping download.") else: print("Downloading FEX emotion model.") try: fex_emotion_model = "https://github.com/cosanlab/feat/releases/download/v0.1/best_ferModel.pth" wget.download(fex_emotion_model, get_resource_path(), bar=None) # except: # try: # fex_emotion_model = "https://www.dropbox.com/s/d3yhtsqggqcrjl2/fer_emotion_model.h5?dl=1" # wget.download(fex_emotion_model, get_resource_path(), bar=None) except: print("FeX emotion model failed to download") if os.path.exists(os.path.join(get_resource_path(), "best_ferModel.pth")): print("\nFEX emotion model downloaded successfully.\n") else: print("Something went wrong. Model not found in directory.")