def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename = sample.fname bb = self.get_adjusted_bounding_box(sample.x, sample.y, sample.w, sample.h) face_id = int(sample.name) if self.crop_source == 'lm_ground_truth': landmarks_for_crop = lm98_to_lm68( sample.landmarks.astype(np.float32)) else: # no landmarks -> crop using bounding boxes landmarks_for_crop = None try: crop, landmarks, pose, cropper = self.face_extractor.get_face( filename, self.fullsize_img_dir, self.cropped_img_dir, crop_type=self.crop_type, landmarks=landmarks_for_crop, bb=bb, use_cache=self.use_cache, id=face_id, aligned=self.align_face_orientation) except: print('Could not load image {}'.format(filename)) raise landmarks_gt = sample.landmarks.astype(np.float32) landmarks_gt = cropper.apply_to_landmarks(landmarks_gt)[0] cropped_sample = { 'image': crop, 'landmarks': landmarks_gt, 'pose': np.zeros(3, dtype=np.float32) } item = self.transform(cropped_sample) em_val_ar = np.array([[-1, 0, 0]], dtype=np.float32) if self.crop_type != 'fullsize': result = self.crop_to_tensor(item) else: result = item result.update({'fnames': filename, 'expression': em_val_ar, 'bb': bb}) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps and self.crop_type != 'fullsize': result['lm_heatmaps'] = create_landmark_heatmaps( result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename, id = sample[0], sample.ID bb = None landmarks_for_crop = None if self.crop_source == 'bb_ground_truth': bb = self.get_bounding_box(sample) pose = np.zeros(3, dtype=np.float32) else: of_conf, landmarks_for_crop = sample.of_conf, sample.landmarks_of pose = sample.pose try: crop, landmarks, pose, cropper = self.face_extractor.get_face(filename+'.jpg', self.fullsize_img_dir, self.cropped_img_dir, landmarks=landmarks_for_crop, bb=bb, pose=pose, use_cache=self.use_cache, detect_face=False, crop_type='tight', aligned=self.align_face_orientation) except: print(filename) raise # return self.__getitem__(random.randint(0,len(self)-1)) try: landmarks, _ = cropper.apply_to_landmarks(sample.landmarks) except AttributeError: landmarks = np.zeros((68,2)) # vis.show_landmarks(crop, landmarks, title='lms', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks.astype(np.float32), 'pose': pose} item = self.transform(cropped_sample) transforms = [fp.CenterCrop(cfg.INPUT_SIZE)] transforms += [fp.ToTensor() ] transforms += [fp.Normalize([0.518, 0.418, 0.361], [1, 1, 1])] # VGGFace(2) transforms = tf.Compose(transforms) result = transforms(item) result.update({ 'id': id, 'fnames': filename, 'expression': np.array([[0,0,0]], dtype=np.float32), }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = transforms(crop_occ) result['image_mod'] = crop_occ # add landmark heatmaps if landmarks enabled if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(item['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): sample = self.annotations.iloc[idx] # assert sample.fname == sample.fname_full face_id = sample.ra filename = sample.fname bb = self.get_bounding_box(sample) try: crop, landmarks, pose, cropper = self.face_extractor.get_face(filename, self.fullsize_img_dir, self.cropped_img_dir, bb=bb, use_cache=self.use_cache, id=face_id) except: print(filename) raise landmarks, _ = cropper.apply_to_landmarks(sample.landmarks_full) # vis.show_landmarks(crop, landmarks, title='lms aflw', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks.astype(np.float32), # 'landmarks': np.zeros((68,2), dtype=np.float32), 'pose': np.zeros(3, dtype=np.float32)} item = self.transform(cropped_sample) em_val_ar = np.array([[-1,0,0]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({ 'fnames': filename, 'expression': em_val_ar, 'id': 0 }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename = sample.subDirectory_filePath pose = sample.pose bb = None landmarks_for_crop = None landmarks_to_return = self.parse_landmarks(sample.facial_landmarks) if self.crop_source == 'bb_ground_truth': bb = self.get_bounding_box(sample) elif self.crop_source == 'lm_ground_truth': landmarks_for_crop = landmarks_to_return elif self.crop_source == 'lm_openface': of_conf, landmarks_for_crop = sample.conf, sample.landmarks_of # if OpenFace didn't detect a face, fall back to AffectNet landmarks if sample.conf <= 0.1: try: landmarks_for_crop = self.parse_landmarks( sample.facial_landmarks) except ValueError: pass try: crop, landmarks, pose, cropper = self.face_extractor.get_face( filename, self.fullsize_img_dir, self.cropped_img_dir, landmarks=landmarks_for_crop, bb=bb, pose=pose, use_cache=self.use_cache, detect_face=False, crop_type=self.crop_type, aligned=self.align_face_orientation) except AssertionError: print(filename) raise landmarks, _ = cropper.apply_to_landmarks(landmarks_to_return) # vis.show_landmarks(crop, landmarks, title='lms affectnet', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks, 'pose': pose} item = self.transform(cropped_sample) em_val_ar = np.array( [[sample.expression, sample.valence, sample.arousal]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({'id': 0, 'fnames': filename, 'expression': em_val_ar}) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps( result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): def get_landmarks_for_crop(): pose = np.zeros(3, dtype=np.float32) if self.crop_source == 'lm_openface': openface_filepath = os.path.join(self.feature_dir_of, os.path.splitext(filename)[0]) est_face_center = landmarks_gt.mean(axis=0) of_conf, landmarks_of, pose = ds_utils.read_openface_detection(openface_filepath, expected_face_center=est_face_center) if of_conf < 0.01: landmarks_of = landmarks_gt else: # landmarks_of, pose = self.cropper.apply_crop_to_landmarks(landmarks_of, pose) landmarks_of[:,0] -= cfg.CROP_BORDER landmarks_of[:,1] -= cfg.CROP_BORDER landmarks = landmarks_of elif self.crop_source == 'lm_cnn': try: landmarks = np.load(os.path.join(self.feature_dir_cnn, os.path.splitext(filename)[0]+'.npy')) except FileNotFoundError: landmarks = None elif self.crop_source == 'lm_ground_truth': landmarks = landmarks_gt else: # no landmarks -> crop using bounding boxes landmarks = None return landmarks, pose sample = self.annotations.iloc[idx] filename = sample.fname landmarks_gt = sample.landmarks.astype(np.float32) bbox = self.get_bounding_box(sample) if not self.split == '300w' else None landmarks_for_crop, pose = get_landmarks_for_crop() crop, landmarks, pose, cropper = self.face_extractor.get_face(filename, self.fullsize_img_dir, self.cropped_img_dir, bb=bbox, landmarks=landmarks_for_crop, pose=pose, use_cache=self.use_cache, detect_face=self.detect_face, crop_type=self.crop_type, aligned=self.align_face_orientation) landmarks_gt, _ = cropper.apply_to_landmarks(landmarks_gt) # self.show_landmarks(crop, landmarks_gt) cropped_sample = {'image': crop, 'landmarks': landmarks_gt, 'pose': pose} item = self.transform(cropped_sample) em_val_ar = np.array([[-1,0,0]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({ 'fnames': filename, 'expression': em_val_ar }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ # add landmark heatmaps if landmarks enabled if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids) return result