def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename = sample.fname bb = self.get_adjusted_bounding_box(sample.x, sample.y, sample.w, sample.h) face_id = int(sample.name) if self.crop_source == 'lm_ground_truth': landmarks_for_crop = lm98_to_lm68( sample.landmarks.astype(np.float32)) else: # no landmarks -> crop using bounding boxes landmarks_for_crop = None try: crop, landmarks, pose, cropper = self.face_extractor.get_face( filename, self.fullsize_img_dir, self.cropped_img_dir, crop_type=self.crop_type, landmarks=landmarks_for_crop, bb=bb, use_cache=self.use_cache, id=face_id, aligned=self.align_face_orientation) except: print('Could not load image {}'.format(filename)) raise landmarks_gt = sample.landmarks.astype(np.float32) landmarks_gt = cropper.apply_to_landmarks(landmarks_gt)[0] cropped_sample = { 'image': crop, 'landmarks': landmarks_gt, 'pose': np.zeros(3, dtype=np.float32) } item = self.transform(cropped_sample) em_val_ar = np.array([[-1, 0, 0]], dtype=np.float32) if self.crop_type != 'fullsize': result = self.crop_to_tensor(item) else: result = item result.update({'fnames': filename, 'expression': em_val_ar, 'bb': bb}) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps and self.crop_type != 'fullsize': result['lm_heatmaps'] = create_landmark_heatmaps( result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename, id = sample[0], sample.ID bb = None landmarks_for_crop = None if self.crop_source == 'bb_ground_truth': bb = self.get_bounding_box(sample) pose = np.zeros(3, dtype=np.float32) else: of_conf, landmarks_for_crop = sample.of_conf, sample.landmarks_of pose = sample.pose try: crop, landmarks, pose, cropper = self.face_extractor.get_face(filename+'.jpg', self.fullsize_img_dir, self.cropped_img_dir, landmarks=landmarks_for_crop, bb=bb, pose=pose, use_cache=self.use_cache, detect_face=False, crop_type='tight', aligned=self.align_face_orientation) except: print(filename) raise # return self.__getitem__(random.randint(0,len(self)-1)) try: landmarks, _ = cropper.apply_to_landmarks(sample.landmarks) except AttributeError: landmarks = np.zeros((68,2)) # vis.show_landmarks(crop, landmarks, title='lms', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks.astype(np.float32), 'pose': pose} item = self.transform(cropped_sample) transforms = [fp.CenterCrop(cfg.INPUT_SIZE)] transforms += [fp.ToTensor() ] transforms += [fp.Normalize([0.518, 0.418, 0.361], [1, 1, 1])] # VGGFace(2) transforms = tf.Compose(transforms) result = transforms(item) result.update({ 'id': id, 'fnames': filename, 'expression': np.array([[0,0,0]], dtype=np.float32), }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = transforms(crop_occ) result['image_mod'] = crop_occ # add landmark heatmaps if landmarks enabled if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(item['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def get_sample(self, filename, bb, landmarks): def transform(image, bb, image_size): return image try: image = cv2.imread(filename) if len(image.shape) == 3: image = image[:, :, 0] imgsize = image.shape except: raise IOError("\tError: Could not load image {}".format(filename)) image = image[bb[1]:bb[3], bb[0]:bb[2]] try: image = cv2.resize(image, self.image_size, interpolation=cv2.INTER_CUBIC) except: raise IOError( "An error occurred when resizing image {} with size {}.". format(filename, image.shape)) image = np.expand_dims(image, axis=0) image = image.astype(np.float32) image /= 256 sample = {"image": image, "landmarks": landmarks} sample = self.crop_to_tensor(sample) if self.return_landmark_heatmaps: heatmap_size = lmcfg.HEATMAP_SIZE _landmarks = [] for each in landmarks: each[0] -= bb[0] each[1] -= bb[1] each[0] = int(each[0] * self.image_size[0] / (bb[2] - bb[0])) each[1] = int(each[1] * self.image_size[1] / (bb[3] - bb[1])) _landmarks.append(each) landmarks = np.array(_landmarks) lm_heatmaps = lmutils.create_landmark_heatmaps( landmarks, self.landmark_sigma, self.ALL_LANDMARKS, lmcfg.HEATMAP_SIZE, self.image_size[0], ) sample.update({"lm_heatmaps": lm_heatmaps}) sample.update({"filename": filename}) return sample
def __getitem__(self, idx): sample = self.annotations.iloc[idx] # assert sample.fname == sample.fname_full face_id = sample.ra filename = sample.fname bb = self.get_bounding_box(sample) try: crop, landmarks, pose, cropper = self.face_extractor.get_face(filename, self.fullsize_img_dir, self.cropped_img_dir, bb=bb, use_cache=self.use_cache, id=face_id) except: print(filename) raise landmarks, _ = cropper.apply_to_landmarks(sample.landmarks_full) # vis.show_landmarks(crop, landmarks, title='lms aflw', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks.astype(np.float32), # 'landmarks': np.zeros((68,2), dtype=np.float32), 'pose': np.zeros(3, dtype=np.float32)} item = self.transform(cropped_sample) em_val_ar = np.array([[-1,0,0]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({ 'fnames': filename, 'expression': em_val_ar, 'id': 0 }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def get_sample(self, filename, bb=None, landmarks_for_crop=None, id=None, landmarks_to_return=None): try: crop_mode = 'landmarks' if landmarks_for_crop is not None else 'bounding_box' crop_params = { 'landmarks': landmarks_for_crop, 'bb': bb, 'id': id, 'aligned': self.align_face_orientation, 'mode': crop_mode } image = self.loader.load_crop(filename, **crop_params) except: print('Could not load image {}'.format(filename)) raise relative_landmarks = self._crop_landmarks(landmarks_to_return) \ if landmarks_to_return is not None else self.empty_landmarks # self.show_landmarks(image, landmarks) sample = { 'image': image, 'landmarks': relative_landmarks, 'pose': np.zeros(3, dtype=np.float32) } if self.transform is not None: sample = self.transform(sample) target = self.target_transform( sample) if self.target_transform else None # self.show_landmarks(sample['image'], sample['landmarks']) if self.crop_type != 'fullsize': sample = self.crop_to_tensor(sample) if target is not None: target = self.crop_to_tensor(target) sample.update({ 'fnames': filename, 'bb': bb if bb is not None else [0, 0, 0, 0], # 'expression':self._get_expression(sample), # 'id': self._get_identity(sample), }) if target is not None: sample['target'] = target if self.return_landmark_heatmaps and self.crop_type != 'fullsize': from landmarks import lmconfig as lmcfg heatmap_size = lmcfg.HEATMAP_SIZE scaled_landmarks = sample['landmarks'] * (heatmap_size / self.image_size) sample['lm_heatmaps'] = lmutils.create_landmark_heatmaps( scaled_landmarks, self.landmark_sigma, self.ALL_LANDMARKS, heatmap_size) return sample
def __getitem__(self, idx): sample = self.annotations.iloc[idx] filename = sample.subDirectory_filePath pose = sample.pose bb = None landmarks_for_crop = None landmarks_to_return = self.parse_landmarks(sample.facial_landmarks) if self.crop_source == 'bb_ground_truth': bb = self.get_bounding_box(sample) elif self.crop_source == 'lm_ground_truth': landmarks_for_crop = landmarks_to_return elif self.crop_source == 'lm_openface': of_conf, landmarks_for_crop = sample.conf, sample.landmarks_of # if OpenFace didn't detect a face, fall back to AffectNet landmarks if sample.conf <= 0.1: try: landmarks_for_crop = self.parse_landmarks( sample.facial_landmarks) except ValueError: pass try: crop, landmarks, pose, cropper = self.face_extractor.get_face( filename, self.fullsize_img_dir, self.cropped_img_dir, landmarks=landmarks_for_crop, bb=bb, pose=pose, use_cache=self.use_cache, detect_face=False, crop_type=self.crop_type, aligned=self.align_face_orientation) except AssertionError: print(filename) raise landmarks, _ = cropper.apply_to_landmarks(landmarks_to_return) # vis.show_landmarks(crop, landmarks, title='lms affectnet', wait=0, color=(0,0,255)) cropped_sample = {'image': crop, 'landmarks': landmarks, 'pose': pose} item = self.transform(cropped_sample) em_val_ar = np.array( [[sample.expression, sample.valence, sample.arousal]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({'id': 0, 'fnames': filename, 'expression': em_val_ar}) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps( result['landmarks'], self.landmark_sigma, self.landmark_ids) return result
def __getitem__(self, idx): def get_landmarks_for_crop(): pose = np.zeros(3, dtype=np.float32) if self.crop_source == 'lm_openface': openface_filepath = os.path.join(self.feature_dir_of, os.path.splitext(filename)[0]) est_face_center = landmarks_gt.mean(axis=0) of_conf, landmarks_of, pose = ds_utils.read_openface_detection(openface_filepath, expected_face_center=est_face_center) if of_conf < 0.01: landmarks_of = landmarks_gt else: # landmarks_of, pose = self.cropper.apply_crop_to_landmarks(landmarks_of, pose) landmarks_of[:,0] -= cfg.CROP_BORDER landmarks_of[:,1] -= cfg.CROP_BORDER landmarks = landmarks_of elif self.crop_source == 'lm_cnn': try: landmarks = np.load(os.path.join(self.feature_dir_cnn, os.path.splitext(filename)[0]+'.npy')) except FileNotFoundError: landmarks = None elif self.crop_source == 'lm_ground_truth': landmarks = landmarks_gt else: # no landmarks -> crop using bounding boxes landmarks = None return landmarks, pose sample = self.annotations.iloc[idx] filename = sample.fname landmarks_gt = sample.landmarks.astype(np.float32) bbox = self.get_bounding_box(sample) if not self.split == '300w' else None landmarks_for_crop, pose = get_landmarks_for_crop() crop, landmarks, pose, cropper = self.face_extractor.get_face(filename, self.fullsize_img_dir, self.cropped_img_dir, bb=bbox, landmarks=landmarks_for_crop, pose=pose, use_cache=self.use_cache, detect_face=self.detect_face, crop_type=self.crop_type, aligned=self.align_face_orientation) landmarks_gt, _ = cropper.apply_to_landmarks(landmarks_gt) # self.show_landmarks(crop, landmarks_gt) cropped_sample = {'image': crop, 'landmarks': landmarks_gt, 'pose': pose} item = self.transform(cropped_sample) em_val_ar = np.array([[-1,0,0]], dtype=np.float32) result = self.crop_to_tensor(item) result.update({ 'fnames': filename, 'expression': em_val_ar }) if self.return_modified_images: mod_transforms = tf.Compose([fp.RandomOcclusion()]) crop_occ = mod_transforms(item['image']) crop_occ = self.crop_to_tensor(crop_occ) result['image_mod'] = crop_occ # add landmark heatmaps if landmarks enabled if self.return_landmark_heatmaps: result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids) return result