def __getitem__(self, idx):
sample = self.annotations.iloc[idx]
filename = sample.fname
bb = self.get_adjusted_bounding_box(sample.x, sample.y, sample.w,
sample.h)
face_id = int(sample.name)
if self.crop_source == 'lm_ground_truth':
landmarks_for_crop = lm98_to_lm68(
sample.landmarks.astype(np.float32))
else:
# no landmarks -> crop using bounding boxes
landmarks_for_crop = None
try:
crop, landmarks, pose, cropper = self.face_extractor.get_face(
filename,
self.fullsize_img_dir,
self.cropped_img_dir,
crop_type=self.crop_type,
landmarks=landmarks_for_crop,
bb=bb,
use_cache=self.use_cache,
id=face_id,
aligned=self.align_face_orientation)
except:
print('Could not load image {}'.format(filename))
raise
landmarks_gt = sample.landmarks.astype(np.float32)
landmarks_gt = cropper.apply_to_landmarks(landmarks_gt)[0]
cropped_sample = {
'image': crop,
'landmarks': landmarks_gt,
'pose': np.zeros(3, dtype=np.float32)
}
item = self.transform(cropped_sample)
em_val_ar = np.array([[-1, 0, 0]], dtype=np.float32)
if self.crop_type != 'fullsize':
result = self.crop_to_tensor(item)
else:
result = item
result.update({'fnames': filename, 'expression': em_val_ar, 'bb': bb})
if self.return_modified_images:
mod_transforms = tf.Compose([fp.RandomOcclusion()])
crop_occ = mod_transforms(item['image'])
crop_occ = self.crop_to_tensor(crop_occ)
result['image_mod'] = crop_occ
if self.return_landmark_heatmaps and self.crop_type != 'fullsize':
result['lm_heatmaps'] = create_landmark_heatmaps(
result['landmarks'], self.landmark_sigma, self.landmark_ids)
return result
def __getitem__(self, idx):
sample = self.annotations.iloc[idx]
filename, id = sample[0], sample.ID
bb = None
landmarks_for_crop = None
if self.crop_source == 'bb_ground_truth':
bb = self.get_bounding_box(sample)
pose = np.zeros(3, dtype=np.float32)
else:
of_conf, landmarks_for_crop = sample.of_conf, sample.landmarks_of
pose = sample.pose
try:
crop, landmarks, pose, cropper = self.face_extractor.get_face(filename+'.jpg', self.fullsize_img_dir,
self.cropped_img_dir, landmarks=landmarks_for_crop,
bb=bb, pose=pose, use_cache=self.use_cache,
detect_face=False, crop_type='tight',
aligned=self.align_face_orientation)
except:
print(filename)
raise
# return self.__getitem__(random.randint(0,len(self)-1))
try:
landmarks, _ = cropper.apply_to_landmarks(sample.landmarks)
except AttributeError:
landmarks = np.zeros((68,2))
# vis.show_landmarks(crop, landmarks, title='lms', wait=0, color=(0,0,255))
cropped_sample = {'image': crop, 'landmarks': landmarks.astype(np.float32), 'pose': pose}
item = self.transform(cropped_sample)
transforms = [fp.CenterCrop(cfg.INPUT_SIZE)]
transforms += [fp.ToTensor() ]
transforms += [fp.Normalize([0.518, 0.418, 0.361], [1, 1, 1])] # VGGFace(2)
transforms = tf.Compose(transforms)
result = transforms(item)
result.update({
'id': id,
'fnames': filename,
'expression': np.array([[0,0,0]], dtype=np.float32),
})
if self.return_modified_images:
mod_transforms = tf.Compose([fp.RandomOcclusion()])
crop_occ = mod_transforms(item['image'])
crop_occ = transforms(crop_occ)
result['image_mod'] = crop_occ
# add landmark heatmaps if landmarks enabled
if self.return_landmark_heatmaps:
result['lm_heatmaps'] = create_landmark_heatmaps(item['landmarks'], self.landmark_sigma, self.landmark_ids)
return result
def get_sample(self, filename, bb, landmarks):
def transform(image, bb, image_size):
return image
try:
image = cv2.imread(filename)
if len(image.shape) == 3:
image = image[:, :, 0]
imgsize = image.shape
except:
raise IOError("\tError: Could not load image {}".format(filename))
image = image[bb[1]:bb[3], bb[0]:bb[2]]
try:
image = cv2.resize(image,
self.image_size,
interpolation=cv2.INTER_CUBIC)
except:
raise IOError(
"An error occurred when resizing image {} with size {}.".
format(filename, image.shape))
image = np.expand_dims(image, axis=0)
image = image.astype(np.float32)
image /= 256
sample = {"image": image, "landmarks": landmarks}
sample = self.crop_to_tensor(sample)
if self.return_landmark_heatmaps:
heatmap_size = lmcfg.HEATMAP_SIZE
_landmarks = []
for each in landmarks:
each[0] -= bb[0]
each[1] -= bb[1]
each[0] = int(each[0] * self.image_size[0] / (bb[2] - bb[0]))
each[1] = int(each[1] * self.image_size[1] / (bb[3] - bb[1]))
_landmarks.append(each)
landmarks = np.array(_landmarks)
lm_heatmaps = lmutils.create_landmark_heatmaps(
landmarks,
self.landmark_sigma,
self.ALL_LANDMARKS,
lmcfg.HEATMAP_SIZE,
self.image_size[0],
)
sample.update({"lm_heatmaps": lm_heatmaps})
sample.update({"filename": filename})
return sample
def __getitem__(self, idx):
sample = self.annotations.iloc[idx]
# assert sample.fname == sample.fname_full
face_id = sample.ra
filename = sample.fname
bb = self.get_bounding_box(sample)
try:
crop, landmarks, pose, cropper = self.face_extractor.get_face(filename,
self.fullsize_img_dir,
self.cropped_img_dir,
bb=bb,
use_cache=self.use_cache,
id=face_id)
except:
print(filename)
raise
landmarks, _ = cropper.apply_to_landmarks(sample.landmarks_full)
# vis.show_landmarks(crop, landmarks, title='lms aflw', wait=0, color=(0,0,255))
cropped_sample = {'image': crop,
'landmarks': landmarks.astype(np.float32),
# 'landmarks': np.zeros((68,2), dtype=np.float32),
'pose': np.zeros(3, dtype=np.float32)}
item = self.transform(cropped_sample)
em_val_ar = np.array([[-1,0,0]], dtype=np.float32)
result = self.crop_to_tensor(item)
result.update({
'fnames': filename,
'expression': em_val_ar,
'id': 0
})
if self.return_modified_images:
mod_transforms = tf.Compose([fp.RandomOcclusion()])
crop_occ = mod_transforms(item['image'])
crop_occ = self.crop_to_tensor(crop_occ)
result['image_mod'] = crop_occ
if self.return_landmark_heatmaps:
result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids)
return result
def get_sample(self,
filename,
bb=None,
landmarks_for_crop=None,
id=None,
landmarks_to_return=None):
try:
crop_mode = 'landmarks' if landmarks_for_crop is not None else 'bounding_box'
crop_params = {
'landmarks': landmarks_for_crop,
'bb': bb,
'id': id,
'aligned': self.align_face_orientation,
'mode': crop_mode
}
image = self.loader.load_crop(filename, **crop_params)
except:
print('Could not load image {}'.format(filename))
raise
relative_landmarks = self._crop_landmarks(landmarks_to_return) \
if landmarks_to_return is not None else self.empty_landmarks
# self.show_landmarks(image, landmarks)
sample = {
'image': image,
'landmarks': relative_landmarks,
'pose': np.zeros(3, dtype=np.float32)
}
if self.transform is not None:
sample = self.transform(sample)
target = self.target_transform(
sample) if self.target_transform else None
# self.show_landmarks(sample['image'], sample['landmarks'])
if self.crop_type != 'fullsize':
sample = self.crop_to_tensor(sample)
if target is not None:
target = self.crop_to_tensor(target)
sample.update({
'fnames': filename,
'bb': bb if bb is not None else [0, 0, 0, 0],
# 'expression':self._get_expression(sample),
# 'id': self._get_identity(sample),
})
if target is not None:
sample['target'] = target
if self.return_landmark_heatmaps and self.crop_type != 'fullsize':
from landmarks import lmconfig as lmcfg
heatmap_size = lmcfg.HEATMAP_SIZE
scaled_landmarks = sample['landmarks'] * (heatmap_size /
self.image_size)
sample['lm_heatmaps'] = lmutils.create_landmark_heatmaps(
scaled_landmarks, self.landmark_sigma, self.ALL_LANDMARKS,
heatmap_size)
return sample
def __getitem__(self, idx):
sample = self.annotations.iloc[idx]
filename = sample.subDirectory_filePath
pose = sample.pose
bb = None
landmarks_for_crop = None
landmarks_to_return = self.parse_landmarks(sample.facial_landmarks)
if self.crop_source == 'bb_ground_truth':
bb = self.get_bounding_box(sample)
elif self.crop_source == 'lm_ground_truth':
landmarks_for_crop = landmarks_to_return
elif self.crop_source == 'lm_openface':
of_conf, landmarks_for_crop = sample.conf, sample.landmarks_of
# if OpenFace didn't detect a face, fall back to AffectNet landmarks
if sample.conf <= 0.1:
try:
landmarks_for_crop = self.parse_landmarks(
sample.facial_landmarks)
except ValueError:
pass
try:
crop, landmarks, pose, cropper = self.face_extractor.get_face(
filename,
self.fullsize_img_dir,
self.cropped_img_dir,
landmarks=landmarks_for_crop,
bb=bb,
pose=pose,
use_cache=self.use_cache,
detect_face=False,
crop_type=self.crop_type,
aligned=self.align_face_orientation)
except AssertionError:
print(filename)
raise
landmarks, _ = cropper.apply_to_landmarks(landmarks_to_return)
# vis.show_landmarks(crop, landmarks, title='lms affectnet', wait=0, color=(0,0,255))
cropped_sample = {'image': crop, 'landmarks': landmarks, 'pose': pose}
item = self.transform(cropped_sample)
em_val_ar = np.array(
[[sample.expression, sample.valence, sample.arousal]],
dtype=np.float32)
result = self.crop_to_tensor(item)
result.update({'id': 0, 'fnames': filename, 'expression': em_val_ar})
if self.return_modified_images:
mod_transforms = tf.Compose([fp.RandomOcclusion()])
crop_occ = mod_transforms(item['image'])
crop_occ = self.crop_to_tensor(crop_occ)
result['image_mod'] = crop_occ
if self.return_landmark_heatmaps:
result['lm_heatmaps'] = create_landmark_heatmaps(
result['landmarks'], self.landmark_sigma, self.landmark_ids)
return result
def __getitem__(self, idx):
def get_landmarks_for_crop():
pose = np.zeros(3, dtype=np.float32)
if self.crop_source == 'lm_openface':
openface_filepath = os.path.join(self.feature_dir_of, os.path.splitext(filename)[0])
est_face_center = landmarks_gt.mean(axis=0)
of_conf, landmarks_of, pose = ds_utils.read_openface_detection(openface_filepath, expected_face_center=est_face_center)
if of_conf < 0.01:
landmarks_of = landmarks_gt
else:
# landmarks_of, pose = self.cropper.apply_crop_to_landmarks(landmarks_of, pose)
landmarks_of[:,0] -= cfg.CROP_BORDER
landmarks_of[:,1] -= cfg.CROP_BORDER
landmarks = landmarks_of
elif self.crop_source == 'lm_cnn':
try:
landmarks = np.load(os.path.join(self.feature_dir_cnn, os.path.splitext(filename)[0]+'.npy'))
except FileNotFoundError:
landmarks = None
elif self.crop_source == 'lm_ground_truth':
landmarks = landmarks_gt
else:
# no landmarks -> crop using bounding boxes
landmarks = None
return landmarks, pose
sample = self.annotations.iloc[idx]
filename = sample.fname
landmarks_gt = sample.landmarks.astype(np.float32)
bbox = self.get_bounding_box(sample) if not self.split == '300w' else None
landmarks_for_crop, pose = get_landmarks_for_crop()
crop, landmarks, pose, cropper = self.face_extractor.get_face(filename, self.fullsize_img_dir, self.cropped_img_dir,
bb=bbox, landmarks=landmarks_for_crop, pose=pose,
use_cache=self.use_cache,
detect_face=self.detect_face,
crop_type=self.crop_type,
aligned=self.align_face_orientation)
landmarks_gt, _ = cropper.apply_to_landmarks(landmarks_gt)
# self.show_landmarks(crop, landmarks_gt)
cropped_sample = {'image': crop, 'landmarks': landmarks_gt, 'pose': pose}
item = self.transform(cropped_sample)
em_val_ar = np.array([[-1,0,0]], dtype=np.float32)
result = self.crop_to_tensor(item)
result.update({
'fnames': filename,
'expression': em_val_ar
})
if self.return_modified_images:
mod_transforms = tf.Compose([fp.RandomOcclusion()])
crop_occ = mod_transforms(item['image'])
crop_occ = self.crop_to_tensor(crop_occ)
result['image_mod'] = crop_occ
# add landmark heatmaps if landmarks enabled
if self.return_landmark_heatmaps:
result['lm_heatmaps'] = create_landmark_heatmaps(result['landmarks'], self.landmark_sigma, self.landmark_ids)
return result