def _pre_process_input_image(image):
"""
Pre-processes an image for ESR-9.
:param image: (ndarray)
:return: (ndarray) image
"""
image = uimage.resize(image, ESR.INPUT_IMAGE_SIZE)
image = Image.fromarray(image)
image = transforms.Normalize(mean=ESR.INPUT_IMAGE_NORMALIZATION_MEAN,
std=ESR.INPUT_IMAGE_NORMALIZATION_STD)(
transforms.ToTensor()(image)).unsqueeze(0)
return image
def _draw_input_container(self, is_blank):
self._input_container = self._get_container(0, 0,
self._container_height,
self._container_width)
if is_blank:
uimage.draw_text(self._input_container, FERDemo._TEXT_BLANK_INPUT,
self._container_center_position - 60,
FERDemo._COLOUR_BGR_WHITE,
FERDemo._TEXT_PARAM_SCALE[self._screen_size],
FERDemo._TEXT_PARAM_THICKNESS[self._screen_size])
else:
# Compute resize factor 'f'
h, w, c = self._fer.input_image.shape
h_c, w_c, c_c = self._input_container.shape
h_ratio = h / h_c
w_ratio = w / w_c
if h_ratio > w_ratio:
if h < (self._container_height *
FERDemo._INPUT_IMAGE_SCALE_MIN):
f = (self._container_height *
FERDemo._INPUT_IMAGE_SCALE_MIN) / float(h)
else:
f = (self._container_height *
FERDemo._INPUT_IMAGE_SCALE_MAX) / float(h)
else:
if w < (self._container_height *
FERDemo._INPUT_IMAGE_SCALE_MIN):
f = (self._container_width *
FERDemo._INPUT_IMAGE_SCALE_MIN) / float(w)
else:
f = (self._container_width *
FERDemo._INPUT_IMAGE_SCALE_MAX) / float(w)
# Resize input image
self._input_image = uimage.resize(self._fer.input_image, f=f)
# Set input image to the container
h, w, c = self._input_image.shape
x = int((self._container_height // 2) - (h // 2))
y = int((self._container_width // 2) - (w // 2))
self._input_container[x:(x + h), y:(y + w), :] = self._input_image
def detect_face(image, face_detection_method=_ID_FACE_DETECTOR_DLIB):
"""
Detects faces in an image.
:param image: (ndarray) Raw input image.
:param face_detection_method: (int) (1) haar cascade classifiers or (2) Dlib face detection method.
:return: (list) Tuples with coordinates of a detected face.
"""
face_coordinates = []
# Converts to greyscale
greyscale_image = uimage.convert_bgr_to_grey(image)
if face_detection_method == _ID_FACE_DETECTOR_HAAR_CASCADE:
face_coordinates = _haar_cascade_face_detection(
greyscale_image, _HAAR_SCALE_FACTOR, _HAAR_NEIGHBORS,
_HAAR_MIN_SIZE)
elif face_detection_method == _ID_FACE_DETECTOR_DLIB:
# If input image is large, upper-bound of the scale factor is 0.5
scale_factors = _DLIB_SCALE_FACTOR_LARGE_IMAGES if (
greyscale_image.size > _DLIB_SCALE_FACTOR_THRESHOLD
) else _DLIB_SCALE_FACTOR_SMALL_IMAGES
# Down-sample the image to speed-up face detection
for scale in scale_factors:
greyscale_image_re_scaled = uimage.resize(greyscale_image, f=scale)
face_coordinates = _dlib_face_detection(greyscale_image_re_scaled)
# If found a face, then stop iterating
if len(face_coordinates) > 0:
face_coordinates = ((1 / scale) * face_coordinates).astype(int)
break
else: # Standard Dlib
face_coordinates = _dlib_face_detection(greyscale_image).astype(int)
# Returns None if no face is detected
return face_coordinates[0] if (len(face_coordinates) > 0 and
(np.sum(face_coordinates[0]) > 0)) else None
def _draw_background(self):
if (self._fer is None) or (self._fer.input_image is None):
self._background = np.ones(
(self._height, self._width, 3),
dtype=np.uint8) * FERDemo._COLOUR_G_DARK_GREY
else:
# Resize
self._background = uimage.resize(
self._fer.input_image,
f=np.maximum(
np.maximum(self._fer.input_image.shape[0] / self._height,
self._fer.input_image.shape[1] / self._width),
np.maximum(self._height / self._fer.input_image.shape[0],
self._width / self._fer.input_image.shape[1])
))[:self._height, :self._width, :]
# Blur
self._background = uimage.blur(uimage.blur(self._background, 40),
20)
# Brightness
mean = np.mean(self._background)
gamma = 0.75 if mean > 100 else 1.5
mean = mean if mean > 50 else 100
self._background = np.clip((gamma * self._background) + mean, 0,
255).astype(np.uint8)
def _draw_output_container(self, is_blank):
self._output_container = self._get_container(
0, self._output_container_initial_position[1],
self._container_height, self._container_width)
if is_blank:
uimage.draw_text(self._output_container, FERDemo._TEXT_BLANK_INPUT,
self._container_center_position - 60,
FERDemo._COLOUR_BGR_WHITE,
FERDemo._TEXT_PARAM_SCALE[self._screen_size],
FERDemo._TEXT_PARAM_THICKNESS[self._screen_size])
else:
if self._fer.face_image is None:
uimage.draw_text(
self._output_container, FERDemo._TEXT_NO_FACE,
self._container_center_position - 210,
FERDemo._COLOUR_BGR_BLACK,
FERDemo._TEXT_PARAM_SCALE[self._screen_size],
FERDemo._TEXT_PARAM_THICKNESS[self._screen_size])
else:
# Display ensemble and individual classifications
if self._display_individual_classification:
# Resize face image
face_image = uimage.resize(
self._fer.face_image,
FERDemo._BLOCK_IMAGE_SIZE[self._screen_size])
# Generate block of the ensemble prediction
block = self._generate_block(
FERDemo._TEXT_ENSEMBLE,
self._fer.list_emotion[-1],
self._fer.list_affect[-1][0],
self._fer.list_affect[-1][1],
face_image=face_image,
x=0,
y=self._output_container_initial_position[1])
# Draw block ot the ensemble prediction
uimage.draw_image(self._output_container, block, (0, 0))
# Branches
for branch in range(len(self._fer.list_emotion) - 1):
# Superimpose saliency map on input face image
grad_cam = self._fer.get_grad_cam(branch)
if not (grad_cam is None):
grad_cam = uimage.superimpose(grad_cam, face_image)
# Generate block of the branch prediction
block = self._generate_block(
FERDemo._TEXT_BRANCH.format(branch + 1),
self._fer.list_emotion[branch],
self._fer.list_affect[branch][0],
self._fer.list_affect[branch][1],
grad_cam,
x=self._output_block_height * (branch + 1),
y=self._output_container_initial_position[1])
# Draw block of the branch prediction
uimage.draw_image(self._output_container, block,
(self._output_block_height *
(branch + 1), 0))
# Display ensemble classification in detail
else:
# Ensemble
face_image = uimage.resize(
self._fer.face_image,
FERDemo._BLOCK_IMAGE_SIZE_ENSEMBLE[self._screen_size])
block = self._generate_block_ensemble(
FERDemo._TEXT_ENSEMBLE,
self._fer.list_emotion[-1],
self._fer.list_affect[-1][0],
self._fer.list_affect[-1][1],
face_image=face_image,
x=0,
y=self._output_container_initial_position[1])
uimage.draw_image(self._output_container, block, (0, 0))
def pre_process_affect_net(base_path_to_images, base_path_to_annotations,
set_index):
"""
Pre-process the AffectNet dataset. Faces are cropped and resized to 96 x 96 pixels.
The images are organized in folders with 500 images each. The test set had not been released
when this experiment was carried out.
:param base_path_to_images: (string) Path to images.
:param base_path_to_annotations: (string) Path to annotations.
:param set_index: (int = {0, 1, 2}) set_index = 0 process the automatically annotated images.
set_index = 1 process the manually annotated images: training set.
set_index = 2 process the manually annotated images: validation set.
:return: (void)
"""
assert ((set_index < 3)
and (set_index >= 0)), "set_index must be 0, 1 or 2."
annotation_folders = [
'Automatically_Annotated_Images/', 'Manually_Annotated_Images/',
'Manually_Annotated_Images/'
]
destination_set_folders = [
'AffectNet/Training_Unlabeled/', 'AffectNet/Training_Labeled/',
'AffectNet/Validation/'
]
annotation_file_names = [
'automatically_annotated.csv', 'Manually_training.csv',
'Manually_validation.csv'
]
image_id = 0
error_image_id = []
img_size = (96, 96)
num_images_per_folder = 500
annotation_file = pandas.read_csv(
path.join(base_path_to_annotations, annotation_file_names[set_index]))
for line in range(image_id, annotation_file.shape[0]):
try:
# Read image
img_file_name = annotation_file.get('subDirectory_filePath')[line]
img_full_path = path.join(base_path_to_images,
annotation_folders[set_index],
img_file_name)
img = uimage.read(img_full_path)
# Crop face
x = int(annotation_file.get('face_x')[line])
y = int(annotation_file.get('face_y')[line])
w = int(annotation_file.get('face_width')[line])
h = int(annotation_file.get('face_height')[line])
img = img[x:x + w, y:y + h, :]
# Resize image
img = uimage.resize(img, img_size)
# Save image
folder = str(image_id // num_images_per_folder)
exp = annotation_file.get('expression')[line]
val = annotation_file.get('valence')[line]
aro = annotation_file.get('arousal')[line]
file_name = _generate_single_file_name(image_id, exp, val, aro)
uimage.write(
img,
path.join(base_path_to_images,
destination_set_folders[set_index], folder),
file_name)
image_id += 1
except Exception:
print('ERROR: The image ID %d is corrupted.' % image_id)
error_image_id.append(image_id)
print('Dataset has been processed.')
print('Images successfully processed: %d' %
(image_id - len(error_image_id)))
print('Images processed with error: %d' % len(error_image_id))
print('Image IDs processed with error: %s' % error_image_id)
def _load(self):
csv_label = []
data, labels = [], []
counter_loaded_images_per_label = [0 for _ in range(self.num_labels)]
path_folders_images = path.join(self.base_path_to_FER_plus, 'Images',
self.fer_sets[self.idx_set])
path_folders_labels = path.join(self.base_path_to_FER_plus, 'Labels',
self.fer_sets[self.idx_set])
with open(path_folders_labels + '/label.csv') as csvfile:
lines = csv.reader(csvfile)
for row in lines:
csv_label.append(row)
# Shuffle training set
if self.idx_set == 0:
np.random.shuffle(csv_label)
for l in csv_label:
emotion_raw = list(map(float, l[2:len(l)]))
emotion = self._process_data(emotion_raw)
emotion = emotion[:-2]
try:
emotion = [float(i) / sum(emotion) for i in emotion]
emotion = self._parse_to_label(emotion)
except ZeroDivisionError:
emotion = 9
if (emotion < self.num_labels) and (
counter_loaded_images_per_label[int(emotion)] <
self.max_loaded_images_per_label):
counter_loaded_images_per_label[int(emotion)] += 1
img = np.array(
uimage.read(path.join(path_folders_images, l[0])),
np.uint8)
box = list(map(int, l[1][1:-1].split(',')))
if box[-1] != 48:
print("[INFO] Face is not centralized.")
print(path.join(path_folders_images, l[0]))
print(box)
exit(-1)
img = img[box[0]:box[2], box[1]:box[3], :]
img = uimage.resize(img, (96, 96))
data.append(img)
labels.append(emotion)
has_loading_finished = (
np.sum(counter_loaded_images_per_label) >=
(self.max_loaded_images_per_label * self.num_labels))
if has_loading_finished:
break
return [np.array(data), np.array(labels)]
