def _pre_process_input_image(image): """ Pre-processes an image for ESR-9. :param image: (ndarray) :return: (ndarray) image """ image = uimage.resize(image, ESR.INPUT_IMAGE_SIZE) image = Image.fromarray(image) image = transforms.Normalize(mean=ESR.INPUT_IMAGE_NORMALIZATION_MEAN, std=ESR.INPUT_IMAGE_NORMALIZATION_STD)( transforms.ToTensor()(image)).unsqueeze(0) return image
def _draw_input_container(self, is_blank): self._input_container = self._get_container(0, 0, self._container_height, self._container_width) if is_blank: uimage.draw_text(self._input_container, FERDemo._TEXT_BLANK_INPUT, self._container_center_position - 60, FERDemo._COLOUR_BGR_WHITE, FERDemo._TEXT_PARAM_SCALE[self._screen_size], FERDemo._TEXT_PARAM_THICKNESS[self._screen_size]) else: # Compute resize factor 'f' h, w, c = self._fer.input_image.shape h_c, w_c, c_c = self._input_container.shape h_ratio = h / h_c w_ratio = w / w_c if h_ratio > w_ratio: if h < (self._container_height * FERDemo._INPUT_IMAGE_SCALE_MIN): f = (self._container_height * FERDemo._INPUT_IMAGE_SCALE_MIN) / float(h) else: f = (self._container_height * FERDemo._INPUT_IMAGE_SCALE_MAX) / float(h) else: if w < (self._container_height * FERDemo._INPUT_IMAGE_SCALE_MIN): f = (self._container_width * FERDemo._INPUT_IMAGE_SCALE_MIN) / float(w) else: f = (self._container_width * FERDemo._INPUT_IMAGE_SCALE_MAX) / float(w) # Resize input image self._input_image = uimage.resize(self._fer.input_image, f=f) # Set input image to the container h, w, c = self._input_image.shape x = int((self._container_height // 2) - (h // 2)) y = int((self._container_width // 2) - (w // 2)) self._input_container[x:(x + h), y:(y + w), :] = self._input_image
def detect_face(image, face_detection_method=_ID_FACE_DETECTOR_DLIB): """ Detects faces in an image. :param image: (ndarray) Raw input image. :param face_detection_method: (int) (1) haar cascade classifiers or (2) Dlib face detection method. :return: (list) Tuples with coordinates of a detected face. """ face_coordinates = [] # Converts to greyscale greyscale_image = uimage.convert_bgr_to_grey(image) if face_detection_method == _ID_FACE_DETECTOR_HAAR_CASCADE: face_coordinates = _haar_cascade_face_detection( greyscale_image, _HAAR_SCALE_FACTOR, _HAAR_NEIGHBORS, _HAAR_MIN_SIZE) elif face_detection_method == _ID_FACE_DETECTOR_DLIB: # If input image is large, upper-bound of the scale factor is 0.5 scale_factors = _DLIB_SCALE_FACTOR_LARGE_IMAGES if ( greyscale_image.size > _DLIB_SCALE_FACTOR_THRESHOLD ) else _DLIB_SCALE_FACTOR_SMALL_IMAGES # Down-sample the image to speed-up face detection for scale in scale_factors: greyscale_image_re_scaled = uimage.resize(greyscale_image, f=scale) face_coordinates = _dlib_face_detection(greyscale_image_re_scaled) # If found a face, then stop iterating if len(face_coordinates) > 0: face_coordinates = ((1 / scale) * face_coordinates).astype(int) break else: # Standard Dlib face_coordinates = _dlib_face_detection(greyscale_image).astype(int) # Returns None if no face is detected return face_coordinates[0] if (len(face_coordinates) > 0 and (np.sum(face_coordinates[0]) > 0)) else None
def _draw_background(self): if (self._fer is None) or (self._fer.input_image is None): self._background = np.ones( (self._height, self._width, 3), dtype=np.uint8) * FERDemo._COLOUR_G_DARK_GREY else: # Resize self._background = uimage.resize( self._fer.input_image, f=np.maximum( np.maximum(self._fer.input_image.shape[0] / self._height, self._fer.input_image.shape[1] / self._width), np.maximum(self._height / self._fer.input_image.shape[0], self._width / self._fer.input_image.shape[1]) ))[:self._height, :self._width, :] # Blur self._background = uimage.blur(uimage.blur(self._background, 40), 20) # Brightness mean = np.mean(self._background) gamma = 0.75 if mean > 100 else 1.5 mean = mean if mean > 50 else 100 self._background = np.clip((gamma * self._background) + mean, 0, 255).astype(np.uint8)
def _draw_output_container(self, is_blank): self._output_container = self._get_container( 0, self._output_container_initial_position[1], self._container_height, self._container_width) if is_blank: uimage.draw_text(self._output_container, FERDemo._TEXT_BLANK_INPUT, self._container_center_position - 60, FERDemo._COLOUR_BGR_WHITE, FERDemo._TEXT_PARAM_SCALE[self._screen_size], FERDemo._TEXT_PARAM_THICKNESS[self._screen_size]) else: if self._fer.face_image is None: uimage.draw_text( self._output_container, FERDemo._TEXT_NO_FACE, self._container_center_position - 210, FERDemo._COLOUR_BGR_BLACK, FERDemo._TEXT_PARAM_SCALE[self._screen_size], FERDemo._TEXT_PARAM_THICKNESS[self._screen_size]) else: # Display ensemble and individual classifications if self._display_individual_classification: # Resize face image face_image = uimage.resize( self._fer.face_image, FERDemo._BLOCK_IMAGE_SIZE[self._screen_size]) # Generate block of the ensemble prediction block = self._generate_block( FERDemo._TEXT_ENSEMBLE, self._fer.list_emotion[-1], self._fer.list_affect[-1][0], self._fer.list_affect[-1][1], face_image=face_image, x=0, y=self._output_container_initial_position[1]) # Draw block ot the ensemble prediction uimage.draw_image(self._output_container, block, (0, 0)) # Branches for branch in range(len(self._fer.list_emotion) - 1): # Superimpose saliency map on input face image grad_cam = self._fer.get_grad_cam(branch) if not (grad_cam is None): grad_cam = uimage.superimpose(grad_cam, face_image) # Generate block of the branch prediction block = self._generate_block( FERDemo._TEXT_BRANCH.format(branch + 1), self._fer.list_emotion[branch], self._fer.list_affect[branch][0], self._fer.list_affect[branch][1], grad_cam, x=self._output_block_height * (branch + 1), y=self._output_container_initial_position[1]) # Draw block of the branch prediction uimage.draw_image(self._output_container, block, (self._output_block_height * (branch + 1), 0)) # Display ensemble classification in detail else: # Ensemble face_image = uimage.resize( self._fer.face_image, FERDemo._BLOCK_IMAGE_SIZE_ENSEMBLE[self._screen_size]) block = self._generate_block_ensemble( FERDemo._TEXT_ENSEMBLE, self._fer.list_emotion[-1], self._fer.list_affect[-1][0], self._fer.list_affect[-1][1], face_image=face_image, x=0, y=self._output_container_initial_position[1]) uimage.draw_image(self._output_container, block, (0, 0))
def pre_process_affect_net(base_path_to_images, base_path_to_annotations, set_index): """ Pre-process the AffectNet dataset. Faces are cropped and resized to 96 x 96 pixels. The images are organized in folders with 500 images each. The test set had not been released when this experiment was carried out. :param base_path_to_images: (string) Path to images. :param base_path_to_annotations: (string) Path to annotations. :param set_index: (int = {0, 1, 2}) set_index = 0 process the automatically annotated images. set_index = 1 process the manually annotated images: training set. set_index = 2 process the manually annotated images: validation set. :return: (void) """ assert ((set_index < 3) and (set_index >= 0)), "set_index must be 0, 1 or 2." annotation_folders = [ 'Automatically_Annotated_Images/', 'Manually_Annotated_Images/', 'Manually_Annotated_Images/' ] destination_set_folders = [ 'AffectNet/Training_Unlabeled/', 'AffectNet/Training_Labeled/', 'AffectNet/Validation/' ] annotation_file_names = [ 'automatically_annotated.csv', 'Manually_training.csv', 'Manually_validation.csv' ] image_id = 0 error_image_id = [] img_size = (96, 96) num_images_per_folder = 500 annotation_file = pandas.read_csv( path.join(base_path_to_annotations, annotation_file_names[set_index])) for line in range(image_id, annotation_file.shape[0]): try: # Read image img_file_name = annotation_file.get('subDirectory_filePath')[line] img_full_path = path.join(base_path_to_images, annotation_folders[set_index], img_file_name) img = uimage.read(img_full_path) # Crop face x = int(annotation_file.get('face_x')[line]) y = int(annotation_file.get('face_y')[line]) w = int(annotation_file.get('face_width')[line]) h = int(annotation_file.get('face_height')[line]) img = img[x:x + w, y:y + h, :] # Resize image img = uimage.resize(img, img_size) # Save image folder = str(image_id // num_images_per_folder) exp = annotation_file.get('expression')[line] val = annotation_file.get('valence')[line] aro = annotation_file.get('arousal')[line] file_name = _generate_single_file_name(image_id, exp, val, aro) uimage.write( img, path.join(base_path_to_images, destination_set_folders[set_index], folder), file_name) image_id += 1 except Exception: print('ERROR: The image ID %d is corrupted.' % image_id) error_image_id.append(image_id) print('Dataset has been processed.') print('Images successfully processed: %d' % (image_id - len(error_image_id))) print('Images processed with error: %d' % len(error_image_id)) print('Image IDs processed with error: %s' % error_image_id)
def _load(self): csv_label = [] data, labels = [], [] counter_loaded_images_per_label = [0 for _ in range(self.num_labels)] path_folders_images = path.join(self.base_path_to_FER_plus, 'Images', self.fer_sets[self.idx_set]) path_folders_labels = path.join(self.base_path_to_FER_plus, 'Labels', self.fer_sets[self.idx_set]) with open(path_folders_labels + '/label.csv') as csvfile: lines = csv.reader(csvfile) for row in lines: csv_label.append(row) # Shuffle training set if self.idx_set == 0: np.random.shuffle(csv_label) for l in csv_label: emotion_raw = list(map(float, l[2:len(l)])) emotion = self._process_data(emotion_raw) emotion = emotion[:-2] try: emotion = [float(i) / sum(emotion) for i in emotion] emotion = self._parse_to_label(emotion) except ZeroDivisionError: emotion = 9 if (emotion < self.num_labels) and ( counter_loaded_images_per_label[int(emotion)] < self.max_loaded_images_per_label): counter_loaded_images_per_label[int(emotion)] += 1 img = np.array( uimage.read(path.join(path_folders_images, l[0])), np.uint8) box = list(map(int, l[1][1:-1].split(','))) if box[-1] != 48: print("[INFO] Face is not centralized.") print(path.join(path_folders_images, l[0])) print(box) exit(-1) img = img[box[0]:box[2], box[1]:box[3], :] img = uimage.resize(img, (96, 96)) data.append(img) labels.append(emotion) has_loading_finished = ( np.sum(counter_loaded_images_per_label) >= (self.max_loaded_images_per_label * self.num_labels)) if has_loading_finished: break return [np.array(data), np.array(labels)]