def fr_experiments(params, show_results): """ Execute face recognition experiments :type params: dictionary :param params: configuration parameters to be used for the experiment (see table) :type show_results: boolean :param show_results: show (True) or do not show (False) images with detected faces ============================================ ======================================== ============================== Key Value Default value ============================================ ======================================== ============================== check_eye_positions If True, check eye positions True classifiers_dir_path Path of directory with OpenCV cascade classifiers eye_detection_classifier Classifier for eye detection 'haarcascade_mcs_lefteye.xml' face_detection_algorithm Classifier for face detection 'HaarCascadeFrontalFaceAlt2' ('HaarCascadeFrontalFaceAlt', 'HaarCascadeFrontalFaceAltTree', 'HaarCascadeFrontalFaceAlt2', 'HaarCascadeFrontalFaceDefault', 'HaarCascadeProfileFace', 'HaarCascadeFrontalAndProfileFaces', 'HaarCascadeFrontalAndProfileFaces2', 'LBPCascadeFrontalface', 'LBPCascadeProfileFace' or 'LBPCascadeFrontalAndProfileFaces') flags Flags used in face detection 'DoCannyPruning' ('DoCannyPruning', 'ScaleImage', 'FindBiggestObject', 'DoRoughSearch'). If 'DoCannyPruning' is used, regions that do not contain lines are discarded. If 'ScaleImage' is used, image instead of the detector is scaled (it can be advantegeous in terms of memory and cache use). If 'FindBiggestObject' is used, only the biggest object is returned by the detector. 'DoRoughSearch', used together with 'FindBiggestObject', terminates the search as soon as the first candidate object is found min_neighbors Mininum number of neighbor bounding 5 boxes for retaining face detection min_size_height Minimum height of face detection 20 bounding box (in pixels) min_size_width Minimum width of face detection 20 bounding box (in pixels) scale_factor Scale factor between two scans 1.1 in face detection max_eye_angle Maximum inclination of the line 0.125 connecting the eyes (in % of pi radians) min_eye_distance Minimum distance between eyes 0.25 (in % of the width of the face bounding box) nose_detection_classifier Classifier for nose detection 'haarcascade_mcs_nose.xml' test_set_path path of directory containing test set use_nose_pos_in_detection If True, detections with no good False nose position are discarded aligned_faces_path Default path of directory for aligned faces cropped_face_height Height of aligned faces (in pixels) 400 cropped_face_width Width of aligned faces (in pixels) 200 dataset_already_divided If True, dataset is already divided False between training and test set dataset_path Path of whole dataset, used if dataset is not already divided between training and test set db_name Name of single file containing face models db_models_path Path of directory containing face models face_model_algorithm Algorithm for face recognition 'LBP' ('Eigenfaces', 'Fisherfaces' or 'LBP') face_recognition_results_path Path of directory where test results will be saved test_set_path Path of directory containing test set training_set_path Path of directory containing training set LBP_grid_x Number of columns in grid 4 used for calculating LBP LBP_grid_y Number of columns in grid 8 used for calculating LBP LBP_neighbors Number of neighbors 8 used for calculating LBP LBP_radius Radius used 1 for calculating LBP (in pixels) offset_pct_x % of the image to keep next to 0.20 the eyes in the horizontal direction offset_pct_y % of the image to keep next to 0.50 the eyes in the vertical direction software_test_file Path of image to be used for software test training_images_nr Number of images per person used in training set use_eye_detection If True, use eye detection for detecting True eye position for aligning faces in test images use_eye_detection_in_training If True, use eye detection for detecting True eye position for aligning faces in training images use_eyes_position If True, align faces in test images True by using eye positions use_eyes_position_in_training If True, align faces in training images True by using eye positions use_face_detection_in_training If True, use face detection False for images in training set use_NBNN If True, False use Naive Bayes Nearest Neighbor use_one_file_for_face_models If True, use one file for face models True use_resizing If True, resize images True use_weighted_regions If True, use weighted LBP False ============================================ ======================================== ============================== """ rec_images_nr = 0 # Number of correctly recognized images test_images_nr = 0 # Number of total test images mean_rec_time = 0 # List of confidence values for true positives true_pos_confidence_list = [] # List of confidence values for false positives false_pos_confidence_list = [] fm = FaceModels(params) training_images_nr = ce.TRAINING_IMAGES_NR if params is not None: # Number of images for each person to be used for the training training_images_nr = params[ce.TRAINING_IMAGES_NR_KEY] # Number of people people_nr = fm.get_people_nr() rec_dict = {} # Dictionary containing all results for this experiment # List used for creating YAML file with list of images images_list_for_YAML = [] # List used for creating YAML file with list of people people_list_for_YAML = [] # List containing recognition rates rec_rate_list = [] # Initialize dictionaries with people people_true_positives_dict = {} people_false_positives_dict = {} people_test_images_nr_dict = {} tags = fm.get_tags() for tag in tags: people_true_positives_dict[tag] = 0 people_false_positives_dict[tag] = 0 people_test_images_nr_dict[tag] = 0 dataset_already_divided = ce.DATASET_ALREADY_DIVIDED # directory with test set test_set_path = ce.FACE_RECOGNITION_TEST_SET_PATH if not dataset_already_divided: test_set_path = ce.FACE_RECOGNITION_DATASET_PATH results_path = ce.FACE_RECOGNITION_RESULTS_PATH if params is not None: # Get path of directories with used files from params if ce.DATASET_ALREADY_DIVIDED_KEY in params: dataset_already_divided = params[ce.DATASET_ALREADY_DIVIDED_KEY] if dataset_already_divided: if ce.TEST_SET_PATH_KEY in params: test_set_path = params[ce.TEST_SET_PATH_KEY] else: if ce.DATASET_PATH_KEY in params: test_set_path = params[ce.DATASET_PATH_KEY] # directory with results if ce.FACE_RECOGNITION_RESULTS_PATH_KEY in params: results_path = params[ce.FACE_RECOGNITION_RESULTS_PATH_KEY] # Iterate over all directories with images images_dirs = os.listdir(test_set_path) total_test_images_nr = 0 for images_dir in images_dirs: ann_face_tag = images_dir images_dir_complete_path = os.path.join(test_set_path, images_dir) # Iterate over all images in this directory image_counter = 0 person_test_images = 0 person_rec_images = 0 for image in os.listdir(images_dir_complete_path): # If dataset is not already divided, # first training_images_nr images are used for training, # the remaining for test if(dataset_already_divided or (image_counter >= training_images_nr)): total_test_images_nr += 1 person_test_images += 1 # Complete path of image image_complete_path = os.path.join( images_dir_complete_path, image) try: assigned_tag = 'Undefined' confidence = -1 if USE_FACEEXTRACTOR: fe = FaceExtractor(fm, params) handle = fe.extract_faces_from_image( image_complete_path) results = fe.get_results(handle) faces = results[c.FACES_KEY] if len(faces) != 0: face = faces[0] mean_rec_time = ( mean_rec_time + results[c.ELAPSED_CPU_TIME_KEY]) assigned_tag = face[c.ASSIGNED_TAG_KEY] confidence = face[c.CONFIDENCE_KEY] else: face = cv2.imread( image_complete_path, cv2.IMREAD_GRAYSCALE) sz = None use_resizing = ce.USE_RESIZING use_eyes_position = c.USE_EYES_POSITION use_eye_detection = ce.USE_EYE_DETECTION offset_pct_x = c.OFFSET_PCT_X offset_pct_y = c.OFFSET_PCT_Y if params is not None: if ce.USE_RESIZING_KEY in params: use_resizing = params[ce.USE_RESIZING_KEY] if c.USE_EYES_POSITION_KEY in params: use_eyes_position = ( params[c.USE_EYES_POSITION_KEY]) if ce.USE_EYE_DETECTION_KEY in params: use_eye_detection = ( params[ce.USE_EYE_DETECTION_KEY]) if c.OFFSET_PCT_X_KEY in params: offset_pct_x = params[c.OFFSET_PCT_X_KEY] if c.OFFSET_PCT_Y_KEY in params: offset_pct_y = params[c.OFFSET_PCT_Y_KEY] if use_resizing: width = c.CROPPED_FACE_WIDTH height = c.CROPPED_FACE_HEIGHT if params is not None: if c.CROPPED_FACE_WIDTH_KEY in params: width = params[c.CROPPED_FACE_WIDTH_KEY] if c.CROPPED_FACE_HEIGHT_KEY in params: height = params[c.CROPPED_FACE_HEIGHT_KEY] sz = (width, height) if use_eyes_position: align_path = c.ALIGNED_FACES_PATH if params is not None: align_path = params[c.ALIGNED_FACES_PATH_KEY] if use_eye_detection: face = fd.get_cropped_face( image_complete_path, align_path, params, return_always_face=False) else: face = fd.get_cropped_face_using_fixed_eye_pos( image_complete_path, align_path, offset_pct=(offset_pct_x, offset_pct_y), dest_size=sz) if face is not None: face = face[c.FACE_KEY] else: if sz is not None: face = cv2.resize(face, sz) if face is not None: rec_results = recognize_face( face, fm, params, show_results) assigned_tag = rec_results[c.ASSIGNED_TAG_KEY] confidence = rec_results[c.CONFIDENCE_KEY] # Add recognition time to total mean_rec_time = ( mean_rec_time + rec_results[c.ELAPSED_CPU_TIME_KEY]) image_dict = {ce.IMAGE_KEY: image, c.ANN_TAG_KEY: images_dir, c.ASSIGNED_TAG_KEY: assigned_tag, c.CONFIDENCE_KEY: confidence} if assigned_tag == ann_face_tag: image_dict[ce.PERSON_CHECK_KEY] = 'TP' people_true_positives_dict[assigned_tag] += 1 rec_images_nr += 1 true_pos_confidence_list.append(confidence) person_rec_images += 1 else: image_dict[ce.PERSON_CHECK_KEY] = 'FP' if assigned_tag != 'Undefined': people_false_positives_dict[assigned_tag] += 1 false_pos_confidence_list.append(confidence) image_dict_extended = {ce.IMAGE_KEY: image_dict} images_list_for_YAML.append(image_dict_extended) except IOError, (errno, strerror): print "I/O error({0}): {1}".format(errno, strerror) except: print "Unexpected error:", sys.exc_info()[0] raise
def fr_video_experiments(params, show_results): """ Execute face recognition experiments on video files :type params: dictionary :param params: configuration parameters to be used for the experiment :type show_results: boolean :param show_results: show (True) or do not show (False) image with detected faces ============================================ ======================================== ============== Key Value Default value ============================================ ======================================== ============== annotations_path Path of directory containing the manual annotations for the images check_eye_positions If True, check eye positions True classifiers_dir_path Path of directory with OpenCV cascade classifiers eye_detection_classifier Classifier for eye detection 'haarcascade_mcs_lefteye.xml' face_detection_algorithm Classifier for face detection 'HaarCascadeFrontalFaceAlt2' ('HaarCascadeFrontalFaceAlt', 'HaarCascadeFrontalFaceAltTree', 'HaarCascadeFrontalFaceAlt2', 'HaarCascadeFrontalFaceDefault', 'HaarCascadeProfileFace', 'HaarCascadeFrontalAndProfileFaces', 'HaarCascadeFrontalAndProfileFaces2', 'LBPCascadeFrontalface', 'LBPCascadeProfileFace' or 'LBPCascadeFrontalAndProfileFaces') flags Flags used in face detection 'DoCannyPruning' ('DoCannyPruning', 'ScaleImage', 'FindBiggestObject', 'DoRoughSearch'). If 'DoCannyPruning' is used, regions that do not contain lines are discarded. If 'ScaleImage' is used, image instead of the detector is scaled (it can be advantegeous in terms of memory and cache use). If 'FindBiggestObject' is used, only the biggest object is returned by the detector. 'DoRoughSearch', used together with 'FindBiggestObject', terminates the search as soon as the first candidate object is found min_neighbors Mininum number of neighbor bounding 5 boxes for retaining face detection min_size_height Minimum height of face detection 20 bounding box (in pixels) min_size_width Minimum width of face detection 20 bounding box (in pixels) face_detection_results_path Path of directory where test results will be saved scale_factor Scale factor between two scans 1.1 in face detection max_eye_angle Maximum inclination of the line 0.125 connecting the eyes (in % of pi radians) min_eye_distance Minimum distance between eyes 0.25 (in % of the width of the face bounding box) nose_detection_classifier Classifier for nose detection 'haarcascade_mcs_nose.xml' software_test_file Path of image to be used for software test test_set_path path of directory containing test set use_nose_pos_in_detection If True, detections with no good False nose position are discarded aligned_faces_path Default path of directory for aligned faces cropped_face_height Height of aligned faces (in pixels) 400 cropped_face_width Width of aligned faces (in pixels) 200 dataset_already_divided If True, dataset is already divided False between training and test set dataset_path Path of whole dataset, used if dataset is not already divided between training and test set db_name Name of single file containing face models db_models_path Path of directory containing face models face_model_algorithm Algorithm for face recognition 'LBP' ('Eigenfaces', 'Fisherfaces' or 'LBP') face_recognition_results_path Path of directory where test results will be saved test_set_path Path of directory containing test set training_set_path Path of directory containing training set LBP_grid_x Number of columns in grid 4 used for calculating LBP LBP_grid_y Number of columns in grid 8 used for calculating LBP LBP_neighbors Number of neighbors 8 used for calculating LBP LBP_radius Radius used 1 for calculating LBP (in pixels) max_frames_with_missed_detections Maximum number of frames with 5 missed detection that does not interrupt tracking offset_pct_x % of the image to keep next to 0.20 the eyes in the horizontal direction offset_pct_y % of the image to keep next to 0.50 the eyes in the vertical direction sim_tracking If True, results from all frames in False video are aggregated sliding_window_size Size of sliding window in seconds 5.0 software_test_file Path of image to be used for software test test_video_path Path of test video training_images_nr Number of images per person used in training set use_captions If True, training set False for face recognition is built on the base of captions use_eye_detection If True, use eye detection for detecting True eye position for aligning faces in test images use_eye_detection_in_training If True, use eye detection for detecting True eye position for aligning faces in training images use_eyes_position If True, align faces in test images True by using eye positions use_eyes_position_in_training If True, align faces in training images True by using eye positions use_face_detection_in_training If True, use face detection False for images in training set use_majority_rule If True, in aggregating results True from several frames, final tag is the tag that was assigned to the majority of frames use_mean_confidence_rule If True, in aggregating results False from several frames, final tag is the tag that received the minimum value for the mean of confidences among frames use_min_confidence_rule If True, in aggregating results True from several frames, final tag is the tag that received the minimum confidence value use_NBNN If True, False use Naive Bayes Nearest Neighbor use_one_file_for_face_models If True, use one file for face models True use_original_fps If True, original frame rate is used False use_original_fps_in_training If True, use original frame rate False of video when creating training set use_resizing If True, resize images True use_sliding_window If True, use sliding window False in face extraction use_tracking If True, False use tracking in face extraction use_weighted_regions If True, use weighted LBP False used_fps Frame rate at which video 5.0 is analyzed (in frames per second) used_fps_in_training Frame rate at which video is analyzed 1.0 in training from captions (in frames per second) ============================================ ======================================== ============== """ # Folder with results results_path = ce.FACE_RECOGNITION_RESULTS_PATH sim_tracking = ce.SIM_TRACKING # Folder with test files test_set_path = ce.FACE_RECOGNITION_TEST_SET_PATH use_captions = ce.USE_CAPTIONS use_tracking = ce.USE_TRACKING video_path = ce.TEST_VIDEO_PATH if params is not None: if ce.RESULTS_PATH_KEY in params: results_path = params[ce.RESULTS_PATH_KEY] if ce.SIM_TRACKING_KEY in params: sim_tracking = params[ce.SIM_TRACKING_KEY] if ce.TEST_SET_PATH_KEY in params: test_set_path = params[ce.TEST_SET_PATH_KEY] if ce.USE_CAPTIONS_KEY in params: use_captions = params[ce.USE_CAPTIONS_KEY] if use_captions: if ce.TEST_VIDEO_PATH_KEY in params: video_path = params[ce.TEST_VIDEO_PATH_KEY] else: video_path = None if ce.USE_TRACKING_KEY in params: use_tracking = params[ce.USE_TRACKING_KEY] tot_rec_frames_nr = 0 # Number of correctly recognized frames tot_test_frames_nr = 0 # Number of total test frames mean_rec_time = 0 # Mean recognition time for videos # List of confidence values for true positives true_pos_confidence_list = [] # List of confidence values for false positives false_pos_confidence_list = [] # List of tags of people that appear in test set tested_people_tag_list = [] fm = FaceModels(params=params, video_path=video_path) # Number of people people_nr = fm.get_people_nr() # Initialize dictionaries with people people_true_positives_dict = {} people_false_positives_dict = {} people_test_frames_nr_dict = {} tags = fm.get_tags() for tag in tags: people_true_positives_dict[tag] = 0 people_false_positives_dict[tag] = 0 people_test_frames_nr_dict[tag] = 0 # Iterate over all videos experiment_dict_list = [] number_of_anal_video = 0 for video in os.listdir(test_set_path): # Dictionary for YAML file with results experiment_dict = {} experiment_dict_frames = [] assigned_tag = 'Undefined' final_confidence = -1 video_delta_xs = [] video_delta_ys = [] video_delta_ws = [] ann_face_tag, file_ext = os.path.splitext(video) print('Annotated face tag: ', ann_face_tag) tested_people_tag_list.append(ann_face_tag) video_complete_path = test_set_path + video print(video_complete_path) try: fe = FaceExtractor(fm) handle = fe.extract_faces_from_video(video_complete_path) results = fe.get_results(handle) error = results[c.ERROR_KEY] if error: print('Warning') print(error) else: video_test_frames_nr = results[ce.TOT_FRAMES_NR_KEY] tot_test_frames_nr = tot_test_frames_nr + video_test_frames_nr people_test_frames_nr_dict[ann_face_tag] = video_test_frames_nr mean_rec_time = mean_rec_time + results[c.ELAPSED_CPU_TIME_KEY] if use_tracking: segments = results[c.SEGMENTS_KEY] tot_segments_frames_nr = 0 true_positives_in_segment = 0 for segment in segments: segment_frames = segment[c.FRAMES_KEY] frames_nr = segment[c.SEGMENT_TOT_FRAMES_NR_KEY] assigned_tag = segment[c.ASSIGNED_TAG_KEY] print('assigned_tag = ' + assigned_tag) final_confidence = segment[c.CONFIDENCE_KEY] tot_segments_frames_nr = tot_segments_frames_nr + frames_nr if assigned_tag == ann_face_tag: people_true_positives_dict[assigned_tag] += frames_nr true_positives_in_segment = true_positives_in_segment + frames_nr tot_rec_frames_nr = tot_rec_frames_nr + frames_nr true_pos_confidence_list.append(final_confidence) else: if assigned_tag != 'Undefined': people_false_positives_dict[assigned_tag] = people_false_positives_dict[assigned_tag] + frames_nr false_pos_confidence_list.append(final_confidence) # Tot number of frames in segments # cannot be more than number of frames in video if true_positives_in_segment > video_test_frames_nr: print('### WARNING! ###') print 'true_positives_in_segment = ' + str( true_positives_in_segment) print 'video_test_frames_nr = ' + str( video_test_frames_nr) people_false_positives_dict[ ann_face_tag] += (tot_segments_frames_nr - video_test_frames_nr) tot_rec_frames_nr -= (tot_segments_frames_nr - video_test_frames_nr) elif sim_tracking: # Simulate tracking # (every frame of this video contains the same person) frames = results[c.FRAMES_KEY] [assigned_tag, final_confidence] = aggregate_frame_results_in_sim_tracking(frames, fm) print('assigned_tag = ' + assigned_tag) if assigned_tag == ann_face_tag: people_true_positives_dict[assigned_tag] += len(frames) tot_rec_frames_nr += len(frames) true_pos_confidence_list.append(final_confidence) else: if assigned_tag != 'Undefined': people_false_positives_dict[assigned_tag] += len( frames) false_pos_confidence_list.append(final_confidence) else: frames = results[c.FRAMES_KEY] frame_counter = 0 prev_frame_counter = -1 for frame in frames: assigned_tag = 'Undefined' confidence = -1 faces = frame[c.FACES_KEY] time_stamp = frame[c.ELAPSED_VIDEO_TIME_KEY] experiment_dict_frame = { c.ELAPSED_VIDEO_TIME_KEY: time_stamp} if len(faces) != 0: face = faces[0] assigned_tag = face[c.ASSIGNED_TAG_KEY] experiment_dict_faces = [] experiment_dict_face = { c.ASSIGNED_TAG_KEY: assigned_tag} experiment_dict_faces.append(experiment_dict_face) if len(faces) > 1: for face_counter in range(1, len(faces)): other_face = faces[face_counter] other_face_assigned_tag = other_face[c.ASSIGNED_TAG_KEY] experiment_dict_face = { c.ASSIGNED_TAG_KEY: other_face_assigned_tag} experiment_dict_faces.append(experiment_dict_face) confidence = face[c.CONFIDENCE_KEY] bbox = face[c.BBOX_KEY] if((frame_counter > 0) and (frame_counter <= (prev_frame_counter + c.MAX_FR_WITH_MISSED_DET + 1))): bbox_x = bbox[0] bbox_y = bbox[1] bbox_w = bbox[2] prev_bbox_x = prev_bbox[0] prev_bbox_y = prev_bbox[1] prev_bbox_w = prev_bbox[2] delta_x = (abs(bbox_x - prev_bbox_x) / float(prev_bbox_w)) delta_y = (abs(bbox_x - prev_bbox_x) / float(prev_bbox_w)) delta_w = (abs(bbox_w - prev_bbox_w) / float(prev_bbox_w)) video_delta_xs.append(delta_x) video_delta_ys.append(delta_y) video_delta_ws.append(delta_w) if((delta_x > ce.MAX_DELTA_PCT_X) or (delta_y > ce.MAX_DELTA_PCT_Y) and (delta_w > ce.MAX_DELTA_PCT_W)): print('delta_x: ', delta_x) print('delta_y: ', delta_y) print('delta_w: ', delta_w) prev_frame_counter = frame_counter prev_bbox = bbox experiment_dict_frame[c.FACES_KEY] = experiment_dict_faces experiment_dict_frame[c.ASSIGNED_TAG_KEY] = assigned_tag else: experiment_dict_frame[c.FACES_KEY] = "No face detected" experiment_dict_frame[c.ASSIGNED_TAG_KEY] = "No face detected" experiment_dict_frame[c.CONFIDENCE_KEY] = confidence experiment_dict_frames.append(experiment_dict_frame) if assigned_tag == ann_face_tag: people_true_positives_dict[assigned_tag] += 1 tot_rec_frames_nr += 1 true_pos_confidence_list.append(confidence) else: if assigned_tag != 'Undefined': people_false_positives_dict[assigned_tag] += 1 false_pos_confidence_list.append(confidence) frame_counter += 1 experiment_dict[ce.VIDEO_COUNTER_KEY] = number_of_anal_video experiment_dict[c.ANN_TAG_KEY] = ann_face_tag if sim_tracking: experiment_dict[c.ASSIGNED_TAG_KEY] = assigned_tag experiment_dict[c.CONFIDENCE_KEY] = final_confidence else: experiment_dict[c.FRAMES_KEY] = experiment_dict_frames save_YAML_file( results_path + ann_face_tag + ".yml", experiment_dict) experiment_dict_list.append(experiment_dict) number_of_anal_video += 1 except IOError, (errno, strerror): print "I/O error({0}): {1}".format(errno, strerror) except: