def fr_experiments(params, show_results):
"""
Execute face recognition experiments
:type params: dictionary
:param params: configuration parameters
to be used for the experiment (see table)
:type show_results: boolean
:param show_results: show (True) or do not show (False)
images with detected faces
============================================ ======================================== ==============================
Key Value Default value
============================================ ======================================== ==============================
check_eye_positions If True, check eye positions True
classifiers_dir_path Path of directory with OpenCV
cascade classifiers
eye_detection_classifier Classifier for eye detection 'haarcascade_mcs_lefteye.xml'
face_detection_algorithm Classifier for face detection 'HaarCascadeFrontalFaceAlt2'
('HaarCascadeFrontalFaceAlt',
'HaarCascadeFrontalFaceAltTree',
'HaarCascadeFrontalFaceAlt2',
'HaarCascadeFrontalFaceDefault',
'HaarCascadeProfileFace',
'HaarCascadeFrontalAndProfileFaces',
'HaarCascadeFrontalAndProfileFaces2',
'LBPCascadeFrontalface',
'LBPCascadeProfileFace' or
'LBPCascadeFrontalAndProfileFaces')
flags Flags used in face detection 'DoCannyPruning'
('DoCannyPruning', 'ScaleImage',
'FindBiggestObject', 'DoRoughSearch').
If 'DoCannyPruning' is used, regions
that do not contain lines are discarded.
If 'ScaleImage' is used, image instead
of the detector is scaled
(it can be advantegeous in terms of
memory and cache use).
If 'FindBiggestObject' is used,
only the biggest object is returned
by the detector.
'DoRoughSearch', used together with
'FindBiggestObject',
terminates the search as soon as
the first candidate object is found
min_neighbors Mininum number of neighbor bounding 5
boxes for retaining face detection
min_size_height Minimum height of face detection 20
bounding box (in pixels)
min_size_width Minimum width of face detection 20
bounding box (in pixels)
scale_factor Scale factor between two scans 1.1
in face detection
max_eye_angle Maximum inclination of the line 0.125
connecting the eyes
(in % of pi radians)
min_eye_distance Minimum distance between eyes 0.25
(in % of the width of the face
bounding box)
nose_detection_classifier Classifier for nose detection 'haarcascade_mcs_nose.xml'
test_set_path path of directory
containing test set
use_nose_pos_in_detection If True, detections with no good False
nose position are discarded
aligned_faces_path Default path of directory
for aligned faces
cropped_face_height Height of aligned faces (in pixels) 400
cropped_face_width Width of aligned faces (in pixels) 200
dataset_already_divided If True, dataset is already divided False
between training and test set
dataset_path Path of whole dataset, used if dataset
is not already divided between
training and test set
db_name Name of single file
containing face models
db_models_path Path of directory containing face models
face_model_algorithm Algorithm for face recognition 'LBP'
('Eigenfaces', 'Fisherfaces' or 'LBP')
face_recognition_results_path Path of directory where
test results will be saved
test_set_path Path of directory containing
test set
training_set_path Path of directory containing
training set
LBP_grid_x Number of columns in grid 4
used for calculating LBP
LBP_grid_y Number of columns in grid 8
used for calculating LBP
LBP_neighbors Number of neighbors 8
used for calculating LBP
LBP_radius Radius used 1
for calculating LBP (in pixels)
offset_pct_x % of the image to keep next to 0.20
the eyes in the horizontal direction
offset_pct_y % of the image to keep next to 0.50
the eyes in the vertical direction
software_test_file Path of image to be used for
software test
training_images_nr Number of images per person used in
training set
use_eye_detection If True, use eye detection for detecting True
eye position for aligning faces in
test images
use_eye_detection_in_training If True, use eye detection for detecting True
eye position for aligning faces in
training images
use_eyes_position If True, align faces in test images True
by using eye positions
use_eyes_position_in_training If True, align faces in training images True
by using eye positions
use_face_detection_in_training If True, use face detection False
for images in training set
use_NBNN If True, False
use Naive Bayes Nearest Neighbor
use_one_file_for_face_models If True, use one file for face models True
use_resizing If True, resize images True
use_weighted_regions If True, use weighted LBP False
============================================ ======================================== ==============================
"""
rec_images_nr = 0 # Number of correctly recognized images
test_images_nr = 0 # Number of total test images
mean_rec_time = 0
# List of confidence values for true positives
true_pos_confidence_list = []
# List of confidence values for false positives
false_pos_confidence_list = []
fm = FaceModels(params)
training_images_nr = ce.TRAINING_IMAGES_NR
if params is not None:
# Number of images for each person to be used for the training
training_images_nr = params[ce.TRAINING_IMAGES_NR_KEY]
# Number of people
people_nr = fm.get_people_nr()
rec_dict = {} # Dictionary containing all results for this experiment
# List used for creating YAML file with list of images
images_list_for_YAML = []
# List used for creating YAML file with list of people
people_list_for_YAML = []
# List containing recognition rates
rec_rate_list = []
# Initialize dictionaries with people
people_true_positives_dict = {}
people_false_positives_dict = {}
people_test_images_nr_dict = {}
tags = fm.get_tags()
for tag in tags:
people_true_positives_dict[tag] = 0
people_false_positives_dict[tag] = 0
people_test_images_nr_dict[tag] = 0
dataset_already_divided = ce.DATASET_ALREADY_DIVIDED
# directory with test set
test_set_path = ce.FACE_RECOGNITION_TEST_SET_PATH
if not dataset_already_divided:
test_set_path = ce.FACE_RECOGNITION_DATASET_PATH
results_path = ce.FACE_RECOGNITION_RESULTS_PATH
if params is not None:
# Get path of directories with used files from params
if ce.DATASET_ALREADY_DIVIDED_KEY in params:
dataset_already_divided = params[ce.DATASET_ALREADY_DIVIDED_KEY]
if dataset_already_divided:
if ce.TEST_SET_PATH_KEY in params:
test_set_path = params[ce.TEST_SET_PATH_KEY]
else:
if ce.DATASET_PATH_KEY in params:
test_set_path = params[ce.DATASET_PATH_KEY]
# directory with results
if ce.FACE_RECOGNITION_RESULTS_PATH_KEY in params:
results_path = params[ce.FACE_RECOGNITION_RESULTS_PATH_KEY]
# Iterate over all directories with images
images_dirs = os.listdir(test_set_path)
total_test_images_nr = 0
for images_dir in images_dirs:
ann_face_tag = images_dir
images_dir_complete_path = os.path.join(test_set_path, images_dir)
# Iterate over all images in this directory
image_counter = 0
person_test_images = 0
person_rec_images = 0
for image in os.listdir(images_dir_complete_path):
# If dataset is not already divided,
# first training_images_nr images are used for training,
# the remaining for test
if(dataset_already_divided
or (image_counter >= training_images_nr)):
total_test_images_nr += 1
person_test_images += 1
# Complete path of image
image_complete_path = os.path.join(
images_dir_complete_path, image)
try:
assigned_tag = 'Undefined'
confidence = -1
if USE_FACEEXTRACTOR:
fe = FaceExtractor(fm, params)
handle = fe.extract_faces_from_image(
image_complete_path)
results = fe.get_results(handle)
faces = results[c.FACES_KEY]
if len(faces) != 0:
face = faces[0]
mean_rec_time = (
mean_rec_time +
results[c.ELAPSED_CPU_TIME_KEY])
assigned_tag = face[c.ASSIGNED_TAG_KEY]
confidence = face[c.CONFIDENCE_KEY]
else:
face = cv2.imread(
image_complete_path, cv2.IMREAD_GRAYSCALE)
sz = None
use_resizing = ce.USE_RESIZING
use_eyes_position = c.USE_EYES_POSITION
use_eye_detection = ce.USE_EYE_DETECTION
offset_pct_x = c.OFFSET_PCT_X
offset_pct_y = c.OFFSET_PCT_Y
if params is not None:
if ce.USE_RESIZING_KEY in params:
use_resizing = params[ce.USE_RESIZING_KEY]
if c.USE_EYES_POSITION_KEY in params:
use_eyes_position = (
params[c.USE_EYES_POSITION_KEY])
if ce.USE_EYE_DETECTION_KEY in params:
use_eye_detection = (
params[ce.USE_EYE_DETECTION_KEY])
if c.OFFSET_PCT_X_KEY in params:
offset_pct_x = params[c.OFFSET_PCT_X_KEY]
if c.OFFSET_PCT_Y_KEY in params:
offset_pct_y = params[c.OFFSET_PCT_Y_KEY]
if use_resizing:
width = c.CROPPED_FACE_WIDTH
height = c.CROPPED_FACE_HEIGHT
if params is not None:
if c.CROPPED_FACE_WIDTH_KEY in params:
width = params[c.CROPPED_FACE_WIDTH_KEY]
if c.CROPPED_FACE_HEIGHT_KEY in params:
height = params[c.CROPPED_FACE_HEIGHT_KEY]
sz = (width, height)
if use_eyes_position:
align_path = c.ALIGNED_FACES_PATH
if params is not None:
align_path = params[c.ALIGNED_FACES_PATH_KEY]
if use_eye_detection:
face = fd.get_cropped_face(
image_complete_path, align_path, params,
return_always_face=False)
else:
face = fd.get_cropped_face_using_fixed_eye_pos(
image_complete_path, align_path,
offset_pct=(offset_pct_x, offset_pct_y),
dest_size=sz)
if face is not None:
face = face[c.FACE_KEY]
else:
if sz is not None:
face = cv2.resize(face, sz)
if face is not None:
rec_results = recognize_face(
face, fm, params, show_results)
assigned_tag = rec_results[c.ASSIGNED_TAG_KEY]
confidence = rec_results[c.CONFIDENCE_KEY]
# Add recognition time to total
mean_rec_time = (
mean_rec_time +
rec_results[c.ELAPSED_CPU_TIME_KEY])
image_dict = {ce.IMAGE_KEY: image,
c.ANN_TAG_KEY: images_dir,
c.ASSIGNED_TAG_KEY: assigned_tag,
c.CONFIDENCE_KEY: confidence}
if assigned_tag == ann_face_tag:
image_dict[ce.PERSON_CHECK_KEY] = 'TP'
people_true_positives_dict[assigned_tag] += 1
rec_images_nr += 1
true_pos_confidence_list.append(confidence)
person_rec_images += 1
else:
image_dict[ce.PERSON_CHECK_KEY] = 'FP'
if assigned_tag != 'Undefined':
people_false_positives_dict[assigned_tag] += 1
false_pos_confidence_list.append(confidence)
image_dict_extended = {ce.IMAGE_KEY: image_dict}
images_list_for_YAML.append(image_dict_extended)
except IOError, (errno, strerror):
print "I/O error({0}): {1}".format(errno, strerror)
except:
print "Unexpected error:", sys.exc_info()[0]
raise
def fr_video_experiments(params, show_results):
"""
Execute face recognition experiments on video files
:type params: dictionary
:param params: configuration parameters to be used for the experiment
:type show_results: boolean
:param show_results: show (True) or do not show (False)
image with detected faces
============================================ ======================================== ==============
Key Value Default value
============================================ ======================================== ==============
annotations_path Path of directory containing the
manual annotations for the images
check_eye_positions If True, check eye positions True
classifiers_dir_path Path of directory with OpenCV
cascade classifiers
eye_detection_classifier Classifier for eye detection 'haarcascade_mcs_lefteye.xml'
face_detection_algorithm Classifier for face detection 'HaarCascadeFrontalFaceAlt2'
('HaarCascadeFrontalFaceAlt',
'HaarCascadeFrontalFaceAltTree',
'HaarCascadeFrontalFaceAlt2',
'HaarCascadeFrontalFaceDefault',
'HaarCascadeProfileFace',
'HaarCascadeFrontalAndProfileFaces',
'HaarCascadeFrontalAndProfileFaces2',
'LBPCascadeFrontalface',
'LBPCascadeProfileFace' or
'LBPCascadeFrontalAndProfileFaces')
flags Flags used in face detection 'DoCannyPruning'
('DoCannyPruning', 'ScaleImage',
'FindBiggestObject', 'DoRoughSearch').
If 'DoCannyPruning' is used, regions
that do not contain lines are discarded.
If 'ScaleImage' is used, image instead
of the detector is scaled
(it can be advantegeous in terms of
memory and cache use).
If 'FindBiggestObject' is used,
only the biggest object is returned
by the detector.
'DoRoughSearch', used together with
'FindBiggestObject',
terminates the search as soon as
the first candidate object is found
min_neighbors Mininum number of neighbor bounding 5
boxes for retaining face detection
min_size_height Minimum height of face detection 20
bounding box (in pixels)
min_size_width Minimum width of face detection 20
bounding box (in pixels)
face_detection_results_path Path of directory where
test results will be saved
scale_factor Scale factor between two scans 1.1
in face detection
max_eye_angle Maximum inclination of the line 0.125
connecting the eyes
(in % of pi radians)
min_eye_distance Minimum distance between eyes 0.25
(in % of the width of the face
bounding box)
nose_detection_classifier Classifier for nose detection 'haarcascade_mcs_nose.xml'
software_test_file Path of image to be used for
software test
test_set_path path of directory
containing test set
use_nose_pos_in_detection If True, detections with no good False
nose position are discarded
aligned_faces_path Default path of directory
for aligned faces
cropped_face_height Height of aligned faces (in pixels) 400
cropped_face_width Width of aligned faces (in pixels) 200
dataset_already_divided If True, dataset is already divided False
between training and test set
dataset_path Path of whole dataset, used if dataset
is not already divided between
training and test set
db_name Name of single file
containing face models
db_models_path Path of directory containing face models
face_model_algorithm Algorithm for face recognition 'LBP'
('Eigenfaces', 'Fisherfaces' or 'LBP')
face_recognition_results_path Path of directory where
test results will be saved
test_set_path Path of directory containing
test set
training_set_path Path of directory containing
training set
LBP_grid_x Number of columns in grid 4
used for calculating LBP
LBP_grid_y Number of columns in grid 8
used for calculating LBP
LBP_neighbors Number of neighbors 8
used for calculating LBP
LBP_radius Radius used 1
for calculating LBP (in pixels)
max_frames_with_missed_detections Maximum number of frames with 5
missed detection that
does not interrupt tracking
offset_pct_x % of the image to keep next to 0.20
the eyes in the horizontal direction
offset_pct_y % of the image to keep next to 0.50
the eyes in the vertical direction
sim_tracking If True, results from all frames in False
video are aggregated
sliding_window_size Size of sliding window in seconds 5.0
software_test_file Path of image to be used for
software test
test_video_path Path of test video
training_images_nr Number of images per person used in
training set
use_captions If True, training set False
for face recognition is built
on the base of captions
use_eye_detection If True, use eye detection for detecting True
eye position for aligning faces in
test images
use_eye_detection_in_training If True, use eye detection for detecting True
eye position for aligning faces in
training images
use_eyes_position If True, align faces in test images True
by using eye positions
use_eyes_position_in_training If True, align faces in training images True
by using eye positions
use_face_detection_in_training If True, use face detection False
for images in training set
use_majority_rule If True, in aggregating results True
from several frames,
final tag is the tag that was
assigned to the majority of frames
use_mean_confidence_rule If True, in aggregating results False
from several frames,
final tag is the tag that received
the minimum value for the mean of
confidences among frames
use_min_confidence_rule If True, in aggregating results True
from several frames,
final tag is the tag that received
the minimum confidence value
use_NBNN If True, False
use Naive Bayes Nearest Neighbor
use_one_file_for_face_models If True, use one file for face models True
use_original_fps If True, original frame rate is used False
use_original_fps_in_training If True, use original frame rate False
of video when creating training set
use_resizing If True, resize images True
use_sliding_window If True, use sliding window False
in face extraction
use_tracking If True, False
use tracking in face extraction
use_weighted_regions If True, use weighted LBP False
used_fps Frame rate at which video 5.0
is analyzed (in frames per second)
used_fps_in_training Frame rate at which video is analyzed 1.0
in training from captions
(in frames per second)
============================================ ======================================== ==============
"""
# Folder with results
results_path = ce.FACE_RECOGNITION_RESULTS_PATH
sim_tracking = ce.SIM_TRACKING
# Folder with test files
test_set_path = ce.FACE_RECOGNITION_TEST_SET_PATH
use_captions = ce.USE_CAPTIONS
use_tracking = ce.USE_TRACKING
video_path = ce.TEST_VIDEO_PATH
if params is not None:
if ce.RESULTS_PATH_KEY in params:
results_path = params[ce.RESULTS_PATH_KEY]
if ce.SIM_TRACKING_KEY in params:
sim_tracking = params[ce.SIM_TRACKING_KEY]
if ce.TEST_SET_PATH_KEY in params:
test_set_path = params[ce.TEST_SET_PATH_KEY]
if ce.USE_CAPTIONS_KEY in params:
use_captions = params[ce.USE_CAPTIONS_KEY]
if use_captions:
if ce.TEST_VIDEO_PATH_KEY in params:
video_path = params[ce.TEST_VIDEO_PATH_KEY]
else:
video_path = None
if ce.USE_TRACKING_KEY in params:
use_tracking = params[ce.USE_TRACKING_KEY]
tot_rec_frames_nr = 0 # Number of correctly recognized frames
tot_test_frames_nr = 0 # Number of total test frames
mean_rec_time = 0 # Mean recognition time for videos
# List of confidence values for true positives
true_pos_confidence_list = []
# List of confidence values for false positives
false_pos_confidence_list = []
# List of tags of people that appear in test set
tested_people_tag_list = []
fm = FaceModels(params=params, video_path=video_path)
# Number of people
people_nr = fm.get_people_nr()
# Initialize dictionaries with people
people_true_positives_dict = {}
people_false_positives_dict = {}
people_test_frames_nr_dict = {}
tags = fm.get_tags()
for tag in tags:
people_true_positives_dict[tag] = 0
people_false_positives_dict[tag] = 0
people_test_frames_nr_dict[tag] = 0
# Iterate over all videos
experiment_dict_list = []
number_of_anal_video = 0
for video in os.listdir(test_set_path):
# Dictionary for YAML file with results
experiment_dict = {}
experiment_dict_frames = []
assigned_tag = 'Undefined'
final_confidence = -1
video_delta_xs = []
video_delta_ys = []
video_delta_ws = []
ann_face_tag, file_ext = os.path.splitext(video)
print('Annotated face tag: ', ann_face_tag)
tested_people_tag_list.append(ann_face_tag)
video_complete_path = test_set_path + video
print(video_complete_path)
try:
fe = FaceExtractor(fm)
handle = fe.extract_faces_from_video(video_complete_path)
results = fe.get_results(handle)
error = results[c.ERROR_KEY]
if error:
print('Warning')
print(error)
else:
video_test_frames_nr = results[ce.TOT_FRAMES_NR_KEY]
tot_test_frames_nr = tot_test_frames_nr + video_test_frames_nr
people_test_frames_nr_dict[ann_face_tag] = video_test_frames_nr
mean_rec_time = mean_rec_time + results[c.ELAPSED_CPU_TIME_KEY]
if use_tracking:
segments = results[c.SEGMENTS_KEY]
tot_segments_frames_nr = 0
true_positives_in_segment = 0
for segment in segments:
segment_frames = segment[c.FRAMES_KEY]
frames_nr = segment[c.SEGMENT_TOT_FRAMES_NR_KEY]
assigned_tag = segment[c.ASSIGNED_TAG_KEY]
print('assigned_tag = ' + assigned_tag)
final_confidence = segment[c.CONFIDENCE_KEY]
tot_segments_frames_nr = tot_segments_frames_nr + frames_nr
if assigned_tag == ann_face_tag:
people_true_positives_dict[assigned_tag] += frames_nr
true_positives_in_segment = true_positives_in_segment + frames_nr
tot_rec_frames_nr = tot_rec_frames_nr + frames_nr
true_pos_confidence_list.append(final_confidence)
else:
if assigned_tag != 'Undefined':
people_false_positives_dict[assigned_tag] = people_false_positives_dict[assigned_tag] + frames_nr
false_pos_confidence_list.append(final_confidence)
# Tot number of frames in segments
# cannot be more than number of frames in video
if true_positives_in_segment > video_test_frames_nr:
print('### WARNING! ###')
print 'true_positives_in_segment = ' + str(
true_positives_in_segment)
print 'video_test_frames_nr = ' + str(
video_test_frames_nr)
people_false_positives_dict[
ann_face_tag] += (tot_segments_frames_nr -
video_test_frames_nr)
tot_rec_frames_nr -= (tot_segments_frames_nr -
video_test_frames_nr)
elif sim_tracking:
# Simulate tracking
# (every frame of this video contains the same person)
frames = results[c.FRAMES_KEY]
[assigned_tag, final_confidence] = aggregate_frame_results_in_sim_tracking(frames, fm)
print('assigned_tag = ' + assigned_tag)
if assigned_tag == ann_face_tag:
people_true_positives_dict[assigned_tag] += len(frames)
tot_rec_frames_nr += len(frames)
true_pos_confidence_list.append(final_confidence)
else:
if assigned_tag != 'Undefined':
people_false_positives_dict[assigned_tag] += len(
frames)
false_pos_confidence_list.append(final_confidence)
else:
frames = results[c.FRAMES_KEY]
frame_counter = 0
prev_frame_counter = -1
for frame in frames:
assigned_tag = 'Undefined'
confidence = -1
faces = frame[c.FACES_KEY]
time_stamp = frame[c.ELAPSED_VIDEO_TIME_KEY]
experiment_dict_frame = {
c.ELAPSED_VIDEO_TIME_KEY: time_stamp}
if len(faces) != 0:
face = faces[0]
assigned_tag = face[c.ASSIGNED_TAG_KEY]
experiment_dict_faces = []
experiment_dict_face = {
c.ASSIGNED_TAG_KEY: assigned_tag}
experiment_dict_faces.append(experiment_dict_face)
if len(faces) > 1:
for face_counter in range(1, len(faces)):
other_face = faces[face_counter]
other_face_assigned_tag = other_face[c.ASSIGNED_TAG_KEY]
experiment_dict_face = {
c.ASSIGNED_TAG_KEY: other_face_assigned_tag}
experiment_dict_faces.append(experiment_dict_face)
confidence = face[c.CONFIDENCE_KEY]
bbox = face[c.BBOX_KEY]
if((frame_counter > 0) and
(frame_counter <= (prev_frame_counter + c.MAX_FR_WITH_MISSED_DET + 1))):
bbox_x = bbox[0]
bbox_y = bbox[1]
bbox_w = bbox[2]
prev_bbox_x = prev_bbox[0]
prev_bbox_y = prev_bbox[1]
prev_bbox_w = prev_bbox[2]
delta_x = (abs(bbox_x - prev_bbox_x) /
float(prev_bbox_w))
delta_y = (abs(bbox_x - prev_bbox_x) /
float(prev_bbox_w))
delta_w = (abs(bbox_w - prev_bbox_w) /
float(prev_bbox_w))
video_delta_xs.append(delta_x)
video_delta_ys.append(delta_y)
video_delta_ws.append(delta_w)
if((delta_x > ce.MAX_DELTA_PCT_X) or
(delta_y > ce.MAX_DELTA_PCT_Y) and
(delta_w > ce.MAX_DELTA_PCT_W)):
print('delta_x: ', delta_x)
print('delta_y: ', delta_y)
print('delta_w: ', delta_w)
prev_frame_counter = frame_counter
prev_bbox = bbox
experiment_dict_frame[c.FACES_KEY] = experiment_dict_faces
experiment_dict_frame[c.ASSIGNED_TAG_KEY] = assigned_tag
else:
experiment_dict_frame[c.FACES_KEY] = "No face detected"
experiment_dict_frame[c.ASSIGNED_TAG_KEY] = "No face detected"
experiment_dict_frame[c.CONFIDENCE_KEY] = confidence
experiment_dict_frames.append(experiment_dict_frame)
if assigned_tag == ann_face_tag:
people_true_positives_dict[assigned_tag] += 1
tot_rec_frames_nr += 1
true_pos_confidence_list.append(confidence)
else:
if assigned_tag != 'Undefined':
people_false_positives_dict[assigned_tag] += 1
false_pos_confidence_list.append(confidence)
frame_counter += 1
experiment_dict[ce.VIDEO_COUNTER_KEY] = number_of_anal_video
experiment_dict[c.ANN_TAG_KEY] = ann_face_tag
if sim_tracking:
experiment_dict[c.ASSIGNED_TAG_KEY] = assigned_tag
experiment_dict[c.CONFIDENCE_KEY] = final_confidence
else:
experiment_dict[c.FRAMES_KEY] = experiment_dict_frames
save_YAML_file(
results_path + ann_face_tag + ".yml", experiment_dict)
experiment_dict_list.append(experiment_dict)
number_of_anal_video += 1
except IOError, (errno, strerror):
print "I/O error({0}): {1}".format(errno, strerror)
except: