def verify(img1_path, img2_path): backend = 'opencv' img1 = functions.preprocess_face(img1_path, target_size=(h, w), detector_backend=backend) img2 = functions.preprocess_face(img2_path, target_size=(h, w), detector_backend=backend) #----------------------------------------------------- img1_embedding = model.predict(img1)[0] img2_embedding = model.predict(img2)[0] #----------------------------------------------------- #we might need to change this logic: http://cseweb.ucsd.edu/~mkchandraker/classes/CSE252C/Spring2020/Homeworks/hw2-CSE252C-Sol.html #print(np.argmax(img1_embedding), np.argmax(img2_embedding)) return (dst.findEuclideanDistance(img1_embedding, img2_embedding), dst.findEuclideanDistance(dst.l2_normalize(img1_embedding), dst.l2_normalize(img2_embedding)), dst.findCosineDistance(img1_embedding, img2_embedding))
def get_embedding(self, image): """Takes an image with only the desired face""" image = self.get_image_from_url(image.url) preprocessed_face = functions.preprocess_face( image, target_size=self.target_size, detector_backend="mtcnn" ) return self.model.predict(preprocessed_face)[0]
def verify_face(key): embedding = redis.lrange('embedding:'+key, 0, -1) #print(embedding) #print(np.array(embedding).astype('float')) distance = findEuclideanDistance(target_embedding, np.array(embedding).astype('float')) print("Distance is ",distance) img_name = redis.get('photo:'+key).decode() source_img = functions.preprocess_face(img_name) #------------------------------------ fig = plt.figure(figsize = (7, 7)) ax1 = fig.add_subplot(1,2,1) plt.imshow(target_img[0][:, :, ::-1]) plt.axis('off') ax2 = fig.add_subplot(1,2,2) plt.imshow(source_img[0][:, :, ::-1]) plt.axis('off') plt.show() #------------------------------------ if distance <= 10: print("this is "+key) else: print("this IS NOT "+key)
def detectFace(img_path, detector_backend='mtcnn'): functions.initialize_detector(detector_backend=detector_backend) img = functions.preprocess_face( img=img_path, detector_backend=detector_backend)[ 0] #preprocess_face returns (1, 224, 224, 3) return img[:, :, ::-1] #bgr to rgb
def main(): """ Our App """ st.title("Find what indian actor/actress you look like") st.text( "Built with streamlit and deepface, please wait for 20 seconds for the result" ) st.set_option('deprecation.showfileUploaderEncoding', False) image_file = st.file_uploader("Upload Image", type=["jpg", "png", "jpeg", "webp"]) if image_file is not None: st.write("Upload Successful") image = Image.open(image_file) open_cv_test_image = np.array(image) open_cv_test_image = open_cv_test_image[:, :, ::-1].copy( ) # Convert RGB to BGR input_shape = model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] demography = DeepFace.analyze(open_cv_test_image, actions=["gender"]) if demography["gender"] == "Man": entries = actorso else: entries = actress test_preds = model.predict( functions.preprocess_face(img=open_cv_test_image, target_size=input_shape, enforce_detection=False, detector_backend='ssd'))[0, :] least_diff = dst.findEuclideanDistance(dst.l2_normalize(entries[0][1]), dst.l2_normalize(test_preds)) least_img = entries[0][0] print(least_img) for rep in entries: diff = dst.findEuclideanDistance(dst.l2_normalize(rep[1]), dst.l2_normalize(test_preds)) print(rep[0]) if (diff < least_diff): least_diff = diff least_img = rep[0] st.write("You look like ", least_img[8:][:-4]) pil_result_im = Image.open(least_img) st.image(pil_result_im, width=140)
def represent(img_path, model_name='VGG-Face', model=None, enforce_detection=True, detector_backend='opencv', align=True, normalization='base'): """ This function represents facial images as vectors. Parameters: img_path: exact image path, numpy array (BGR) or based64 encoded images could be passed. model_name (string): VGG-Face, Facenet, OpenFace, DeepFace, DeepID, Dlib, ArcFace. model: Built deepface model. A face recognition model is built every call of verify function. You can pass pre-built face recognition model optionally if you will call verify function several times. Consider to pass model if you are going to call represent function in a for loop. model = DeepFace.build_model('VGG-Face') enforce_detection (boolean): If any face could not be detected in an image, then verify function will return exception. Set this to False not to have this exception. This might be convenient for low resolution images. detector_backend (string): set face detector backend as retinaface, mtcnn, opencv, ssd or dlib normalization (string): normalize the input image before feeding to model Returns: Represent function returns a multidimensional vector. The number of dimensions is changing based on the reference model. E.g. FaceNet returns 128 dimensional vector; VGG-Face returns 2622 dimensional vector. """ if model is None: model = build_model(model_name) #--------------------------------- #decide input shape input_shape_x, input_shape_y = functions.find_input_shape(model) #detect and align img = functions.preprocess_face(img=img_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend, align=align) #--------------------------------- #custom normalization img = functions.normalize_input(img=img, normalization=normalization) #--------------------------------- #represent embedding = model.predict(img)[0].tolist() return embedding
def embedded(db_path, model_name, model, input_shape=(224, 224)): file_name = "representations_%s.pkl" % (model_name) ## input_shape = (224, 224) input_shape_x = input_shape[0] input_shape_y = input_shape[1] text_color = (255, 255, 255) employees = [] #check passed db folder exists if os.path.isdir(db_path) == True: for r, d, f in os.walk(db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file): #exact_path = os.path.join(r, file) exact_path = r + "/" + file #print(exact_path) employees.append(exact_path) if len(employees) == 0: print("WARNING: There is no image in this path ( ", db_path, ") . Face recognition will not be performed.") tic = time.time() pbar = tqdm(range(0, len(employees)), desc='Finding embeddings') embeddings = [] #for employee in employees: for index in pbar: employee = employees[index] pbar.set_description("Finding embedding for %s" % (employee.split("/")[-1])) embedding = [] img = functions.preprocess_face(img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=False) img_representation = model.predict(img)[0, :] embedding.append(employee) embedding.append(img_representation) embeddings.append(embedding) f = open(db_path + '/' + file_name, "wb") pickle.dump(embeddings, f) f.close() toc = time.time() print("Embeddings found for given data set in ", toc - tic, " seconds")
def detectFace(img_path, detector_backend = 'opencv', enforce_detection = True): """ This function applies pre-processing stages of a face recognition pipeline including detection and alignment Parameters: img_path: exact image path, numpy array or base64 encoded image detector_backend (string): face detection backends are retinaface, mtcnn, opencv, ssd or dlib Returns: deteced and aligned face in numpy format """ img = functions.preprocess_face(img = img_path, detector_backend = detector_backend , enforce_detection = enforce_detection)[0] #preprocess_face returns (1, 224, 224, 3) return img[:, :, ::-1] #bgr to rgb
def get_features_vec(pic) -> np.ndarray: """ :argument pic: input PIL.Image.open object :return features_vec: features vec np array """ pic = np.array(pic) model = Facenet.loadModel() input_shape = functions.find_input_shape(model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] img = functions.preprocess_face(pic, target_size=(input_shape_y, input_shape_x), enforce_detection=True, detector_backend='mtcnn') features_vec = model.predict(img)[0, :] return features_vec
def detectFace(img_path, detector_backend='mtcnn'): """ This function applies pre-processing stages of a face recognition pipeline including detection and alignment Parameters: img_path: exact image path, numpy array or base64 encoded image detector_backend (string): face detection backends are mtcnn, opencv, ssd or dlib Returns: deteced and aligned face in numpy format """ functions.initialize_detector(detector_backend=detector_backend) imgs = functions.preprocess_face( img=img_path, detector_backend=detector_backend ) #preprocess_face returns (1, 224, 224, 3) return imgs #bgr to rgb
print("VGG loaded") model["Facenet"] = Facenet.loadModel() print("Facenet loaded") model["OpenFace"] = OpenFace.loadModel() print("OpenFace loaded") model["DeepFace"] = FbDeepFace.loadModel() print("DeepFace loaded") df = DeepFace.find("dataset/img1.jpg", db_path = "dataset", model_name = 'Ensemble', model=model, enforce_detection=False) print(df) #----------------------------------- print("--------------------------") print("Different face detector backends") backends = ['opencv', 'ssd', 'dlib', 'mtcnn'] for backend in backends: tic = time.time() processed_img = functions.preprocess_face(img = "dataset/img11.jpg", detector_backend = backend) toc = time.time() print("Backend ", backend, " is done in ", toc-tic," seconds") #----------------------------------- print("--------------------------")
def find(img_path, db_path, model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='mtcnn'): tic = time.time() img_paths, bulkProcess = initialize_input(img_path) functions.initialize_detector(detector_backend=detector_backend) #------------------------------- #model metric pairs for ensemble model_names = ['VGG-Face', 'Facenet', 'OpenFace', 'DeepFace'] metric_names = ['cosine', 'euclidean', 'euclidean_l2'] #------------------------------- if os.path.isdir(db_path) == True: #--------------------------------------- if model == None: if model_name == 'Ensemble': print("Ensemble learning enabled") #TODO: include DeepID in ensemble method import lightgbm as lgb #lightgbm==2.3.1 models = {} pbar = tqdm(range(0, len(model_names)), desc='Face recognition models') for index in pbar: if index == 0: pbar.set_description("Loading VGG-Face") models['VGG-Face'] = build_model('VGG-Face') elif index == 1: pbar.set_description("Loading FaceNet") models['Facenet'] = build_model('Facenet') elif index == 2: pbar.set_description("Loading OpenFace") models['OpenFace'] = build_model('OpenFace') elif index == 3: pbar.set_description("Loading DeepFace") models['DeepFace'] = build_model('DeepFace') else: #model is not ensemble model = build_model(model_name) else: #model != None print("Already built model is passed") if model_name == 'Ensemble': import lightgbm as lgb #lightgbm==2.3.1 #validate model dictionary because it might be passed from input as pre-trained found_models = [] for key, value in model.items(): found_models.append(key) if ('VGG-Face' in found_models) and ('Facenet' in found_models) and ( 'OpenFace' in found_models) and ('DeepFace' in found_models): print("Ensemble learning will be applied for ", found_models, " models") else: raise ValueError( "You would like to apply ensemble learning and pass pre-built models but models must contain [VGG-Face, Facenet, OpenFace, DeepFace] but you passed " + found_models) models = model.copy() #threshold = functions.findThreshold(model_name, distance_metric) #--------------------------------------- file_name = "representations_%s.pkl" % (model_name) file_name = file_name.replace("-", "_").lower() if path.exists(db_path + "/" + file_name): print( "WARNING: Representations for images in ", db_path, " folder were previously stored in ", file_name, ". If you added new instances after this file creation, then please delete this file and call find function again. It will create it again." ) f = open(db_path + '/' + file_name, 'rb') representations = pickle.load(f) print("There are ", len(representations), " representations found in ", file_name) else: employees = [] for r, d, f in os.walk( db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file): exact_path = r + "/" + file employees.append(exact_path) if len(employees) == 0: raise ValueError("There is no image in ", db_path, " folder!") #------------------------ #find representations for db images representations = [] pbar = tqdm(range(0, len(employees)), desc='Finding representations') #for employee in employees: for index in pbar: employee = employees[index] if model_name != 'Ensemble': #input_shape = model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] input_shape_x = input_shape[0] input_shape_y = input_shape[1] img = functions.preprocess_face( img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) representation = model.predict(img)[0, :] instance = [] instance.append(employee) instance.append(representation) else: #ensemble learning instance = [] instance.append(employee) for j in model_names: ensemble_model = models[j] #input_shape = model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = ensemble_model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] input_shape_x = input_shape[0] input_shape_y = input_shape[1] img = functions.preprocess_face( img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) representation = ensemble_model.predict(img)[0, :] instance.append(representation) #------------------------------- representations.append(instance) f = open(db_path + '/' + file_name, "wb") pickle.dump(representations, f) f.close() print( "Representations stored in ", db_path, "/", file_name, " file. Please delete this file when you add new identities in your database." ) #---------------------------- #we got representations for database if model_name != 'Ensemble': df = pd.DataFrame(representations, columns=["identity", "representation"]) else: #ensemble learning df = pd.DataFrame(representations, columns=[ "identity", "VGG-Face_representation", "Facenet_representation", "OpenFace_representation", "DeepFace_representation" ]) df_base = df.copy() resp_obj = [] global_pbar = tqdm(range(0, len(img_paths)), desc='Analyzing') for j in global_pbar: img_path = img_paths[j] #find representation for passed image if model_name == 'Ensemble': for j in model_names: ensemble_model = models[j] #input_shape = ensemble_model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = ensemble_model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] img = functions.preprocess_face( img=img_path, target_size=input_shape, enforce_detection=enforce_detection, detector_backend=detector_backend) target_representation = ensemble_model.predict(img)[0, :] for k in metric_names: distances = [] for index, instance in df.iterrows(): source_representation = instance[ "%s_representation" % (j)] if k == 'cosine': distance = dst.findCosineDistance( source_representation, target_representation) elif k == 'euclidean': distance = dst.findEuclideanDistance( source_representation, target_representation) elif k == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(source_representation), dst.l2_normalize(target_representation)) distances.append(distance) if j == 'OpenFace' and k == 'euclidean': continue else: df["%s_%s" % (j, k)] = distances #---------------------------------- feature_names = [] for j in model_names: for k in metric_names: if j == 'OpenFace' and k == 'euclidean': continue else: feature = '%s_%s' % (j, k) feature_names.append(feature) #print(df[feature_names].head()) x = df[feature_names].values #---------------------------------- #lightgbm model home = str(Path.home()) if os.path.isfile( home + '/.deepface/weights/face-recognition-ensemble-model.txt' ) != True: print( "face-recognition-ensemble-model.txt will be downloaded..." ) url = 'https://raw.githubusercontent.com/serengil/deepface/master/deepface/models/face-recognition-ensemble-model.txt' output = home + '/.deepface/weights/face-recognition-ensemble-model.txt' gdown.download(url, output, quiet=False) ensemble_model_path = home + '/.deepface/weights/face-recognition-ensemble-model.txt' deepface_ensemble = lgb.Booster(model_file=ensemble_model_path) y = deepface_ensemble.predict(x) verified_labels = [] scores = [] for i in y: verified = np.argmax(i) == 1 score = i[np.argmax(i)] verified_labels.append(verified) scores.append(score) df['verified'] = verified_labels df['score'] = scores df = df[df.verified == True] #df = df[df.score > 0.99] #confidence score df = df.sort_values(by=["score"], ascending=False).reset_index(drop=True) df = df[['identity', 'verified', 'score']] resp_obj.append(df) df = df_base.copy() #restore df for the next iteration #---------------------------------- if model_name != 'Ensemble': #input_shape = model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] input_shape_x = input_shape[0] input_shape_y = input_shape[1] #------------------------ img = functions.preprocess_face( img=img_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) target_representation = model.predict(img)[0, :] distances = [] for index, instance in df.iterrows(): source_representation = instance["representation"] if distance_metric == 'cosine': distance = dst.findCosineDistance( source_representation, target_representation) elif distance_metric == 'euclidean': distance = dst.findEuclideanDistance( source_representation, target_representation) elif distance_metric == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(source_representation), dst.l2_normalize(target_representation)) else: raise ValueError("Invalid distance_metric passed - ", distance_metric) distances.append(distance) threshold = functions.findThreshold(model_name, distance_metric) df["distance"] = distances df = df.drop(columns=["representation"]) df = df[df.distance <= threshold] df = df.sort_values(by=["distance"], ascending=True).reset_index(drop=True) resp_obj.append(df) df = df_base.copy() #restore df for the next iteration toc = time.time() print("find function lasts ", toc - tic, " seconds") if len(resp_obj) == 1: return resp_obj[0] return resp_obj else: raise ValueError("Passed db_path does not exist!") return None
def verify(img1_path, img2_path='', model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='mtcnn'): tic = time.time() img_list, bulkProcess = initialize_input(img1_path, img2_path) functions.initialize_detector(detector_backend=detector_backend) resp_objects = [] if model_name == 'Ensemble': print("Ensemble learning enabled") import lightgbm as lgb #lightgbm==2.3.1 if model == None: model = {} model_pbar = tqdm(range(0, 4), desc='Face recognition models') for index in model_pbar: if index == 0: model_pbar.set_description("Loading VGG-Face") model["VGG-Face"] = build_model('VGG-Face') elif index == 1: model_pbar.set_description("Loading Google FaceNet") model["Facenet"] = build_model('Facenet') elif index == 2: model_pbar.set_description("Loading OpenFace") model["OpenFace"] = build_model('OpenFace') elif index == 3: model_pbar.set_description("Loading Facebook DeepFace") model["DeepFace"] = build_model('DeepFace') #-------------------------- #validate model dictionary because it might be passed from input as pre-trained found_models = [] for key, value in model.items(): found_models.append(key) if ('VGG-Face' in found_models) and ('Facenet' in found_models) and ( 'OpenFace' in found_models) and ('DeepFace' in found_models): print("Ensemble learning will be applied for ", found_models, " models") else: raise ValueError( "You would like to apply ensemble learning and pass pre-built models but models must contain [VGG-Face, Facenet, OpenFace, DeepFace] but you passed " + found_models) #-------------------------- model_names = ["VGG-Face", "Facenet", "OpenFace", "DeepFace"] metrics = ["cosine", "euclidean", "euclidean_l2"] pbar = tqdm(range(0, len(img_list)), desc='Verification') #for instance in img_list: for index in pbar: instance = img_list[index] if type(instance) == list and len(instance) >= 2: img1_path = instance[0] img2_path = instance[1] ensemble_features = [] ensemble_features_string = "[" for i in model_names: custom_model = model[i] #input_shape = custom_model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = custom_model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] img1 = functions.preprocess_face( img=img1_path, target_size=input_shape, enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=input_shape, enforce_detection=enforce_detection, detector_backend=detector_backend) img1_representation = custom_model.predict(img1)[0, :] img2_representation = custom_model.predict(img2)[0, :] for j in metrics: if j == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif j == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif j == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) if i == 'OpenFace' and j == 'euclidean': #this returns same with OpenFace - euclidean_l2 continue else: ensemble_features.append(distance) if len(ensemble_features) > 1: ensemble_features_string += ", " ensemble_features_string += str(distance) #print("ensemble_features: ", ensemble_features) ensemble_features_string += "]" #------------------------------- #find deepface path home = str(Path.home()) if os.path.isfile( home + '/.deepface/weights/face-recognition-ensemble-model.txt' ) != True: print( "face-recognition-ensemble-model.txt will be downloaded..." ) url = 'https://raw.githubusercontent.com/serengil/deepface/master/deepface/models/face-recognition-ensemble-model.txt' output = home + '/.deepface/weights/face-recognition-ensemble-model.txt' gdown.download(url, output, quiet=False) ensemble_model_path = home + '/.deepface/weights/face-recognition-ensemble-model.txt' #print(ensemble_model_path) #------------------------------- deepface_ensemble = lgb.Booster(model_file=ensemble_model_path) prediction = deepface_ensemble.predict( np.expand_dims(np.array(ensemble_features), axis=0))[0] verified = np.argmax(prediction) == 1 score = prediction[np.argmax(prediction)] #print("verified: ", verified,", score: ", score) resp_obj = { "verified": verified, "score": score, "distance": ensemble_features_string, "model": ["VGG-Face", "Facenet", "OpenFace", "DeepFace"], "similarity_metric": ["cosine", "euclidean", "euclidean_l2"] } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj #------------------------------- if bulkProcess == True: resp_obj = {} for i in range(0, len(resp_objects)): resp_item = resp_objects[i] resp_obj["pair_%d" % (i + 1)] = resp_item return resp_obj return None #ensemble learning block end #-------------------------------- #ensemble learning disabled if model == None: model = build_model(model_name) """else: #model != None print("Already built model is passed")""" #------------------------------ #face recognition models have different size of inputs #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] input_shape_x = input_shape[0] input_shape_y = input_shape[1] #------------------------------ #tuned thresholds for model and metric pair threshold = functions.findThreshold(model_name, distance_metric) #------------------------------ #calling deepface in a for loop causes lots of progress bars. this prevents it. disable_option = False if len(img_list) > 1 else True pbar = tqdm(range(0, len(img_list)), desc='Verification', disable=disable_option) #for instance in img_list: for index in pbar: instance = img_list[index] if type(instance) == list and len(instance) >= 2: img1_path = instance[0] img2_path = instance[1] #---------------------- #crop and align faces img1 = functions.preprocess_face( img=img1_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) #---------------------- #find embeddings img1_representation = model.predict(img1)[0, :] img2_representation = model.predict(img2)[0, :] #---------------------- #find distances between embeddings if distance_metric == 'cosine': distance = dst.findCosineDistance(img1_representation, img2_representation) elif distance_metric == 'euclidean': distance = dst.findEuclideanDistance(img1_representation, img2_representation) elif distance_metric == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) else: raise ValueError("Invalid distance_metric passed - ", distance_metric) #---------------------- #decision if distance <= threshold: identified = True else: identified = False #---------------------- #response object resp_obj = { "verified": identified, "distance": distance, "max_threshold_to_verify": threshold, "model": model_name, "similarity_metric": distance_metric } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj #---------------------- else: raise ValueError("Invalid arguments passed to verify function: ", instance) #------------------------- toc = time.time() #print("identification lasts ",toc-tic," seconds") if bulkProcess == True: resp_obj = "{" for i in range(0, len(resp_objects)): resp_item = json.dumps(resp_objects[i]) if i > 0: resp_obj += ", " resp_obj += "\"pair_" + str(i + 1) + "\": " + resp_item resp_obj += "}" resp_obj = json.loads(resp_obj) return resp_obj
def analyze(img_path, actions=[], models={}, enforce_detection=True, detector_backend='mtcnn'): img_paths, bulkProcess = initialize_input(img_path) functions.initialize_detector(detector_backend=detector_backend) #--------------------------------- #if a specific target is not passed, then find them all if len(actions) == 0: actions = ['emotion', 'age', 'gender', 'race'] #print("Actions to do: ", actions) #--------------------------------- if 'emotion' in actions: if 'emotion' in models: print("already built emotion model is passed") emotion_model = models['emotion'] else: emotion_model = build_model('Emotion') if 'age' in actions: if 'age' in models: #print("already built age model is passed") age_model = models['age'] else: age_model = build_model('Age') if 'gender' in actions: if 'gender' in models: print("already built gender model is passed") gender_model = models['gender'] else: gender_model = build_model('Gender') if 'race' in actions: if 'race' in models: print("already built race model is passed") race_model = models['race'] else: race_model = build_model('Race') #--------------------------------- resp_objects = [] disable_option = False if len(img_paths) > 1 else True global_pbar = tqdm(range(0, len(img_paths)), desc='Analyzing', disable=disable_option) #for img_path in img_paths: for j in global_pbar: img_path = img_paths[j] resp_obj = {} disable_option = False if len(actions) > 1 else True pbar = tqdm(range(0, len(actions)), desc='Finding actions', disable=disable_option) action_idx = 0 img_224 = None # Set to prevent re-detection #for action in actions: for index in pbar: action = actions[index] pbar.set_description("Action: %s" % (action)) if action == 'emotion': emotion_labels = [ 'angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral' ] img = functions.preprocess_face( img=img_path, target_size=(48, 48), grayscale=True, enforce_detection=enforce_detection, detector_backend=detector_backend) emotion_predictions = emotion_model.predict(img)[0, :] sum_of_predictions = emotion_predictions.sum() resp_obj["emotion"] = {} for i in range(0, len(emotion_labels)): emotion_label = emotion_labels[i] emotion_prediction = 100 * emotion_predictions[ i] / sum_of_predictions resp_obj["emotion"][emotion_label] = emotion_prediction resp_obj["dominant_emotion"] = emotion_labels[np.argmax( emotion_predictions)] elif action == 'age': if img_224 is None: img_224 = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend ) #just emotion model expects grayscale images #print("age prediction") age_predictions = age_model.predict(img_224)[0, :] apparent_age = Age.findApparentAge(age_predictions) resp_obj["age"] = str(int(apparent_age)) elif action == 'gender': if img_224 is None: img_224 = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend ) #just emotion model expects grayscale images #print("gender prediction") gender_prediction = gender_model.predict(img_224)[0, :] if np.argmax(gender_prediction) == 0: gender = "Woman" elif np.argmax(gender_prediction) == 1: gender = "Man" resp_obj["gender"] = gender elif action == 'race': if img_224 is None: img_224 = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend ) #just emotion model expects grayscale images race_predictions = race_model.predict(img_224)[0, :] race_labels = [ 'asian', 'indian', 'black', 'white', 'middle eastern', 'latino hispanic' ] sum_of_predictions = race_predictions.sum() resp_obj["race"] = {} for i in range(0, len(race_labels)): race_label = race_labels[i] race_prediction = 100 * race_predictions[ i] / sum_of_predictions resp_obj["race"][race_label] = race_prediction resp_obj["dominant_race"] = race_labels[np.argmax( race_predictions)] action_idx = action_idx + 1 if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj if bulkProcess == True: resp_obj = {} for i in range(0, len(resp_objects)): resp_item = resp_objects[i] resp_obj["instance_%d" % (i + 1)] = resp_item return resp_obj
def verify(img1_path, img2_path='', model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='mtcnn'): tic = time.time() img_list, bulkProcess = initialize_input(img1_path, img2_path) functions.initialize_detector(detector_backend=detector_backend) resp_objects = [] #-------------------------------- if model_name == 'Ensemble': model_names = ["VGG-Face", "Facenet", "OpenFace", "DeepFace"] metrics = ["cosine", "euclidean", "euclidean_l2"] else: model_names = [] metrics = [] model_names.append(model_name) metrics.append(distance_metric) #-------------------------------- if model == None: if model_name == 'Ensemble': models = Boosting.loadModel() else: model = build_model(model_name) models = {} models[model_name] = model else: if model_name == 'Ensemble': Boosting.validate_model(model) else: models = {} models[model_name] = model #------------------------------ #calling deepface in a for loop causes lots of progress bars. this prevents it. disable_option = False if len(img_list) > 1 else True pbar = tqdm(range(0, len(img_list)), desc='Verification', disable=disable_option) for index in pbar: instance = img_list[index] if type(instance) == list and len(instance) >= 2: img1_path = instance[0] img2_path = instance[1] ensemble_features = [] for i in model_names: custom_model = models[i] #decide input shape input_shape = functions.find_input_shape(custom_model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #---------------------- #detect and align faces img1 = functions.preprocess_face( img=img1_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) #---------------------- #find embeddings img1_representation = custom_model.predict(img1)[0, :] img2_representation = custom_model.predict(img2)[0, :] #---------------------- #find distances between embeddings for j in metrics: if j == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif j == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif j == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) else: raise ValueError("Invalid distance_metric passed - ", distance_metric) #---------------------- #decision if model_name != 'Ensemble': threshold = dst.findThreshold(i, j) if distance <= threshold: identified = True else: identified = False resp_obj = { "verified": identified, "distance": distance, "max_threshold_to_verify": threshold, "model": model_name, "similarity_metric": distance_metric } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj else: #Ensemble #this returns same with OpenFace - euclidean_l2 if i == 'OpenFace' and j == 'euclidean': continue else: ensemble_features.append(distance) #---------------------- if model_name == 'Ensemble': boosted_tree = Boosting.build_gbm() prediction = boosted_tree.predict( np.expand_dims(np.array(ensemble_features), axis=0))[0] verified = np.argmax(prediction) == 1 score = prediction[np.argmax(prediction)] resp_obj = { "verified": verified, "score": score, "distance": ensemble_features, "model": ["VGG-Face", "Facenet", "OpenFace", "DeepFace"], "similarity_metric": ["cosine", "euclidean", "euclidean_l2"] } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj #---------------------- else: raise ValueError("Invalid arguments passed to verify function: ", instance) #------------------------- toc = time.time() if bulkProcess == True: resp_obj = {} for i in range(0, len(resp_objects)): resp_item = resp_objects[i] resp_obj["pair_%d" % (i + 1)] = resp_item return resp_obj
model = VGGFace.loadModel() #model = Facenet.loadModel() #model = OpenFace.loadModel() #model = FbDeepFace.loadModel() input_shape = model.layers[0].input_shape[1:3] print("model input shape: ", model.layers[0].input_shape[1:]) print("model output shape: ", model.layers[-1].input_shape[-1]) #---------------------------------------------- #load images and find embeddings #img1 = functions.detectFace("dataset/img1.jpg", input_shape) img1 = functions.preprocess_face("dataset/img1.jpg", input_shape) img1_representation = model.predict(img1)[0, :] #img2 = functions.detectFace("dataset/img3.jpg", input_shape) img2 = functions.preprocess_face("dataset/img3.jpg", input_shape) img2_representation = model.predict(img2)[0, :] #---------------------------------------------- #distance between two images distance_vector = np.square(img1_representation - img2_representation) #print(distance_vector) distance = np.sqrt(distance_vector.sum()) print("Euclidean distance: ", distance)
def verify(img1_path, img2_path='', model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='opencv'): tic = time.time() if isinstance(img1_path, list): bulkProcess = True img_list = img1_path.copy() else: bulkProcess = False img_list = [[img1_path, img2_path]] #------------------------------ if detector_backend == 'mtcnn': functions.load_mtcnn() resp_objects = [] if model_name == 'Ensemble': print("Ensemble learning enabled") import lightgbm as lgb if model is None: model = {} model_pbar = tqdm(range(0, 4), desc='Face recognition models') for index in model_pbar: if index == 0: model_pbar.set_description("Loading VGG-Face") model["VGG-Face"] = models_loader["VGG-Face"]() elif index == 1: model_pbar.set_description("Loading Google FaceNet") model["Facenet"] = models_loader["Facenet"]() elif index == 2: model_pbar.set_description("Loading OpenFace") model["OpenFace"] = models_loader["OpenFace"]() elif index == 3: model_pbar.set_description("Loading Facebook DeepFace") model["DeepFace"] = models_loader["DeepFace"]() #-------------------------- #validate model dictionary because it might be passed from input as pre-trained found_models = [] for key, value in model.items(): found_models.append(key) if ('VGG-Face' in found_models) and ('Facenet' in found_models) and ( 'OpenFace' in found_models) and ('DeepFace' in found_models): print("Ensemble learning will be applied for ", found_models, " models") else: raise ValueError( "You would like to apply ensemble learning and pass pre-built models but models must contain [VGG-Face, Facenet, OpenFace, DeepFace] but you passed " + found_models) #-------------------------- model_names = ["VGG-Face", "Facenet", "OpenFace", "DeepFace"] metrics = ["cosine", "euclidean", "euclidean_l2"] pbar = tqdm(range(0, len(img_list)), desc='Verification') #for instance in img_list: for index in pbar: instance = img_list[index] if type(instance) == list and len(instance) >= 2: img1_path = instance[0] img2_path = instance[1] ensemble_features = [] ensemble_features_string = "[" for i in model_names: custom_model = model[i] #input_shape = custom_model.layers[0].input_shape[1:3] #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. input_shape = custom_model.layers[0].input_shape if type(input_shape) == list: input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] img1 = functions.preprocess_face( img=img1_path, target_size=input_shape, enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=input_shape, enforce_detection=enforce_detection, detector_backend=detector_backend) both_imgs = np.vstack([img1, img2]) img1_representation, img2_representation = custom_model.predict( both_imgs) for j in metrics: if j == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif j == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif j == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) else: raise ValueError( 'Only cosine, euclidean and euclidean_l2 are supported as a metric but {} was passed' .format(j)) if i == 'OpenFace' and j == 'euclidean': #this returns same with OpenFace - euclidean_l2 continue else: ensemble_features.append(distance) if len(ensemble_features) > 1: ensemble_features_string += ", " ensemble_features_string += str(distance) #print("ensemble_features: ", ensemble_features) ensemble_features_string += "]" #------------------------------- #find deepface path home = str(Path.home()) if os.path.isfile( home + '/.deepface/weights/face-recognition-ensemble-model.txt' ) != True: print( "face-recognition-ensemble-model.txt will be downloaded..." ) url = 'https://raw.githubusercontent.com/serengil/deepface/master/deepface/models/face-recognition-ensemble-model.txt' output = home + '/.deepface/weights/face-recognition-ensemble-model.txt' gdown.download(url, output, quiet=False) ensemble_model_path = home + '/.deepface/weights/face-recognition-ensemble-model.txt' #print(ensemble_model_path) #------------------------------- deepface_ensemble = lgb.Booster(model_file=ensemble_model_path) prediction = deepface_ensemble.predict( np.expand_dims(np.array(ensemble_features), axis=0))[0] verified = np.argmax(prediction) == 1 if verified: identified = "true" else: identified = "false" score = prediction[np.argmax(prediction)] #print("verified: ", verified,", score: ", score) resp_obj = "{" resp_obj += "\"verified\": " + identified resp_obj += ", \"score\": " + str(score) resp_obj += ", \"distance\": " + ensemble_features_string resp_obj += ", \"model\": [\"VGG-Face\", \"Facenet\", \"OpenFace\", \"DeepFace\"]" resp_obj += ", \"similarity_metric\": [\"cosine\", \"euclidean\", \"euclidean_l2\"]" resp_obj += "}" #print(resp_obj) resp_obj = json.loads(resp_obj) #string to json if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj #------------------------------- if bulkProcess == True: resp_obj = "{" for i in range(0, len(resp_objects)): resp_item = json.dumps(resp_objects[i]) if i > 0: resp_obj += ", " resp_obj += "\"pair_" + str(i + 1) + "\": " + resp_item resp_obj += "}" resp_obj = json.loads(resp_obj) return resp_obj return None #ensemble learning block end #-------------------------------- #ensemble learning disabled if model is None: model = models_loader.get(model_name) if model: model = model() print('Using {} model backend and {} distance'.format( model_name, distance_metric)) else: raise ValueError( 'Invalid model_name passed - {}'.format(model_name)) else: pass # print("Already built model is passed") #------------------------------ #face recognition models have different size of inputs #my environment returns (None, 224, 224, 3) but some people mentioned that they got [(None, 224, 224, 3)]. I think this is because of version issue. if model_name == 'Dlib': #this is not a regular keras model input_shape = (150, 150, 3) else: #keras based models input_shape = model.layers[0].input_shape if isinstance(input_shape, list): input_shape = input_shape[0][1:3] else: input_shape = input_shape[1:3] input_shape_x, input_shape_y = input_shape #------------------------------ #tuned thresholds for model and metric pair threshold = functions.findThreshold(model_name, distance_metric) #------------------------------ #calling deepface in a for loop causes lots of progress bars. this prevents it. disable_option = False if len(img_list) > 1 else True pbar = tqdm(range(0, len(img_list)), desc='Verification', disable=disable_option) #for instance in img_list: for index in pbar: instance = img_list[index] if isinstance(instance, list) and len(instance) == 2: img1_path, img2_path = instance #---------------------- #crop and align faces # img1, img2 = Parallel(n_jobs=2)(delayed(functions.preprocess_face)(img=img_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) for img_path in (img1_path, img2_path)) img1 = functions.preprocess_face( img=img1_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) #---------------------- #find embeddings both_imgs = np.vstack([img1, img2]) img1_representation, img2_representation = model.predict(both_imgs) #---------------------- #find distances between embeddings if distance_metric == 'cosine': distance = dst.findCosineDistance(img1_representation, img2_representation) elif distance_metric == 'euclidean': distance = dst.findEuclideanDistance(img1_representation, img2_representation) elif distance_metric == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) else: raise ValueError("Invalid distance_metric passed - ", distance_metric) #---------------------- #decision if distance <= threshold: identified = "true" else: identified = "false" #---------------------- #response object resp_obj = "{" resp_obj += "\"verified\": " + identified resp_obj += ", \"distance\": " + str(distance) resp_obj += ", \"max_threshold_to_verify\": " + str(threshold) resp_obj += ", \"model\": \"" + model_name + "\"" resp_obj += ", \"similarity_metric\": \"" + distance_metric + "\"" resp_obj += "}" resp_obj = json.loads(resp_obj) #string to json if bulkProcess == True: resp_objects.append(resp_obj) else: #K.clear_session() return resp_obj #---------------------- else: raise ValueError("Invalid arguments passed to verify function: ", instance) #------------------------- toc = time.time() #print("identification lasts ",toc-tic," seconds") if bulkProcess == True: resp_obj = "{" for i in range(0, len(resp_objects)): resp_item = json.dumps(resp_objects[i]) if i > 0: resp_obj += ", " resp_obj += "\"pair_" + str(i + 1) + "\": " + resp_item resp_obj += "}" resp_obj = json.loads(resp_obj) return resp_obj
model = OpenFace.loadModel() elif args.model == 'fbdeepface': model = FbDeepFace.loadModel() else: print('Invalid model choice. Exiting...') exit() input_shape = model.layers[0].input_shape[0][1:3] #---------------------------------------------- #load images and find embeddings dir = [args.dir if args.dir[-1] == '/' else args.dir + '/'][0] img_fnames = glob.glob(dir + '*.png') + glob.glob(dir + '*.jpg') + glob.glob( dir + '*.jpeg') embeddings = {} for fname in img_fnames: feat = model.predict( functions.preprocess_face(fname, target_size=input_shape, enforce_detection=False))[0] embeddings[fname] = feat with open(args.embeddings, 'wb') as f: pickle.dump(embeddings, f) print('Results saved to {}'.format(args.embeddings)) exit() #----------------------------------------------
#accuracy calculation for positive samples log_file.write("Accuracy Calculation for Positive Samples \n") true_count = 0 total_sample_count = 0 no_face_count = 0 for imageName in os.listdir(base_path + anchor_path): input_folder = base_path + imageName.split('_anchor')[0] for sampleImage in os.listdir(input_folder+'\\'): print(sampleImage) input_foto_path = input_folder +'\\' + sampleImage anchor_foto_path = base_path + anchor_path + imageName #---------------------- img1, face_flag = functions.preprocess_face(img=anchor_foto_path, target_size=(input_shape_y, input_shape_x), enforce_detection = False, detector_backend = detector) if face_flag == False: print("no face detected on" + anchor_foto_path + "\n") log_file.write("no face detected on" + anchor_foto_path + "\n") no_face_count += 1 continue img2,face_flag = functions.preprocess_face(img=input_foto_path, target_size=(input_shape_y, input_shape_x), enforce_detection = False, detector_backend = detector) if face_flag == False: print("no face detected on" + input_foto_path + "\n") log_file.write("no face detected on" + input_foto_path + "\n") no_face_count += 1 continue #---------------------- #find embeddings img1_representation = model.predict(img1)[0,:]
def find(img_path, db_path, model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='mtcnn'): """ This function applies verification several times and find an identity in a database Parameters: img_path: exact image path, numpy array or based64 encoded image. If you are going to find several identities, then you should pass img_path as array instead of calling find function in a for loop. e.g. img_path = ["img1.jpg", "img2.jpg"] db_path (string): You should store some .jpg files in a folder and pass the exact folder path to this. model_name (string): VGG-Face, Facenet, OpenFace, DeepFace, DeepID, Dlib or Ensemble distance_metric (string): cosine, euclidean, euclidean_l2 model: built deepface model. A face recognition models are built in every call of find function. You can pass pre-built models to speed the function up. model = DeepFace.build_model('VGG-Face') enforce_detection (boolean): The function throws exception if a face could not be detected. Set this to True if you don't want to get exception. This might be convenient for low resolution images. detector_backend (string): set face detector backend as mtcnn, opencv, ssd or dlib Returns: This function returns pandas data frame. If a list of images is passed to img_path, then it will return list of pandas data frame. """ tic = time.time() img_paths, bulkProcess = functions.initialize_input(img_path) functions.initialize_detector(detector_backend=detector_backend) #------------------------------- if os.path.isdir(db_path) == True: if model == None: if model_name == 'Ensemble': print("Ensemble learning enabled") models = Boosting.loadModel() else: #model is not ensemble model = build_model(model_name) models = {} models[model_name] = model else: #model != None print("Already built model is passed") if model_name == 'Ensemble': Boosting.validate_model(model) models = model.copy() else: models = {} models[model_name] = model #--------------------------------------- if model_name == 'Ensemble': model_names = ['VGG-Face', 'Facenet', 'OpenFace', 'DeepFace'] metric_names = ['cosine', 'euclidean', 'euclidean_l2'] elif model_name != 'Ensemble': model_names = [] metric_names = [] model_names.append(model_name) metric_names.append(distance_metric) #--------------------------------------- file_name = "representations_%s.pkl" % (model_name) file_name = file_name.replace("-", "_").lower() if path.exists(db_path + "/" + file_name): # print("WARNING: Representations for images in ",db_path," folder were previously stored in ", file_name, ". If you added new instances after this file creation, then please delete this file and call find function again. It will create it again.") f = open(db_path + '/' + file_name, 'rb') representations = pickle.load(f) print("There are ", len(representations), " representations found in ", file_name) else: #create representation.pkl from scratch employees = [] for r, d, f in os.walk( db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file.lower()) or ('.png' in file.lower()): exact_path = r + "/" + file employees.append(exact_path) if len(employees) == 0: raise ValueError( "There is no image in ", db_path, " folder! Validate .jpg or .png files exist in this path.") #------------------------ #find representations for db images representations = [] pbar = tqdm(range(0, len(employees)), desc='Finding representations') #for employee in employees: for index in pbar: employee = employees[index] instance = [] instance.append(employee) for j in model_names: custom_model = models[j] #---------------------------------- #decide input shape input_shape = functions.find_input_shape(custom_model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #---------------------------------- img = functions.preprocess_face( img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) representation = custom_model.predict(img)[0, :] instance.append(representation) #------------------------------- representations.append(instance) f = open(db_path + '/' + file_name, "wb") pickle.dump(representations, f) f.close() print( "Representations stored in ", db_path, "/", file_name, " file. Please delete this file when you add new identities in your database." ) #---------------------------- #now, we got representations for facial database if model_name != 'Ensemble': df = pd.DataFrame( representations, columns=["identity", "%s_representation" % (model_name)]) else: #ensemble learning columns = ['identity'] [columns.append('%s_representation' % i) for i in model_names] df = pd.DataFrame(representations, columns=columns) df_base = df.copy( ) #df will be filtered in each img. we will restore it for the next item. resp_obj = [] global_pbar = tqdm(range(0, len(img_paths)), desc='Analyzing') for j in global_pbar: img_path = img_paths[j] #find representation for passed image for j in model_names: custom_model = models[j] #-------------------------------- #decide input shape input_shape = functions.find_input_shape(custom_model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #-------------------------------- img = functions.preprocess_face( img=img_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) target_representation = custom_model.predict(img)[0, :] for k in metric_names: distances = [] for index, instance in df.iterrows(): source_representation = instance["%s_representation" % (j)] if k == 'cosine': distance = dst.findCosineDistance( source_representation, target_representation) elif k == 'euclidean': distance = dst.findEuclideanDistance( source_representation, target_representation) elif k == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(source_representation), dst.l2_normalize(target_representation)) distances.append(distance) #--------------------------- if model_name == 'Ensemble' and j == 'OpenFace' and k == 'euclidean': continue else: df["%s_%s" % (j, k)] = distances if model_name != 'Ensemble': threshold = dst.findThreshold(j, k) df = df.drop(columns=["%s_representation" % (j)]) df = df[df["%s_%s" % (j, k)] <= threshold] df = df.sort_values( by=["%s_%s" % (j, k)], ascending=True).reset_index(drop=True) resp_obj.append(df) df = df_base.copy( ) #restore df for the next iteration #---------------------------------- if model_name == 'Ensemble': feature_names = [] for j in model_names: for k in metric_names: if model_name == 'Ensemble' and j == 'OpenFace' and k == 'euclidean': continue else: feature = '%s_%s' % (j, k) feature_names.append(feature) #print(df.head()) x = df[feature_names].values #-------------------------------------- boosted_tree = Boosting.build_gbm() y = boosted_tree.predict(x) verified_labels = [] scores = [] for i in y: verified = np.argmax(i) == 1 score = i[np.argmax(i)] verified_labels.append(verified) scores.append(score) df['verified'] = verified_labels df['score'] = scores df = df[df.verified == True] #df = df[df.score > 0.99] #confidence score df = df.sort_values(by=["score"], ascending=False).reset_index(drop=True) df = df[['identity', 'verified', 'score']] resp_obj.append(df) df = df_base.copy() #restore df for the next iteration #---------------------------------- toc = time.time() print("find function lasts ", toc - tic, " seconds") if len(resp_obj) == 1: return resp_obj[0] return resp_obj else: raise ValueError("Passed db_path does not exist!") return None
def analyze(img_path, actions=['emotion', 'age', 'gender', 'race'], models={}, enforce_detection=True, detector_backend='mtcnn'): """ This function analyzes facial attributes including age, gender, emotion and race Parameters: img_path: exact image path, numpy array or base64 encoded image could be passed. If you are going to analyze lots of images, then set this to list. e.g. img_path = ['img1.jpg', 'img2.jpg'] actions (list): The default is ['age', 'gender', 'emotion', 'race']. You can drop some of those attributes. models: facial attribute analysis models are built in every call of analyze function. You can pass pre-built models to speed the function up. models = {} models['age'] = DeepFace.build_model('Age') models['gender'] = DeepFace.build_model('Gender') models['emotion'] = DeepFace.build_model('Emotion') models['race'] = DeepFace.build_model('race') enforce_detection (boolean): The function throws exception if a face could not be detected. Set this to True if you don't want to get exception. This might be convenient for low resolution images. detector_backend (string): set face detector backend as mtcnn, opencv, ssd or dlib. Returns: The function returns a dictionary. If img_path is a list, then it will return list of dictionary. { "region": {'x': 230, 'y': 120, 'w': 36, 'h': 45}, "age": 28.66, "gender": "woman", "dominant_emotion": "neutral", "emotion": { 'sad': 37.65260875225067, 'angry': 0.15512987738475204, 'surprise': 0.0022171278033056296, 'fear': 1.2489334680140018, 'happy': 4.609785228967667, 'disgust': 9.698561953541684e-07, 'neutral': 56.33133053779602 } "dominant_race": "white", "race": { 'indian': 0.5480832420289516, 'asian': 0.7830780930817127, 'latino hispanic': 2.0677512511610985, 'black': 0.06337375962175429, 'middle eastern': 3.088453598320484, 'white': 93.44925880432129 } } """ img_paths, bulkProcess = functions.initialize_input(img_path) functions.initialize_detector(detector_backend=detector_backend) #--------------------------------- built_models = list(models.keys()) #--------------------------------- #pre-trained models passed but it doesn't exist in actions if len(built_models) > 0: if 'emotion' in built_models and 'emotion' not in actions: actions.append('emotion') if 'age' in built_models and 'age' not in actions: actions.append('age') if 'gender' in built_models and 'gender' not in actions: actions.append('gender') if 'race' in built_models and 'race' not in actions: actions.append('race') #--------------------------------- if 'emotion' in actions and 'emotion' not in built_models: models['emotion'] = build_model('Emotion') if 'age' in actions and 'age' not in built_models: models['age'] = build_model('Age') if 'gender' in actions and 'gender' not in built_models: models['gender'] = build_model('Gender') if 'race' in actions and 'race' not in built_models: models['race'] = build_model('Race') #--------------------------------- resp_objects = [] disable_option = False if len(img_paths) > 1 else True global_pbar = tqdm(range(0, len(img_paths)), desc='Analyzing', disable=disable_option) for j in global_pbar: img_path = img_paths[j] resp_obj = {} disable_option = False if len(actions) > 1 else True pbar = tqdm(range(0, len(actions)), desc='Finding actions', disable=disable_option) img_224 = None # Set to prevent re-detection region = [] # x, y, w, h of the detected face region region_labels = ['x', 'y', 'w', 'h'] #facial attribute analysis for index in pbar: action = actions[index] pbar.set_description("Action: %s" % (action)) if action == 'emotion': emotion_labels = [ 'angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral' ] img, region = functions.preprocess_face( img=img_path, target_size=(48, 48), grayscale=True, enforce_detection=enforce_detection, detector_backend=detector_backend, return_region=True) resp_obj["region"] = {} for i, parameter in enumerate(region_labels): resp_obj["region"][parameter] = region[i] emotion_predictions = models['emotion'].predict(img)[0, :] sum_of_predictions = emotion_predictions.sum() resp_obj["emotion"] = {} for i in range(0, len(emotion_labels)): emotion_label = emotion_labels[i] emotion_prediction = 100 * emotion_predictions[ i] / sum_of_predictions resp_obj["emotion"][emotion_label] = emotion_prediction resp_obj["dominant_emotion"] = emotion_labels[np.argmax( emotion_predictions)] elif action == 'age': if img_224 is None: img_224, region = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend, return_region=True) resp_obj["region"] = {} for i, parameter in enumerate(region_labels): resp_obj["region"][parameter] = region[i] age_predictions = models['age'].predict(img_224)[0, :] apparent_age = Age.findApparentAge(age_predictions) resp_obj["age"] = int(apparent_age) elif action == 'gender': if img_224 is None: img_224, region = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend, return_region=True) resp_obj["region"] = {} for i, parameter in enumerate(region_labels): resp_obj["region"][parameter] = region[i] gender_prediction = models['gender'].predict(img_224)[0, :] if np.argmax(gender_prediction) == 0: gender = "Woman" elif np.argmax(gender_prediction) == 1: gender = "Man" resp_obj["gender"] = gender elif action == 'race': if img_224 is None: img_224, region = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend, return_region=True ) #just emotion model expects grayscale images race_predictions = models['race'].predict(img_224)[0, :] race_labels = [ 'asian', 'indian', 'black', 'white', 'middle eastern', 'latino hispanic' ] resp_obj["region"] = {} for i, parameter in enumerate(region_labels): resp_obj["region"][parameter] = region[i] sum_of_predictions = race_predictions.sum() resp_obj["race"] = {} for i in range(0, len(race_labels)): race_label = race_labels[i] race_prediction = 100 * race_predictions[ i] / sum_of_predictions resp_obj["race"][race_label] = race_prediction resp_obj["dominant_race"] = race_labels[np.argmax( race_predictions)] #--------------------------------- if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj if bulkProcess == True: resp_obj = {} for i in range(0, len(resp_objects)): resp_item = resp_objects[i] resp_obj["instance_%d" % (i + 1)] = resp_item return resp_obj
import matplotlib.pyplot as plt from deepface import DeepFace from deepface.commons import functions #!pip install redis import redis #---------------------------- #model model = DeepFace.build_model("Facenet") input_shape = (160, 160) #---------------------------- #target target_img_path = "target.png" target_img = functions.preprocess_face(target_img_path, target_size = (160, 160)) plt.imshow(target_img[0][:,:,::-1]) plt.axis('off') plt.show() target_embedding = model.predict(target_img)[0].tolist() #---------------------------- #redis server #redis = redis.Redis(host='localhost', port=6379, db=0) redis = redis.StrictRedis(host='localhost', port=6379, db=0) for key in redis.scan_iter("embedding:*"): redis.delete(key)
img_paths = [] for path in os.listdir(REALPATH): img_folder = REALPATH + path for face in os.listdir(img_folder): img_path = img_folder + '/' + face img_paths.append(img_path) for i in img_paths: img_path = i out_path = i + '.tiff' detector_backend = 'opencv' enforce_detection = False img, region = functions.preprocess_face( img=img_path, target_size=(48, 48), grayscale=True, enforce_detection=enforce_detection, detector_backend=detector_backend, return_region=True) img.resize(48, 48) cv2.imwrite(out_path, img) print("image done", img) # for i in video_paths: # originalImage = cv2.imread(image) # grey_image = cv2.cvtColor(originalImage, cv2.COLOR_BGR2GRAY) # grey_array = np.asarray(grey_image) # colour_to_grayscale.append(grey_array) # X_test_gen = get_datagen(REALPATH) #
def detectFace(img_path, detector_backend='opencv'): imgs = functions.preprocess_face( img=img_path, detector_backend=detector_backend)[ 'original'] #preprocess_face returns (1, 224, 224, 3) return imgs
def analyze(img_path, actions=[], models={}, enforce_detection=True, detector_backend='opencv'): if type(img_path) == list: img_paths = img_path.copy() bulkProcess = True else: img_paths = [img_path] bulkProcess = False #--------------------------------- #if a specific target is not passed, then find them all if len(actions) == 0: actions = ['emotion', 'age', 'gender', 'race'] print("Actions to do: ", actions) #--------------------------------- if 'emotion' in actions: if 'emotion' in models: print("already built emotion model is passed") emotion_model = models['emotion'] else: emotion_model = Emotion.loadModel() if 'age' in actions: if 'age' in models: print("already built age model is passed") age_model = models['age'] else: age_model = Age.loadModel() if 'gender' in actions: if 'gender' in models: print("already built gender model is passed") gender_model = models['gender'] else: gender_model = Gender.loadModel() if 'race' in actions: if 'race' in models: print("already built race model is passed") race_model = models['race'] else: race_model = Race.loadModel() #--------------------------------- resp_objects = [] global_pbar = tqdm(range(0, len(img_paths)), desc='Analyzing') #for img_path in img_paths: for j in global_pbar: img_path = img_paths[j] resp_obj = "{" #TO-DO: do this in parallel pbar = tqdm(range(0, len(actions)), desc='Finding actions') action_idx = 0 img_224 = None # Set to prevent re-detection #for action in actions: for index in pbar: action = actions[index] pbar.set_description("Action: %s" % (action)) if action_idx > 0: resp_obj += ", " if action == 'emotion': emotion_labels = [ 'angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral' ] img = functions.preprocess_face( img=img_path, target_size=(48, 48), grayscale=True, enforce_detection=enforce_detection, detector_backend=detector_backend) emotion_predictions = emotion_model.predict(img)[0, :] sum_of_predictions = emotion_predictions.sum() emotion_obj = "\"emotion\": {" for i in range(0, len(emotion_labels)): emotion_label = emotion_labels[i] emotion_prediction = 100 * emotion_predictions[ i] / sum_of_predictions if i > 0: emotion_obj += ", " emotion_obj += "\"%s\": %s" % (emotion_label, emotion_prediction) emotion_obj += "}" emotion_obj += ", \"dominant_emotion\": \"%s\"" % ( emotion_labels[np.argmax(emotion_predictions)]) resp_obj += emotion_obj elif action == 'age': if img_224 is None: img_224 = functions.preprocess_face( img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection ) #just emotion model expects grayscale images #print("age prediction") age_predictions = age_model.predict(img_224)[0, :] apparent_age = Age.findApparentAge(age_predictions) resp_obj += "\"age\": %s" % (apparent_age) elif action == 'gender': if img_224 is None: img_224 = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend ) #just emotion model expects grayscale images #print("gender prediction") gender_prediction = gender_model.predict(img_224)[0, :] if np.argmax(gender_prediction) == 0: gender = "Woman" elif np.argmax(gender_prediction) == 1: gender = "Man" resp_obj += "\"gender\": \"%s\"" % (gender) elif action == 'race': if img_224 is None: img_224 = functions.preprocess_face( img=img_path, target_size=(224, 224), grayscale=False, enforce_detection=enforce_detection, detector_backend=detector_backend ) #just emotion model expects grayscale images race_predictions = race_model.predict(img_224)[0, :] race_labels = [ 'asian', 'indian', 'black', 'white', 'middle eastern', 'latino hispanic' ] sum_of_predictions = race_predictions.sum() race_obj = "\"race\": {" for i in range(0, len(race_labels)): race_label = race_labels[i] race_prediction = 100 * race_predictions[ i] / sum_of_predictions if i > 0: race_obj += ", " race_obj += "\"%s\": %s" % (race_label, race_prediction) race_obj += "}" race_obj += ", \"dominant_race\": \"%s\"" % ( race_labels[np.argmax(race_predictions)]) resp_obj += race_obj action_idx = action_idx + 1 resp_obj += "}" resp_obj = json.loads(resp_obj) if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj if bulkProcess == True: resp_obj = "{" for i in range(0, len(resp_objects)): resp_item = json.dumps(resp_objects[i]) if i > 0: resp_obj += ", " resp_obj += "\"instance_" + str(i + 1) + "\": " + resp_item resp_obj += "}" resp_obj = json.loads(resp_obj) return resp_obj
def analysis(img, db_path, input_shape, model_name, distance_metric, model=None, enable_face_analysis=True): if (model == None): model, input_shape = create_model(db_path, model_name) file_name = "representations_%s.pkl" % (model_name) f = open(db_path + '/' + file_name, 'rb') embeddings = pickle.load(f) # print(embeddings) input_shape_x = input_shape[0] input_shape_y = input_shape[1] time_threshold = 5 frame_threshold = 5 pivot_img_size = 112 #face recognition result image #----------------------- opencv_path = functions.get_opencv_path() face_detector_path = opencv_path + "haarcascade_frontalface_default.xml" face_cascade = cv2.CascadeClassifier(face_detector_path) #----------------------- freeze = False face_detected = False face_included_frames = 0 #freeze screen if face detected sequantially 5 frames freezed_frame = 0 tic = time.time() text_color = (255, 255, 255) employees = [] #check passed db folder exists if os.path.isdir(db_path) == True: for r, d, f in os.walk(db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file): #exact_path = os.path.join(r, file) exact_path = r + "/" + file #print(exact_path) employees.append(exact_path) df = pd.DataFrame(embeddings, columns=['employee', 'embedding']) df['distance_metric'] = distance_metric # cap = VideoStream(src=0,usePiCamera=True,framerate=32).start() threshold = functions.findThreshold(model_name, distance_metric) # time.sleep(2) i = 1 while i: label = 'None' i = 0 start = time.time() # img = cap.read() faces = face_cascade.detectMultiScale(img, 1.3, 5) for (x, y, w, h) in faces: custom_face = functions.preprocess_face( img=img, target_size=(input_shape_y, input_shape_x), enforce_detection=False) #check preprocess_face function handled if custom_face.shape[1:3] == input_shape: if df.shape[ 0] > 0: #if there are images to verify, apply face recognition img1_representation = model.predict(custom_face)[0, :] ## print(img1_representation) #print(freezed_frame," - ",img1_representation[0:5]) def findDistance(row): distance_metric = row['distance_metric'] img2_representation = row['embedding'] distance = 1000 #initialize very large value if distance_metric == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif distance_metric == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif distance_metric == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) return distance df['distance'] = df.apply(findDistance, axis=1) df = df.sort_values(by=["distance"]) candidate = df.iloc[0] employee_name = candidate['employee'] best_distance = candidate['distance'] ## print(candidate[['employee', 'distance']].values) ## print(threshold) #if True: if best_distance <= threshold: #print(employee_name) end = time.time() display_img = cv2.imread(employee_name) display_img = cv2.resize( display_img, (pivot_img_size, pivot_img_size)) label = employee_name.split("/")[-2].replace( ".jpg", "") ## print(label) # hex_string = "0x5C"[2:] # bytes_object = bytes.fromhex(hex_string) # ascii_string = bytes_object.decode("ASCII") # label = label.split(ascii_string)[-1] print(best_distance, "--->", label, "----->", str(end - start)) # cv2.putText(img, str(label), (100,60), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2) else: label = "Unkown" # cv2.putText(img, str(label), (100,60), cv2.FONT_HERSHEY_SIMPLEX, 2, (255, 0, 0), 2) # cv2.rectangle(img, (x,y), (x+w,y+h), (67,167,67), 1) return label
def verify(img1_path, img2_path='', model_name='VGG-Face', distance_metric='cosine', model=None, enforce_detection=True, detector_backend='mtcnn'): """ This function verifies an image pair is same person or different persons. Parameters: img1_path, img2_path: exact image path, numpy array or based64 encoded images could be passed. If you are going to call verify function for a list of image pairs, then you should pass an array instead of calling the function in for loops. e.g. img1_path = [ ['img1.jpg', 'img2.jpg'], ['img2.jpg', 'img3.jpg'] ] model_name (string): VGG-Face, Facenet, OpenFace, DeepFace, DeepID, Dlib, ArcFace or Ensemble distance_metric (string): cosine, euclidean, euclidean_l2 model: Built deepface model. A face recognition model is built every call of verify function. You can pass pre-built face recognition model optionally if you will call verify function several times. model = DeepFace.build_model('VGG-Face') enforce_detection (boolean): If any face could not be detected in an image, then verify function will return exception. Set this to False not to have this exception. This might be convenient for low resolution images. detector_backend (string): set face detector backend as mtcnn, opencv, ssd or dlib Returns: Verify function returns a dictionary. If img1_path is a list of image pairs, then the function will return list of dictionary. { "verified": True , "distance": 0.2563 , "max_threshold_to_verify": 0.40 , "model": "VGG-Face" , "similarity_metric": "cosine" } """ tic = time.time() img_list, bulkProcess = functions.initialize_input(img1_path, img2_path) functions.initialize_detector(detector_backend=detector_backend) resp_objects = [] #-------------------------------- if model_name == 'Ensemble': model_names = ["VGG-Face", "Facenet", "OpenFace", "DeepFace"] metrics = ["cosine", "euclidean", "euclidean_l2"] else: model_names = [] metrics = [] model_names.append(model_name) metrics.append(distance_metric) #-------------------------------- if model == None: if model_name == 'Ensemble': models = Boosting.loadModel() else: model = build_model(model_name) models = {} models[model_name] = model else: if model_name == 'Ensemble': Boosting.validate_model(model) models = model.copy() else: models = {} models[model_name] = model #------------------------------ #calling deepface in a for loop causes lots of progress bars. this prevents it. disable_option = False if len(img_list) > 1 else True pbar = tqdm(range(0, len(img_list)), desc='Verification', disable=disable_option) for index in pbar: instance = img_list[index] if type(instance) == list and len(instance) >= 2: img1_path = instance[0] img2_path = instance[1] ensemble_features = [] for i in model_names: custom_model = models[i] #decide input shape input_shape = functions.find_input_shape(custom_model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #---------------------- #detect and align faces img1 = functions.preprocess_face( img=img1_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) img2 = functions.preprocess_face( img=img2_path, target_size=(input_shape_y, input_shape_x), enforce_detection=enforce_detection, detector_backend=detector_backend) #---------------------- #find embeddings img1_representation = custom_model.predict(img1)[0, :] img2_representation = custom_model.predict(img2)[0, :] #---------------------- #find distances between embeddings for j in metrics: if j == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif j == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif j == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize(img1_representation), dst.l2_normalize(img2_representation)) else: raise ValueError("Invalid distance_metric passed - ", distance_metric) distance = np.float64( distance ) #causes trobule for euclideans in api calls if this is not set (issue #175) #---------------------- #decision if model_name != 'Ensemble': threshold = dst.findThreshold(i, j) if distance <= threshold: identified = True else: identified = False resp_obj = { "verified": identified, "distance": distance, "max_threshold_to_verify": threshold, "model": model_name, "similarity_metric": distance_metric } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj else: #Ensemble #this returns same with OpenFace - euclidean_l2 if i == 'OpenFace' and j == 'euclidean': continue else: ensemble_features.append(distance) #---------------------- if model_name == 'Ensemble': boosted_tree = Boosting.build_gbm() prediction = boosted_tree.predict( np.expand_dims(np.array(ensemble_features), axis=0))[0] verified = np.argmax(prediction) == 1 score = prediction[np.argmax(prediction)] resp_obj = { "verified": verified, "score": score, "distance": ensemble_features, "model": ["VGG-Face", "Facenet", "OpenFace", "DeepFace"], "similarity_metric": ["cosine", "euclidean", "euclidean_l2"] } if bulkProcess == True: resp_objects.append(resp_obj) else: return resp_obj #---------------------- else: raise ValueError("Invalid arguments passed to verify function: ", instance) #------------------------- toc = time.time() if bulkProcess == True: resp_obj = {} for i in range(0, len(resp_objects)): resp_item = resp_objects[i] resp_obj["pair_%d" % (i + 1)] = resp_item return resp_obj
def detectFace(img_path, detector_backend='opencv'): img = functions.preprocess_face( img=img_path, detector_backend=detector_backend)[ 0] #preprocess_face returns (1, 224, 224, 3) return img[:, :, ::-1] #bgr to rgb
def analysis(db_path, model_name, distance_metric, enable_face_analysis=True, source=0, time_threshold=5, frame_threshold=5): input_shape = (224, 224) input_shape_x = input_shape[0] input_shape_y = input_shape[1] text_color = (255, 255, 255) employees = [] #check passed db folder exists if os.path.isdir(db_path) == True: for r, d, f in os.walk(db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file): #exact_path = os.path.join(r, file) exact_path = r + "/" + file #print(exact_path) employees.append(exact_path) if len(employees) == 0: print("WARNING: There is no image in this path ( ", db_path, ") . Face recognition will not be performed.") #------------------------ if len(employees) > 0: model = DeepFace.build_model(model_name) print(model_name, " is built") #------------------------ input_shape = functions.find_input_shape(model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #tuned thresholds for model and metric pair threshold = dst.findThreshold(model_name, distance_metric) #------------------------ #facial attribute analysis models if enable_face_analysis == True: tic = time.time() emotion_model = DeepFace.build_model('Emotion') print("Emotion model loaded") age_model = DeepFace.build_model('Age') print("Age model loaded") gender_model = DeepFace.build_model('Gender') print("Gender model loaded") toc = time.time() print("Facial attibute analysis models loaded in ", toc - tic, " seconds") #------------------------ #find embeddings for employee list tic = time.time() pbar = tqdm(range(0, len(employees)), desc='Finding embeddings') embeddings = [] #for employee in employees: for index in pbar: employee = employees[index] pbar.set_description("Finding embedding for %s" % (employee.split("/")[-1])) embedding = [] img = functions.preprocess_face(img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=False) img_representation = model.predict(img)[0, :] embedding.append(employee) embedding.append(img_representation) embeddings.append(embedding) df = pd.DataFrame(embeddings, columns=['employee', 'embedding']) df['distance_metric'] = distance_metric toc = time.time() print("Embeddings found for given data set in ", toc - tic, " seconds") #----------------------- pivot_img_size = 112 #face recognition result image #----------------------- opencv_path = functions.get_opencv_path() face_detector_path = opencv_path + "haarcascade_frontalface_default.xml" face_cascade = cv2.CascadeClassifier(face_detector_path) #----------------------- freeze = False face_detected = False face_included_frames = 0 #freeze screen if face detected sequantially 5 frames freezed_frame = 0 tic = time.time() cap = cv2.VideoCapture(source) #webcam while (True): ret, img = cap.read() #cv2.namedWindow('img', cv2.WINDOW_FREERATIO) #cv2.setWindowProperty('img', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) raw_img = img.copy() resolution = img.shape resolution_x = img.shape[1] resolution_y = img.shape[0] if freeze == False: faces = face_cascade.detectMultiScale(img, 1.3, 5) if len(faces) == 0: face_included_frames = 0 else: faces = [] detected_faces = [] face_index = 0 for (x, y, w, h) in faces: if w > 130: #discard small detected faces face_detected = True if face_index == 0: face_included_frames = face_included_frames + 1 #increase frame for a single face cv2.rectangle(img, (x, y), (x + w, y + h), (67, 67, 67), 1) #draw rectangle to main image cv2.putText(img, str(frame_threshold - face_included_frames), (int(x + w / 4), int(y + h / 1.5)), cv2.FONT_HERSHEY_SIMPLEX, 4, (255, 255, 255), 2) detected_face = img[int(y):int(y + h), int(x):int(x + w)] #crop detected face #------------------------------------- detected_faces.append((x, y, w, h)) face_index = face_index + 1 #------------------------------------- if face_detected == True and face_included_frames == frame_threshold and freeze == False: freeze = True #base_img = img.copy() base_img = raw_img.copy() detected_faces_final = detected_faces.copy() tic = time.time() if freeze == True: toc = time.time() if (toc - tic) < time_threshold: if freezed_frame == 0: freeze_img = base_img.copy() #freeze_img = np.zeros(resolution, np.uint8) #here, np.uint8 handles showing white area issue for detected_face in detected_faces_final: x = detected_face[0] y = detected_face[1] w = detected_face[2] h = detected_face[3] cv2.rectangle(freeze_img, (x, y), (x + w, y + h), (67, 67, 67), 1) #draw rectangle to main image #------------------------------- #apply deep learning for custom_face custom_face = base_img[y:y + h, x:x + w] #------------------------------- #facial attribute analysis if enable_face_analysis == True: gray_img = functions.preprocess_face( img=custom_face, target_size=(48, 48), grayscale=True, enforce_detection=False) emotion_labels = [ 'Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral' ] emotion_predictions = emotion_model.predict( gray_img)[0, :] sum_of_predictions = emotion_predictions.sum() mood_items = [] for i in range(0, len(emotion_labels)): mood_item = [] emotion_label = emotion_labels[i] emotion_prediction = 100 * emotion_predictions[ i] / sum_of_predictions mood_item.append(emotion_label) mood_item.append(emotion_prediction) mood_items.append(mood_item) emotion_df = pd.DataFrame( mood_items, columns=["emotion", "score"]) emotion_df = emotion_df.sort_values( by=["score"], ascending=False).reset_index(drop=True) #background of mood box #transparency overlay = freeze_img.copy() opacity = 0.4 if x + w + pivot_img_size < resolution_x: #right cv2.rectangle( freeze_img #, (x+w,y+20) , (x + w, y), (x + w + pivot_img_size, y + h), (64, 64, 64), cv2.FILLED) cv2.addWeighted(overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) elif x - pivot_img_size > 0: #left cv2.rectangle( freeze_img #, (x-pivot_img_size,y+20) , (x - pivot_img_size, y), (x, y + h), (64, 64, 64), cv2.FILLED) cv2.addWeighted(overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) for index, instance in emotion_df.iterrows(): emotion_label = "%s " % (instance['emotion']) emotion_score = instance['score'] / 100 bar_x = 35 #this is the size if an emotion is 100% bar_x = int(bar_x * emotion_score) if x + w + pivot_img_size < resolution_x: text_location_y = y + 20 + (index + 1) * 20 text_location_x = x + w if text_location_y < y + h: cv2.putText( freeze_img, emotion_label, (text_location_x, text_location_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) cv2.rectangle( freeze_img, (x + w + 70, y + 13 + (index + 1) * 20), (x + w + 70 + bar_x, y + 13 + (index + 1) * 20 + 5), (255, 255, 255), cv2.FILLED) elif x - pivot_img_size > 0: text_location_y = y + 20 + (index + 1) * 20 text_location_x = x - pivot_img_size if text_location_y <= y + h: cv2.putText( freeze_img, emotion_label, (text_location_x, text_location_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) cv2.rectangle( freeze_img, (x - pivot_img_size + 70, y + 13 + (index + 1) * 20), (x - pivot_img_size + 70 + bar_x, y + 13 + (index + 1) * 20 + 5), (255, 255, 255), cv2.FILLED) #------------------------------- face_224 = functions.preprocess_face( img=custom_face, target_size=(224, 224), grayscale=False, enforce_detection=False) age_predictions = age_model.predict(face_224)[0, :] apparent_age = Age.findApparentAge(age_predictions) #------------------------------- gender_prediction = gender_model.predict(face_224)[ 0, :] if np.argmax(gender_prediction) == 0: gender = "W" elif np.argmax(gender_prediction) == 1: gender = "M" #print(str(int(apparent_age))," years old ", dominant_emotion, " ", gender) analysis_report = str( int(apparent_age)) + " " + gender #------------------------------- info_box_color = (46, 200, 255) #top if y - pivot_img_size + int( pivot_img_size / 5) > 0: triangle_coordinates = np.array([ (x + int(w / 2), y), (x + int(w / 2) - int(w / 10), y - int(pivot_img_size / 3)), (x + int(w / 2) + int(w / 10), y - int(pivot_img_size / 3)) ]) cv2.drawContours(freeze_img, [triangle_coordinates], 0, info_box_color, -1) cv2.rectangle( freeze_img, (x + int(w / 5), y - pivot_img_size + int(pivot_img_size / 5)), (x + w - int(w / 5), y - int(pivot_img_size / 3)), info_box_color, cv2.FILLED) cv2.putText(freeze_img, analysis_report, (x + int(w / 3.5), y - int(pivot_img_size / 2.1)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 111, 255), 2) #bottom elif y + h + pivot_img_size - int( pivot_img_size / 5) < resolution_y: triangle_coordinates = np.array([ (x + int(w / 2), y + h), (x + int(w / 2) - int(w / 10), y + h + int(pivot_img_size / 3)), (x + int(w / 2) + int(w / 10), y + h + int(pivot_img_size / 3)) ]) cv2.drawContours(freeze_img, [triangle_coordinates], 0, info_box_color, -1) cv2.rectangle( freeze_img, (x + int(w / 5), y + h + int(pivot_img_size / 3)), (x + w - int(w / 5), y + h + pivot_img_size - int(pivot_img_size / 5)), info_box_color, cv2.FILLED) cv2.putText(freeze_img, analysis_report, (x + int(w / 3.5), y + h + int(pivot_img_size / 1.5)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 111, 255), 2) #------------------------------- #face recognition custom_face = functions.preprocess_face( img=custom_face, target_size=(input_shape_y, input_shape_x), enforce_detection=False) #check preprocess_face function handled if custom_face.shape[1:3] == input_shape: if df.shape[ 0] > 0: #if there are images to verify, apply face recognition img1_representation = model.predict( custom_face)[0, :] #print(freezed_frame," - ",img1_representation[0:5]) def findDistance(row): distance_metric = row['distance_metric'] img2_representation = row['embedding'] distance = 1000 #initialize very large value if distance_metric == 'cosine': distance = dst.findCosineDistance( img1_representation, img2_representation) elif distance_metric == 'euclidean': distance = dst.findEuclideanDistance( img1_representation, img2_representation) elif distance_metric == 'euclidean_l2': distance = dst.findEuclideanDistance( dst.l2_normalize( img1_representation), dst.l2_normalize( img2_representation)) return distance df['distance'] = df.apply(findDistance, axis=1) df = df.sort_values(by=["distance"]) candidate = df.iloc[0] employee_name = candidate['employee'] best_distance = candidate['distance'] #print(candidate[['employee', 'distance']].values) #if True: if best_distance <= threshold: #print(employee_name) display_img = cv2.imread(employee_name) display_img = cv2.resize( display_img, (pivot_img_size, pivot_img_size)) label = employee_name.split( "/")[-1].replace(".jpg", "") label = re.sub('[0-9]', '', label) try: if y - pivot_img_size > 0 and x + w + pivot_img_size < resolution_x: #top right freeze_img[ y - pivot_img_size:y, x + w:x + w + pivot_img_size] = display_img overlay = freeze_img.copy() opacity = 0.4 cv2.rectangle( freeze_img, (x + w, y), (x + w + pivot_img_size, y + 20), (46, 200, 255), cv2.FILLED) cv2.addWeighted( overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) cv2.putText( freeze_img, label, (x + w, y + 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1) #connect face and text cv2.line(freeze_img, (x + int(w / 2), y), (x + 3 * int(w / 4), y - int(pivot_img_size / 2)), (67, 67, 67), 1) cv2.line(freeze_img, (x + 3 * int(w / 4), y - int(pivot_img_size / 2)), (x + w, y - int(pivot_img_size / 2)), (67, 67, 67), 1) elif y + h + pivot_img_size < resolution_y and x - pivot_img_size > 0: #bottom left freeze_img[ y + h:y + h + pivot_img_size, x - pivot_img_size:x] = display_img overlay = freeze_img.copy() opacity = 0.4 cv2.rectangle( freeze_img, (x - pivot_img_size, y + h - 20), (x, y + h), (46, 200, 255), cv2.FILLED) cv2.addWeighted( overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) cv2.putText( freeze_img, label, (x - pivot_img_size, y + h - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1) #connect face and text cv2.line(freeze_img, (x + int(w / 2), y + h), (x + int(w / 2) - int(w / 4), y + h + int(pivot_img_size / 2)), (67, 67, 67), 1) cv2.line(freeze_img, (x + int(w / 2) - int(w / 4), y + h + int(pivot_img_size / 2)), (x, y + h + int(pivot_img_size / 2)), (67, 67, 67), 1) elif y - pivot_img_size > 0 and x - pivot_img_size > 0: #top left freeze_img[ y - pivot_img_size:y, x - pivot_img_size:x] = display_img overlay = freeze_img.copy() opacity = 0.4 cv2.rectangle( freeze_img, (x - pivot_img_size, y), (x, y + 20), (46, 200, 255), cv2.FILLED) cv2.addWeighted( overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) cv2.putText( freeze_img, label, (x - pivot_img_size, y + 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1) #connect face and text cv2.line( freeze_img, (x + int(w / 2), y), (x + int(w / 2) - int(w / 4), y - int(pivot_img_size / 2)), (67, 67, 67), 1) cv2.line( freeze_img, (x + int(w / 2) - int(w / 4), y - int(pivot_img_size / 2)), (x, y - int(pivot_img_size / 2)), (67, 67, 67), 1) elif x + w + pivot_img_size < resolution_x and y + h + pivot_img_size < resolution_y: #bottom righ freeze_img[ y + h:y + h + pivot_img_size, x + w:x + w + pivot_img_size] = display_img overlay = freeze_img.copy() opacity = 0.4 cv2.rectangle( freeze_img, (x + w, y + h - 20), (x + w + pivot_img_size, y + h), (46, 200, 255), cv2.FILLED) cv2.addWeighted( overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) cv2.putText( freeze_img, label, (x + w, y + h - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, text_color, 1) #connect face and text cv2.line(freeze_img, (x + int(w / 2), y + h), (x + int(w / 2) + int(w / 4), y + h + int(pivot_img_size / 2)), (67, 67, 67), 1) cv2.line(freeze_img, (x + int(w / 2) + int(w / 4), y + h + int(pivot_img_size / 2)), (x + w, y + h + int(pivot_img_size / 2)), (67, 67, 67), 1) except Exception as err: print(str(err)) tic = time.time( ) #in this way, freezed image can show 5 seconds #------------------------------- time_left = int(time_threshold - (toc - tic) + 1) cv2.rectangle(freeze_img, (10, 10), (90, 50), (67, 67, 67), -10) cv2.putText(freeze_img, str(time_left), (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1) cv2.imshow('img', freeze_img) freezed_frame = freezed_frame + 1 else: face_detected = False face_included_frames = 0 freeze = False freezed_frame = 0 else: cv2.imshow('img', img) if cv2.waitKey(1) & 0xFF == ord('q'): #press q to quit break #kill open cv things cap.release() cv2.destroyAllWindows()
def analysis(db_path, model_name, distance_metric, enable_face_analysis=False, source=0, time_threshold=5, frame_threshold=5): #Intitalize the input shape #input_shape = (224, 224); input_shape_x = input_shape[0]; input_shape_y = input_shape[1] employees = [] #check passed db folder exists if os.path.isdir(db_path) == True: for r, d, f in os.walk(db_path): # r=root, d=directories, f = files for file in f: if ('.jpg' in file): #exact_path = os.path.join(r, file) exact_path = r + "/" + file #print(exact_path) employees.append(exact_path) #------------------------ if len(employees) > 0: model = DeepFace.build_model(model_name) print(model_name, " is built") #------------------------ input_shape = functions.find_input_shape(model) input_shape_x = input_shape[0] input_shape_y = input_shape[1] #------------------------ #facial attribute analysis models if enable_face_analysis == True: tic = time.time() emotion_model = DeepFace.build_model('Emotion') #sg.popup_quick_message('Emotion Model Loaded') print("Emotion model loaded") toc = time.time() print("Facial attibute analysis models loaded in ", toc - tic, " seconds") #------------------------ #find embeddings for employee list tic = time.time() pbar = tqdm(range(0, len(employees)), desc='Finding embeddings') embeddings = [] #for employee in employees: for index in pbar: employee = employees[index] pbar.set_description("Finding embedding for %s" % (employee.split("/")[-1])) embedding = [] captured_frame = functions.preprocess_face(img=employee, target_size=(input_shape_y, input_shape_x), enforce_detection=False) img_representation = model.predict(captured_frame)[0, :] embedding.append(employee) embedding.append(img_representation) embeddings.append(embedding) df = pd.DataFrame(embeddings, columns=['employee', 'embedding']) df['distance_metric'] = distance_metric toc = time.time() print("Embeddings found for given data set in ", toc - tic, " seconds") #----------------------- pivot_img_size = 112 #face recognition result image #----------------------- opencv_path = functions.get_opencv_path() face_detector_path = opencv_path + "haarcascade_frontalface_default.xml" face_cascade = cv2.CascadeClassifier(face_detector_path) #-----------------------Declare Variables freeze = False face_detected = False face_included_frames = 0 #freeze screen if face detected sequantially 5 frames freezed_frame = 0 tic = time.time() # START CAMERA win_started = False captured_video = cv2.VideoCapture(source) #Start webcam while (True): ret, captured_frame = captured_video.read() #cv2.namedWindow('captured_frame', cv2.WINDOW_FREERATIO) #cv2.setWindowProperty('captured_frame', cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) raw_img = captured_frame.copy() resolution = captured_frame.shape resolution_x = captured_frame.shape[1] resolution_y = captured_frame.shape[0] if freeze == False: faces = face_cascade.detectMultiScale(captured_frame, 1.3, 5) if len(faces) == 0: face_included_frames = 0 else: faces = [] detected_faces = [] face_index = 0 for (x, y, w, h) in faces: if w > 130: #discard small detected faces face_detected = True if face_index == 0: face_included_frames = face_included_frames + 1 #increase frame for a single face cv2.rectangle(captured_frame, (x, y), (x + w, y + h), (67, 67, 67), 1) #draw rectangle to main image cv2.putText(captured_frame, str(frame_threshold - face_included_frames), (int(x + w / 4), int(y + h / 1.5)), cv2.FONT_HERSHEY_SIMPLEX, 4, (255, 255, 255), 2) detected_face = captured_frame[int(y):int(y + h), int(x):int( x + w)] #crop detected face #------------------------------------- detected_faces.append((x, y, w, h)) face_index = face_index + 1 #------------------------------------- if face_detected == True and face_included_frames == frame_threshold and freeze == False: freeze = True #base_img = captured_frame.copy() base_img = raw_img.copy() detected_faces_final = detected_faces.copy() tic = time.time() if freeze == True: toc = time.time() if (toc - tic) < time_threshold: if freezed_frame == 0: freeze_img = base_img.copy() #freeze_img = np.zeros(resolution, np.uint8) #here, np.uint8 handles showing white area issue for detected_face in detected_faces_final: x = detected_face[0] y = detected_face[1] w = detected_face[2] h = detected_face[3] cv2.rectangle(freeze_img, (x, y), (x + w, y + h), (67, 67, 67), 1) #draw rectangle to main image #------------------------------- #apply deep learning for custom_face custom_face = base_img[y:y + h, x:x + w] #------------------------------- #facial attribute analysis if enable_face_analysis == True: gray_img = functions.preprocess_face( img=custom_face, target_size=(48, 48), grayscale=True, enforce_detection=False) emotion_labels = [ 'Angry', 'Disgust', 'Fear', 'Happy', 'Sad', 'Surprise', 'Neutral' ] emotion_predictions = emotion_model.predict( gray_img)[0, :] sum_of_predictions = emotion_predictions.sum() mood_items = [] for i in range(0, len(emotion_labels)): mood_item = [] emotion_label = emotion_labels[i] emotion_prediction = 100 * emotion_predictions[ i] / sum_of_predictions mood_item.append(emotion_label) mood_item.append(emotion_prediction) mood_items.append(mood_item) emotion_df = pd.DataFrame( mood_items, columns=["emotion", "score"]) emotion_df = emotion_df.sort_values( by=["score"], ascending=False).reset_index(drop=True) #background of mood box #transparency overlay = freeze_img.copy() opacity = 0.1 if x + w + pivot_img_size < resolution_x: #right cv2.rectangle( freeze_img #, (x+w,y+20) , (x + w, y), (x + w + pivot_img_size, y + h), (208, 189, 0), cv2.FILLED) #TEAL cv2.addWeighted( overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img, ) elif x - pivot_img_size > 0: #left cv2.rectangle( freeze_img #, (x-pivot_img_size,y+20) , (x - pivot_img_size, y), (x, y + h), (208, 189, 0), cv2.FILLED) #TEAL cv2.addWeighted(overlay, opacity, freeze_img, 1 - opacity, 0, freeze_img) for index, instance in emotion_df.iterrows(): emotion_label = "%s " % (instance['emotion']) emotion_score = instance['score'] / 100 bar_x = 35 #this is the size if an emotion is 100% bar_x = int(bar_x * emotion_score) if x + w + pivot_img_size < resolution_x: text_location_y = y + 20 + (index + 1) * 20 text_location_x = x + w if text_location_y < y + h: cv2.putText( freeze_img, emotion_label, (text_location_x, text_location_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) cv2.rectangle( freeze_img, (x + w + 70, y + 13 + (index + 1) * 20), (x + w + 70 + bar_x, y + 13 + (index + 1) * 20 + 5), (255, 255, 255), cv2.FILLED) elif x - pivot_img_size > 0: text_location_y = y + 20 + (index + 1) * 20 text_location_x = x - pivot_img_size if text_location_y <= y + h: cv2.putText( freeze_img, emotion_label, (text_location_x, text_location_y), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 1) cv2.rectangle( freeze_img, (x - pivot_img_size + 70, y + 13 + (index + 1) * 20), (x - pivot_img_size + 70 + bar_x, y + 13 + (index + 1) * 20 + 5), (255, 255, 255), cv2.FILLED) #------------------------------- # time_left = int(time_threshold - (toc - tic) + 1) # countdown box # cv2.rectangle(freeze_img, (10, 10), (90, 50), (208,189,0), -10) # cv2.putText(freeze_img, str(time_left), (40, 40), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 1) cv2.imshow('Analysis Report', freeze_img) freezed_frame = freezed_frame + 1 else: face_detected = False face_included_frames = 0 freeze = False freezed_frame = 0 imgbytes = cv2.imencode('.png', captured_frame)[1].tobytes( ) # Have to add this and also chnage parrameter # ---------------------------- THE GUI ---------------------------- sg.theme('Reddit') if not win_started: win_started = True layout = [ [sg.Text('Facial Analysis in Progress', size=(30, 1))], [sg.Image(data=imgbytes, key='_IMAGE_')], # THIS IS THE ACTUAL CV FEED WINDOW [sg.Button('Stop')] ] window = sg.Window('Emotion Detection', layout, default_element_size=(14, 1), text_justification='left', auto_size_text=False, font='helvetica', icon=icon_logo64, finalize=True) image_elem = window['_IMAGE_'] else: image_elem.update(data=imgbytes) event, values = window.read(timeout=0) if event is None or event == 'Stop': break window.close() #kill open cv things captured_video.release() cv2.destroyAllWindows()