class DeepFace: def __init__(self): self.pre_model = create_model() self.pre_model.load_weights('./deep/weights/nn4.small2.v1.h5') self.alignment = AlignDlib('./deep/models/landmarks.dat') self.metadata = None self.embedded = None self.classifier = None def pre_train(self, path): self.metadata = load_metadata(path) self.embedded = np.zeros((self.metadata.shape[0], 128)) for i, m in enumerate(self.metadata): img = load_image(m.image_path()) img = self.align_image(img) if img is None: continue # scale RGB values to interval [0,1] img = (img / 255.).astype(np.float32) # obtain embedding vector for image self.embedded[i] = self.pre_model.predict( np.expand_dims(img, axis=0))[0] targets = np.array([m.name for m in self.metadata]) # LabelEncoder可以将标签分配一个0—n_classes - 1 之间的编码 self.encoder = LabelEncoder() self.encoder.fit(targets) # 将各种标签分配一个可数的连续编号 self.y = self.encoder.transform(targets) def align_image(self, img): return self.alignment.align( 96, img, self.alignment.getLargestFaceBoundingBox(img), landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) def train(self, classifier=LinearSVC): X_train = self.embedded y_train = self.y self.classifier = globals()[classifier]() self.classifier.fit(X_train, y_train) def predict(self, img): img = self.align_image(img) if img is None: return str("UnKnown") img = (img / 255.).astype(np.float32) one_embedded = self.pre_model.predict(np.expand_dims(img, axis=0))[0] example_prediction = self.classifier.predict([one_embedded]) example_identity = self.encoder.inverse_transform( example_prediction)[0] return str(example_identity)
def load_image_alignment(path): alignment = AlignDlib('models/landmarks.dat') img = cv2.imread(path, 1) # detect face and return bounding box bb = alignment.getLargestFaceBoundingBox(img) jc_aligned = alignment.align(96, img, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) return jc_aligned
def main_2(): # Initialize the OpenFace face alignment utility alignment = AlignDlib('models/landmarks.dat') cap = cv2.VideoCapture(0) ret, frame = cap.read() while (True): #b,g,r = cv2.split(frame) #frame = cv2.merge((r,g,b)) #frame = frame[...,::-1] #frame = cv2.resize(frame, (250,250)) # cv2.imwrite('img.jpg', frame) frame = load_image('img.jpg') bb = alignment.getLargestFaceBoundingBox(frame) # Transform image using specified face landmark indices and crop image to 96x96 jc_aligned = alignment.align( 96, frame, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # Show original image plt.subplot(131) plt.imshow(frame) # Show original image with bounding box plt.subplot(132) plt.imshow(frame) plt.gca().add_patch( patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red')) # Show aligned image plt.subplot(133) plt.imshow(jc_aligned) plt.show() if cv2.waitKey(1) & 0xFF == ord('y'): #save on pressing 'y' cv2.destroyAllWindows() break _, frame = cap.read()
def plot_sample(): metadata = load_metadata('../data/chapter7/images/') # 初始化OpenFace人脸对齐工具,使用Dlib提供的68个关键点 alignment = AlignDlib('../data/chapter7/landmarks.dat') # 加载一张训练图像 img = load_image(metadata[45].get_image_path()) # 检测人脸并返回边框 bounding_box = alignment.getLargestFaceBoundingBox(img) # 使用指定的人脸关键点转换图像并截取96*96的人脸图像 aligned_img = alignment.align(96, img, bounding_box, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # 绘制原图 plt.subplot(1, 3, 1) plt.imshow(img) plt.xticks([]) plt.yticks([]) # 绘制带人脸边框的原图 plt.subplot(1, 3, 2) plt.imshow(img) plt.gca().add_patch(patches.Rectangle( (bounding_box.left(), bounding_box.top()), bounding_box.width(), bounding_box.height(), fill=False, color='red' )) plt.xticks([]) plt.yticks([]) # 绘制对齐后截取的96*96人脸图像 plt.subplot(1, 3, 3) plt.imshow(aligned_img) plt.xticks([]) plt.yticks([]) plt.show()
metadata = load_metadata('images') # 人脸检测、对齐和提取 import matplotlib.pyplot as plt import matplotlib.patches as patches from align import AlignDlib # 初始化 OpenFace 人脸对齐工具,使用 Dlib 提供的 68 个关键点 alignment = AlignDlib('landmarks.dat') # 加载一张训练图像 img = load_image(metadata[0].image_path()) # 检测人脸并返回边框 bb = alignment.getLargestFaceBoundingBox(img) # 使用指定的人脸关键点转换图像并截取 96x96 的人脸图像 aligned_img = alignment.align(96, img, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # 绘制原图 # plt.figure(1) # plt.subplot(131) # plt.imshow(img) # plt.xticks([]) # plt.yticks([]) # # 绘制带人脸边框的原图 # plt.subplot(132) # plt.imshow(img) # plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red'))
class WebcamTest(): def __init__(self): # Initialize the dlib face alignment utility self.alignment = AlignDlib('shape_predictor_68_face_landmarks.dat') self.detector = dlib.get_frontal_face_detector() # Load the model self.model = load_model('weights_final.hdf5', custom_objects={ 'triplet_loss': self.triplet_loss, 'tf': tf }) # Get the web camera feed self.cap = cv2.VideoCapture(0) # Defining variables self.threshold = 0.7 self.base_images = [] self.distances = [] self.set_new_person = False self.saving = False self.pressed = 0 self.next_base_image = 0 self.names = [] self.saved_images = [] self.counter = 0 # Defining the path for image saving self.path = os.getcwd() + '\persons' # Delete previous directories if os.path.isdir(self.path): shutil.rmtree(self.path, onerror=self.onerror) # Create the 'persons' directory if not os.path.isdir(self.path): os.mkdir(self.path) def onerror(self, func, path, exc_info): """ Error handler for `shutil.rmtree`. If the error is due to an access error (read only file) it attempts to add write permission and then retries. If the error is for another reason it re-raises the error. Usage : `shutil.rmtree(path, onerror=onerror)` """ import stat if not os.access(path, os.W_OK): # Is the error an access error ? os.chmod(path, stat.S_IWUSR) func(path) else: raise # take a bounding predicted by dlib and convert it # to the format (x, y, w, h) as we would normally do # with OpenCV def rect_to_bb(self, rect): x = rect.left() y = rect.top() w = rect.right() - x h = rect.bottom() - y # return a tuple of (x, y, w, h) return (x, y, w, h) # The model calculates the distance between the two images def prediction(self, network, pic_1, pic_2): # increase img dimensions img1 = np.expand_dims(pic_1, axis=0) img1 = np.expand_dims(img1, axis=3) img2 = np.expand_dims(pic_2, axis=0) img2 = np.expand_dims(img2, axis=3) # calculate the network's prediction on img1 and img2 preds = network.predict([img1, img2, img1])[0] pred1 = preds[:128] pred2 = preds[128:256] # calculate the distance between the two images dist = np.sum(np.square(pred1 - pred2)) # print("distance between pictures: {:2.4f}".format(dist)) return dist def triplet_loss(self, y_true, y_pred, alpha=0.2): """ Implementation of the triplet loss function Arguments: y_true -- true labels, required when you define a loss in Keras, you don't need it in this function. y_pred -- python list containing three objects: anchor -- the encodings for the anchor data positive -- the encodings for the positive data (similar to anchor) negative -- the encodings for the negative data (different from anchor) Returns: loss -- real number, value of the loss """ anchor = y_pred[:, :128] positive = y_pred[:, 128:256] negative = y_pred[:, 256:] # distance between the anchor and the positive pos_dist = K.sum(K.square(anchor - positive), axis=1) # distance between the anchor and the negative neg_dist = K.sum(K.square(anchor - negative), axis=1) # compute loss basic_loss = pos_dist - neg_dist + alpha loss = K.maximum(basic_loss, 0.0) return loss # Aligns the given image def image_processing(self, image): # Detect face and return bounding box rect = self.alignment.getLargestFaceBoundingBox(image) img_aligned = self.alignment.align( 64, image, rect, landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP) return img_aligned # Saves the given image with the given name def save_pictures(self, image, path, name, extension): img_item = path + '\\' + name + "." + extension cv2.imwrite(img_item, image) # The main part of the class def cycle(self): # Capture frame-by-frame ret, frame = self.cap.read() # Turn the image into a greyscale gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Detect the faces rects = self.detector(gray, 1) for rect in rects: (x, y, w, h) = self.rect_to_bb(rect) roi_gray = gray[y:y + h, x:x + w] roi_color = frame[y:y + h, x:x + w] # Align the image aligned_image = self.alignment.align( 64, gray, rect, landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP) # If the alignment was succesful if aligned_image is not None: # If there is a new recording save the recorded images (the saving part is done in the 'webacm_app.py') if self.set_new_person: self.counter += 1 if self.counter % 10 == 0: if len(self.saved_images) < 10: self.saved_images.append(aligned_image) else: self.counter = 0 # Create a directory for the new person and save the person's images if self.saving: self.path_person = self.path + '\\' + self.names[-1] os.mkdir(self.path_person) for (i, picture) in enumerate(self.base_images[-1]): self.save_pictures(picture, self.path_person, self.names[-1] + '_' + str(i), 'jpg') self.saving = False # Calculate the average distance from the save distance for i in range(len(self.base_images)): average_distance = 0 for base_image in self.base_images[i]: average_distance += self.prediction( self.model, aligned_image / 255.0, base_image / 255.0) self.distances[i] = average_distance / len( self.base_images[i]) # If the picture alignment was unsucessful, then the distance belonging to the picture is four else: for i in range(len(self.distances)): self.distances[i] = 4 if self.set_new_person: color = (0, 255, 0) else: color = (255, 0, 0) # BGR stroke = 2 end_cord_x = x + w end_cord_y = y + h cv2.rectangle(frame, (x, y), (end_cord_x, end_cord_y), color, stroke) min_index = 0 # Calculate the minimum distances from all person for (i, distance) in enumerate(self.distances): if distance < self.distances[min_index]: min_index = i # Display the distance values on the feed if len(self.distances ) > 0 and self.distances[min_index] < self.threshold: font = cv2.FONT_HERSHEY_SIMPLEX name = self.names[min_index] + "{:2.2f}".format( self.distances[min_index]) color = (255, 255, 255) stroke = 1 cv2.putText(frame, name, (x, y), font, 1, color, stroke, cv2.LINE_AA) else: font = cv2.FONT_HERSHEY_SIMPLEX color = (255, 255, 255) stroke = 1 if len(self.distances) > 0: name = "{:2.2f}".format(self.distances[min_index]) cv2.putText(frame, name, (x, y), font, 1, color, stroke, cv2.LINE_AA) return frame
y2=1 font=cv2.FONT_HERSHEY_SIMPLEX embedded = 0; cap = cv2.VideoCapture(0) cap.open(0) while(True): try: # Capture frame-by-frame ret, frame = cap.read() flipped=cv2.flip(frame,1) bb = alignment.getLargestFaceBoundingBox(flipped) y1=bb.top() x1=bb.left() y2=bb.top()+bb.height() x2=bb.left()+bb.width() x3=bb.left() y3=y2+50 aligned = alignment.align(96, flipped, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) img = (aligned / 255.).astype(np.float32) embedded= nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0] example_prediction = svc.predict([embedded]) example_identity = encoder.inverse_transform(example_prediction)[0] cv2.putText(flipped,example_identity,(x3,y3),font,1,(0,0,255),1,cv2.LINE_AA)
def findd(): import cv2 import matplotlib.pyplot as plt import matplotlib.patches as patches import os os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' import PIL.Image as Image import numpy as np from model import create_model from keras import backend as K import tensorflow as tf from align import AlignDlib from keras import backend as K tf.keras.backend.clear_session() K.clear_session() alignment = AlignDlib('models/landmarks.dat') # img= load_image('E:\\projectImplementation\\projectFiles\\imageee.png') img = cv2.imread('E:\\projectImplementation\\projectFiles\\imageee.png', 1) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) bb = alignment.getLargestFaceBoundingBox(img) im_aligned = alignment.align(96, img, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) anjani = np.load('anjani.npy') aman = np.load('aman.npy') rapaka = np.load('rapaka.npy') akc = np.load('akc.npy') abhi = np.load('abhishek.npy') hemanth = np.load('hemanth.npy') subham = np.load('subham.npy') vivek = np.load('vivek.npy') nn4_small2_pretrained = create_model() nn4_small2_pretrained.load_weights('nn4.small2.v1.h5') im_aligned = (im_aligned / 255.).astype(np.float32) im_aligned = np.expand_dims(im_aligned, axis=0) embedded = nn4_small2_pretrained.predict(im_aligned)[0] # print(embedded) # tf.keras.backend.clear_session() # K.clear_session() distances = [] distances.append(np.linalg.norm(embedded - anjani)) distances.append(np.linalg.norm(embedded - abhi)) distances.append(np.linalg.norm(embedded - subham)) distances.append(np.linalg.norm(embedded - rapaka)) distances.append(np.linalg.norm(embedded - hemanth)) distances.append(np.linalg.norm(embedded - aman)) distances.append(np.linalg.norm(embedded - akc)) fin = min(distances) if (fin == distances[0]): return "Anjani" if (fin == distances[1]): return "Abhishek" if (fin == distances[2]): return "Subham" if (fin == distances[3]): return "Rapaka" if (fin == distances[4]): return "Hemanth" if (fin == distances[5]): return "Aman" if (fin == distances[6]): return "Akshay"
from align import AlignDlib from urllib.request import urlopen from model import create_model from keras.models import model_from_json import numpy as np cap = cv2.VideoCapture("rtsp://192.168.7.137:10085") loaded_model = create_model() loaded_model.load_weights("./weights/model.h5") while (True): # Capture frame-by-frame alignment = AlignDlib('models/landmarks.dat') ret, frame = cap.read() #img = frame # Our operations on the frame come here gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) bb = alignment.getLargestFaceBoundingBox(frame) jc_aligned = alignment.align(96, frame, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) #Display the resulting frame jc_aligned = np.expand_dims(jc_aligned, axis=0) print(jc_aligned.shape) a = np.argmax(loaded_model.predict(jc_aligned)) + 1 cv2.rectangle(frame, (bb.left(), bb.top()), (bb.right(), bb.right() + (bb.top() - bb.left())), (0, 0, 255), 10) cv2.imshow('anh thuoc nhom:' + str(a), frame) if cv2.waitKey(1) & 0xFF == ord('q'): break # When everything done, release the capture
class Trainer(): def __init__(self): # self.image_dir = sys.argv[1] # self.image_dir = "/home/webwerks/projects/project-face-recognition/23-07/face_detection/uploads" self.image_dir = "/home/webwerks/projects/project-face-recognition/23-07/face_detection/src/images" self.model = self.create_model() self.alignment = AlignDlib('models/landmarks.dat') def create_model(self): nn4_small2_pretrained = create_model() nn4_small2_pretrained.load_weights('weights/nn4.small2.v1.h5') return nn4_small2_pretrained def load_metadata(self, path): print("loading metadata ...") metadata = [] for i in sorted(os.listdir(path)): for f in sorted(os.listdir(os.path.join(path, i))): # Check file extension. Allow only jpg/jpeg' files. ext = os.path.splitext(f)[1].lower() if ext == '.jpg' or ext == '.jpeg' or ext == '.png': metadata.append(IdentityMetadata(path, i, f)) return np.array(metadata) def load_image(self, path): img = cv2.imread(path, 1) return img def align_image(self, img): return self.alignment.align( 96, img, self.alignment.getLargestFaceBoundingBox(img), landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) def train(self): metadata = self.load_metadata(self.image_dir) np.save('train-files/metadata.npy', metadata) print("creating embeddings ...") embedded = {} for i, m in enumerate(metadata): img = self.load_image(m.image_path()) img = self.align_image(img) if img is not None: # scale RGB values to interval [0,1] img = (img / 255.).astype(np.float32) # obtain embedding vector for image embedded[m.image_path()] = self.model.predict( np.expand_dims(img, axis=0))[0] pickle.dump(embedded, open("train-files/embeddings.pkl", "wb")) def train_classifier(self): print("training classifier ...") embedded = pickle.load(open("train-files/embeddings.pkl", "rb")) metadata = np.load('train-files/metadata.npy') targets = np.array([m.name for m in metadata]) encoder = LabelEncoder() encoder.fit(targets) pickle.dump(encoder, open("train-files/encoder.pkl", "wb")) # Numerical encoding of identities y = encoder.transform(targets) train_idx = np.arange(metadata.shape[0]) % 2 != 0 test_idx = np.arange(metadata.shape[0]) % 2 == 0 X_train = embedded[train_idx] X_test = embedded[test_idx] y_train = y[train_idx] y_test = y[test_idx] svc = LinearSVC() svc.fit(X_train, y_train) joblib.dump(svc, 'train-files/svc.pkl') acc_svc = accuracy_score(y_test, svc.predict(X_test)) print(f'SVM accuracy = {acc_svc}')
def align_image(img): alignment = AlignDlib('models/landmarks.dat') return alignment.align(96, img, alignment.getLargestFaceBoundingBox(img), landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
class FaceRecognizer(): def __init__(self): dst_dir = 'models' dst_file = os.path.join(dst_dir, 'landmarks.dat') if not os.path.exists(dst_file): os.makedirs(dst_dir) download_landmarks(dst_file) # Create CNN model and load pretrained weights (OpenFace nn4.small2) self.nn4_small2_pretrained = create_model() self.nn4_small2_pretrained.load_weights('models/nn4.small2.v1.h5') self.metadata = self.load_metadata('faces') # Initialize the OpenFace face alignment utility self.alignment = AlignDlib('models/landmarks.dat') # Get embedding vectorsf # self.embedded = np.zeros((self.metadata.shape[0], 128)) self.embedded = np.zeros((0, 128)) # Train images custom_metadata = self.load_metadata("faces") self.metadata = np.append(self.metadata, custom_metadata) self.update_embeddings() self.train_images() # Download dlib face detection landmarks file def download_landmarks(self, dst_file): url = 'http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2' decompressor = bz2.BZ2Decompressor() with urlopen(url) as src, open(dst_file, 'wb') as dst: data = src.read(1024) while len(data) > 0: dst.write(decompressor.decompress(data)) data = src.read(1024) def load_metadata(self, path): ds_store = ".DS_Store" metadata = [] dirs = os.listdir(path) if ds_store in dirs: dirs.remove(ds_store) for i in dirs: subdirs = os.listdir(os.path.join(path, i)) if ds_store in subdirs: subdirs.remove(ds_store) for f in subdirs: metadata.append(IdentityMetadata(path, i, f)) return np.array(metadata) # Align helper functions def get_face_thumbnail(self, img): return self.alignment.getLargestFaceThumbnail( 96, img, self.alignment.getLargestFaceBoundingBox(img), landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) def get_all_face_thumbnails_and_scores(self, img): return self.alignment.getAllFaceThumbnailsAndScores( 96, img, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) def get_face_vector(self, img, is_thumbnail=False): if not is_thumbnail: img = self.get_face_thumbnail(img) # scale RGB values to interval [0,1] img = (img / 255.).astype(np.float32) # obtain embedding vector for image return self.nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0] def get_face_vectors(self, img): face_thumbnails, scores, face_types = self.get_all_face_thumbnails_and_scores( img) face_vectors = [] for face_img in face_thumbnails: # scale RGB values to interval [0,1] face_img = (face_img / 255.).astype(np.float32) # obtain embedding vector for image vector = self.nn4_small2_pretrained.predict( np.expand_dims(face_img, axis=0))[0] face_vectors.append(vector) return face_vectors, face_thumbnails, scores, face_types # Train classifier models def train_images(self, train_with_all_samples=False): self.targets = np.array([m.name for m in self.metadata]) start = time.time() self.encoder = LabelEncoder() self.encoder.fit(self.targets) # Numerical encoding of identities y = self.encoder.transform(self.targets) if train_with_all_samples == False: train_idx = np.arange(self.metadata.shape[0]) % 2 != 0 else: train_idx = np.full(self.metadata.shape[0], True) self.test_idx = np.arange(self.metadata.shape[0]) % 2 == 0 # 50 train examples of 10 identities (5 examples each) X_train = self.embedded[train_idx] # 50 test examples of 10 identities (5 examples each) X_test = self.embedded[self.test_idx] y_train = y[train_idx] y_test = y[self.test_idx] self.knn = KNeighborsClassifier(n_neighbors=1, metric='euclidean') self.svc = LinearSVC() #class_weight='balanced') self.knn.fit(X_train, y_train) self.svc.fit(X_train, y_train) acc_knn = accuracy_score(y_test, self.knn.predict(X_test)) acc_svc = accuracy_score(y_test, self.svc.predict(X_test)) if train_with_all_samples == False: print(f'KNN accuracy = {acc_knn}, SVM accuracy = {acc_svc}') else: print('Trained classification models with all image samples') end = time.time() print("train_images took {} secs".format(end - start)) def update_embeddings(self): for i, m in enumerate(self.metadata): print("loading image from {}".format(m.image_path())) img = load_image(m.image_path()) is_thumbnail = "customer_" in m.image_path() vector = self.get_face_vector(img, is_thumbnail) vector = vector.reshape(1, 128) self.embedded = np.append(self.embedded, vector, axis=0) def label_cv2_image_faces(self, rgb_img, face_bbs, identities): # Convert RGB back to cv2 RBG format img = rgb_img[:, :, ::-1] for i, bb in enumerate(face_bbs): # Draw bounding rectangle around face cv2.rectangle(img, (bb.left(), bb.top()), (bb.right(), bb.bottom()), (0, 0, 255), 2) # Draw a label with a name below the face cv2.rectangle(img, (bb.left(), bb.bottom() - 35), (bb.right(), bb.bottom()), (0, 0, 255), cv2.FILLED) font = cv2.FONT_HERSHEY_DUPLEX cv2.putText(img, identities[i], (bb.left() + 6, bb.bottom() - 6), font, 1.0, (255, 255, 255), 1) return img def display_cv2_image_faces(self, rgb_img, face_bbs, identities): img = label_cv2_image_faces(rgb_img, face_bbs, identities) display_cv2_image(img) def display_plt_image_faces(self, img, face_bbs, identities, subplot=111): plt.subplot(subplot) plt.figure() plt.imshow(img) for bb in face_bbs: plt.gca().add_patch( patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red')) # TODO: Print identities in correct order plt.title(f'Recognized as {identities}') def save_unknown_face(self, face_vector, face_thumbnail): print("Saving unknown face...") dirs = os.listdir("faces") customer_dirs = [dir for dir in dirs if "customer_" in dir] if len(customer_dirs) > 0: dir_indexes = [int(dir.split("_")[1]) for dir in customer_dirs] curr_index = max(dir_indexes) + 1 else: curr_index = 1 # Save image to customer dir # TODO: Remove requirement for double-creation of all data customer_dir = "customer_{}".format(curr_index) dir_path = os.path.join("faces", customer_dir) os.mkdir(dir_path) for i in range(0, 8): customer_file = "customer_{}_{}.jpg".format(curr_index, i + 1) file_path = os.path.join(dir_path, customer_file) imageio.imwrite(file_path, face_thumbnail) metadata = np.append( self.metadata, IdentityMetadata("custom", customer_dir, customer_file)) embedded = np.append(self.embedded, face_vector.reshape(1, 128), axis=0) print("Saved unknown face") def distance(self, emb1, emb2): return np.sum(np.square(emb1 - emb2)) def get_distances(self, vector): distances = [] for embed in self.embedded: distances.append(distance(embed, vector)) return distances def get_sharpness_level(self, image): grey = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) # compute the Laplacian of the image and then return the focus # measure, which is simply the variance of the Laplacian return cv2.Laplacian(grey, cv2.CV_64F).var() def identify_image_faces(self, example_image): vectors, thumbnails, dlib_scores, face_types = self.get_face_vectors( example_image) identities = [] saved_unknown = False for i, vector in enumerate(vectors): vector = vector.reshape(1, 128) confidence_scores = self.svc.decision_function(vector) if (confidence_scores.max() < MIN_CONFIDENCE_SCORE): sharpness_level = self.get_sharpness_level(thumbnails[i]) example_identity = "Unknown" #example_identity = "Unknown ({:0.2f}, {}, {:0.2f})".format(dlib_scores[i], face_types[i], sharpness_level) print( "Unknown face - dlib score={:0.2f}, face_type={}, sharpness_level={:0.2f}" .format(dlib_scores[i], face_types[i], sharpness_level)) if RECOGNIZE_UNKNOWN_FACES: # Only save (and train) a good-quality and front-facing face if dlib_scores[i] >= MIN_DLIB_SCORE and face_types[ i] == 0 and sharpness_level >= MIN_SHARPNESS_LEVEL: saved_unknown = True print("Saving unknown face") save_unknown_face(vector, thumbnails[i]) else: example_prediction = self.svc.predict(vector) example_identity = self.encoder.inverse_transform( example_prediction)[0] identities.append(example_identity) # Add to training model if any unknown faces were saved if saved_unknown: train_images() # Detect faces and return bounding boxes face_bbs = self.alignment.getAllFaceBoundingBoxes(example_image) return face_bbs, identities def display_unknown_image(self, image_path): img = load_image(image_path) face_bbs, identities = self.identify_image_faces(img) #display_cv2_image_faces(img, face_bbs, identities) self.display_plt_image_faces(img, face_bbs, identities) def display_image_prediction(self, example_idx): example_image = load_image( self.metadata[self.test_idx][example_idx].image_path()) example_prediction = self.knn.predict( [self.embedded[self.test_idx][example_idx]]) example_identity = self.encoder.inverse_transform( example_prediction)[0] # Detect face and return bounding box #bb = alignment.getLargestFaceBoundingBox(example_image) plt.imshow(example_image) #plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red')) plt.title(f'Recognized as {example_identity}') def visualize_dataset(self): X_embedded = TSNE(n_components=2).fit_transform(self.embedded) plt.figure() for i, t in enumerate(set(self.targets)): idx = self.targets == t plt.scatter(X_embedded[idx, 0], X_embedded[idx, 1], label=t) plt.legend(bbox_to_anchor=(1, 1))
# encoder = pickle.load(open(encoderfile, 'rb')) alignment = AlignDlib('models/landmarks.dat') # modelfile = "svm_one.sav" # svc = LinearSVC() # svc = pickle.load(open(modelfile, 'rb')) cap = cv2.VideoCapture(-1) loop = 5e10 while(True): try: # Capture frame-by-frame ret, jc_orig = cap.read() #jc_orig = cv2.resize(jc_orig, dim, interpolation = cv2.INTER_AREA) bb = alignment.getLargestFaceBoundingBox(jc_orig)##------------------------return this print(bb) #if(bb==None): # continue #print(bb.left()) x = (bb.left() + bb.right())//2 y = (bb.top() + bb.bottom())//2 #w = bb.right() - bb.left() #h = bb.bottom() - bb.top() #centre = (jc_orig.shape[0] + bb[0], jc_orig.shape[1] + bb[1]) image= cv2.circle (jc_orig, (x,y), radius = 1, color = (0,0,255), thickness = -1) # jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # jc_aligned = (jc_aligned /255.).astype(np.float32) # embedded = nn4_small2_pretrained.predict(np.expand_dims(jc_aligned, axis=0)) # prediction = svc.predict(embedded)
def main(predict_eg): nn4_small2 = create_model() # Input for anchor, positive and negative images input_a = Input(shape=(96, 96, 3)) input_p = Input(shape=(96, 96, 3)) input_n = Input(shape=(96, 96, 3)) # Output for anchor, positive and negative embedding vectors emb_a = nn4_small2(input_a) emb_p = nn4_small2(input_p) emb_n = nn4_small2(input_n) # Layer that computes the triplet loss from anchor, positive and negative embedding vectors triplet_loss_layer = TripletLossLayer.TripletLossLayer( alpha=0.2, name='triplet_loss_layer')([emb_a, emb_p, emb_n]) # Model that can be trained with anchor, positive negative images nn4_small2_train = Model([input_a, input_p, input_n], triplet_loss_layer) # triplet_generator() creates a generator that continuously returns # ([a_batch, p_batch, n_batch], None) tuples where a_batch, p_batch # and n_batch are batches of anchor, positive and negative RGB images # each having a shape of (batch_size, 96, 96, 3). generatorImage = triplet_generator() nn4_small2_train.compile(loss=None, optimizer='adam') nn4_small2_train.fit_generator(generatorImage, epochs=10, steps_per_epoch=100) # nn4_small2_train.fit_generator(generatorImage, epochs=2, steps_per_epoch=100) # Please note that the current implementation of the generator only generates # random image data. The main goal of this code snippet is to demonstrate # the general setup for model training. In the following, we will anyway # use a pre-trained model so we don't need a generator here that operates # on real training data. I'll maybe provide a fully functional generator # later. nn4_small2_pretrained = create_model() nn4_small2_pretrained.load_weights('weights/nn4.small2.v1.h5') metadata = load_metadata('images') # Initialize the OpenFace face alignment utility alignment = AlignDlib('models/landmarks.dat') # Load any random image jc_orig = load_image(metadata[77].image_path()) # Detect face and return bounding box bb = alignment.getLargestFaceBoundingBox(jc_orig) # Transform image using specified face landmark indices and crop image to 96x96 jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # Show original image plt.subplot(131) plt.imshow(jc_orig) plt.title("Orig Image") # plt.show() # Show original image with bounding box plt.subplot(132) plt.imshow(jc_orig) plt.title("with BB") # plt.show() plt.gca().add_patch( patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red')) # Show aligned image plt.subplot(133) plt.imshow(jc_aligned) plt.title("Aligned Image") plt.show() def align_image(img): return alignment.align(96, img, alignment.getLargestFaceBoundingBox(img), landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) embedded = np.zeros((metadata.shape[0], 128)) for i, m in enumerate(metadata): img = load_image(m.image_path()) img = align_image(img) # scale RGB values to interval [0,1] img = (img / 255.).astype(np.float32) # obtain embedding vector for image embedded[i] = nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0] def show_pair(idx1, idx2): plt.figure(figsize=(8, 3)) plt.suptitle( f'Distance = {distance(embedded[idx1], embedded[idx2]):.2f}') plt.subplot(121) plt.imshow(load_image(metadata[idx1].image_path())) # plt.show() plt.subplot(122) plt.imshow(load_image(metadata[idx2].image_path())) plt.show() show_pair(77, 78) show_pair(77, 50) distances = [] # squared L2 distance between pairs identical = [] # 1 if same identity, 0 otherwise num = len(metadata) for i in range(num - 1): for j in range(i + 1, num): distances.append(distance(embedded[i], embedded[j])) identical.append(1 if metadata[i].name == metadata[j].name else 0) distances = np.array(distances) identical = np.array(identical) thresholds = np.arange(0.3, 1.0, 0.01) #F1 Score f1_scores = [f1_score(identical, distances < t) for t in thresholds] acc_scores = [accuracy_score(identical, distances < t) for t in thresholds] opt_idx = np.argmax(f1_scores) # Threshold at maximal F1 score opt_tau = thresholds[opt_idx] # Accuracy at maximal F1 score opt_acc = accuracy_score(identical, distances < opt_tau) # Plot F1 score and accuracy as function of distance threshold plt.plot(thresholds, f1_scores, label='F1 score') plt.plot(thresholds, acc_scores, label='Accuracy') plt.axvline(x=opt_tau, linestyle='--', lw=1, c='lightgrey', label='Threshold') plt.title(f'Accuracy at threshold {opt_tau:.2f} = {opt_acc:.3f}') plt.xlabel('Distance threshold') plt.legend() plt.show() dist_pos = distances[identical == 1] dist_neg = distances[identical == 0] plt.figure(figsize=(12, 4)) plt.subplot(121) plt.hist(dist_pos) plt.axvline(x=opt_tau, linestyle='--', lw=1, c='lightgrey', label='Threshold') plt.title('Distances (+ve pairs)') plt.legend() plt.subplot(122) plt.hist(dist_neg) plt.axvline(x=opt_tau, linestyle='--', lw=1, c='lightgrey', label='Threshold') plt.title('Distances (-ve pairs)') plt.legend() plt.show() targets = np.array([m.name for m in metadata]) encoder = LabelEncoder() encoder.fit(targets) # Numerical encoding of identities y = encoder.transform(targets) train_idx = np.arange(metadata.shape[0]) % 2 != 0 test_idx = np.arange(metadata.shape[0]) % 2 == 0 # 50 train examples of 10 identities (5 examples each) X_train = embedded[train_idx] # 50 test examples of 10 identities (5 examples each) X_test = embedded[test_idx] y_train = y[train_idx] y_test = y[test_idx] knn = KNeighborsClassifier(n_neighbors=1, metric='euclidean') svc = LinearSVC() knn.fit(X_train, y_train) svc.fit(X_train, y_train) acc_knn = accuracy_score(y_test, knn.predict(X_test)) acc_svc = accuracy_score(y_test, svc.predict(X_test)) print(f'KNN accuracy = {acc_knn}, SVM accuracy = {acc_svc}') warnings.filterwarnings('ignore') example_idx = predict_eg print("example_idx", example_idx) example_image = load_image(metadata[test_idx][example_idx].image_path()) example_prediction = svc.predict([embedded[test_idx][example_idx]]) example_identity = encoder.inverse_transform(example_prediction)[0] plt.imshow(example_image) plt.title(f'Recognized as {example_identity}') plt.show() X_embedded = TSNE(n_components=2).fit_transform(embedded) for i, t in enumerate(set(targets)): idx = targets == t plt.scatter(X_embedded[idx, 0], X_embedded[idx, 1], label=t) plt.legend(bbox_to_anchor=(1, 1)) plt.show()
#get_ipython().run_line_magic('matplotlib', 'inline') def load_image(path): img = cv2.imread(path, 1) # OpenCV loads images with color channels # in BGR order. So we need to reverse them return img[...,::-1] # Initialize the OpenFace face alignment utility alignment = AlignDlib('models/landmarks.dat') # Load an image of Jacques Chirac jc_orig = load_image(metadata[6].image_path()) # Detect face and return bounding box bb = alignment.getLargestFaceBoundingBox(jc_orig) # Transform image using specified face landmark indices and crop image to 96x96 jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # Show original image # plt.subplot(131) # plt.imshow(jc_orig) # # Show original image with bounding box # plt.subplot(132) # plt.imshow(jc_orig) # plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red')) # # Show aligned image # plt.subplot(133)
data_num = hdf5_read.root.images.shape[0] prev_label = hdf5_read.root.labels[0] print("Evaluating %s ."%(hdf5_path)) # Blur limit blur_limit = 100 blur = 0 # Iterates through the pictures in a dataset. for i in range(data_num): # Reading the image and the corresponding label. image = hdf5_read.root.images[i] label = hdf5_read.root.labels[i] print("Evaluation: %.3f %%"%(float(i)/float(data_num-1)*100),end="\r", flush=True) # Detect face and return bounding box rect = alignment.getLargestFaceBoundingBox(image) # Transform image using specified face landmark indices and crop image to 64x64 img_aligned = alignment.align(64, image, rect, landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP) # If the aligning was successful if img_aligned is not None: # Check if the picture has to much black (empty) pixels, which would indicate a bad image, because # a face at the edge of a picture could be recognized and aligned even if it's missing part of the face black = 0 # Counting black (empty pixels) for l in range(img_aligned.shape[0]): for k in range(img_aligned.shape[1]): if img_aligned[l,k] == 0: black+=1 # Checking the threshold for black (empty) pixels if black<100: # Calculating blur value.
# open('weights/nn4.small2.myTrain.h5', 'r') # nn4_small2_train.load_weights('weights/nn4.small2.myTrain.h5') # except FileNotFoundError: # # nn4_small2_train = train_model() metadata = load_metadata('images') # Initialize the OpenFace face alignment utility alignment = AlignDlib('models/landmarks.dat') # Load an image of Schwarzenegger jc_orig = load_image(metadata[92].image_path()) # Detect face and return bounding box bb = alignment.getLargestFaceBoundingBox(jc_orig) # Transform image using specified face landmark indices and crop image to 96x96 jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE) # Show original image plt.subplot(131) plt.imshow(jc_orig) # Show original image with bounding box plt.subplot(132) plt.imshow(jc_orig) plt.gca().add_patch(