class DeepFace:
def __init__(self):
self.pre_model = create_model()
self.pre_model.load_weights('./deep/weights/nn4.small2.v1.h5')
self.alignment = AlignDlib('./deep/models/landmarks.dat')
self.metadata = None
self.embedded = None
self.classifier = None
def pre_train(self, path):
self.metadata = load_metadata(path)
self.embedded = np.zeros((self.metadata.shape[0], 128))
for i, m in enumerate(self.metadata):
img = load_image(m.image_path())
img = self.align_image(img)
if img is None:
continue
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
self.embedded[i] = self.pre_model.predict(
np.expand_dims(img, axis=0))[0]
targets = np.array([m.name for m in self.metadata])
# LabelEncoder可以将标签分配一个0—n_classes - 1 之间的编码
self.encoder = LabelEncoder()
self.encoder.fit(targets)
# 将各种标签分配一个可数的连续编号
self.y = self.encoder.transform(targets)
def align_image(self, img):
return self.alignment.align(
96,
img,
self.alignment.getLargestFaceBoundingBox(img),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
def train(self, classifier=LinearSVC):
X_train = self.embedded
y_train = self.y
self.classifier = globals()[classifier]()
self.classifier.fit(X_train, y_train)
def predict(self, img):
img = self.align_image(img)
if img is None:
return str("UnKnown")
img = (img / 255.).astype(np.float32)
one_embedded = self.pre_model.predict(np.expand_dims(img, axis=0))[0]
example_prediction = self.classifier.predict([one_embedded])
example_identity = self.encoder.inverse_transform(
example_prediction)[0]
return str(example_identity)
def load_image_alignment(path):
alignment = AlignDlib('models/landmarks.dat')
img = cv2.imread(path, 1)
# detect face and return bounding box
bb = alignment.getLargestFaceBoundingBox(img)
jc_aligned = alignment.align(96,
img,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
return jc_aligned
def main_2():
# Initialize the OpenFace face alignment utility
alignment = AlignDlib('models/landmarks.dat')
cap = cv2.VideoCapture(0)
ret, frame = cap.read()
while (True):
#b,g,r = cv2.split(frame)
#frame = cv2.merge((r,g,b))
#frame = frame[...,::-1]
#frame = cv2.resize(frame, (250,250))
# cv2.imwrite('img.jpg', frame)
frame = load_image('img.jpg')
bb = alignment.getLargestFaceBoundingBox(frame)
# Transform image using specified face landmark indices and crop image to 96x96
jc_aligned = alignment.align(
96, frame, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# Show original image
plt.subplot(131)
plt.imshow(frame)
# Show original image with bounding box
plt.subplot(132)
plt.imshow(frame)
plt.gca().add_patch(
patches.Rectangle((bb.left(), bb.top()),
bb.width(),
bb.height(),
fill=False,
color='red'))
# Show aligned image
plt.subplot(133)
plt.imshow(jc_aligned)
plt.show()
if cv2.waitKey(1) & 0xFF == ord('y'): #save on pressing 'y'
cv2.destroyAllWindows()
break
_, frame = cap.read()
def plot_sample():
metadata = load_metadata('../data/chapter7/images/')
# 初始化OpenFace人脸对齐工具，使用Dlib提供的68个关键点
alignment = AlignDlib('../data/chapter7/landmarks.dat')
# 加载一张训练图像
img = load_image(metadata[45].get_image_path())
# 检测人脸并返回边框
bounding_box = alignment.getLargestFaceBoundingBox(img)
# 使用指定的人脸关键点转换图像并截取96*96的人脸图像
aligned_img = alignment.align(96, img, bounding_box, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# 绘制原图
plt.subplot(1, 3, 1)
plt.imshow(img)
plt.xticks([])
plt.yticks([])
# 绘制带人脸边框的原图
plt.subplot(1, 3, 2)
plt.imshow(img)
plt.gca().add_patch(patches.Rectangle(
(bounding_box.left(), bounding_box.top()),
bounding_box.width(),
bounding_box.height(),
fill=False,
color='red'
))
plt.xticks([])
plt.yticks([])
# 绘制对齐后截取的96*96人脸图像
plt.subplot(1, 3, 3)
plt.imshow(aligned_img)
plt.xticks([])
plt.yticks([])
plt.show()
metadata = load_metadata('images')
# 人脸检测、对齐和提取
import matplotlib.pyplot as plt
import matplotlib.patches as patches
from align import AlignDlib
# 初始化 OpenFace 人脸对齐工具，使用 Dlib 提供的 68 个关键点
alignment = AlignDlib('landmarks.dat')
# 加载一张训练图像
img = load_image(metadata[0].image_path())
# 检测人脸并返回边框
bb = alignment.getLargestFaceBoundingBox(img)
# 使用指定的人脸关键点转换图像并截取 96x96 的人脸图像
aligned_img = alignment.align(96,
img,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# 绘制原图
# plt.figure(1)
# plt.subplot(131)
# plt.imshow(img)
# plt.xticks([])
# plt.yticks([])
# # 绘制带人脸边框的原图
# plt.subplot(132)
# plt.imshow(img)
# plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red'))
class WebcamTest():
def __init__(self):
# Initialize the dlib face alignment utility
self.alignment = AlignDlib('shape_predictor_68_face_landmarks.dat')
self.detector = dlib.get_frontal_face_detector()
# Load the model
self.model = load_model('weights_final.hdf5',
custom_objects={
'triplet_loss': self.triplet_loss,
'tf': tf
})
# Get the web camera feed
self.cap = cv2.VideoCapture(0)
# Defining variables
self.threshold = 0.7
self.base_images = []
self.distances = []
self.set_new_person = False
self.saving = False
self.pressed = 0
self.next_base_image = 0
self.names = []
self.saved_images = []
self.counter = 0
# Defining the path for image saving
self.path = os.getcwd() + '\persons'
# Delete previous directories
if os.path.isdir(self.path):
shutil.rmtree(self.path, onerror=self.onerror)
# Create the 'persons' directory
if not os.path.isdir(self.path):
os.mkdir(self.path)
def onerror(self, func, path, exc_info):
"""
Error handler for `shutil.rmtree`.
If the error is due to an access error (read only file)
it attempts to add write permission and then retries.
If the error is for another reason it re-raises the error.
Usage : `shutil.rmtree(path, onerror=onerror)`
"""
import stat
if not os.access(path, os.W_OK):
# Is the error an access error ?
os.chmod(path, stat.S_IWUSR)
func(path)
else:
raise
# take a bounding predicted by dlib and convert it
# to the format (x, y, w, h) as we would normally do
# with OpenCV
def rect_to_bb(self, rect):
x = rect.left()
y = rect.top()
w = rect.right() - x
h = rect.bottom() - y
# return a tuple of (x, y, w, h)
return (x, y, w, h)
# The model calculates the distance between the two images
def prediction(self, network, pic_1, pic_2):
# increase img dimensions
img1 = np.expand_dims(pic_1, axis=0)
img1 = np.expand_dims(img1, axis=3)
img2 = np.expand_dims(pic_2, axis=0)
img2 = np.expand_dims(img2, axis=3)
# calculate the network's prediction on img1 and img2
preds = network.predict([img1, img2, img1])[0]
pred1 = preds[:128]
pred2 = preds[128:256]
# calculate the distance between the two images
dist = np.sum(np.square(pred1 - pred2))
# print("distance between pictures: {:2.4f}".format(dist))
return dist
def triplet_loss(self, y_true, y_pred, alpha=0.2):
"""
Implementation of the triplet loss function
Arguments:
y_true -- true labels, required when you define a loss in Keras, you don't need it in this function.
y_pred -- python list containing three objects:
anchor -- the encodings for the anchor data
positive -- the encodings for the positive data (similar to anchor)
negative -- the encodings for the negative data (different from anchor)
Returns:
loss -- real number, value of the loss
"""
anchor = y_pred[:, :128]
positive = y_pred[:, 128:256]
negative = y_pred[:, 256:]
# distance between the anchor and the positive
pos_dist = K.sum(K.square(anchor - positive), axis=1)
# distance between the anchor and the negative
neg_dist = K.sum(K.square(anchor - negative), axis=1)
# compute loss
basic_loss = pos_dist - neg_dist + alpha
loss = K.maximum(basic_loss, 0.0)
return loss
# Aligns the given image
def image_processing(self, image):
# Detect face and return bounding box
rect = self.alignment.getLargestFaceBoundingBox(image)
img_aligned = self.alignment.align(
64,
image,
rect,
landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
return img_aligned
# Saves the given image with the given name
def save_pictures(self, image, path, name, extension):
img_item = path + '\\' + name + "." + extension
cv2.imwrite(img_item, image)
# The main part of the class
def cycle(self):
# Capture frame-by-frame
ret, frame = self.cap.read()
# Turn the image into a greyscale
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Detect the faces
rects = self.detector(gray, 1)
for rect in rects:
(x, y, w, h) = self.rect_to_bb(rect)
roi_gray = gray[y:y + h, x:x + w]
roi_color = frame[y:y + h, x:x + w]
# Align the image
aligned_image = self.alignment.align(
64,
gray,
rect,
landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
# If the alignment was succesful
if aligned_image is not None:
# If there is a new recording save the recorded images (the saving part is done in the 'webacm_app.py')
if self.set_new_person:
self.counter += 1
if self.counter % 10 == 0:
if len(self.saved_images) < 10:
self.saved_images.append(aligned_image)
else:
self.counter = 0
# Create a directory for the new person and save the person's images
if self.saving:
self.path_person = self.path + '\\' + self.names[-1]
os.mkdir(self.path_person)
for (i, picture) in enumerate(self.base_images[-1]):
self.save_pictures(picture, self.path_person,
self.names[-1] + '_' + str(i),
'jpg')
self.saving = False
# Calculate the average distance from the save distance
for i in range(len(self.base_images)):
average_distance = 0
for base_image in self.base_images[i]:
average_distance += self.prediction(
self.model, aligned_image / 255.0,
base_image / 255.0)
self.distances[i] = average_distance / len(
self.base_images[i])
# If the picture alignment was unsucessful, then the distance belonging to the picture is four
else:
for i in range(len(self.distances)):
self.distances[i] = 4
if self.set_new_person:
color = (0, 255, 0)
else:
color = (255, 0, 0) # BGR
stroke = 2
end_cord_x = x + w
end_cord_y = y + h
cv2.rectangle(frame, (x, y), (end_cord_x, end_cord_y), color,
stroke)
min_index = 0
# Calculate the minimum distances from all person
for (i, distance) in enumerate(self.distances):
if distance < self.distances[min_index]:
min_index = i
# Display the distance values on the feed
if len(self.distances
) > 0 and self.distances[min_index] < self.threshold:
font = cv2.FONT_HERSHEY_SIMPLEX
name = self.names[min_index] + "{:2.2f}".format(
self.distances[min_index])
color = (255, 255, 255)
stroke = 1
cv2.putText(frame, name, (x, y), font, 1, color, stroke,
cv2.LINE_AA)
else:
font = cv2.FONT_HERSHEY_SIMPLEX
color = (255, 255, 255)
stroke = 1
if len(self.distances) > 0:
name = "{:2.2f}".format(self.distances[min_index])
cv2.putText(frame, name, (x, y), font, 1, color, stroke,
cv2.LINE_AA)
return frame
y2=1
font=cv2.FONT_HERSHEY_SIMPLEX
embedded = 0;
cap = cv2.VideoCapture(0)
cap.open(0)
while(True):
try:
# Capture frame-by-frame
ret, frame = cap.read()
flipped=cv2.flip(frame,1)
bb = alignment.getLargestFaceBoundingBox(flipped)
y1=bb.top()
x1=bb.left()
y2=bb.top()+bb.height()
x2=bb.left()+bb.width()
x3=bb.left()
y3=y2+50
aligned = alignment.align(96, flipped, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
img = (aligned / 255.).astype(np.float32)
embedded= nn4_small2_pretrained.predict(np.expand_dims(img, axis=0))[0]
example_prediction = svc.predict([embedded])
example_identity = encoder.inverse_transform(example_prediction)[0]
cv2.putText(flipped,example_identity,(x3,y3),font,1,(0,0,255),1,cv2.LINE_AA)
def findd():
import cv2
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
import PIL.Image as Image
import numpy as np
from model import create_model
from keras import backend as K
import tensorflow as tf
from align import AlignDlib
from keras import backend as K
tf.keras.backend.clear_session()
K.clear_session()
alignment = AlignDlib('models/landmarks.dat')
# img= load_image('E:\\projectImplementation\\projectFiles\\imageee.png')
img = cv2.imread('E:\\projectImplementation\\projectFiles\\imageee.png', 1)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
bb = alignment.getLargestFaceBoundingBox(img)
im_aligned = alignment.align(96,
img,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
anjani = np.load('anjani.npy')
aman = np.load('aman.npy')
rapaka = np.load('rapaka.npy')
akc = np.load('akc.npy')
abhi = np.load('abhishek.npy')
hemanth = np.load('hemanth.npy')
subham = np.load('subham.npy')
vivek = np.load('vivek.npy')
nn4_small2_pretrained = create_model()
nn4_small2_pretrained.load_weights('nn4.small2.v1.h5')
im_aligned = (im_aligned / 255.).astype(np.float32)
im_aligned = np.expand_dims(im_aligned, axis=0)
embedded = nn4_small2_pretrained.predict(im_aligned)[0]
# print(embedded)
# tf.keras.backend.clear_session()
# K.clear_session()
distances = []
distances.append(np.linalg.norm(embedded - anjani))
distances.append(np.linalg.norm(embedded - abhi))
distances.append(np.linalg.norm(embedded - subham))
distances.append(np.linalg.norm(embedded - rapaka))
distances.append(np.linalg.norm(embedded - hemanth))
distances.append(np.linalg.norm(embedded - aman))
distances.append(np.linalg.norm(embedded - akc))
fin = min(distances)
if (fin == distances[0]):
return "Anjani"
if (fin == distances[1]):
return "Abhishek"
if (fin == distances[2]):
return "Subham"
if (fin == distances[3]):
return "Rapaka"
if (fin == distances[4]):
return "Hemanth"
if (fin == distances[5]):
return "Aman"
if (fin == distances[6]):
return "Akshay"
from align import AlignDlib
from urllib.request import urlopen
from model import create_model
from keras.models import model_from_json
import numpy as np
cap = cv2.VideoCapture("rtsp://192.168.7.137:10085")
loaded_model = create_model()
loaded_model.load_weights("./weights/model.h5")
while (True):
# Capture frame-by-frame
alignment = AlignDlib('models/landmarks.dat')
ret, frame = cap.read()
#img = frame
# Our operations on the frame come here
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
bb = alignment.getLargestFaceBoundingBox(frame)
jc_aligned = alignment.align(96,
frame,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
#Display the resulting frame
jc_aligned = np.expand_dims(jc_aligned, axis=0)
print(jc_aligned.shape)
a = np.argmax(loaded_model.predict(jc_aligned)) + 1
cv2.rectangle(frame, (bb.left(), bb.top()),
(bb.right(), bb.right() + (bb.top() - bb.left())),
(0, 0, 255), 10)
cv2.imshow('anh thuoc nhom:' + str(a), frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# When everything done, release the capture
class Trainer():
def __init__(self):
# self.image_dir = sys.argv[1]
# self.image_dir = "/home/webwerks/projects/project-face-recognition/23-07/face_detection/uploads"
self.image_dir = "/home/webwerks/projects/project-face-recognition/23-07/face_detection/src/images"
self.model = self.create_model()
self.alignment = AlignDlib('models/landmarks.dat')
def create_model(self):
nn4_small2_pretrained = create_model()
nn4_small2_pretrained.load_weights('weights/nn4.small2.v1.h5')
return nn4_small2_pretrained
def load_metadata(self, path):
print("loading metadata ...")
metadata = []
for i in sorted(os.listdir(path)):
for f in sorted(os.listdir(os.path.join(path, i))):
# Check file extension. Allow only jpg/jpeg' files.
ext = os.path.splitext(f)[1].lower()
if ext == '.jpg' or ext == '.jpeg' or ext == '.png':
metadata.append(IdentityMetadata(path, i, f))
return np.array(metadata)
def load_image(self, path):
img = cv2.imread(path, 1)
return img
def align_image(self, img):
return self.alignment.align(
96,
img,
self.alignment.getLargestFaceBoundingBox(img),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
def train(self):
metadata = self.load_metadata(self.image_dir)
np.save('train-files/metadata.npy', metadata)
print("creating embeddings ...")
embedded = {}
for i, m in enumerate(metadata):
img = self.load_image(m.image_path())
img = self.align_image(img)
if img is not None:
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
embedded[m.image_path()] = self.model.predict(
np.expand_dims(img, axis=0))[0]
pickle.dump(embedded, open("train-files/embeddings.pkl", "wb"))
def train_classifier(self):
print("training classifier ...")
embedded = pickle.load(open("train-files/embeddings.pkl", "rb"))
metadata = np.load('train-files/metadata.npy')
targets = np.array([m.name for m in metadata])
encoder = LabelEncoder()
encoder.fit(targets)
pickle.dump(encoder, open("train-files/encoder.pkl", "wb"))
# Numerical encoding of identities
y = encoder.transform(targets)
train_idx = np.arange(metadata.shape[0]) % 2 != 0
test_idx = np.arange(metadata.shape[0]) % 2 == 0
X_train = embedded[train_idx]
X_test = embedded[test_idx]
y_train = y[train_idx]
y_test = y[test_idx]
svc = LinearSVC()
svc.fit(X_train, y_train)
joblib.dump(svc, 'train-files/svc.pkl')
acc_svc = accuracy_score(y_test, svc.predict(X_test))
print(f'SVM accuracy = {acc_svc}')
def align_image(img):
alignment = AlignDlib('models/landmarks.dat')
return alignment.align(96,
img,
alignment.getLargestFaceBoundingBox(img),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
class FaceRecognizer():
def __init__(self):
dst_dir = 'models'
dst_file = os.path.join(dst_dir, 'landmarks.dat')
if not os.path.exists(dst_file):
os.makedirs(dst_dir)
download_landmarks(dst_file)
# Create CNN model and load pretrained weights (OpenFace nn4.small2)
self.nn4_small2_pretrained = create_model()
self.nn4_small2_pretrained.load_weights('models/nn4.small2.v1.h5')
self.metadata = self.load_metadata('faces')
# Initialize the OpenFace face alignment utility
self.alignment = AlignDlib('models/landmarks.dat')
# Get embedding vectorsf
# self.embedded = np.zeros((self.metadata.shape[0], 128))
self.embedded = np.zeros((0, 128))
# Train images
custom_metadata = self.load_metadata("faces")
self.metadata = np.append(self.metadata, custom_metadata)
self.update_embeddings()
self.train_images()
# Download dlib face detection landmarks file
def download_landmarks(self, dst_file):
url = 'http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2'
decompressor = bz2.BZ2Decompressor()
with urlopen(url) as src, open(dst_file, 'wb') as dst:
data = src.read(1024)
while len(data) > 0:
dst.write(decompressor.decompress(data))
data = src.read(1024)
def load_metadata(self, path):
ds_store = ".DS_Store"
metadata = []
dirs = os.listdir(path)
if ds_store in dirs:
dirs.remove(ds_store)
for i in dirs:
subdirs = os.listdir(os.path.join(path, i))
if ds_store in subdirs:
subdirs.remove(ds_store)
for f in subdirs:
metadata.append(IdentityMetadata(path, i, f))
return np.array(metadata)
# Align helper functions
def get_face_thumbnail(self, img):
return self.alignment.getLargestFaceThumbnail(
96,
img,
self.alignment.getLargestFaceBoundingBox(img),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
def get_all_face_thumbnails_and_scores(self, img):
return self.alignment.getAllFaceThumbnailsAndScores(
96, img, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
def get_face_vector(self, img, is_thumbnail=False):
if not is_thumbnail:
img = self.get_face_thumbnail(img)
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
return self.nn4_small2_pretrained.predict(np.expand_dims(img,
axis=0))[0]
def get_face_vectors(self, img):
face_thumbnails, scores, face_types = self.get_all_face_thumbnails_and_scores(
img)
face_vectors = []
for face_img in face_thumbnails:
# scale RGB values to interval [0,1]
face_img = (face_img / 255.).astype(np.float32)
# obtain embedding vector for image
vector = self.nn4_small2_pretrained.predict(
np.expand_dims(face_img, axis=0))[0]
face_vectors.append(vector)
return face_vectors, face_thumbnails, scores, face_types
# Train classifier models
def train_images(self, train_with_all_samples=False):
self.targets = np.array([m.name for m in self.metadata])
start = time.time()
self.encoder = LabelEncoder()
self.encoder.fit(self.targets)
# Numerical encoding of identities
y = self.encoder.transform(self.targets)
if train_with_all_samples == False:
train_idx = np.arange(self.metadata.shape[0]) % 2 != 0
else:
train_idx = np.full(self.metadata.shape[0], True)
self.test_idx = np.arange(self.metadata.shape[0]) % 2 == 0
# 50 train examples of 10 identities (5 examples each)
X_train = self.embedded[train_idx]
# 50 test examples of 10 identities (5 examples each)
X_test = self.embedded[self.test_idx]
y_train = y[train_idx]
y_test = y[self.test_idx]
self.knn = KNeighborsClassifier(n_neighbors=1, metric='euclidean')
self.svc = LinearSVC() #class_weight='balanced')
self.knn.fit(X_train, y_train)
self.svc.fit(X_train, y_train)
acc_knn = accuracy_score(y_test, self.knn.predict(X_test))
acc_svc = accuracy_score(y_test, self.svc.predict(X_test))
if train_with_all_samples == False:
print(f'KNN accuracy = {acc_knn}, SVM accuracy = {acc_svc}')
else:
print('Trained classification models with all image samples')
end = time.time()
print("train_images took {} secs".format(end - start))
def update_embeddings(self):
for i, m in enumerate(self.metadata):
print("loading image from {}".format(m.image_path()))
img = load_image(m.image_path())
is_thumbnail = "customer_" in m.image_path()
vector = self.get_face_vector(img, is_thumbnail)
vector = vector.reshape(1, 128)
self.embedded = np.append(self.embedded, vector, axis=0)
def label_cv2_image_faces(self, rgb_img, face_bbs, identities):
# Convert RGB back to cv2 RBG format
img = rgb_img[:, :, ::-1]
for i, bb in enumerate(face_bbs):
# Draw bounding rectangle around face
cv2.rectangle(img, (bb.left(), bb.top()),
(bb.right(), bb.bottom()), (0, 0, 255), 2)
# Draw a label with a name below the face
cv2.rectangle(img, (bb.left(), bb.bottom() - 35),
(bb.right(), bb.bottom()), (0, 0, 255), cv2.FILLED)
font = cv2.FONT_HERSHEY_DUPLEX
cv2.putText(img, identities[i], (bb.left() + 6, bb.bottom() - 6),
font, 1.0, (255, 255, 255), 1)
return img
def display_cv2_image_faces(self, rgb_img, face_bbs, identities):
img = label_cv2_image_faces(rgb_img, face_bbs, identities)
display_cv2_image(img)
def display_plt_image_faces(self, img, face_bbs, identities, subplot=111):
plt.subplot(subplot)
plt.figure()
plt.imshow(img)
for bb in face_bbs:
plt.gca().add_patch(
patches.Rectangle((bb.left(), bb.top()),
bb.width(),
bb.height(),
fill=False,
color='red'))
# TODO: Print identities in correct order
plt.title(f'Recognized as {identities}')
def save_unknown_face(self, face_vector, face_thumbnail):
print("Saving unknown face...")
dirs = os.listdir("faces")
customer_dirs = [dir for dir in dirs if "customer_" in dir]
if len(customer_dirs) > 0:
dir_indexes = [int(dir.split("_")[1]) for dir in customer_dirs]
curr_index = max(dir_indexes) + 1
else:
curr_index = 1
# Save image to customer dir
# TODO: Remove requirement for double-creation of all data
customer_dir = "customer_{}".format(curr_index)
dir_path = os.path.join("faces", customer_dir)
os.mkdir(dir_path)
for i in range(0, 8):
customer_file = "customer_{}_{}.jpg".format(curr_index, i + 1)
file_path = os.path.join(dir_path, customer_file)
imageio.imwrite(file_path, face_thumbnail)
metadata = np.append(
self.metadata,
IdentityMetadata("custom", customer_dir, customer_file))
embedded = np.append(self.embedded,
face_vector.reshape(1, 128),
axis=0)
print("Saved unknown face")
def distance(self, emb1, emb2):
return np.sum(np.square(emb1 - emb2))
def get_distances(self, vector):
distances = []
for embed in self.embedded:
distances.append(distance(embed, vector))
return distances
def get_sharpness_level(self, image):
grey = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
# compute the Laplacian of the image and then return the focus
# measure, which is simply the variance of the Laplacian
return cv2.Laplacian(grey, cv2.CV_64F).var()
def identify_image_faces(self, example_image):
vectors, thumbnails, dlib_scores, face_types = self.get_face_vectors(
example_image)
identities = []
saved_unknown = False
for i, vector in enumerate(vectors):
vector = vector.reshape(1, 128)
confidence_scores = self.svc.decision_function(vector)
if (confidence_scores.max() < MIN_CONFIDENCE_SCORE):
sharpness_level = self.get_sharpness_level(thumbnails[i])
example_identity = "Unknown"
#example_identity = "Unknown ({:0.2f}, {}, {:0.2f})".format(dlib_scores[i], face_types[i], sharpness_level)
print(
"Unknown face - dlib score={:0.2f}, face_type={}, sharpness_level={:0.2f}"
.format(dlib_scores[i], face_types[i], sharpness_level))
if RECOGNIZE_UNKNOWN_FACES:
# Only save (and train) a good-quality and front-facing face
if dlib_scores[i] >= MIN_DLIB_SCORE and face_types[
i] == 0 and sharpness_level >= MIN_SHARPNESS_LEVEL:
saved_unknown = True
print("Saving unknown face")
save_unknown_face(vector, thumbnails[i])
else:
example_prediction = self.svc.predict(vector)
example_identity = self.encoder.inverse_transform(
example_prediction)[0]
identities.append(example_identity)
# Add to training model if any unknown faces were saved
if saved_unknown:
train_images()
# Detect faces and return bounding boxes
face_bbs = self.alignment.getAllFaceBoundingBoxes(example_image)
return face_bbs, identities
def display_unknown_image(self, image_path):
img = load_image(image_path)
face_bbs, identities = self.identify_image_faces(img)
#display_cv2_image_faces(img, face_bbs, identities)
self.display_plt_image_faces(img, face_bbs, identities)
def display_image_prediction(self, example_idx):
example_image = load_image(
self.metadata[self.test_idx][example_idx].image_path())
example_prediction = self.knn.predict(
[self.embedded[self.test_idx][example_idx]])
example_identity = self.encoder.inverse_transform(
example_prediction)[0]
# Detect face and return bounding box
#bb = alignment.getLargestFaceBoundingBox(example_image)
plt.imshow(example_image)
#plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red'))
plt.title(f'Recognized as {example_identity}')
def visualize_dataset(self):
X_embedded = TSNE(n_components=2).fit_transform(self.embedded)
plt.figure()
for i, t in enumerate(set(self.targets)):
idx = self.targets == t
plt.scatter(X_embedded[idx, 0], X_embedded[idx, 1], label=t)
plt.legend(bbox_to_anchor=(1, 1))
# encoder = pickle.load(open(encoderfile, 'rb'))
alignment = AlignDlib('models/landmarks.dat')
# modelfile = "svm_one.sav"
# svc = LinearSVC()
# svc = pickle.load(open(modelfile, 'rb'))
cap = cv2.VideoCapture(-1)
loop = 5e10
while(True):
try:
# Capture frame-by-frame
ret, jc_orig = cap.read()
#jc_orig = cv2.resize(jc_orig, dim, interpolation = cv2.INTER_AREA)
bb = alignment.getLargestFaceBoundingBox(jc_orig)##------------------------return this
print(bb)
#if(bb==None):
# continue
#print(bb.left())
x = (bb.left() + bb.right())//2
y = (bb.top() + bb.bottom())//2
#w = bb.right() - bb.left()
#h = bb.bottom() - bb.top()
#centre = (jc_orig.shape[0] + bb[0], jc_orig.shape[1] + bb[1])
image= cv2.circle (jc_orig, (x,y), radius = 1, color = (0,0,255), thickness = -1)
# jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# jc_aligned = (jc_aligned /255.).astype(np.float32)
# embedded = nn4_small2_pretrained.predict(np.expand_dims(jc_aligned, axis=0))
# prediction = svc.predict(embedded)
def main(predict_eg):
nn4_small2 = create_model()
# Input for anchor, positive and negative images
input_a = Input(shape=(96, 96, 3))
input_p = Input(shape=(96, 96, 3))
input_n = Input(shape=(96, 96, 3))
# Output for anchor, positive and negative embedding vectors
emb_a = nn4_small2(input_a)
emb_p = nn4_small2(input_p)
emb_n = nn4_small2(input_n)
# Layer that computes the triplet loss from anchor, positive and negative embedding vectors
triplet_loss_layer = TripletLossLayer.TripletLossLayer(
alpha=0.2, name='triplet_loss_layer')([emb_a, emb_p, emb_n])
# Model that can be trained with anchor, positive negative images
nn4_small2_train = Model([input_a, input_p, input_n], triplet_loss_layer)
# triplet_generator() creates a generator that continuously returns
# ([a_batch, p_batch, n_batch], None) tuples where a_batch, p_batch
# and n_batch are batches of anchor, positive and negative RGB images
# each having a shape of (batch_size, 96, 96, 3).
generatorImage = triplet_generator()
nn4_small2_train.compile(loss=None, optimizer='adam')
nn4_small2_train.fit_generator(generatorImage,
epochs=10,
steps_per_epoch=100)
# nn4_small2_train.fit_generator(generatorImage, epochs=2, steps_per_epoch=100)
# Please note that the current implementation of the generator only generates
# random image data. The main goal of this code snippet is to demonstrate
# the general setup for model training. In the following, we will anyway
# use a pre-trained model so we don't need a generator here that operates
# on real training data. I'll maybe provide a fully functional generator
# later.
nn4_small2_pretrained = create_model()
nn4_small2_pretrained.load_weights('weights/nn4.small2.v1.h5')
metadata = load_metadata('images')
# Initialize the OpenFace face alignment utility
alignment = AlignDlib('models/landmarks.dat')
# Load any random image
jc_orig = load_image(metadata[77].image_path())
# Detect face and return bounding box
bb = alignment.getLargestFaceBoundingBox(jc_orig)
# Transform image using specified face landmark indices and crop image to 96x96
jc_aligned = alignment.align(96,
jc_orig,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# Show original image
plt.subplot(131)
plt.imshow(jc_orig)
plt.title("Orig Image")
# plt.show()
# Show original image with bounding box
plt.subplot(132)
plt.imshow(jc_orig)
plt.title("with BB")
# plt.show()
plt.gca().add_patch(
patches.Rectangle((bb.left(), bb.top()),
bb.width(),
bb.height(),
fill=False,
color='red'))
# Show aligned image
plt.subplot(133)
plt.imshow(jc_aligned)
plt.title("Aligned Image")
plt.show()
def align_image(img):
return alignment.align(96,
img,
alignment.getLargestFaceBoundingBox(img),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
embedded = np.zeros((metadata.shape[0], 128))
for i, m in enumerate(metadata):
img = load_image(m.image_path())
img = align_image(img)
# scale RGB values to interval [0,1]
img = (img / 255.).astype(np.float32)
# obtain embedding vector for image
embedded[i] = nn4_small2_pretrained.predict(np.expand_dims(img,
axis=0))[0]
def show_pair(idx1, idx2):
plt.figure(figsize=(8, 3))
plt.suptitle(
f'Distance = {distance(embedded[idx1], embedded[idx2]):.2f}')
plt.subplot(121)
plt.imshow(load_image(metadata[idx1].image_path()))
# plt.show()
plt.subplot(122)
plt.imshow(load_image(metadata[idx2].image_path()))
plt.show()
show_pair(77, 78)
show_pair(77, 50)
distances = [] # squared L2 distance between pairs
identical = [] # 1 if same identity, 0 otherwise
num = len(metadata)
for i in range(num - 1):
for j in range(i + 1, num):
distances.append(distance(embedded[i], embedded[j]))
identical.append(1 if metadata[i].name == metadata[j].name else 0)
distances = np.array(distances)
identical = np.array(identical)
thresholds = np.arange(0.3, 1.0, 0.01)
#F1 Score
f1_scores = [f1_score(identical, distances < t) for t in thresholds]
acc_scores = [accuracy_score(identical, distances < t) for t in thresholds]
opt_idx = np.argmax(f1_scores)
# Threshold at maximal F1 score
opt_tau = thresholds[opt_idx]
# Accuracy at maximal F1 score
opt_acc = accuracy_score(identical, distances < opt_tau)
# Plot F1 score and accuracy as function of distance threshold
plt.plot(thresholds, f1_scores, label='F1 score')
plt.plot(thresholds, acc_scores, label='Accuracy')
plt.axvline(x=opt_tau,
linestyle='--',
lw=1,
c='lightgrey',
label='Threshold')
plt.title(f'Accuracy at threshold {opt_tau:.2f} = {opt_acc:.3f}')
plt.xlabel('Distance threshold')
plt.legend()
plt.show()
dist_pos = distances[identical == 1]
dist_neg = distances[identical == 0]
plt.figure(figsize=(12, 4))
plt.subplot(121)
plt.hist(dist_pos)
plt.axvline(x=opt_tau,
linestyle='--',
lw=1,
c='lightgrey',
label='Threshold')
plt.title('Distances (+ve pairs)')
plt.legend()
plt.subplot(122)
plt.hist(dist_neg)
plt.axvline(x=opt_tau,
linestyle='--',
lw=1,
c='lightgrey',
label='Threshold')
plt.title('Distances (-ve pairs)')
plt.legend()
plt.show()
targets = np.array([m.name for m in metadata])
encoder = LabelEncoder()
encoder.fit(targets)
# Numerical encoding of identities
y = encoder.transform(targets)
train_idx = np.arange(metadata.shape[0]) % 2 != 0
test_idx = np.arange(metadata.shape[0]) % 2 == 0
# 50 train examples of 10 identities (5 examples each)
X_train = embedded[train_idx]
# 50 test examples of 10 identities (5 examples each)
X_test = embedded[test_idx]
y_train = y[train_idx]
y_test = y[test_idx]
knn = KNeighborsClassifier(n_neighbors=1, metric='euclidean')
svc = LinearSVC()
knn.fit(X_train, y_train)
svc.fit(X_train, y_train)
acc_knn = accuracy_score(y_test, knn.predict(X_test))
acc_svc = accuracy_score(y_test, svc.predict(X_test))
print(f'KNN accuracy = {acc_knn}, SVM accuracy = {acc_svc}')
warnings.filterwarnings('ignore')
example_idx = predict_eg
print("example_idx", example_idx)
example_image = load_image(metadata[test_idx][example_idx].image_path())
example_prediction = svc.predict([embedded[test_idx][example_idx]])
example_identity = encoder.inverse_transform(example_prediction)[0]
plt.imshow(example_image)
plt.title(f'Recognized as {example_identity}')
plt.show()
X_embedded = TSNE(n_components=2).fit_transform(embedded)
for i, t in enumerate(set(targets)):
idx = targets == t
plt.scatter(X_embedded[idx, 0], X_embedded[idx, 1], label=t)
plt.legend(bbox_to_anchor=(1, 1))
plt.show()
#get_ipython().run_line_magic('matplotlib', 'inline')
def load_image(path):
img = cv2.imread(path, 1)
# OpenCV loads images with color channels
# in BGR order. So we need to reverse them
return img[...,::-1]
# Initialize the OpenFace face alignment utility
alignment = AlignDlib('models/landmarks.dat')
# Load an image of Jacques Chirac
jc_orig = load_image(metadata[6].image_path())
# Detect face and return bounding box
bb = alignment.getLargestFaceBoundingBox(jc_orig)
# Transform image using specified face landmark indices and crop image to 96x96
jc_aligned = alignment.align(96, jc_orig, bb, landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# Show original image
# plt.subplot(131)
# plt.imshow(jc_orig)
# # Show original image with bounding box
# plt.subplot(132)
# plt.imshow(jc_orig)
# plt.gca().add_patch(patches.Rectangle((bb.left(), bb.top()), bb.width(), bb.height(), fill=False, color='red'))
# # Show aligned image
# plt.subplot(133)
data_num = hdf5_read.root.images.shape[0]
prev_label = hdf5_read.root.labels[0]
print("Evaluating %s ."%(hdf5_path))
# Blur limit
blur_limit = 100
blur = 0
# Iterates through the pictures in a dataset.
for i in range(data_num):
# Reading the image and the corresponding label.
image = hdf5_read.root.images[i]
label = hdf5_read.root.labels[i]
print("Evaluation: %.3f %%"%(float(i)/float(data_num-1)*100),end="\r", flush=True)
# Detect face and return bounding box
rect = alignment.getLargestFaceBoundingBox(image)
# Transform image using specified face landmark indices and crop image to 64x64
img_aligned = alignment.align(64, image, rect, landmarkIndices=AlignDlib.INNER_EYES_AND_BOTTOM_LIP)
# If the aligning was successful
if img_aligned is not None:
# Check if the picture has to much black (empty) pixels, which would indicate a bad image, because
# a face at the edge of a picture could be recognized and aligned even if it's missing part of the face
black = 0
# Counting black (empty pixels)
for l in range(img_aligned.shape[0]):
for k in range(img_aligned.shape[1]):
if img_aligned[l,k] == 0:
black+=1
# Checking the threshold for black (empty) pixels
if black<100:
# Calculating blur value.
# open('weights/nn4.small2.myTrain.h5', 'r')
# nn4_small2_train.load_weights('weights/nn4.small2.myTrain.h5')
# except FileNotFoundError:
#
# nn4_small2_train = train_model()
metadata = load_metadata('images')
# Initialize the OpenFace face alignment utility
alignment = AlignDlib('models/landmarks.dat')
# Load an image of Schwarzenegger
jc_orig = load_image(metadata[92].image_path())
# Detect face and return bounding box
bb = alignment.getLargestFaceBoundingBox(jc_orig)
# Transform image using specified face landmark indices and crop image to 96x96
jc_aligned = alignment.align(96,
jc_orig,
bb,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
# Show original image
plt.subplot(131)
plt.imshow(jc_orig)
# Show original image with bounding box
plt.subplot(132)
plt.imshow(jc_orig)
plt.gca().add_patch(
