def detect_and_align_faces(capturedImage):
image = capturedImage
face_aligner = AlignDlib(predictor_model)
face_detector = dlib.get_frontal_face_detector()
img = np.uint8(image)
# Run the HOG face detector on the image data.
# The result will be the bounding boxes of the faces in our image.
detected_faces = face_detector(img, 1)
print("I found {} faces".format(len(detected_faces)))
aligned_faces = []
boundary_boxes = []
#Loop through each face we found in our image
for i, face_rect in enumerate(detected_faces):
# Detected faces are returned as an object with the coordinates of the top, left, right and bottom edges
print(
"- Face #{} found at Left: {} Top: {} Right: {} Bottom: {}".format(
i + 1, face_rect.left(), face_rect.top(), face_rect.right(),
face_rect.bottom()))
# Aligning the face in correct position.
alignedFace = face_aligner.align(
imgDim=234,
rgbImg=img,
bb=face_rect,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
aligned_faces.append(alignedFace)
boundary_boxes.append(face_rect)
return aligned_faces, boundary_boxes
def get_aligned_face(self, np_img, sq_img_dim, bounding_box):
openface = AlignDlib(self.dlib_predictor_path)
aligned_img = openface.align(sq_img_dim,
np_img,
skipMulti=True,
bb=bounding_box)
return aligned_img
def process(person_id, video_path, output_path):
ad = AlignDlib("/data/shape_predictor_68_face_landmarks.dat")
catch_pic_num = 5
cap = cv2.VideoCapture(video_path)
color = (0, 255, 0)
num = 0
try_num = 0
while cap.isOpened():
print("进入检测")
ok, frame = cap.read() # 读取一帧数据
if not ok:
try_num += 1
print("未读取到")
break
if try_num > 100:
print("连续失败100次，退出")
break
size = frame.shape[0]
output_path = output_path + '%s_0000.jpg' % (person_id)
if (ad.align(size, frame) is not None):
cv2.imwrite(output_path, frame)
break
frame90 = np.rot90(frame)
if (ad.align(size, frame90) is not None):
cv2.imwrite(output_path, frame90)
break
frame180 = np.rot90(np.rot90(frame))
if (ad.align(size, frame180) is not None):
cv2.imwrite(output_path, frame180)
break
frame270 = np.rot90(np.rot90(np.rot90(frame)))
if (ad.align(size, frame270) is not None):
cv2.imwrite(output_path, frame270)
break
print("未提取成功，等待下一帧")
c = cv2.waitKey(10)
if c & 0xFF == ord('q'):
break
cap.release()
def crop_align_cam():
minsize = 20
caffe_model_path = "/media/arthur/work/smart_card_snd/deepid2_wuqianliang/deepid2_caffe.git/trunk/test_lfw/mtcnn_model"
threshold = [0.6, 0.7, 0.7]
factor = 0.709
caffe.set_mode_gpu()
PNet = caffe.Net(caffe_model_path + "/det1.prototxt",
caffe_model_path + "/det1.caffemodel", caffe.TEST)
RNet = caffe.Net(caffe_model_path + "/det2.prototxt",
caffe_model_path + "/det2.caffemodel", caffe.TEST)
ONet = caffe.Net(caffe_model_path + "/det3.prototxt",
caffe_model_path + "/det3.caffemodel", caffe.TEST)
align = AlignDlib('./shape_predictor_68_face_landmarks.dat')
cap = cv2.VideoCapture(0)
start = time()
while True:
ret, img = cap.read()
img_matlab = img.copy()
tmp = img_matlab[:, :, 2].copy()
img_matlab[:, :, 2] = img_matlab[:, :, 0]
img_matlab[:, :, 0] = tmp #BGR TO RGB
# check rgb position
tic()
boundingboxes, points = detect_face(img_matlab, minsize, PNet, RNet,
ONet, threshold, False, factor)
toc()
for i in range(len(boundingboxes)):
left = int(boundingboxes[i][0])
top = int(boundingboxes[i][1])
right = int(boundingboxes[i][2])
bottom = int(boundingboxes[i][3])
alignedFace = align.align(
96, # 96x96x3
img_matlab,
dlib.rectangle(left, top, right, bottom),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
cv2.imshow('crop align face', alignedFace)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
def main(args):
#sleep(random.random())
output_dir = os.path.expanduser(args.output_dir)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Store some git revision info in a text file in the log directory
src_path,_ = os.path.split(os.path.realpath(__file__))
#dataset = facenet.get_dataset(args.input_dir)
align_dlib = AlignDlib('/home/arthur/facenet.git/trunk/src/align/shape_predictor_68_face_landmarks.dat')
DATA_BASE = "/home/arthur/caffe-master/data/lfw/";
with tf.Graph().as_default():
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=args.gpu_memory_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options, log_device_placement=False))
with sess.as_default():
pnet, rnet, onet = align.detect_face.create_mtcnn(sess, None)
print('Creating networks and loading parameters')
###################tensorflow predictor init#############################
FLAGS = utils.load_tf_flags()
facenet.load_model(FLAGS.model_dir)
facenet.store_revision_info(src_path, output_dir, ' '.join(sys.argv))
embeddings = tf.get_default_graph().get_tensor_by_name("embeddings:0")
phase_train_placeholder = tf.get_default_graph().get_tensor_by_name("phase_train:0")
images_placeholder = tf.get_default_graph().get_tensor_by_name("input:0")
###########################################################
minsize = 10 # minimum size of face
threshold = [ 0.6, 0.7, 0.7 ] # three steps's threshold
factor = 0.709 # scale factor
# Add a random key to the filename to allow alignment using multiple processes
random_key = np.random.randint(0, high=99999)
# Read video file frame
#cap = cv2.VideoCapture('/home/wuqianliang/test/VID_20171013_121412.mp4')
# start capture video
print('start capture video.......................................')
cap = cv2.VideoCapture(0)
while(cap.isOpened()):
ret, img = cap.read()
# Add code###########
img = img[:,:,0:3]
#img = cv2.resize(img, (780, 364), interpolation=cv2.INTER_AREA)
bounding_boxes, _ = align.detect_face.detect_face(img, minsize, pnet, rnet, onet, threshold, factor)
nrof_faces = bounding_boxes.shape[0]
if nrof_faces > 0:
print('detected faces ',nrof_faces)
for bbox in bounding_boxes:
det = bbox[0:5]
detect_confidence = det[4]
if detect_confidence > 0.8:
#######################################################################################
left =int(bbox[0])
top =int(bbox[1])
right =int(bbox[2])
bottom =int(bbox[3])
alignedFace=align_dlib.align(
160, # 96x96x3
img,
dlib.rectangle(left,top,right,bottom),
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
#######################################################################################
#cv2.rectangle(img,(int(det[0]),int(det[1])),(int(det[2]),int(det[3])),(55,255,155),2)
#cropped = img[int(det[1]):int(det[3]),int(det[0]):int(det[2]),:]
try:
#cropped = cv2.resize(cropped, (160, 160), interpolation=cv2.INTER_CUBIC )
cv2.imshow('cropped detected face',alignedFace)
# here can add more cropped image to set a batch to accelarate
cropped=cropped.reshape((1,160,160,3))
except:
continue
#######################caculate embeddings
emb_dict = list(sess.run([embeddings], feed_dict={images_placeholder: np.array(cropped), phase_train_placeholder: False })[0])
print(emb_dict)
######################
cv2.imshow('image detected face', img)
k = cv2.waitKey(20)
if (k & 0xff == ord('q')):
break
cap.release()
cv2.destroyAllWindows()
class RecogServerProtocol(WebSocketServerProtocol):
def __init__(self):
#conexao onde buscara imagem para ser classificada
self.mdb = None
#classe usada para processamento e classificacao
self.classif = None
# alocado globalmente o extrador dlib
self.align = AlignDlib(os.path.join(basedir,"models","dlib", "shape_predictor_68_face_landmarks.dat"))
self.net = openface.TorchNeuralNet(os.path.join(basedir,"models", 'openface', #'treinado-jun16.t7'))
'nn4.small2.v1.t7')
,imgDim=96,
cuda=False)
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onMessage(self, payload, isBinary):
raw = payload.decode('utf8')
msg = json.loads(raw)
print msg['type']
if msg['type'] == "MONGOSRVINI":
# inicializar o servidor mongodb onde sera baixado os arquivos
resposta = self.initMongo(msg)
elif msg['type'] == "MONGOPOSEDB":
resposta = self.initMongoPose(msg)
elif msg['type'] == "QUERYIMGREP":
# processar uma imagem
resposta = self.processImgRep(msg)
elif msg['type'] == "QUERYIDREP":
# processar as imagens de uma determinada identidade
resposta = self.processIdRep(msg)
elif msg['type'] == "QUERYIMGINFO":
resposta = self.processImgInfo(msg)
elif msg['type'] == "QUERYIDINFO":
resposta = self.processIdInfo(msg)
else:
resposta = {'REASON':"UNKNOW parameters {}".format(msg.keys()),'RESPONSE':'ERROR'}
resposta['type']=msg['type']
self.sendMessage(json.dumps(resposta))
def initMongo(self,msg):
#resposta = resposta = {'RESPONSE':'ERROR','REASON':'MONGOSRVINI:Unknow parameters'+msg.keys()}
if 'host' not in msg.keys() or 'port' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have host or port field'}
else:
if 'base' not in msg.keys():
msg['base']='facecoleta' # default database from libColeta
print "mongodb://{}:{}".format(msg['host'],msg['port'])
self.mdb = MongoConn(url="mongodb://{}:{}".format(msg['host'],msg['port']),
dbs=msg['base'])
if self.mdb is not None:
resposta = {'RESPONSE':'OK'}
else:
resposta = {'RESPONSE':'ERROR'}
return resposta
def initMongoPose(self,msg):
#resposta = resposta = {'RESPONSE':'ERROR','REASON':'MONGOSRVINI:Unknow parameters'+msg.keys()}
if 'host' not in msg.keys() or 'port' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have host or port field'}
else:
texto = "Load face database classifier from mongodb://{}:{}".format(msg['host'],msg['port'])
print texto
try:
self.classif=Classificador(self.align,self.net,host=msg['host'],port=msg['port'])
except pymongo.errors.ServerSelectionTimeoutError:
print "Can not connect to server mongodb://{}:{}".format(msg['host'],msg['port'])
self.classif=None
if self.classif is not None:
resposta = {'RESPONSE':'OK','INFO':texto}
try:
self.classif.loadClassif()
except pymongo.errors.ServerSelectionTimeoutError:
texto= "Can not load data from server mongodb://{}:{}".format(msg['host'],msg['port'])
self.classif=None
resposta = {'RESPONSE':'ERROR','REASON':texto}
else:
resposta = {'RESPONSE':'ERROR'}
return resposta
def processImgRep(self,msg):
#resposta = {'RESPONSE':'ERROR','REASON':'QUERYIMGREP:Unknow parameters:'+msg.keys()}
if self.mdb is None:
resposta = {'RESPONSE':'ERROR','REASON':'mongodb serve is not connected'}
else:
if 'filename' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have filename field'}
else:
keys={'filename':msg['filename']}
imgs=self.mdb.readImFromDB(keys=keys)
if imgs is None:
resposta = {'RESPONSE':'ERROR','REASON':'Something wrong in mongodb'}
elif len(imgs)<1:
resposta = {'RESPONSE':'ERROR','REASON':'Do not have file in db'}
elif len(imgs) == 1:
for doc,img in imgs:
bb = self.align.getLargestFaceBoundingBox(img)
if bb is None:
print "O arquivo {} nao detectou face".format(doc["filename"])
resposta = {'RESPONSE':'ERROR','REASON':'Not found face'}
break
landmarks = self.align.findLandmarks(img, bb)
angcab,angvcab,pp = RepUtil.calcHVAngRosto(landmarks[0],landmarks[16],landmarks[27])
alignedFace = self.align.align(96, img, bb,
landmarks=landmarks,
landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE)
rep = self.net.forward(alignedFace)
resposta = {'RESPONSE':'OK','rep':rep.tolist(),
'identity':doc['identity'],
'angh':angcab,
'angv':angvcab}
else:
resposta = {'RESPONSE':'ERROR','REASON':'Multiples results docs'}
return resposta
def processImgInfo(self,msg):
#resposta = {'RESPONSE':'ERROR','REASON':'QUERYIMGREP:Unknow parameters:'+msg.keys()}
if self.mdb is None:
resposta = {'RESPONSE':'ERROR','REASON':'mongodb serve is not connected'}
else:
if 'filename' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have filename field'}
else:
keys={'filename':msg['filename']}
imgs=self.mdb.readImFromDB(keys=keys)
if imgs is None:
resposta = {'RESPONSE':'ERROR','REASON':'Something wrong in mongodb'}
elif len(imgs)<1:
resposta = {'RESPONSE':'ERROR','REASON':'Do not have file in db'}
elif len(imgs) == 1:
for doc,img in imgs:
bb = self.align.getLargestFaceBoundingBox(img)
if bb is None:
print "O arquivo {} nao detectou face".format(doc["filename"])
resposta = {'RESPONSE':'ERROR','REASON':'Not found face'}
break
landmarks = self.align.findLandmarks(img, bb)
angcab,angvcab,pp = RepUtil.calcHVAngRosto(landmarks[0],landmarks[16],landmarks[27])
alignedFace = self.align.align(96, img, bb,
landmarks=landmarks,
landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE)
rep = self.net.forward(alignedFace)
#repF = self.net.forward(cv2.flip(alignedFace))
# Somente sobre a imagem base se o espelhamento
# Reduz a metade a possibilidades
resp,nc = self.classif.buscaCandidato(rep, angcab, angvcab)
resposta = {'RESPONSE':'OK','rep':rep.tolist(),
'identity':doc['identity'],
'angh':angcab,
'angv':angvcab}
countMat = 0
for nome,pb,reffile,tipoc,angs in resp:
if nome != "Desconhecido":
resposta["name{:02d}".format(countMat)]=nome
resposta["prob{:02d}".format(countMat)]=pb
resposta["tycl{:02d}".format(countMat)]=tipoc
anghe,angve = angs
resposta["angh{:02d}".format(countMat)]=anghe
resposta["angv{:02d}".format(countMat)]=angve
print "A imagem {} pode ser de {} com prob {:4.1f}".format(doc["filename"],nome,pb*100)
countMat += 1
resposta['nclassif']=countMat
else:
resposta = {'RESPONSE':'ERROR','REASON':'Multiples results docs'}
return resposta
def processIdRep(self,msg):
#resposta = {'RESPONSE':'ERROR','REASON':'QUERYIDREP:Unknow parameters:'+msg.keys()}
if self.mdb is None:
resposta = {'RESPONSE':'ERROR','REASON':'mongodb serve is not connected'}
else:
if 'identity' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have filename field'}
else:
keys={'identity':msg['identity']}
imgs=self.mdb.readImFromDB(keys=keys)
if imgs is None:
resposta = {'RESPONSE':'ERROR','REASON':'Something wrong in mongodb'}
elif len(imgs)<1:
resposta = {'RESPONSE':'ERROR','REASON':'Do not have file in db'}
else:
resposta = {'RESPONSE':'OK'}
contap = 0
contaf = 0
seqf=[]
for doc,img in imgs:
bb = self.align.getLargestFaceBoundingBox(img)
if bb is None:
print "O arquivo {} nao detectou face".format(doc["filename"])
seqf.append(doc['seq'])
contaf += 1
else:
if "rep{:03d}".format(doc['seq']) in doc.keys():
resposta["rep{:03d}".format(doc['seq'])] = doc["rep{:03d}".format(doc['seq'])]
resposta["angh{:03d}".format(doc['seq'])] = doc["angh{:03d}".format(doc['seq'])]
resposta["angv{:03d}".format(doc['seq'])] = doc["angv{:03d}".format(doc['seq'])]
else:
landmarks = self.align.findLandmarks(img, bb)
angcab,angvcab,pp = RepUtil.calcHVAngRosto(landmarks[0],landmarks[16],landmarks[27])
alignedFace = self.align.align(96, img, bb,
landmarks=landmarks,
landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE)
rep = self.net.forward(alignedFace)
resposta["rep{:03d}".format(doc['seq'])] = rep.tolist()
resposta["angh{:03d}".format(doc['seq'])] = angcab
resposta["angv{:03d}".format(doc['seq'])] = angvcab
self.mdb.db.fs.files.update({'_id':doc['_id']},{ "$set":
{"rep{:03d}".format(doc['seq']):rep.tolist(),
"angh{:03d}".format(doc['seq']):angcab,
"angv{:03d}".format(doc['seq']):angvcab}})
contap += 1
if (contap+contaf)%10 == 0:
texto = "Processed {} images".format((contap+contaf))
print texto
parcial={'RESPONSE':'INPROGRESS',
'INFO':texto}
self.sendMessage(json.dumps(parcial))
resposta['countp']=contap
resposta['seqf']=np.array(seqf).tolist()
resposta['countn']=contaf
resposta['identity']=doc['identity']
return resposta
def processIdInfo(self,msg):
#resposta = {'RESPONSE':'ERROR','REASON':'QUERYIDREP:Unknow parameters:'+msg.keys()}
if self.mdb is None:
resposta = {'RESPONSE':'ERROR','REASON':'mongodb serve is not connected'}
else:
if 'identity' not in msg.keys():
resposta = {'RESPONSE':'ERROR','REASON':'Not have filename field'}
else:
keys={'identity':msg['identity']}
imgs=self.mdb.readImFromDB(keys=keys)
if imgs is None:
resposta = {'RESPONSE':'ERROR','REASON':'Something wrong in mongodb'}
elif len(imgs)<1:
resposta = {'RESPONSE':'ERROR','REASON':'Do not have file in db'}
else:
resposta = {'RESPONSE':'OK'}
contap = 0
contaf = 0
seqf=[]
for doc,img in imgs:
bb = self.align.getLargestFaceBoundingBox(img)
if bb is None:
print "O arquivo {} nao detectou face".format(doc["filename"])
seqf.append(doc['seq'])
contaf += 1
else:
landmarks = self.align.findLandmarks(img, bb)
angcab,angvcab,pp = RepUtil.calcHVAngRosto(landmarks[0],landmarks[16],landmarks[27])
alignedFace = self.align.align(96, img, bb,
landmarks=landmarks,
landmarkIndices=openface.AlignDlib.OUTER_EYES_AND_NOSE)
rep = self.net.forward(alignedFace)
resposta["rep{:03d}".format(doc['seq'])] = rep.tolist()
resposta["angh{:03d}".format(doc['seq'])] = angcab
resposta["angv{:03d}".format(doc['seq'])] = angvcab
resp,nc = self.classif.buscaCandidato(rep, angcab, angvcab)
countMat = 0
for nome,pb,reffile,tipoc,angs in resp:
if nome != "Desconhecido":
resposta["name{:03d}.{:02d}".format(doc['seq'],countMat)]=nome
resposta["prob{:03d}.{:02d}".format(doc['seq'],countMat)]=pb
resposta["tycl{:03d}.{:02d}".format(doc['seq'],countMat)]=tipoc
anghe,angve = angs
resposta["angh{:02d}".format(countMat)]=anghe
resposta["angv{:02d}".format(countMat)]=angve
print "A imagem {} pode ser de {} com prob {:4.1f}".format(doc["filename"],nome,pb*100)
countMat += 1
resposta["nclassif".format(doc['seq'])]=countMat
contap += 1
if (contap+contaf)%10 == 0:
texto = "Processed {} images".format((contap+contaf))
print texto
parcial={'RESPONSE':'INPROGRESS',
'INFO':texto}
self.sendMessage(json.dumps(parcial))
resposta['countp']=contap
resposta['seqf']=np.array(seqf).tolist()
resposta['countn']=contaf
resposta['identity']=doc['identity']
return resposta
def onClose(self,wasClean,code,reason):
print "Connection closed with reason:{} code:{} is clean:{}".format(reason,code,wasClean)
import os
import cv2
from scipy.misc import imread, imsave
from align_dlib import AlignDlib
face_predictor_path = '/Users/Lavector/git-back/facerecognition/server/model/shape_predictor_68_face_landmarks.dat'
flist = os.listdir()
img_name = '/Users/Lavector/git-back/facerecognition/client/my_faces/acbc32c963d52017081414205615711930.jpg'
align = AlignDlib(face_predictor_path)
landmarkIndices = AlignDlib.OUTER_EYES_AND_NOSE
img = imread(img_name, mode='RGB')
aligned = align.align(160,
img, [445, 197, 816, 569],
landmarkIndices=landmarkIndices)
# thumbnail = cv2.warpAffine(img[445:816,197:569], aligned, (160, 160))
imsave('./result.jpg', aligned)
class VideoCapturePlayer(object):
size = (SCREEN_WIDTH, SCREEN_HEIGHT)
def __init__(self, **argd):
self.__dict__.update(**argd)
super(VideoCapturePlayer, self).__init__(**argd)
# create a display image. standard pygame stuff
self.display = pygame.display.set_mode(self.size, 0)
self.init_cams(1)
self.background = pygame.Surface(SCREEN_RECT.size)
Circle.images = [load_image('circle.png')]
# Initialize Game Groups
self.all = pygame.sprite.RenderUpdates()
self.facial_feature_group = pygame.sprite.RenderUpdates()
# assign default groups to each sprite class
Circle.containers = self.all, self.facial_feature_group
self.circles = [Circle() for _ in range(68)]
self.align_dlib_object = AlignDlib()
# HeadPoseEstimator
self.head_pose_estimator = HeadPoseEstimator(CAMERA_INPUT_WIDTH,
CAMERA_INPUT_HEIGHT)
def init_cams(self, which_cam_idx):
# gets a list of available cameras.
self.clist = pygame.camera.list_cameras()
print(self.clist)
if not self.clist:
raise ValueError("Sorry, no cameras detected.")
try:
cam_id = self.clist[which_cam_idx]
except IndexError:
cam_id = self.clist[0]
# creates the camera of the specified size and in RGB colorspace
self.camera = pygame.camera.Camera(
cam_id, (CAMERA_INPUT_WIDTH, CAMERA_INPUT_HEIGHT), "RGB")
# starts the camera
self.camera.start()
self.clock = pygame.time.Clock()
# create a image to capture to. for performance purposes, you want the
# bit depth to be the same as that of the display image.
self.camera_shot_raw = pygame.surface.Surface(
(CAMERA_INPUT_WIDTH, CAMERA_INPUT_HEIGHT), 0, self.display)
self.camera_default_display_location = (SCREEN_WIDTH -
CAMERA_DISPLAY_WIDTH,
SCREEN_HEIGHT -
CAMERA_DISPLAY_HEIGHT)
def get_camera_shot(self):
# For now, only get the camera shot and store it in self.camera_shot_raw.
# if you don't want to tie the framerate to the camera, you can check and
# see if the camera has an image ready. note that while this works
# on most cameras, some will never return true.
# if 0 and self.camera.query_image():
# # capture an image
# self.camera_shot_raw = self.camera.get_image(self.camera_shot_raw)
# if 0:
# self.camera_shot_raw = self.camera.get_image(self.camera_shot_raw)
# # blit it to the display image. simple!
# self.display.blit(self.camera_shot_raw, (0, 0))
# else:
# self.camera_shot_raw = self.camera.get_image(self.display)
self.camera_shot_raw = self.camera.get_image(self.camera_shot_raw)
self.camera_shot = pygame.transform.scale(
self.camera_shot_raw, (SCREEN_WIDTH, SCREEN_HEIGHT))
def blit_camera_shot(self, blit_location):
"""
:param blit_location: tuple with format (x, y)
:return:
"""
self.display.blit(self.camera_shot, blit_location)
def main(
self,
facial_landmark_detector,
):
# First calibrate
while not facial_landmark_detector.calibrate_face(
self.camera_shot_raw, self.head_pose_estimator):
self.get_camera_shot()
self.blit_camera_shot((0, 0))
pygame.display.flip()
do_rotate = False
going = True
while going:
events = pygame.event.get()
for e in events:
if e.type == QUIT or (e.type == KEYDOWN and e.key == K_ESCAPE):
going = False
if e.type == KEYDOWN:
if e.key in range(K_0, K_0 + 10):
self.init_cams(e.key - K_0)
if e.key == K_SPACE:
do_rotate = not do_rotate
print("ROTATE ", do_rotate)
self.get_camera_shot()
# Now use the facial landmark defector
# This step decreases the frame rate from 30 fps to 6fps. So we need to do something about it.
# The speed is directly related to FACIAL_LANDMARK_PREDICTOR_WIDTH.
face_coordinates, facial_features = facial_landmark_detector.get_features(
self.camera_shot_raw)
if len(face_coordinates) > 0:
# print("Detected %d face%s." % (len(face_coordinates), "s" if len(face_coordinates) > 1 else ""))
face_index = facial_landmark_detector.get_largest_face_index(
face_coordinates)
for i in range(68):
self.circles[i].move(facial_features[face_index][i])
# Use head pose estimator
head_pose = self.head_pose_estimator.head_pose_estimation(
facial_features[face_index])
# This is the roll, pitch, and yaw, but it is very dependent on the initial position of the camera.
# So maybe calibrate and set a relative threshold for controlling.
# print(np.array(head_pose[0]) * 180 / np.pi)
# print("Reconstructed 3d of the nose (point 34): ",
# self.hpe.two_d_to_three_d(facial_features[face_index][33],
# head_pose[0], head_pose[1]))
# Get the rotation invariant facial features.
facial_features_3d = self.head_pose_estimator.facial_features_to_3d(
facial_features[face_index], head_pose[0], head_pose[1])
# This is the difference in pose, i.e. yaw, pitch, and roll
pose_diff = facial_landmark_detector.get_pose_diff(
head_pose[0])
# print("left-right difference: %s" % (str(pose_diff[0])))
# print("Pose difference: %s" % (str(pose_diff)))
# is_moving, direction = get_direction_from_pose(pose_diff)
is_moving, direction = get_direction_from_line(head_pose[2][0])
# print("Direction %f" %direction)
# min_y, min_z = np.min(facial_features_3d[:,1:], axis=0).tolist()
# max_y, max_z = np.max(facial_features_3d[:,1:], axis=0).tolist()
# for i in range(68):
# # Get rid of the x axis (depth).
# y = (max_y - facial_features_3d[i][1]) / (max_y - min_y) * SCREEN_WIDTH
# z = (max_z - facial_features_3d[i][2]) / (max_z - min_z) * SCREEN_HEIGHT
# self.circles[i].move((y,z))
# mouth_open_degree = get_mouth_open_score(facial_features_3d)
# if mouth_open_degree >= MOUTH_RATIO_LOWER_THRESHOLD:
# print("Mouth open degree: %f" % (mouth_open_degree))
# else:
# print("Mouth closed degree: %f" % (mouth_open_degree))
# mouth_left_corner_score = get_mouth_left_corner_score(facial_features_3d,
# facial_landmark_detector.norm_mouth_left_corner_to_center_dist)
# mouth_right_corner_score = get_mouth_right_corner_score(facial_features_3d,
# facial_landmark_detector.norm_mouth_right_corner_to_center_dist)
#
# # mouth_left_corner_score = get_mouth_left_corner_to_center_dist(facial_features_3d)
# # mouth_right_corner_score = get_mouth_right_corner_to_center_dist(facial_features_3d)
# # print("Mouth left corner score: %f, right corner score: %f, raw: %f"
# # %(mouth_left_corner_score, mouth_right_corner_score, pose_diff[0]))
# print("%f,%f,%f"
# %(mouth_left_corner_score, mouth_right_corner_score, pose_diff[0]))
# Get blink scores
# left_blink_score = get_left_blink_score(facial_features_3d)
# right_blink_score = get_right_blink_score(facial_features_3d)
# print("Blink left eye score: %f, right eye score: %f"
# %(left_blink_score, right_blink_score))
aligned_face = self.align_dlib_object.align(
128,
rgbImg=imutils.resize(
get_image_from_surface(self.camera_shot_raw)[..., :3],
FACIAL_LANDMARK_PREDICTOR_WIDTH),
)
# clear/erase the last drawn sprites
self.all.clear(self.display, self.background)
self.blit_camera_shot((0, 0))
# update the facial feature sprites only if at least one face is detected.
if len(face_coordinates) > 0:
self.facial_feature_group.update()
# draw the scene
dirty = self.facial_feature_group.draw(self.display)
pygame.display.update(dirty)
if aligned_face is not None:
aligned_face_surface = pygame.surfarray.make_surface(
aligned_face)
self.display.blit(aligned_face_surface, (0, 0))
# For now simply get the first face and update the sprites.
pygame.draw.rect(self.display, (0, 255, 0),
face_coordinates[face_index], 2)
pygame.draw.line(self.display, (255, 0, 0), head_pose[2][0][0],
head_pose[2][0][1])
pygame.draw.line(self.display, (0, 255, 0), head_pose[2][1][0],
head_pose[2][1][1])
pygame.draw.line(self.display, (0, 0, 255), head_pose[2][2][0],
head_pose[2][2][1])
pygame.display.flip()
self.clock.tick()
print(self.clock.get_fps())
class Preprocessor(object):
def __init__(self,
predictor_model="shape_predictor_68_face_landmarks.dat"):
# we use a pretrained model that can be downloaded from http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2
# download it to the repo root and unzip using `bunzip2 [filename]`
self.predictor_model = predictor_model
self.face_detector = dlib.get_frontal_face_detector()
self.face_pose_predictor = dlib.shape_predictor(predictor_model)
self.face_aligner = AlignDlib(predictor_model)
def flatten(self, img):
return img.flatten()
def unflatten(self, img, h=processed_height, w=processed_width):
return np.reshape(img, (h, w))
def process(self, img):
if len(img.shape) > 2 and img.shape[2] != 1:
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
detected_faces = self.face_detector(img, 1)
aligned_faces = []
for face_r in detected_faces:
pose_landmarks = self.face_pose_predictor(img, face_r)
aligned_face = self.face_aligner.align(
processed_height,
img,
face_r,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE)
aligned_face = cv2.resize(aligned_face,
(processed_height, processed_width))
aligned_face = aligned_face.flatten()
aligned_faces.append(aligned_face)
return aligned_faces
def data_from_db(self):
pass
def cache_file(self, hash):
return "{}/{}.npy".format(PREPROCESSOR_CACHE_PATH, hash)
def cache_processed_image(self, processed, hash):
if not os.path.exists(PREPROCESSOR_CACHE_PATH):
print("directory {} doesn't existing creating it".format(
PREPROCESSOR_CACHE_PATH))
os.makedirs(PREPROCESSOR_CACHE_PATH)
try:
f = open(self.cache_file(hash), "wb")
np.save(f, processed)
finally:
f.close()
return
def processed_image_from_cache(self, hash):
if not os.path.exists(self.cache_file(hash)):
return None
res = None
try:
f = open(self.cache_file(hash), "rb")
res = np.load(f)
finally:
f.close()
return res
def hash_arr(self, arr):
a = arr.view(np.uint8)
return hashlib.sha1(a).hexdigest()
def process_raw_images(self, images):
processed_images = []
not_found = []
for i, image in enumerate(images):
image_hash = self.hash_arr(image)
cache = self.processed_image_from_cache(image_hash)
if cache is not None:
processed_images.append(cache)
continue
image = image.astype(np.uint8)
faces = self.process(image)
if len(faces) == 1:
processed_images.append(faces[0])
self.cache_processed_image(faces[0], image_hash)
else:
not_found.append(i)
return np.array(processed_images), not_found
def get_lfw_data(self):
people = fetch_lfw_people('./data',
resize=1.0,
funneled=False,
min_faces_per_person=config['min_faces'])
X, not_found = self.process_raw_images(people.images)
y = people.target
y = np.delete(y, not_found)
target_names = people.target_names
return X, y, target_names
## for one person
def load_test_data(self, dir_path):
if not isdir(dir_path):
return None, None
label = dir_path.split('/')[-1] #name of person
paths = [join(dir_path, f) for f in listdir(dir_path)]
n_pictures = len(paths)
images = []
for p in paths:
images.append(cv2.imread(p))
images = np.array(images)
target_labels = np.array([label for i in range(n_pictures)])
images, not_found = self.process_raw_images(images)
target_labels = np.delete(target_labels, not_found)
return images, target_labels
def get_small_data(self):
person_names, file_paths = [], []
for person_name in sorted(listdir(SMALL_DATA_PATH)):
folder_path = join(SMALL_DATA_PATH, person_name)
if not isdir(folder_path):
continue
paths = [join(folder_path, f) for f in listdir(folder_path)]
n_pictures = len(paths)
person_names.extend([person_name] * n_pictures)
file_paths.extend(paths)
target_names = np.unique(person_names)
target = np.searchsorted(target_names, person_names)
n_faces = len(file_paths)
images = []
for p in file_paths:
images.append(cv2.imread(p))
images = np.array(images)
## network training sensitive to consecutive same labels
indices = np.arange(n_faces)
np.random.RandomState(42).shuffle(indices)
images, target = images[indices], target[indices]
images, not_found = self.process_raw_images(images)
target = np.delete(target, not_found)
return images, target, target_names
def get_data(self):
if SMALL_MODEL:
print("Using training data from small dataset")
X, y, target_names = self.get_small_data()
else:
print("Using training data from sklearn")
X, y, target_names = self.get_lfw_data()
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.25)
return X_train, X_test, y_train, y_test, target_names
yd2 = bb.bottom()
wb = xd2 - xd1
xd1,yd1,xd2,yd2 = (np.array((xd1,yd1,xd2,yd2))/fator).astype(int)
draw_rects(anotado,[(x1+xd1,y1+yd1,xd2-xd1,yd2-yd1)],(0,255,255))
# print "Largest B:{}".format(bb)
landmarks = align.findLandmarks(reduzido, bb)
distl = calcDistPoints(landmarks[0],landmarks[16])
#w = x2 - x1
frac = distl/(w*fator)
soma += frac
contad += 1
print "Valor de distl:{} , w: {} wb: {} e distl/w: {} media: {}".format(distl,w,wb,frac,soma/contad)
alignedFace = align.align(96, reduzido, bb,
landmarks=landmarks,
landmarkIndices=align.OUTER_EYES_AND_NOSE)
cv2.imshow("Extraido",alignedFace)
# #print "Landmarks: {}".format(landmarks)
# reff = [0,8,16,27,30,33,36,45,48,51,54,57,62,66]
# for idxp in landmarks:
# cv2.circle(anotado,center=idxp,radius=3,
# color=(255,204,102),thickness=-1)
#===============================================================
#if len(rects)>0:
# draw_rects(anotado,rects,(0,0,255))
cv2.imshow("Anotado",anotado)
dt = clock() - t
# prediction
confidence = detections[0, 0, 0, 2]
min_confidence = 0
# filter out weak detections
if confidence > min_confidence:
# compute the (x, y)-coordinates of the bounding box for the
# face
box = detections[0, 0, 0, 3:7] * np.array([iw, ih, iw, ih])
(startX, startY, endX, endY) = box.astype("int")
if startX < 0 or startY < 0 or endX > iw or endY > ih:
continue
rect = dlib.rectangle(startX, startY, endX, endY)
face = face_aligner.align(
96,
image,
rect,
landmarkIndices=AlignDlib.OUTER_EYES_AND_NOSE
)
# construct a blob for the face ROI, then pass the blob
# through our face embedding model to obtain the 128-d
# quantification of the face
faceBlob = cv2.dnn.blobFromImage(face, 1.0 / 255,
(96, 96), (0, 0, 0), swapRB=True, crop=False)
inputs = torch.from_numpy(faceBlob).to(device)
vec = embedder.forward(inputs).cpu().numpy()
# add the name of the person + corresponding face
# embedding to their respective lists
knownNames.append(name)
knownEmbeddings.append(vec.flatten())
