curTime = time.time() # calc fps
find_results = []
frame = frame[:, :, 0:3]
boxes, scores = face_detector.detect(frame)
face_boxes = boxes[np.argwhere(scores>0.3).reshape(-1)]
face_scores = scores[np.argwhere(scores>0.3).reshape(-1)]
print('Detected_FaceNum: %d' % len(face_boxes))
if len(face_boxes) > 0:
for i in range(len(face_boxes)):
box = face_boxes[i]
cropped_face = frame[box[0]:box[2], box[1]:box[3], :]
cropped_face = cv2.resize(cropped_face, (160, 160), interpolation=cv2.INTER_AREA)
feature = face_recognition.recognize(cropped_face)
name = face_classfier.classify(feature)
cv2.rectangle(frame, (box[1], box[0]), (box[3], box[2]), (0, 255, 0), 2)
# plot result idx under box
text_x = box[1]
text_y = box[2] + 20
cv2.putText(frame, name, (text_x, text_y), cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 255), thickness=1, lineType=2)
else:
print('Unable to align')
sec = curTime - prevTime
prevTime = curTime
fps = 1 / (sec)
str = 'FPS: %2.3f' % fps
class FaceRecognition(object):
def __init__(self, profile_activated):
self.profile_activated = profile_activated
self.model_path = BASE_DIR + SPERATOR + "facedatabase" + SPERATOR
self.svm_model_name = "trained_classifier.pkl"
self.image_rep_name = "faces.npy"
self.people_rep_name = "people.npy"
self.image_folder_path = "images" + SPERATOR
if self.profile_activated is True:
print("FaceRecognition: initialize face recognizer")
self.svm = None
self.training = False
self.training_id = 0
self.training_name = "Unknown"
self.trainingInProgress = False
self.identify_threshold = 0.8
self.face_classfier = None
if not os.path.exists(self.model_path):
os.makedirs(self.model_path)
if self.profile_activated is True:
print("FaceRecognition: loading prepresentations of kownen persons")
try:
self.images = np.load(self.model_path + SPERATOR + self.image_rep_name,allow_pickle=True).item()
except IOError:
if self.profile_activated is True:
print("Warning: no image representation file found")
self.images = {}
try:
self.people = np.load(self.model_path + SPERATOR + self.people_rep_name,allow_pickle=True).item()
except IOError:
if self.profile_activated is True:
print("Warning: no people representation file found")
self.people = {}
self.people["0"] = "Unknown"
if self.profile_activated is True:
print ("FaceRecognition: try to load saved svm for estimation")
try:
self.face_classfier = FaceClassifier(self.model_path + SPERATOR + self.svm_model_name)
except IOError:
if self.profile_activated is True:
print ("Warning: no svm saved")
self.face_classfier = None
if self.profile_activated is True:
print ("FaceRecognition: initialisation done")
self.tracker_sort = Sort(25,3)
def __del__(self):
if self.profile_activated is True:
print ("FaceRecognition: starting destruktor")
"""
Trains the neural net with all saved images.
Therefore every pic is loaded, cropped and representations are created.
This uses all neuronal nets, so there need to be available (self.loadingfromimages = True)
"""
def learnFromPics(self):
if self.profile_activated is True:
print ("FaceRecognition: loading representations from saved pics")
print ("Folder: " + self.model_path + SPERATOR +self.image_folder_path)
self.trainingInProgress = True
self.people["0"] = "Unknown"
time.sleep(5)
"""
Loop trough every subfolder and get every picture
The folder names provide id and the person name
"""
for folder in glob.glob(self.model_path + SPERATOR + self.image_folder_path + "*"):
args = folder.replace(self.model_path + SPERATOR + self.image_folder_path, "")
args = args.split(SPERATOR)
args = args[0].split("_")
for file in glob.glob(folder + SPERATOR +"*"):
alignedFace = []
image = cv2.imread(file)
image = np.asarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
if image.shape == (160,160,3):
if self.profile_activated is True:
print(file + " [" + str(image.shape) + "]: image of a face")
alignedFace.append(image)
else:
bbs, scores = face_detector.detect(image)
bbs = bbs[np.argwhere(scores > FACE_DETEC_THRESHOLD).reshape(-1)]
scores = scores[np.argwhere(scores > FACE_DETEC_THRESHOLD).reshape(-1)]
if self.profile_activated is True:
print(file + ": found " + str(len(bbs)) + " faces")
if len(bbs) < 1:
continue
for bb in bbs:
cropped_face = image[bb[0]:bb[2], bb[1]:bb[3], :]
alignedFace.append(cv2.resize(cropped_face, (160, 160), interpolation=cv2.INTER_AREA))
for face in alignedFace:
phash = str(imagehash.phash(Image.fromarray(face)))
rep = face_recognition.recognize(face)
self.images[phash] = Face(rep, int(args[0]))
self.people[args[0]] = args[1]
if self.profile_activated is True:
print ("FaceRecognition: finished loading data from saved images")
self.trainSVM()
self.trainingInProgress = False
"""
Saves self.image and self.people
"""
def saveImageDataAndTrainedNet (self):
if self.profile_activated is True:
print ("FaceRecognition: saving image representations and people dict")
if not os.path.exists(self.model_path):
os.makedirs(self.model_path)
np.save(self.model_path + SPERATOR + self.image_rep_name, self.images)
np.save(self.model_path + SPERATOR + self.people_rep_name, self.people)
"""
Get the information from self.images and convert it to two list with reps and labels.
"""
def getData(self):
X = []
y = []
for img in self.images.values():
X.append(img.rep)
y.append(img.identity)
numIdentities = len(set(y + [-1])) - 1
if numIdentities == 0:
return None
X = np.vstack(X)
y = np.array(y)
return (X, y)
"""
Called to train the neural net with all representations currently loaded in self.image.
Saves also everything afterwards.
"""
def trainSVM(self):
self.trainingInProgress = True
if self.profile_activated is True:
print ("FaceRecognition: Training SVM on {} labeled images.".format(len(self.images)))
d = self.getData()
if d is None:
self.face_classfier = None
if self.profile_activated is True:
print ("FaceRecognition: at least 2 persons needed ..")
self.trainingInProgress = False
return
else:
(X, y) = d
numIdentities = len(set(y + [-1]))
if numIdentities <= 1:
self.trainingInProgress = False
return
self.face_classfier = FaceClassifier()
self.face_classfier.train(X, y, model='svm', save_model_path=(self.model_path + SPERATOR + self.svm_model_name))
self.saveImageDataAndTrainedNet()
print ("training done!")
self.trainingInProgress = False
"""
Called to process every frame (basically the main function)
Need a Frame to process, returns die found identities, confidences and a frame with marks
"""
def processframe(self, frame):
#For profiling this script
if self.profile_activated is True:
start_time = time.time()
if self.trainingInProgress is True:
return [],[], frame , caption_frame
identities = []
identities_bb = []
confidences = []
caption_frame = np.zeros(frame.shape, np.uint8)
"""
Searching for everything that look similar to a face
Remove scores that are too low
"""
bbs, scores = face_detector.detect(frame)
bbs = bbs[np.argwhere(scores > 0.25).reshape(-1)]
scores = scores[np.argwhere(scores > 0.25).reshape(-1)]
if self.training is True:
if len(bbs) is not 1:
if self.profile_activated is True:
print ("FaceRecognition: Need to find exacly one Person")
return [],[], frame, caption_frame
else:
if self.profile_activated is True:
print ("FaceRecognition: I want to train.. Taking Picture! Please move to differnt angles")
pass
# For profiling this script
if self.profile_activated is True:
boundingboxes_time = time.time()
loopstart_time = []
alignedFace_time = []
finished_time = []
found_identity_time = []
"""
Deside the identity of every boxes found in the frame
"""
for index,bb in enumerate(bbs):
# For profiling this script
if self.profile_activated is True:
loopstart_time.append(time.time())
confidence = ""
identity = "0"
"""
cropp every face to a 160x160 dimention to pass it into a neuronal net
"""
cropped_face = frame[bb[0]:bb[2], bb[1]:bb[3], :]
alignedFace = cv2.resize(cropped_face, (160, 160), interpolation=cv2.INTER_AREA)
# For profiling this script
if self.profile_activated is True:
alignedFace_time.append(time.time())
if alignedFace is None:
# For profiling this script
if self.profile_activated is True:
finished_time.append(time.time())
continue
"""
We know the identity if the picture was taken before or is equal to a used image.
Otherwise get a feature representation of that face and compare via a neuronal net.
"""
phash = str(imagehash.phash(Image.fromarray(alignedFace)))
if phash in self.images:
identity = str(self.images[phash].identity)
confidence = "1.0000"
# For profiling this script
if self.profile_activated is True:
found_identity_time.append(time.time())
else:
"""if we are training a the face_recognition nn can be occupied!"""
if self.trainingInProgress is True:
return [],[], frame , caption_frame,
rep = face_recognition.recognize(alignedFace)
if self.training is True:
"""
If training is enabled, add the representation to self.images
and save the new image into a subfolder of the image folder for later training runs.
The new image is shown in the top left.
"""
self.images[phash] = Face(rep, self.training_id)
self.people[str(self.training_id)] = self.training_name
person_path = self.model_path + SPERATOR + self.image_folder_path + SPERATOR + str(self.training_id) + "_" + str(self.training_name) + SPERATOR
if not os.path.exists(person_path):
os.makedirs(person_path)
cv2.imwrite( person_path + str(phash) +".png",alignedFace)
frame[0:160, 0:160, 0] = alignedFace[:, :, 2]
frame[0:160, 0:160, 1] = alignedFace[:, :, 1]
frame[0:160, 0:160, 2] = alignedFace[:, :, 0]
cv2.rectangle(frame, (161, 161), (0, 0), color=(255, 50, 50), thickness=4)
# For profiling this script
if self.profile_activated is True:
found_identity_time.append(time.time())
elif self.face_classfier is not None:
"""
If a classifier is existent, get a prediction who this face might be.
Compare the confidence to a threshold to evade to low scores.
"""
rep = rep.reshape(1, -1)
predictions = self.face_classfier.classify(rep).ravel()
maxI = np.argmax(predictions)
confidence = str(predictions[maxI].round(5))
if predictions[maxI] <= self.identify_threshold:
maxI = 0
identity = str(maxI)
# For profiling this script
if self.profile_activated is True:
found_identity_time.append(time.time())
else:
# For profiling this script
if self.profile_activated is True:
found_identity_time.append(time.time())
""" Append the identity in any case with the highest confidence."""
identities.append(identity)
identities_bb.append((bb[1], bb[0], bb[3], bb[2]))
confidences.append(confidence)
""" Mark everyone in the frame """
cv2.rectangle(frame, (bb[1], bb[0]), (bb[3], bb[2]), color=(255,50,50), thickness=2)
cv2.rectangle(caption_frame, (bb[1], bb[0]), (bb[3], bb[2]), color=(255,50,50), thickness=2)
cv2.putText(frame, self.people[identity] , (bb[1], bb[0] - 10),
cv2.FONT_HERSHEY_DUPLEX, fontScale=1,
color=(255, 50, 50), thickness=2)
cv2.putText(frame, confidence, (bb[1], bb[2] + 30),
cv2.FONT_HERSHEY_DUPLEX, fontScale=1,
color=(255, 50, 50), thickness=2)
cv2.putText(caption_frame, self.people[identity] , (bb[1], bb[0] - 10),
cv2.FONT_HERSHEY_DUPLEX, fontScale=1,
color=(255, 50, 50), thickness=2)
cv2.putText(caption_frame, confidence, (bb[1], bb[2] + 30),
cv2.FONT_HERSHEY_DUPLEX, fontScale=1,
color=(255, 50, 50), thickness=2)
# For profiling this script
if self.profile_activated is True:
finished_time.append(time.time())
# For profiling this script
if self.profile_activated is True:
end_time = time.time()
profile_string = "TIMES: bounding boxes: " + str( format(boundingboxes_time - start_time,PROFILE_ROUND ))
profile_string += ", face(aligne, recognize, annotate) {"
for i in range(0, len(loopstart_time)):
profile_string += " (" + str(format(alignedFace_time[i] - loopstart_time[i],PROFILE_ROUND))
profile_string += "," + str(format(found_identity_time[i] - alignedFace_time[i],PROFILE_ROUND))
profile_string += "," + str(format(finished_time[i] - found_identity_time[i],PROFILE_ROUND)) + ")"
profile_string += " }"
profile_string += ", entire: " + str(format(end_time - start_time,PROFILE_ROUND))
profile_string += ", fps: " + str(format(1.0 / (end_time - start_time),PROFILE_ROUND))
print (profile_string)
return identities,identities_bb,confidences,frame, caption_frame
def findFacesTracked(self, frame):
tracking_dets = []
bbs, scores = face_detector.detect(frame)
bbs = bbs[np.argwhere(scores > 0.25).reshape(-1)]
scores = scores[np.argwhere(scores > 0.25).reshape(-1)]
for i in range (0, len(bbs)):
tracking_dets.append([int(bbs[i][1]),int(bbs[i][0]),int(bbs[i][3]),int(bbs[i][2]),int(100 * scores[i])])
#if len(tracking_dets) == 0:
# tracking_dets.append([])
trackers = self.tracker_sort.update(np.asarray(tracking_dets))
int_trackers = []
for tracker in trackers:
if tracker[0] < 0:
tracker[0] = 0
if tracker[1] < 0:
tracker[1] = 0
if tracker[2] > 1079:
tracker[2] = 1079
if tracker[3] > 1919:
tracker[3] = 1919
int_trackers.append([int(tracker[0]),int(tracker[1]),int(tracker[2]),int(tracker[3]),int(tracker[4])])
return int_trackers
def identifyFacesInBB(self, frame, bb):
cropped_face = cv2.UMat(frame[bb[1]:bb[3], bb[0]:bb[2], :])
alignedFace = cv2.UMat.get(cv2.resize(cropped_face, (160, 160), interpolation=cv2.INTER_AREA))
rep = face_recognition.recognize(alignedFace)
rep = rep.reshape(1, -1)
predictions = self.face_classfier.classify(rep).ravel()
maxI = np.argmax(predictions)
if predictions[maxI] <= self.identify_threshold:
maxI = 0
return (self.people[str(maxI)] , maxI, predictions[maxI])
def main(args):
face_detector = FaceDetector()
face_recognition = FaceRecognition(args.model)
face_classfier = FaceClassifier(args.classifier_filename)
video_capture = cv2.VideoCapture(args.video_input)
output_file = './media/result/' + os.path.basename(
args.video_input) + '_result.avi'
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter(
output_file, fourcc, 24.0,
(int(video_capture.get(3)), int(video_capture.get(4))))
print('Start Recognition!')
prevTime = 0
while video_capture.isOpened():
ret, frame = video_capture.read()
curTime = time.time() # calc fps
find_results = []
frame = frame[:, :, 0:3]
boxes, scores = face_detector.detect(frame)
face_boxes = boxes[np.argwhere(scores > 0.3).reshape(-1)]
face_scores = scores[np.argwhere(scores > 0.3).reshape(-1)]
print('Detected_FaceNum: %d' % len(face_boxes))
if len(face_boxes) > 0:
for i in range(len(face_boxes)):
box = face_boxes[i]
cropped_face = frame[box[0]:box[2], box[1]:box[3], :]
cropped_face = cv2.resize(cropped_face, (160, 160),
interpolation=cv2.INTER_AREA)
feature = face_recognition.recognize(cropped_face)
name = face_classfier.classify(feature)
cv2.rectangle(frame, (box[1], box[0]), (box[3], box[2]),
(0, 255, 0), 2)
# plot result idx under box
text_x = box[1]
text_y = box[2] + 20
cv2.putText(frame,
name, (text_x, text_y),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 255),
thickness=1,
lineType=2)
else:
print('Unable to align')
sec = curTime - prevTime
prevTime = curTime
fps = 1 / (sec)
str = 'FPS: %2.3f' % fps
text_fps_x = len(frame[0]) - 150
text_fps_y = 20
cv2.putText(frame,
str, (text_fps_x, text_fps_y),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 0),
thickness=1,
lineType=2)
out.write(frame)
video_capture.release()
out.release()
cv2.destroyAllWindows()
path = os.path.join("media", "train_classifier", 'milind')
attendance_list.append(name)
_update_attendance(attendance_list)
#print(type(path))
if not os.path.exists(path):
os.mkdir(path)
if len(face_boxes) > 0:
box = face_boxes[box_ii]
cropped_face = frame[box[0]:box[2], box[1]:box[3], :]
cropped_face = cv2.resize(cropped_face, (160, 160),
interpolation=cv2.INTER_AREA)
feature = face_recognition.recognize(cropped_face, mobilenet=mobilenet)
print("len features {}".format(len(feature)))
name_features = face_classfier.classify(feature)
#madeachange
print('name features', name_features)
if os.path.exists(os.path.join("media", "train_image_classifier",
name)):
continue
# Expand by 20 pixels
save_face = frame[box[0] - 20:box[2] + 20, box[1] - 20:box[3] + 20, :]
#print(type(save_face))
#cv2.imwrite(os.path.join(path,name+"_"+"%d"%(count)+".jpg"),save_face)
count += 1
cv2.rectangle(frame, (box[1], box[0]), (box[3], box[2]), (0, 255, 0),
2)
frame = frame[:, :, 0:3]
boxes, scores = face_detector.detect(frame)
face_boxes = boxes[np.argwhere(scores > 0.3).reshape(-1)]
face_scores = scores[np.argwhere(scores > 0.3).reshape(-1)]
print('Detected_FaceNum: %d' % len(face_boxes))
if len(face_boxes) > 0:
for i in range(len(face_boxes)):
box = face_boxes[i]
cropped_face = frame[box[0]:box[2], box[1]:box[3], :]
cropped_face = cv2.resize(cropped_face, (160, 160),
interpolation=cv2.INTER_AREA) #face area
feature = face_recognition.recognize(
cropped_face) #get 512 feature
name = face_classfier.classify(feature) #just suspect
# check if acceptable (faster)
# image_name1 = 'media/check/'+ name +'.jpg'
# image1 = scipy.misc.imread(image_name1, mode='RGB')
# cv2.imshow('%d'%i, image1)
# feature2 = face_recognition.recognize(image1)
dist = np.sqrt(np.sum(np.square(feature_dict[name] - feature)))
#dist: distance of this founded face and original face, the smaller the similar
if dist >= 1.05:
name = 'unknown'
#check if acceptable (faster)
cv2.rectangle(frame, (box[1], box[0]), (box[3], box[2]),
(0, 0, 255), 1)
if len(boxes) > 0:
for x1, y1, x2, y2 in boxes:
if x2 - x1 < 30 or y2 - y1 < 30:
continue
cropped_face = frame[y1:y2, x1:x2, :]
# cv2.imshow(f"Deteccao", cropped_face)
# cv2.waitKey(0)
pre_processed_face = face_detector.pre_process(cropped_face)
if pre_processed_face is None:
continue
feature = face_recognition.describe(pre_processed_face)
name, prob = face_classifier.classify(feature)
cv2.rectangle(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
# plot result idx under box
text_x = x1 - 5
text_y = y2 + 15
cv2.putText(frame,
f"{name} ({prob})", (text_x, text_y),
cv2.FONT_HERSHEY_COMPLEX_SMALL,
1, (0, 0, 255),
thickness=1,
lineType=2)
else:
print('Unable to align')