def reverse_face_same_as_cropped_face(self, name, image):
bbs, pts = DETECTOR.detect_face(image)
cropped_face = CropperUtils.crop_face(image, bbs[0])
display_face, padded_bb_str = CropperUtils.crop_display_face(
image, bbs[0])
reversed_face = CropperUtils.reverse_display_face(
display_face, padded_bb_str)
self.assertEqual(cropped_face.shape, reversed_face.shape, msg=name)
self.assertAlmostEqual(
np.sum(cropped_face - reversed_face), 0, msg=name)
def load_and_align_data(face_graph, image_paths, do_display):
detector = MTCNNDetector(face_graph)
nrof_samples = len(image_paths)
img_list = [None] * nrof_samples
for i in range(nrof_samples):
img = misc.imread(os.path.expanduser(image_paths[i]))
bounding_boxes, landmarks = detector.detect_face(img)
all_face_marks_x = landmarks[0:5, :]
all_face_marks_y = landmarks[5:10, :]
face_marks_x = all_face_marks_x[:, 0]
face_marks_y = all_face_marks_y[:, 0]
print(face_marks_x)
print(face_marks_y)
# draw landmarks on image
if do_display:
frame = img.copy()
for x, y in zip(face_marks_x, face_marks_y):
cv2.circle(frame, (int(x), int(y)), 2, (0, 0, 255), -1)
cv2.imshow(image_paths[i], frame)
cv2.waitKey()
bounding_box = bounding_boxes[0]
img_list[i] = CropperUtils.crop_face(img, bounding_box)
images = np.stack(img_list)
return images
def display_face_ratio(self, name, image):
bbs, pts = DETECTOR.detect_face(image)
display_face, _ = CropperUtils.crop_display_face(image, bbs[0])
h, w, _ = display_face.shape
self.assertGreater(h, 0, msg=name)
self.assertGreater(w, 0, msg=name)
self.assertAlmostEqual(h / w - 1.5, 0, delta=0.01, msg=name)
def get_bounding_box(original_path):
restructured_path = original_path + "_restructured"
create_if_not_exist(restructured_path)
face_ids = [
id for id in os.listdir(original_path)
if os.path.isdir(os.path.join(original_path, id))
]
for face_id in face_ids:
id_path = os.path.join(original_path, face_id)
r_id_path = os.path.join(restructured_path, face_id)
create_if_not_exist(r_id_path)
images = [
os.path.join(id_path, image) for image in os.listdir(id_path)
if "jpg" in image or "png" in image
]
for image_name in images:
img = cv2.cvtColor(cv2.imread(image_name), cv2.COLOR_BGR2RGB)
rects, landmarks = detector.detect_face(img)
if len(rects == 1) and FaceAngleUtils.is_acceptable_angle(
landmarks[:, 0]):
origin_bb = rects[0][:4]
display_face, str_padded_box = CropperUtils.crop_display_face(
img, origin_bb)
bbox_str = '_'.join([str(int(num)) for num in origin_bb])
image_id = '{}_{}_{}_{}.jpg'.format(face_id, bbox_str,
time.time(),
str_padded_box)
cv2.imwrite(os.path.join(r_id_path, image_id),
cv2.cvtColor(display_face, cv2.COLOR_RGB2BGR))
print(image_id)
def extract_embs(model_path, data, margin, save_name,
nrof_samples=float('Inf')):
save_name = os.path.expanduser(save_name.rstrip('/'))
if not os.path.exists(os.path.dirname(save_name)):
print('please enter a valid save_name')
data = os.path.expanduser(data.rstrip('/'))
id_dirs = glob.glob(os.path.join(data, '*'))
id_dict = {}
for id_dir in id_dirs:
id_label = os.path.basename(id_dir)
image_paths = glob.glob(os.path.join(id_dir, '*.*'))
use_samples = min(nrof_samples, len(image_paths))
for path in image_paths[:use_samples]:
id_dict[path] = id_label
tf = FaceGraph()
extractor = FacenetExtractor(tf, model_path=model_path)
nrof_imgs = 0
emb_array = np.zeros((len(id_dict), 128))
label_list = []
image_ids = []
print('reading images')
for path, label in id_dict.items():
try:
img = misc.imread(path)
coor_list = os.path.splitext(
os.path.basename(path))[0].split('_')[-4:]
bbox = np.array((coor_list), dtype=int)
face_img = CropperUtils.crop_face(img, bbox, margin)
face_img = default_preprocess(face_img)
emb, coeff = extractor.extract_features(face_img).squeeze()
emb_array[nrof_imgs, :] = emb
except (ValueError, OSError) as e:
print(e)
print('skipping', path)
continue
nrof_imgs += 1
label_list.append(label)
image_ids.append(os.path.basename(path))
emb_array = emb_array[:nrof_imgs]
print('extracted {} images'.format(nrof_imgs))
save = {'embs': emb_array, 'labels': label_list, 'image_ids': image_ids}
save_file = os.path.join(save_name)
print('result saved at', save_file)
with open(save_file, 'wb') as f:
pickle.dump(save, f)
def generate_video_sample(cam_url, area):
'''generating'''
print('Generating... ')
print("Cam URL: {}".format(cam_url))
print("Area: {}".format(area))
# Variables for tracking faces
frame_counter = 0
# Variables holding the correlation trackers and the name per faceid
frame_sample = {}
face_rec_graph = FaceGraph()
face_extractor = FacenetExtractor(face_rec_graph)
detector = MTCNNDetector(face_rec_graph)
preprocessor = Preprocessor()
if args.cam_url is not None:
frame_reader = URLFrameReader(args.cam_url, scale_factor=1)
else:
frame_reader = RabbitFrameReader(rabbit_mq)
try:
while True: # frame_reader.has_next():
frame = frame_reader.next_frame()
frame_sample[frame_counter] = FrameSample()
frame_sample[frame_counter].read_image = frame
if frame is None:
print("Waiting for the new image")
continue
print("Frame ID: %d" % frame_counter)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
origin_bbs, points = detector.detect_face(frame)
frame_sample[frame_counter].origin_bbs = origin_bbs
frame_sample[frame_counter].points = points
for _, origin_bb in enumerate(origin_bbs):
cropped_face = CropperUtils.crop_face(frame, origin_bb)
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
emb_array, coeff = face_extractor.extract_features(
preprocessed_image)
frame_sample[frame_counter].embs.append(emb_array)
frame_counter += 1
except KeyboardInterrupt:
print('Keyboard Interrupt !!! Release All !!!')
print('Saved this video sample as ../session/db/sample.pkl')
PickleUtils.save_pickle('../session/db/sample.pkl', frame_sample)
def gen_frames(root_folder):
dir_list = []
for folder in os.listdir(root_folder):
dir_list.append(folder)
frames = {}
face_rec_graph_face = FaceGraph()
face_rec_graph_coeff = FaceGraph()
face_extractor = FacenetExtractor(face_rec_graph_face,
model_path=Config.FACENET_DIR)
coeff_extractor = FacenetExtractor(face_rec_graph_coeff,
model_path=Config.COEFF_DIR)
preprocessor = Preprocessor()
for dir in dir_list:
for file in glob.glob(os.path.join(root_folder, dir, '*')):
image = imageio.imread(file)
identity = os.path.split(file)[-2]
file_name = os.path.split(file)[-1]
frame_id = file_name.split('_')[5]
origin_bbox = [int(i) for i in file_name.split('_')[1:5]]
if file_name.startswith('BAD-TRACK'):
padding_bbox = [
int(i) for i in file_name.split('.')[0].split('_')[-4:]
]
else:
padding_bbox = [
int(i) for i in file_name.split('.')[1].split('_')[-4:]
]
cropped_face = CropperUtils.crop_face(image, padding_bbox)
preprocessed_image = preprocessor.process(cropped_face)
emb_array, _ = face_extractor.extract_features(preprocessed_image)
_, coeff = coeff_extractor.extract_features(preprocessed_image)
if frame_id in frames:
frames[frame_id].append(
(file, origin_bbox, padding_bbox, identity, emb_array))
else:
frames[frame_id] = [(file, origin_bbox, padding_bbox, identity,
emb_array)]
return frames
def register_imgs(track_id_counter, is_registered, img_path, face_id,
mongodb_dashinfo, preprocessor, face_extractor):
img_dirs = glob.glob(img_path + '/*.jpg')
if img_dirs == []:
print('ERROR: No image in ' + img_path)
raise 'NO IMAGE IN ' + img_path
return False
labels = []
embs = []
for img_dir in img_dirs:
image_id = img_dir.split('/')[-1].replace('.jpg', '')
splitted_image_id = image_id.split('_')
bbox = splitted_image_id[-4:len(splitted_image_id)]
bbox = '_'.join(bbox)
bounding_box = splitted_image_id[1:5]
bounding_box = [int(bb_num) for bb_num in bounding_box]
time_stamp = float(splitted_image_id[5])
img = misc.imread(img_dir)
cropped_face = CropperUtils.reverse_display_face(img, bbox)
# Extract feature
preprocessed_image = preprocessor.process(cropped_face)
emb_array, _ = face_extractor.extract_features(preprocessed_image)
# For Session
mongodb_faceinfo.remove({'image_id': image_id})
mongodb_faceinfo.insert_one({
'track_id': track_id_counter,
'image_id': image_id,
'face_id': face_id,
'time_stamp': time_stamp,
'bounding_box': bounding_box,
'embedding': emb_array.tolist(),
'points': None,
'is_registered': is_registered
})
def extract_embs(tracking_folder, preprocessor, face_extractor, detector):
img_dirs = glob.glob(tracking_folder + '/*.jpg')
embs = []
labels = []
aligner = AlignCustom()
undetectable_faces = 0
for img_dir in img_dirs:
image_id = img_dir.split('/')[-1].replace('.jpg', '')
splitted_image_id = image_id.split('_')
bbox = splitted_image_id[-4:len(splitted_image_id)]
bbox = '_'.join(bbox)
origin_bb = splitted_image_id[1:5]
origin_bb = [int(bbnum) for bbnum in origin_bb]
time_stamp = float(splitted_image_id[5])
img = misc.imread(img_dir)
if detector is not None:
origin_bb, points = detector.detect_face(img)
print('nof detected faces ' + str(len(origin_bb)))
if len(points) == 0 or len(origin_bb) != 1:
undetectable_faces += 1
print(undetectable_faces)
continue
else:
preprocessed_image = preprocessor.process(
img, points[:, 0], aligner, 160)
else:
cropped_face = CropperUtils.reverse_display_face(img, bbox)
preprocessed_image = preprocessor.process(cropped_face)
# Extract feature
emb_array, _ = face_extractor.extract_features(preprocessed_image)
# For Matcher
embs.append(emb_array)
labels.append(image_id)
return embs, labels
def regdict_to_faceinfo():
mongodb_faceinfo.remove({})
reg_dict = PickleUtils.read_pickle(reg_dict_path)
reg_dict_length = len(reg_dict)
for i, i_id in enumerate(reg_dict):
print(reg_dict_length - i)
splitted_image_id = i_id.split('_')
track_id = int(splitted_image_id[0])
image_id = i_id
face_id = reg_dict[i_id]
time_stamp = float(splitted_image_id[5])
bounding_box = splitted_image_id[1:5]
bounding_box = [int(bb_num) for bb_num in bounding_box]
padded_bbox = splitted_image_id[-4:len(splitted_image_id)]
padded_bbox = '_'.join(padded_bbox)
display_img = PickleUtils.read_pickle(live_folder + '/' + i_id +
'.pkl')[0]
display_img = cv2.cvtColor(display_img, cv2.COLOR_BGR2RGB)
cropped_img = CropperUtils.reverse_display_face(
display_img, padded_bbox)
# Extract feature
preprocessed_image = preprocessor.process(cropped_img)
emb_array, _ = face_extractor.extract_features(preprocessed_image)
mongodb_faceinfo.insert_one({
'track_id': track_id,
'image_id': image_id,
'face_id': face_id,
'time_stamp': time_stamp,
'bounding_box': bounding_box,
'embedding': emb_array.tolist(),
'points': None,
'is_registered': True
})
def general_process(lock_id, detector, preprocessor, face_extractor,
blynk_locker):
'''
INPUT: lock_id
'''
# Get locker infomation
# lock_id = 'query from mongo'
locker_info = mongodb_lockersinfo.find({'lock_id': lock_id})[0]
this_locker = mongodb_lockers.find({'lock_id': lock_id})[0]
cam_url = locker_info['cam_url']
status = this_locker['status']
blynk_locker.processing(status)
# init face info
mongodb_faceinfo = mongodb_db[str(lock_id)]
# Variables for tracking faces
frame_counter = 0
start_time = time.time()
acceptable_spoofing = 0
# Variables holding the correlation trackers and the name per faceid
tracking_folder = os.path.join(Config.TRACKING_DIR, str(lock_id))
create_if_not_exist(tracking_folder)
tracking_dirs = glob.glob(tracking_folder + '/*')
if tracking_dirs == []:
number_of_existing_trackers = 0
else:
lof_int_trackid = [
int(tracking_dir.split('/')[-1]) for tracking_dir in tracking_dirs
]
number_of_existing_trackers = max(lof_int_trackid) + 1
tracker_manager = TrackerManager(
'LOCKID' + str(lock_id), current_id=number_of_existing_trackers)
frame_reader = URLFrameReader(cam_url, scale_factor=1)
matcher = FaissMatcher()
if status == 'locked':
embs = []
labels = []
cursors = mongodb_faceinfo.find({
'face_id': this_locker['lock_face_id']
})
for cursor in cursors:
embs.append(np.array(cursor['embedding']))
labels.append(cursor['image_id'])
nof_registered_image_ids = len(labels)
matcher.fit(embs, labels)
while True:
# in case the jerk hits the button
if time.time() - start_time > 4:
with open('../data/locker_{}_log.txt'.format(lock_id), 'a') as f:
f.write('[LOCKER {}] OUT OF TIME! \n\n'.format(lock_id))
frame_reader.release()
blynk_locker.stop_processing(status)
return -1
frame = frame_reader.next_frame()
if frame is None:
print('Invalid Video Source')
break
fps_counter = time.time()
# cv2.imshow('Locker {}'.format(lock_id), frame)
# cv2.waitKey(1)
tracker_manager.update_trackers(frame)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
origin_bbs, points = detector.detect_face(frame)
for i, in_origin_bb in enumerate(origin_bbs):
origin_bb = in_origin_bb[:-1]
display_face, str_padded_bbox = CropperUtils.crop_display_face(
frame, origin_bb)
cropped_face = CropperUtils.crop_face(frame, origin_bb)
# is_spoofing = spoofing_detector.is_face_spoofing(cropped_face)
# if is_spoofing:
# acceptable_spoofing += 1
# with open('../data/spoofing_log.txt', 'a') as f:
# f.write('Spoofing Detected at Locker {}: {}\n'.format(lock_id, is_spoofing))
# if acceptable_spoofing > 5:
# with open('../data/locker_{}_log.txt'.format(lock_id), 'a') as f:
# f.write(
# '[LOCKER {}] STOP PROCESSING. '
# 'SPOOFING DETECTED!\n'.format(lock_id)
# )
# frame_reader.release()
# blynk_locker.stop_processing(status)
# return -1
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
# preprocessed_image = align_preprocessor.process(frame, points[:,i], aligner, 160)
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
face_info = FaceInfo(origin_bb.tolist(), emb_array,
frame_counter, display_face,
str_padded_bbox, points[:, i].tolist())
is_good_face = handle_filters(points[:, i], coeff_extractor,
face_info, preprocessed_image)
face_info.is_good = is_good_face
# TODO: refractor matching_detected_face_with_trackers
matched_track_id = tracker_manager.track(face_info)
if not face_info.is_good:
print('BAD FACE')
continue
# Update tracker_manager
tracker_manager.update(matched_track_id, frame, face_info)
checking_tracker = None
checking_tracker, top_predicted_face_ids, matching_result_dict = \
tracker_manager.check_and_recognize_tracker(
matcher,
matched_track_id,
mongodb_faceinfo,
None)
# handle_results(checking_tracker, matching_result_dict)
if checking_tracker is not None:
dumped_images = checking_tracker.dump_images(
mongodb_faceinfo,
add_new=True,
trackingfolder=tracking_folder)
checking_tracker.represent_image_id = dumped_images[0]
face_url = os.path.join(Config.SEND_RBMQ_HTTP, str(lock_id),
str(checking_tracker.track_id),
checking_tracker.represent_image_id)
face_url += '.jpg'
if status == 'available':
# Save locker, sign up the face
mongodb_lockers.remove({'lock_id': lock_id})
msg_dict = {
'lock_id': lock_id,
'status': 'locked',
'lock_face_url': face_url,
'lock_face_id': checking_tracker.face_id,
'lock_timestamp': time.time(),
'unlock_face_url': None,
'unlock_face_id': None,
'unlock_timestap': None
}
mongodb_lockers.insert_one(msg_dict)
# update logs
msg_dict.update({'log_timestamp': time.time()})
mongodb_logs.insert_one(msg_dict)
with open('../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write(
'[LOCKER {}] REGISTERED FACE AS {}. LOCKED\n'.
format(lock_id, checking_tracker.face_id))
blynk_locker.stop_processing('locked')
elif status == 'locked':
# Release the locker, face verification
# update locker
msg_dict = mongodb_lockers.find(
{
'lock_id': lock_id
}, projection={"_id": False})[0]
msg_dict.update({
'unlock_face': face_url,
'unlock_timestamp': time.time()
})
if this_locker[
'lock_face_id'] == checking_tracker.face_id:
print('UNLOCK!')
blynk_locker.stop_processing('available')
mongodb_lockers.remove({'lock_id': lock_id})
mongodb_lockers.insert_one({
'lock_id': lock_id,
'status': 'available',
'lock_face_id': None,
'lock_face_url': None,
'lock_timestamp': None,
'unlock_face_id': None,
'unlock_face_url': None,
'unlock_timestap': None
})
with open(
'../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write(
'[LOCKER {}] MATCHED WITH FACE ID {}. '
'UNLOCKED. THIS LOCKER IS AVAILABLE NOW!\n'.
format(lock_id, checking_tracker.face_id))
else:
print('NOT MATCH')
blynk_locker.stop_processing('locked')
with open(
'../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write('[LOCKER {}] NOT MATCH. '
'PLEASE TRY AGAIN!\n'.format(lock_id))
# update logs
msg_dict.update({'log_timestamp': time.time()})
mongodb_logs.insert_one(msg_dict)
frame_reader.release()
return 1
tracker_manager.find_and_process_end_track(mongodb_faceinfo)
frame_counter += 1
print("FPS: %f" % (1 / (time.time() - fps_counter)))
def cropped_shape(self, name, image):
bbs, pts = DETECTOR.detect_face(image)
cropped_face = CropperUtils.crop_face(image, bbs[0])
h, w, _ = cropped_face.shape
self.assertEqual(h, 160, msg=name)
self.assertEqual(w, 160, msg=name)
def cam_worker_function(cam_url, area):
'''
Cam worker function
'''
print("Cam URL: {}".format(cam_url))
print("Area: {}".format(area))
# Modify Config
Config.Track.TRACKING_QUEUE_CAM_TO_CENTRAL = True
rabbit_mq = RabbitMQ((Config.Rabbit.USERNAME, Config.Rabbit.PASSWORD),
(Config.Rabbit.IP_ADDRESS, Config.Rabbit.PORT))
frame_counter = 0
# Variables holding the correlation trackers and the name per faceid
list_of_trackers = TrackersList()
face_rec_graph = FaceGraph()
face_extractor = FacenetExtractor(face_rec_graph)
detector = MTCNNDetector(face_rec_graph)
preprocessor = Preprocessor()
matcher = KdTreeMatcher()
if Config.CALC_FPS:
start_time = time.time()
if args.cam_url is not None:
frame_reader = URLFrameReader(args.cam_url, scale_factor=1.5)
else:
frame_reader = RabbitFrameReader(rabbit_mq)
try:
while True: # frame_reader.has_next():
frame = frame_reader.next_frame()
if frame is None:
print("Waiting for the new image")
list_of_trackers.check_delete_trackers(matcher,
rabbit_mq,
history_mode=False)
continue
print("Frame ID: %d" % frame_counter)
if Config.CALC_FPS:
fps_counter = time.time()
list_of_trackers.update_dlib_trackers(frame)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
origin_bbs, points = detector.detect_face(frame)
for i, origin_bb in enumerate(origin_bbs):
display_face, _ = CropperUtils.crop_display_face(
frame, origin_bb)
print("Display face shape")
print(display_face.shape)
if 0 in display_face.shape:
continue
cropped_face = CropperUtils.crop_face(frame, origin_bb)
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
emb_array, coeff = face_extractor.extract_features(
preprocessed_image)
# Calculate angle
angle = FaceAngleUtils.calc_angle(points[:, i])
# TODO: refractor matching_detected_face_with_trackers
matched_fid = list_of_trackers.matching_face_with_trackers(
frame, origin_bb, emb_array)
# Update list_of_trackers
list_of_trackers.update_trackers_list(
matched_fid, origin_bb, display_face, emb_array, angle,
area, frame_counter, matcher, rabbit_mq)
if Config.Track.TRACKING_QUEUE_CAM_TO_CENTRAL:
track_tuple = (matched_fid, display_face, emb_array,
area, time.time(), origin_bb, angle)
rabbit_mq.send_tracking(
track_tuple,
rabbit_mq.RECEIVE_CAM_WORKER_TRACKING_QUEUE)
# Check detete current trackers time
list_of_trackers.check_delete_trackers(matcher,
rabbit_mq,
history_mode=False)
frame_counter += 1
if Config.CALC_FPS:
print("FPS: %f" % (1 / (time.time() - fps_counter)))
except KeyboardInterrupt:
print('Keyboard Interrupt !!! Release All !!!')
if Config.CALC_FPS:
print('Time elapsed: {}'.format(time.time() - start_time))
print('Avg FPS: {}'.format(
(frame_counter + 1) / (time.time() - start_time)))
frame_reader.release()
def main(data_path, dashboard):
#wipe data
mongodb_dashinfo.remove({})
mongodb_faceinfo.remove({})
tracker_paths = [
tracker_path for tracker_path in glob.glob(data_path + '/*')
if os.path.isdir(tracker_path)
]
# face_extractor = FacenetExtractor(face_rec_graph, model_path='../models/am_inception_res_v1_transfer_Vin_5hour_20180701.pb')
preprocessor = Preprocessor()
#get max track id
existing_tracking_paths = [
dir for dir in os.listdir(Config.TRACKING_DIR)
if os.path.isdir(os.path.join(Config.TRACKING_DIR, dir))
]
track_id = max([int(dir) for dir in existing_tracking_paths
]) + 1 if len(existing_tracking_paths) > 0 else 0
for tracker_path in tracker_paths: #assuming that each annotated folder is a tracker
tracker_save_folder = os.path.join(Config.TRACKING_DIR, str(track_id))
preprocessed_images = []
insert_list = []
#create fake face_id
face_id = '{}-{}-{}'.format("Office", track_id, time.time())
display_imgs = [
os.path.basename(_dir)
for _dir in glob.glob(tracker_path + '/*.jpg')
]
#iterate through list of img names
for display_img in display_imgs:
image_id = display_img.replace(".jpg", "")
img = misc.imread(os.path.join(tracker_path, display_img))
#parse image data
data_split = image_id.split('_')
data_split[0] = str(track_id)
image_id = '_'.join(data_split)
bbox = data_split[1:5]
bbox = [int(i) for i in bbox]
padded_bbox = data_split[-4:len(data_split)]
padded_bbox = '_'.join(padded_bbox)
time_stamp = float(data_split[5])
cropped_face = CropperUtils.reverse_display_face(img, padded_bbox)
preprocessed_image = preprocessor.process(cropped_face)
emb_array, _ = face_extractor.extract_features(preprocessed_image)
insert_list.append({
'track_id': track_id,
'image_id': image_id,
'face_id': face_id,
'time_stamp': time_stamp,
'bounding_box': bbox,
'embedding': emb_array.tolist(),
'padded_bbox': padded_bbox,
'points': None,
'is_registered': True
})
#save image to TRACKING DIR
create_if_not_exist(tracker_save_folder)
misc.imsave(os.path.join(tracker_save_folder, image_id + '.jpg'),
img)
print(image_id)
# preprocessed_images.append(preprocessor.process(cropped_face))
# #extract embeddings all at once for performance
# embs_array, _ = face_extractor.extract_features_all_at_once(preprocessed_images)
# #map embedding with its corresponding image id
# for i in range(len(s_insert_list)):
# insert_list[i]['embedding'] = [embs_array[i].tolist()] #embedding is saved as (1,128)
#insert all images at once for performance
mongodb_faceinfo.insert(insert_list)
#add log to dash info
mongodb_dashinfo.remove({'track_id': track_id})
mongodb_dashinfo.insert_one({
'track_id':
track_id,
'represent_image_id':
insert_list[0]['image_id'],
'face_id':
face_id,
'is_registered':
True
})
if dashboard:
#simulate cv <---> web real-time connection
queue_msg = '|'.join([
face_id, Config.SEND_RBMQ_HTTP + '/' + str(track_id) + '/',
insert_list[0]['image_id'],
str(time.time())
])
rabbit_mq.send(Config.Queues.LIVE_RESULT, queue_msg)
track_id += 1 #increment track id
import time
matcher = KdTreeMatcher()
face_graph = FaceGraph()
face_detector = MTCNNDetector(face_graph)
feature_extractor = FacenetExtractor(face_graph)
preprocessor = Preprocessor()
frame_reader = URLFrameReader(cam_url=0, scale_factor=2)
while frame_reader.has_next():
frame = frame_reader.next_frame()
bouncing_boxes, landmarks = face_detector.detect_face(frame)
nrof_faces = len(bouncing_boxes)
start = time.time()
for i in range(nrof_faces):
cropped = CropperUtils.crop_face(frame, bouncing_boxes[i])
display_face, padded_bb_str = CropperUtils.crop_display_face(
frame, bouncing_boxes[i])
reverse_face = CropperUtils.reverse_display_face(
display_face, padded_bb_str)
process_img = preprocessor.process(cropped)
show_frame(reverse_face, 'Reverse')
show_frame(cropped, 'Cropped')
emb, coeff = feature_extractor.extract_features(process_img)
predict_id, top_match_ids = matcher.match(emb)
print('Predict', predict_id)
show_frame(frame, 'nn')
end = time.time()
print('Fps', 1 / (end - start))
start = end
def detect_and_assign(imgs, ids):
embedding_dict = {}
for img_index, img in enumerate(imgs):
bboxes, points = detector.detect_face(img)
embedding_dict[img_index] = []
for bbox in bboxes:
face_img = CropperUtils.crop_face(img, bbox)
face_img = preprocess.whitening(face_img)
display_face_img, padded_bbox_str = CropperUtils.crop_display_face(
img, bbox)
emb, coeff = feature_extractor.extract_features(face_img)
embedding_dict[img_index].append(
(emb, display_face_img, padded_bbox_str, bbox))
id_dict = {}
for img_index, embs_faces in embedding_dict.items():
if img_index == 0:
# not use only top biggest faces here because there maybe big face picture on shirt
embs_array = np.zeros((len(embs_faces), Config.Matcher.EMB_LENGTH))
id_list = [i for i in range(len(embs_faces))]
for face_index, (emb, display_face_img, padded_bbox_str,
bbox) in enumerate(embs_faces):
id_dict[face_index] = [(emb, display_face_img, padded_bbox_str,
get_area(bbox))]
embs_array[face_index] = emb
else:
visited = []
for (emb, display_face_img, padded_bbox_str, bbox) in embs_faces:
dists = np.sum(np.square(np.subtract(embs_array, emb)), 1)
min_index = np.argmin(dists)
found_id = id_list[min_index]
if Config.DEBUG:
print('id_list', id_list)
print('found id:', found_id, '; min dist:',
dists[min_index])
show_frame(display_face_img, wait_time=1000)
if dists[min_index] > Config.MIN_ASSIGN_THRESHOLD:
found_id = len(id_dict)
id_dict[found_id] = []
elif found_id in visited:
continue
id_dict[found_id].append(
(emb, display_face_img, padded_bbox_str, get_area(bbox)))
embs_array = np.append(embs_array, [emb], axis=0)
id_list.append(found_id)
visited.append(found_id)
nrof_ids = len(ids)
nrof_imgs = len(imgs)
# remove id that dont have enough number of face
id_dict_keys = list(id_dict.keys())
for key in id_dict_keys:
if len(id_dict[key]) < nrof_imgs:
id_dict.pop(key, None)
# get top biggest bboxes
bbox_area_dict = dict([(k, id_dict[k][0][-1]) for k in id_dict.keys()])
sorted_bb_areas = sorted(bbox_area_dict.items(),
key=operator.itemgetter(1),
reverse=True)
arg_sorted_bb_areas = [key for (key, _) in sorted_bb_areas]
# check all
if len(id_dict.keys()) < nrof_ids:
key_dict = dict([
(k, v) for k, v in zip(id_dict.keys(), ids[:len(id_dict.keys())])
])
id_dict = dict([(key_dict[k], v) for k, v in id_dict.items()])
elif len(id_dict.keys()) > nrof_ids:
# only get top nrof_ids biggest bounding box
valid_keys = arg_sorted_bb_areas[:nrof_ids]
key_dict = dict([(k, v) for k, v in zip(valid_keys, ids)])
id_dict = dict([(key_dict[k], v) for k, v in id_dict.items()
if k in valid_keys])
else:
key_dict = dict([(k, v) for k, v in zip(id_dict.keys(), ids)])
id_dict = dict([(key_dict[k], v) for k, v in id_dict.items()])
return id_dict
def generic_function(cam_url, area):
'''
This is main function
'''
print("Generic function")
print("Cam URL: {}".format(cam_url))
print("Area: {}".format(area))
# Variables for tracking faces
# Variables holding the correlation trackers and the name per faceid
list_of_trackers = TrackersList()
clear_tracking_folder()
matcher = KdTreeMatcher()
print("Load sample")
frame_sample = PickleUtils.read_pickle('../session/db/sample.pkl')
frame_counter = 0
track_results = TrackerResultsDict()
predict_dict = {}
if Config.CALC_FPS:
start_time = time.time()
if args.cam_url is not None:
frame_reader = URLFrameReader(args.cam_url, scale_factor=1)
else:
frame_reader = RabbitFrameReader(rabbit_mq)
video_out = None
video_out_fps = 24
video_out_w = 1920
video_out_h = 1080
print(video_out_fps, video_out_w, video_out_h)
center = (int(video_out_w / 2), int(video_out_h / 2))
bbox = [
int(center[0] - 0.35 * video_out_w),
int(center[1] - video_out_h * 0.35),
int(center[0] + 0.35 * video_out_w),
int(center[1] + 0.35 * video_out_h)
]
if Config.Track.TRACKING_VIDEO_OUT:
video_out = VideoHandle('../data/tracking_video_out.avi', video_out_fps,
int(video_out_w), int(video_out_h))
try:
while True: # frame_reader.has_next():
frame = frame_sample[frame_counter].read_image
if frame is None:
print("Waiting for the new image")
trackers_return_dict, predict_trackers_dict = \
list_of_trackers.check_delete_trackers(matcher, rabbit_mq)
track_results.update_two_dict(trackers_return_dict)
predict_dict.update(predict_trackers_dict)
continue
print("Frame ID: %d" % frame_counter)
print('Num of ids in matcher: {}'.format(matcher._numofids))
if Config.Track.TRACKING_VIDEO_OUT:
video_out.tmp_video_out(frame)
if Config.CALC_FPS:
fps_counter = time.time()
list_of_trackers.update_dlib_trackers(frame)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
origin_bbs = frame_sample[frame_counter].origin_bbs
points = frame_sample[frame_counter].points
for i, origin_bb in enumerate(origin_bbs):
print(is_inner_bb(bbox, origin_bb))
if not is_inner_bb(bbox, origin_bb):
continue
display_face, _ = CropperUtils.crop_display_face(
frame, origin_bb)
# Calculate embedding
emb_array = frame_sample[frame_counter].embs[i]
# Calculate angle
angle = FaceAngleUtils.calc_angle(points[:, i])
# TODO: refractor matching_detected_face_with_trackers
matched_fid = list_of_trackers.matching_face_with_trackers(
frame, origin_bb, emb_array)
# Update list_of_trackers
list_of_trackers.update_trackers_list(
matched_fid, time.time(), origin_bb, display_face,
emb_array, angle, area, frame_counter, i, matcher,
rabbit_mq)
trackers_return_dict, predict_trackers_dict = \
list_of_trackers.check_delete_trackers(matcher, rabbit_mq)
track_results.update_two_dict(trackers_return_dict)
predict_dict.update(predict_trackers_dict)
# Check extract trackers history time (str(frame_counter) + '_' + str(i))
list_of_trackers.trackers_history.check_time()
frame_counter += 1
if Config.CALC_FPS:
print("FPS: %f" % (1 / (time.time() - fps_counter)))
if Config.Track.TRACKING_VIDEO_OUT:
print('Write track video')
video_out.write_track_video(track_results.tracker_results_dict)
if Config.Track.PREDICT_DICT_OUT:
PickleUtils.save_pickle(Config.PREDICTION_DICT_FILE, predict_dict)
except KeyboardInterrupt:
print('Keyboard Interrupt !!! Release All !!!')
if Config.CALC_FPS:
print('Time elapsed: {}'.format(time.time() - start_time))
print('Avg FPS: {}'.format(
(frame_counter + 1) / (time.time() - start_time)))
frame_reader.release()
if Config.Track.TRACKING_VIDEO_OUT:
print('Write track video')
video_out.write_track_video(track_results.tracker_results_dict)
video_out.release()
if Config.Track.PREDICT_DICT_OUT:
PickleUtils.save_pickle(Config.PREDICTION_DICT_FILE, predict_dict)
def generic_function(cam_url, queue_reader, area, face_extractor_model,
re_source, multi_thread):
'''
This is main function
'''
print("Generic function")
print("Cam URL: {}".format(cam_url))
print("Area: {}".format(area))
# Variables for tracking faces
frame_counter = 0
# Variables holding the correlation trackers and the name per faceid
tracking_dirs = glob.glob(Config.TRACKING_DIR + '/*')
if tracking_dirs == []:
number_of_existing_trackers = 0
else:
lof_int_trackid = [
int(tracking_dir.split('/')[-1]) for tracking_dir in tracking_dirs
]
number_of_existing_trackers = max(lof_int_trackid) + 1
imageid_to_keyid = {}
tracker_manager = TrackerManager(
area,
imageid_to_keyid=imageid_to_keyid,
current_id=number_of_existing_trackers)
if Config.Matcher.CLEAR_SESSION:
clear_session_folder()
global querying_top10_image_ids_queue
# mongodb_faceinfo.remove({})
# reg_dict = PickleUtils.read_pickle(Config.REG_IMAGE_FACE_DICT_FILE)
# if reg_dict is not None:
# for fid in reg_dict:
# mongodb_faceinfo.insert_one({'image_id': fid, 'face_id': reg_dict[fid]})
# print('Saved regdict in mongodb as collection regdict')
matcher = FaissMatcher()
matcher.build(
mongodb_faceinfo, imageid_to_keyid=imageid_to_keyid, use_image_id=True)
svm_matcher = None
if Config.Matcher.CLOSE_SET_SVM:
svm_matcher = SVMMatcher()
svm_matcher.build(mongodb_faceinfo)
track_results = TrackerResultsDict()
if Config.CALC_FPS:
start_time = time.time()
if cam_url is not None:
frame_reader = URLFrameReader(cam_url, scale_factor=1)
elif queue_reader is not None:
frame_reader = RabbitFrameReader(rabbit_mq, queue_reader)
elif args.anno_mode:
frame_reader = URLFrameReader('./nothing.mp4', scale_factor=1)
else:
print('Empty Image Source')
return -1
if not args.anno_mode:
video_out_fps, video_out_w, video_out_h, = frame_reader.get_info()
print(video_out_fps, video_out_w, video_out_h)
bbox = [
int(Config.Frame.ROI_CROP[0] * video_out_w),
int(Config.Frame.ROI_CROP[1] * video_out_h),
int(Config.Frame.ROI_CROP[2] * video_out_w),
int(Config.Frame.ROI_CROP[3] * video_out_h)
]
# bbox = [0, 0, video_out_w, video_out_h]
video_out = None
if Config.Track.TRACKING_VIDEO_OUT:
video_out = VideoHandle(time.time(), video_out_fps, int(video_out_w),
int(video_out_h))
# Turn on querying top 10 from queue
if Config.QUERY_TOP10_MODE:
thread = threading.Thread(target=give_this_id_10_closest_ids)
thread.daemon = True
thread.start()
frame_queue = []
lock = threading.Lock()
if multi_thread:
is_on = [True]
t = threading.Thread(target=(get_frames), args=(frame_queue, frame_reader, re_source, \
cam_url, queue_reader, lock, is_on, ))
t.start()
try:
while True:
ms_msg = rabbit_mq.receive_str(Config.Queues.MERGE)
ms_flag = 'merge'
if ms_msg is None:
ms_msg = rabbit_mq.receive_str(Config.Queues.SPLIT)
ms_flag = 'split'
if ms_msg is not None:
merge_anchor, merge_list = extract_info_from_json(ms_msg)
while merge_list != []:
image_id1 = merge_list.pop()
split_merge_id(ms_flag, image_id1, merge_anchor, matcher,
preprocessor, face_extractor,
tracker_manager, mongodb_dashinfo,
mongodb_faceinfo, mongodb_mslog)
continue
action_msg = rabbit_mq.receive_str(Config.Queues.ACTION)
if action_msg is not None:
return_dict = json.loads(action_msg)
print('Receive: {}'.format(return_dict))
if return_dict['actionType'] == 'getNearest':
lock.acquire()
querying_top10_image_ids_queue.append(return_dict['data'])
lock.release()
continue
if args.anno_mode:
print('Annotation Mode, waiting for new tasks ...')
time.sleep(1)
continue
if multi_thread:
if len(frame_queue) > 0:
lock.acquire()
frame = frame_queue.pop(0)
lock.release()
else:
frame = None
else:
frame = frame_reader.next_frame()
tracker_manager.update_trackers(frame)
#do this before check_and_recognize tracker (sync local matcher vs mongodb)
trackers_return_dict, recognized_results = update_recognition(
self, matcher, svm_matcher, mongodb_faceinfo)
for tracker, matching_result_dict in recognized_results:
handle_results(tracker, matching_result_dict, imageid_to_keyid = imageid_to_keyid, \
dump=False)
# trackers_return_dict = tracker_manager.find_and_process_end_track(mongodb_faceinfo)
track_results.merge(trackers_return_dict)
tracker_manager.long_term_history.check_time(
matcher, mongodb_faceinfo)
if frame is None:
print("Waiting for the new image")
# if Config.Track.RECOGNIZE_FULL_TRACK:
# overtime_track_ids = tracker_manager.find_overtime_current_trackers(
# time_last=Config.Track.CURRENT_EXTRACR_TIMER-5,
# find_unrecognized=True
# )
# for overtime_track_id in overtime_track_ids:
# checking_tracker, top_predicted_face_ids, matching_result_dict = \
# tracker_manager.check_and_recognize_tracker(
# matcher,
# overtime_track_id,
# mongodb_faceinfo,
# svm_matcher)
# handle_results(checking_tracker, matching_result_dict, imageid_to_keyid = imageid_to_keyid,\
# dump=False)
if re_source and not multi_thread:
print('Trying to connect the stream again ...')
if cam_url is not None:
frame_reader = URLFrameReader(cam_url, scale_factor=1)
elif queue_reader is not None:
frame_reader = RabbitFrameReader(
rabbit_mq, queue_reader)
else:
print('Empty Image Source')
return -1
break
print("Frame ID: %d" % frame_counter)
if "_rotate" in video:
# rotate cw
rotation = int(video.split("_")[-1].split(".")[0])
frame = rotate_image_90(frame, rotation)
if Config.Track.TRACKING_VIDEO_OUT:
video_out.tmp_video_out(frame)
if Config.CALC_FPS:
fps_counter = time.time()
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
# crop frame
#frame = frame[bbox[1]:bbox[3], bbox[0]:bbox[2],:]
origin_bbs, points = detector.detect_face(frame)
if len(origin_bbs) > 0:
origin_bbs = [origin_bb[:4] for origin_bb in origin_bbs]
display_and_padded_faces = [
CropperUtils.crop_display_face(frame, origin_bb)
for origin_bb in origin_bbs
]
#cropped_faces = [CropperUtils.crop_face(frame, origin_bb) for origin_bb in origin_bbs]
preprocessed_images = [
preprocessor.process(
CropperUtils.crop_face(frame, origin_bb))
for origin_bb in origin_bbs
]
embeddings_array, _ = face_extractor.extract_features_all_at_once(
preprocessed_images)
for i, origin_bb in enumerate(origin_bbs):
display_face, str_padded_bbox = display_and_padded_faces[i]
#cropped_face = CropperUtils.crop_face(frame, origin_bb)
# Calculate embedding
preprocessed_image = preprocessed_images[i]
# preprocessed_image = align_preprocessor.process(frame, points[:,i], aligner, 160)
emb_array = np.asarray([embeddings_array[i]])
face_info = FaceInfo(
#oigin_bb.tolist(),
#emb_array,
frame_counter,
origin_bb,
points[:, i]
#display_face,
#str_padded_bbox,
#points[:,i].tolist()
)
is_good_face = handle_filters(points[:, i], coeff_extractor,
face_info, preprocessed_image)
face_info.is_good = is_good_face
# TODO: refractor matching_detected_face_with_trackers
matched_track_id = tracker_manager.track(face_info)
if not face_info.is_good:
print('BAD FACE')
continue
# Update tracker_manager
tracker_manager.update(matched_track_id, frame, face_info)
checking_tracker = None
# if not Config.Track.RECOGNIZE_FULL_TRACK:
# checking_tracker, top_predicted_face_ids, matching_result_dict = \
# tracker_manager.check_and_recognize_tracker(
# matcher,
# matched_track_id,
# mongodb_faceinfo,
# svm_matcher)
# handle_results(checking_tracker, matching_result_dict, imageid_to_keyid = imageid_to_keyid, \
# dump=True)
# if Config.Track.RECOGNIZE_FULL_TRACK:
# overtime_track_ids = tracker_manager.find_overtime_current_trackers(
# time_last=Config.Track.CURRENT_EXTRACR_TIMER-5,
# find_unrecognized=True
# )
# for overtime_track_id in overtime_track_ids:
# checking_tracker, top_predicted_face_ids, matching_result_dict = \
# tracker_manager.check_and_recognize_tracker(
# matcher,
# overtime_track_id,
# mongodb_faceinfo,
# svm_matcher)
# handle_results(checking_tracker, matching_result_dict, imageid_to_keyid = imageid_to_keyid, \
# dump=False)
frame_counter += 1
if Config.CALC_FPS:
print("FPS: %f" % (1 / (time.time() - fps_counter)))
if Config.Track.TRACKING_VIDEO_OUT:
print('Write track video')
video_out.write_track_video(track_results.tracker_results_dict)
Config.Track.CURRENT_EXTRACR_TIMER = 0
trackers_return_dict = tracker_manager.find_and_process_end_track(
mongodb_faceinfo)
Config.Track.HISTORY_CHECK_TIMER = 0
Config.Track.HISTORY_EXTRACT_TIMER = 0
tracker_manager.long_term_history.check_time(matcher, mongodb_faceinfo)
except KeyboardInterrupt:
if multi_thread:
is_on[0] = False
t.join()
print('Keyboard Interrupt !!! Release All !!!')
Config.Track.CURRENT_EXTRACR_TIMER = 0
trackers_return_dict = tracker_manager.find_and_process_end_track(
mongodb_faceinfo)
Config.Track.HISTORY_CHECK_TIMER = 0
Config.Track.HISTORY_EXTRACT_TIMER = 0
tracker_manager.long_term_history.check_time(matcher, mongodb_faceinfo)
if Config.CALC_FPS:
print('Time elapsed: {}'.format(time.time() - start_time))
print('Avg FPS: {}'.format(
(frame_counter + 1) / (time.time() - start_time)))
frame_reader.release()
if Config.Track.TRACKING_VIDEO_OUT:
print('Write track video')
video_out.write_track_video(track_results.tracker_results_dict)
video_out.release()
def generic_function(frame_readers, area, session_id, detector, face_extractor,
matcher, register_commands, sent_msg_queue):
'''
This is main function
'''
print("Area: {}".format(area))
print('Thread {} created'.format(session_id))
frame_counter = 0
tracker_manager = TrackerManager(area)
# clear_tracking_folder()
# if Config.Matcher.CLEAR_SESSION:
# clear_session_folder()
if not os.path.exists(Config.SEND_RBMQ_DIR):
os.mkdir(Config.SEND_RBMQ_DIR)
preprocessor = Preprocessor()
# matcher = KdTreeMatcher()
# matcher._match_case = 'TCH'
# face_extractor = components['face_ext']
# detector = components['detector']
# face_rec_graph = FaceGraph()
# detector = MTCNNDetector(face_rec_graph)
# face_cascade = components['face_cascade']
# eye_detector = components['eye_detector']
# mouth_detector = components['mouth_detector']
frame_reader = frame_readers[session_id]
register_command = register_commands[session_id]
if Config.CALC_FPS:
start_time = time.time()
unavailable_counter = time.time()
last_labels = 'empty'
# matcher.build(Config.REG_IMAGE_FACE_DICT_FILE)
try:
while True:
try:
reg_msg_list = register_command.get(False)
except queue.Empty:
reg_msg_list = None
if reg_msg_list is not None:
print(reg_msg_list)
update_message = '{}|register_ko|Register Fail'.format(
session_id)
person_id = reg_msg_list[0] # .lower()
file_url_msg = reg_msg_list[1]
list_of_reg_trackers = TrackerManager(area)
frame_counter = 0
saved_frames = save_frames(file_url_msg)
if saved_frames == []:
print("save frames is None")
update_message = '{}|register_ko|Empty Source or Invalid Format'.format(
session_id)
else:
print('Detecting Faces and Extracting Features ...')
saved_frames.reverse()
for frame in saved_frames:
list_of_reg_trackers.update_dlib_trackers(frame)
origin_bbs, points = detector.detect_face(frame)
if origin_bbs is None:
print('not detect face on frame')
break
for i, origin_bb in enumerate(origin_bbs):
if is_inner_of_range(origin_bb, frame.shape):
continue
display_face, str_padded_bbox = CropperUtils.crop_display_face(
frame, origin_bb)
cropped_face = CropperUtils.crop_face(
frame, origin_bb)
# Calculate embedding
preprocessed_image = preprocessor.process(
cropped_face)
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
face_info = FaceInfo(origin_bb, emb_array,
frame_counter, display_face,
str_padded_bbox)
matched_track_id = list_of_reg_trackers.track(
face_info)
list_of_reg_trackers.update(
matched_track_id, frame, face_info)
frame_counter += 1
if frame_counter > 601:
break
if list_of_reg_trackers.current_trackers != {}:
embs, lbls, result_status = extract_images(
list_of_reg_trackers.current_trackers, person_id)
if result_status == 'ok':
matcher.update(embs, lbls)
registered_ids = set(lbls)
registered_msg = ', '.join(registered_ids)
# send message to rb
update_message = '{}|register_ok|Registered {}'.format(
session_id, registered_msg)
print('REGISTER DONEEEEEEEEEEEEE\n')
elif result_status == 'many_faces':
print(
'REGISTER ERROR: Many faces or your head turns too fast'
)
# send message to rb
update_message = '{}|register_ko|Many faces in the sequence'.format(
session_id)
elif result_status == 'not_good':
update_message = '{}|register_ko|Not enough faces registerd'.format(
session_id)
else:
print('REGISTER ERROR')
# send message to rb
update_message = '{}|register_ko|Register Error'.format(
session_id)
else:
print('No tracker found')
update_message = '{}|register_ko|No Face Detected'.format(
session_id)
sent_msg_queue.put(
('{}-status'.format(Config.DEMO_FOR), update_message))
frame_reader.clear()
# LIVE MODE
frame = frame_reader.next_frame()
if frame is None:
if time.time(
) - unavailable_counter >= Config.TIME_KILL_NON_ACTIVE_PROCESS:
if register_commands[session_id].empty():
frame_readers.pop(session_id, None)
register_commands.pop(session_id, None)
return
time.sleep(1)
tracker_manager.find_and_process_end_track()
# print('Waiting for new frame')
continue
unavailable_counter = time.time()
print("Frame ID: %d" % frame_counter)
fps_counter = time.time()
tracker_manager.update_dlib_trackers(frame)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
# display_frame = frame
print(Config.Frame.FRAME_INTERVAL)
detector.detect_face(frame)
origin_bbs, points = detector.detect_face(frame)
for i, origin_bb in enumerate(origin_bbs):
bb_size = calc_bb_percentage(origin_bb, frame.shape)
# print(bb_size)
if (is_inner_of_range(origin_bb, frame.shape)
and calc_bb_percentage(origin_bb, frame.shape) >
Config.Track.BB_SIZE):
continue
display_face, str_padded_bbox = CropperUtils.crop_display_face(
frame, origin_bb)
cropped_face = CropperUtils.crop_face(frame, origin_bb)
print('pass Crop Utils')
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
print('calculated embedding')
# TODO: refractor matching_detected_face_with_trackers
face_info = FaceInfo(origin_bb, emb_array, frame_counter,
display_face, str_padded_bbox)
matched_track_id = tracker_manager.track(face_info)
tracker_manager.update(matched_track_id, frame, face_info)
tracker_manager.check_and_recognize_tracker(
matcher, matched_track_id, short_term_add_new=False)
matched_tracker = tracker_manager.current_trackers[
matched_track_id]
if matched_tracker.face_id.startswith(
'TCH-{}'.format(area)):
matched_tracker.face_id = Config.Matcher.NEW_FACE
print('update trackers list')
if tracker_manager.current_trackers[
matched_track_id].face_id == last_labels:
continue
last_labels = tracker_manager.current_trackers[
matched_track_id].face_id
image_id = '{}_{}_{}.jpg'.format(
tracker_manager.current_trackers[matched_track_id].
face_id, time.time(), frame_counter)
img_dir = os.path.join(Config.SEND_RBMQ_DIR, image_id)
misc.imsave(img_dir, display_face)
face_msg = '|'.join([
session_id, tracker_manager.
current_trackers[matched_track_id].face_id,
'images/' + img_dir.split('/')[-1]
])
if not Config.Matcher.NEW_FACE in face_msg:
# rabbit_mq.send('{}-result'.format(Config.DEMO_FOR), face_msg)
sent_msg_queue.put(
('{}-result'.format(Config.DEMO_FOR), face_msg))
if matched_tracker.face_id == Config.Matcher.NEW_FACE:
tracker_manager.current_trackers.pop(
matched_track_id, None)
# draw frame
# display_frame = draw_img(
# display_frame,
# origin_bb,
# str(bb_size)
# # track_manager.current_trackers[matched_track_id].face_id
# )
# display_frame = cv2.cvtColor(display_frame, cv2.COLOR_RGB2BGR)
# display_frame = cv2.resize(display_frame, (1280, 720))
# cv2.imshow("FACE TRACKING SYSTEM {}".format(session_id), display_frame)
# key = cv2.waitKey(1)
# if key & 0xFF == ord('q'):
# break
tracker_manager.find_and_process_end_track()
frame_counter += 1
if Config.CALC_FPS:
print("FPS: %f" % (1 / (time.time() - fps_counter)))
except KeyboardInterrupt:
print('Keyboard Interrupt !!! Release All !!!')
tracker_manager.long_term_history.check_time(matcher)
if Config.CALC_FPS:
print('Time elapsed: {}'.format(time.time() - start_time))
print('Avg FPS: {}'.format(
(frame_counter + 1) / (time.time() - start_time)))
def simulate_tracking(root_folder):
Config.Track.FACE_TRACK_IMAGES_OUT = True
Config.Track.SEND_FIRST_STEP_RECOG_API = False
Config.Track.MIN_MATCH_DISTACE_OUT = True
Config.Track.CURRENT_EXTRACR_TIMER = 5
# Load Face Extractor
face_rec_graph = FaceGraph()
face_rec_graph_coeff = FaceGraph()
face_extractor = FacenetExtractor(
face_rec_graph, model_path=Config.FACENET_DIR)
coeff_extractor = FacenetExtractor(
face_rec_graph_coeff, model_path=Config.COEFF_DIR)
# Create empty KDTreeMatcher
matcher = FaissMatcher()
matcher._match_case = 'TCH'
# Preprocessor
preprocessor = Preprocessor()
# Fake rabbit mq
rabbit_mq = FakeMQ()
# Clean up for
clear_tracking_folder()
if Config.Matcher.CLEAR_SESSION:
clear_session_folder()
# Setup result list
list_of_trackers = TrackersList()
track_results = TrackerResultsDict()
predict_dict = {}
confirmed_ids_dict = {}
sim_detections = gen_images_with_time(root_folder)
for detection in sim_detections:
frame = create_fake_frame(detection)
sleep(0.05)
trackers_return_dict, predict_trackers_dict = \
list_of_trackers.check_delete_trackers(matcher, rabbit_mq)
track_results.update_two_dict(trackers_return_dict)
predict_dict.update(predict_trackers_dict)
confirmed_ids_dict = list_of_trackers.trackers_history.confirm_id(
confirmed_ids_dict)
list_of_trackers.trackers_history.check_time(matcher)
list_of_trackers.update_dlib_trackers(frame)
facial_quality = 1
# Crop face for features extraction
origin_bb = detection[2]
display_face, padded_bbox = CropperUtils.crop_display_face(
frame, origin_bb)
cropped_face = CropperUtils.crop_face(frame, origin_bb)
bbox_str = '_'.join(np.array(origin_bb, dtype=np.unicode).tolist())
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
emb_array, _ = face_extractor.extract_features(preprocessed_image)
_, coeff = coeff_extractor.extract_features(preprocessed_image)
if coeff < 0.15:
img_path = '../data/notenoughcoeff/{}_{}_{}.jpg'.format(
detection, bbox_str, coeff)
cv2.imwrite(img_path, cv2.cvtColor(display_face, cv2.COLOR_BGR2RGB))
facial_quality = -1
else:
with open('../data/coeff_log.txt', 'a') as f:
f.write('{}_{}_{}, coeff: {}\n'.format(bbox_str, detection[0],
padded_bbox, coeff))
matched_fid = list_of_trackers.matching_face_with_trackers(
frame, detection[0], origin_bb, emb_array, facial_quality)
if facial_quality == -1 or coeff < 0.15:
continue
list_of_trackers.update_trackers_list(
matched_fid, time.time(), origin_bb, display_face, emb_array, 0,
'VVT', 1, detection[0], padded_bbox, matcher, rabbit_mq)
list_of_trackers.check_recognize_tracker(matcher, rabbit_mq,
matched_fid)
sleep(6)
list_of_trackers.check_delete_trackers(matcher, rabbit_mq)
def load_embeddings_from_path(data_path, json_save_path, from_mongo=True):
try:
with open(json_save_path, "r") as f:
data_structure = json.loads(f.read())
except:
data_structure = {"folder_structure": {}, "regdict": {}}
folder_structure = data_structure["folder_structure"]
reg_dict = data_structure["regdict"]
'''
print folder_structure -->
{
label: {
trackid1 : [imageid1, imageid2,..]
...
}
}
print regdict ---->
{
imageid: {
"face_id" : label
"embedding" : [emb_array]
}
}
'''
labels = [
folder for folder in os.listdir(data_path)
if os.path.isdir(os.path.join(data_path, folder))
]
for label in labels:
trackers = [os.path.join(data_path,label,tracker) for tracker in os.listdir(os.path.join(data_path,label)) \
if os.path.isdir(os.path.join(data_path,label,tracker))]
folder_structure[label] = {}
for tracker in trackers:
folder_structure[label][tracker] = []
image_paths = [
os.path.join(tracker, img_path)
for img_path in os.listdir(tracker)
if "jpg" in img_path
]
for img_path in image_paths:
img = misc.imread(os.path.join(img_path))
image_id = img_path.split('/')[-1].replace(
".jpg", "") #get image id
data_split = image_id.split('_')
padded_bbox = data_split[-4:len(data_split)]
padded_bbox = '_'.join(padded_bbox)
# time_stamp = float(data_split[5])
cropped_face = CropperUtils.reverse_display_face(
img, padded_bbox)
preprocessed_image = preprocessor.process(cropped_face)
if from_mongo:
try:
print(mongodb_faceinfo.find_one({
"image_id": image_id
}))
emb_array = np.asarray(
mongodb_faceinfo.find_one({
"image_id": image_id
})["embedding"])
print("Load from mongodb")
except:
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
else:
print("extract feature normally")
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
print(image_id)
folder_structure[label][tracker].append(image_id)
reg_dict[image_id] = {
"face_id": label,
"embedding": emb_array
}
data_file = open(json_save_path, "w")
data_file.write(json.dumps(data_structure, cls=NumpyEncoder))
data_file.close()
return data_structure