def build_matcher(data_struct):
matcher = FaissMatcher()
embs = []
labels = []
for label in data_struct["folder_structure"]:
for tracker in data_struct["folder_structure"][label]:
for image_id in data_struct["folder_structure"][label][tracker]:
labels.append(image_id)
embs.append(data_struct["regdict"][image_id]["embedding"])
matcher.fit(embs, labels) #build matcher
return matcher
def give_this_id_10_closest_ids():
# init matcher
with open('../data/top10querylog.txt', 'a') as f:
f.write('TOP10 QUERY IS BEING IN PROCESS !!!\n\n')
global querying_top10_image_ids_queue
global mongodb_faceinfo
global mongodb_dashinfo
embs = []
labels = []
cursors = mongodb_dashinfo.find({})
unique_labels = [cursor['represent_image_id'] for cursor in cursors]
cursors = mongodb_faceinfo.find({'image_id': {'$in': unique_labels}})
for cursor in cursors:
embs.append(np.array(cursor['embedding']))
labels.append(cursor['image_id'])
nof_registered_image_ids = len(labels)
matcher = FaissMatcher()
matcher.fit(embs, labels)
with open('../data/top10querylog.txt', 'a') as f:
f.write('MATCHER BUILT!!!\n\n')
while True:
if nof_registered_image_ids < mongodb_dashinfo.find({}).count():
nof_registered_image_ids = mongodb_dashinfo.find({}).count()
print('[Query TOP10] Update new registered image_id ...')
cursors = mongodb_dashinfo.find({
'represent_image_id': {
'$nin': labels
}
})
unique_labels = [cursor['represent_image_id'] for cursor in cursors]
cursors = mongodb_faceinfo.find({
'image_id': {
'$in': unique_labels
}
})
adding_embs = []
adding_labels = []
for cursor in cursors:
adding_embs.append(np.array(cursor['embedding']))
adding_labels.append(cursor['image_id'])
old_embs = embs
old_labels = labels
embs = old_embs + adding_embs
labels = old_labels + adding_labels
matcher.update(adding_embs, adding_labels)
if not len(querying_top10_image_ids_queue) == 0:
lock.acquire()
queue_data = querying_top10_image_ids_queue.pop()
lock.release()
results = {}
session_id = queue_data['sessionId']
image_id = queue_data['imageId']
print('[Query TOP10] image_id: ' + image_id)
with open('../data/top10querylog.txt', 'a') as f:
f.write('image_id: ' + image_id + '\n')
cursors = mongodb_faceinfo.find({'image_id': image_id})
if cursors.count() == 0:
print('[Query TOP10] THIS QUERY IMAGE ID HAS YET TO REGISTER')
with open('../data/top10querylog.txt', 'a') as f:
f.write('THIS QUERY IMAGE ID HAS YET TO REGISTER\n')
face_id = mongodb_dashinfo.find({
'represent_image_id': image_id
})[0]['face_id']
unique_labels = [
cursor['represent_image_id']
for cursor in mongodb_dashinfo.find({
'face_id': face_id
})
]
for label in unique_labels:
results[label] = '0'
else:
query_emb = cursors[0]['embedding']
dists, inds = matcher._classifier.search(
np.array(query_emb).astype('float32'), k=15)
dists = np.squeeze(dists)
inds = np.squeeze(inds)
top_match_ids = [labels[idx] for idx in inds]
for i, top_match_id in enumerate(top_match_ids):
if i < 11 and top_match_id != image_id:
results[top_match_id] = str(dists[i])
msg_results = {
'actionType': 'getNearest',
'sessionId': session_id,
'data': {
'images': results
}
}
with open('../data/top10querylog.txt', 'a') as f:
f.write('Result: \n{}\n\n'.format(results))
print('[Query TOP10] Result: \n{}'.format(results))
rabbit_mq.send_with_exchange(Config.Queues.ACTION_RESULT,
session_id, json.dumps(msg_results))
# Those cmt for querying tracker from the image ids tracker
# query_track_id = int(image_id.split('_')[0])
# query_embs = [cursor['embedding'] for cursor in mongodb_faceinfo.find({'track_id': query_track_id})]
# for emb in query_embs:
# predict_id, _, min_dist = matcher.match(emb, return_min_dist=True)
# if not predict_id in predicted_dict:
# predicted_dict[predict_id] = []
# predicted_dict[predict_id].append(min_dist)
# avg_predicted_dict = {pid: sum(predicted_dict[pid])/float(len(predicted_dict[pid]))
# for pid in predicted_dict}
# sorted_predicted_ids = sorted(avg_predicted_dict.items(), key=lambda kv: kv[1])
# with open('../data/query_top10_log.txt') as f:
# f.write('Query IMAGE_ID: ' + image_id + '\n')
# f.write('Results: {} \n\n'.format(sorted_predicted_ids))
# str_results = []
# for closest_id, dist in sorted_predicted_ids:
# str_results.append(Config.Rabbit.INTRA_SEP.join([closest_id, str(dist)]))
# result_msg = Config.Rabbit.INTER_SEP.join(str_results)
else:
time.sleep(1)
def general_process(lock_id, detector, preprocessor, face_extractor,
blynk_locker):
'''
INPUT: lock_id
'''
# Get locker infomation
# lock_id = 'query from mongo'
locker_info = mongodb_lockersinfo.find({'lock_id': lock_id})[0]
this_locker = mongodb_lockers.find({'lock_id': lock_id})[0]
cam_url = locker_info['cam_url']
status = this_locker['status']
blynk_locker.processing(status)
# init face info
mongodb_faceinfo = mongodb_db[str(lock_id)]
# Variables for tracking faces
frame_counter = 0
start_time = time.time()
acceptable_spoofing = 0
# Variables holding the correlation trackers and the name per faceid
tracking_folder = os.path.join(Config.TRACKING_DIR, str(lock_id))
create_if_not_exist(tracking_folder)
tracking_dirs = glob.glob(tracking_folder + '/*')
if tracking_dirs == []:
number_of_existing_trackers = 0
else:
lof_int_trackid = [
int(tracking_dir.split('/')[-1]) for tracking_dir in tracking_dirs
]
number_of_existing_trackers = max(lof_int_trackid) + 1
tracker_manager = TrackerManager(
'LOCKID' + str(lock_id), current_id=number_of_existing_trackers)
frame_reader = URLFrameReader(cam_url, scale_factor=1)
matcher = FaissMatcher()
if status == 'locked':
embs = []
labels = []
cursors = mongodb_faceinfo.find({
'face_id': this_locker['lock_face_id']
})
for cursor in cursors:
embs.append(np.array(cursor['embedding']))
labels.append(cursor['image_id'])
nof_registered_image_ids = len(labels)
matcher.fit(embs, labels)
while True:
# in case the jerk hits the button
if time.time() - start_time > 4:
with open('../data/locker_{}_log.txt'.format(lock_id), 'a') as f:
f.write('[LOCKER {}] OUT OF TIME! \n\n'.format(lock_id))
frame_reader.release()
blynk_locker.stop_processing(status)
return -1
frame = frame_reader.next_frame()
if frame is None:
print('Invalid Video Source')
break
fps_counter = time.time()
# cv2.imshow('Locker {}'.format(lock_id), frame)
# cv2.waitKey(1)
tracker_manager.update_trackers(frame)
if frame_counter % Config.Frame.FRAME_INTERVAL == 0:
origin_bbs, points = detector.detect_face(frame)
for i, in_origin_bb in enumerate(origin_bbs):
origin_bb = in_origin_bb[:-1]
display_face, str_padded_bbox = CropperUtils.crop_display_face(
frame, origin_bb)
cropped_face = CropperUtils.crop_face(frame, origin_bb)
# is_spoofing = spoofing_detector.is_face_spoofing(cropped_face)
# if is_spoofing:
# acceptable_spoofing += 1
# with open('../data/spoofing_log.txt', 'a') as f:
# f.write('Spoofing Detected at Locker {}: {}\n'.format(lock_id, is_spoofing))
# if acceptable_spoofing > 5:
# with open('../data/locker_{}_log.txt'.format(lock_id), 'a') as f:
# f.write(
# '[LOCKER {}] STOP PROCESSING. '
# 'SPOOFING DETECTED!\n'.format(lock_id)
# )
# frame_reader.release()
# blynk_locker.stop_processing(status)
# return -1
# Calculate embedding
preprocessed_image = preprocessor.process(cropped_face)
# preprocessed_image = align_preprocessor.process(frame, points[:,i], aligner, 160)
emb_array, _ = face_extractor.extract_features(
preprocessed_image)
face_info = FaceInfo(origin_bb.tolist(), emb_array,
frame_counter, display_face,
str_padded_bbox, points[:, i].tolist())
is_good_face = handle_filters(points[:, i], coeff_extractor,
face_info, preprocessed_image)
face_info.is_good = is_good_face
# TODO: refractor matching_detected_face_with_trackers
matched_track_id = tracker_manager.track(face_info)
if not face_info.is_good:
print('BAD FACE')
continue
# Update tracker_manager
tracker_manager.update(matched_track_id, frame, face_info)
checking_tracker = None
checking_tracker, top_predicted_face_ids, matching_result_dict = \
tracker_manager.check_and_recognize_tracker(
matcher,
matched_track_id,
mongodb_faceinfo,
None)
# handle_results(checking_tracker, matching_result_dict)
if checking_tracker is not None:
dumped_images = checking_tracker.dump_images(
mongodb_faceinfo,
add_new=True,
trackingfolder=tracking_folder)
checking_tracker.represent_image_id = dumped_images[0]
face_url = os.path.join(Config.SEND_RBMQ_HTTP, str(lock_id),
str(checking_tracker.track_id),
checking_tracker.represent_image_id)
face_url += '.jpg'
if status == 'available':
# Save locker, sign up the face
mongodb_lockers.remove({'lock_id': lock_id})
msg_dict = {
'lock_id': lock_id,
'status': 'locked',
'lock_face_url': face_url,
'lock_face_id': checking_tracker.face_id,
'lock_timestamp': time.time(),
'unlock_face_url': None,
'unlock_face_id': None,
'unlock_timestap': None
}
mongodb_lockers.insert_one(msg_dict)
# update logs
msg_dict.update({'log_timestamp': time.time()})
mongodb_logs.insert_one(msg_dict)
with open('../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write(
'[LOCKER {}] REGISTERED FACE AS {}. LOCKED\n'.
format(lock_id, checking_tracker.face_id))
blynk_locker.stop_processing('locked')
elif status == 'locked':
# Release the locker, face verification
# update locker
msg_dict = mongodb_lockers.find(
{
'lock_id': lock_id
}, projection={"_id": False})[0]
msg_dict.update({
'unlock_face': face_url,
'unlock_timestamp': time.time()
})
if this_locker[
'lock_face_id'] == checking_tracker.face_id:
print('UNLOCK!')
blynk_locker.stop_processing('available')
mongodb_lockers.remove({'lock_id': lock_id})
mongodb_lockers.insert_one({
'lock_id': lock_id,
'status': 'available',
'lock_face_id': None,
'lock_face_url': None,
'lock_timestamp': None,
'unlock_face_id': None,
'unlock_face_url': None,
'unlock_timestap': None
})
with open(
'../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write(
'[LOCKER {}] MATCHED WITH FACE ID {}. '
'UNLOCKED. THIS LOCKER IS AVAILABLE NOW!\n'.
format(lock_id, checking_tracker.face_id))
else:
print('NOT MATCH')
blynk_locker.stop_processing('locked')
with open(
'../data/locker_{}_log.txt'.format(lock_id),
'a') as f:
f.write('[LOCKER {}] NOT MATCH. '
'PLEASE TRY AGAIN!\n'.format(lock_id))
# update logs
msg_dict.update({'log_timestamp': time.time()})
mongodb_logs.insert_one(msg_dict)
frame_reader.release()
return 1
tracker_manager.find_and_process_end_track(mongodb_faceinfo)
frame_counter += 1
print("FPS: %f" % (1 / (time.time() - fps_counter)))
class FindSimilarFaceThread(Thread):
'''
Find similar faces from dashboard
'''
def __init__(self, **args):
self.nrof_closest_faces = args.get('nrof_closest_faces', 10)
self.database = args.get('database')
self.rabbit_mq = args.get('rabbit_mq')
self.event = threading.Event()
self.setup_matcher()
super(FindSimilarFaceThread, self).__init__()
def join(self, timeout=None):
print('Find similar joint')
self.event.set()
super(FindSimilarFaceThread, self).join()
def setup_matcher(self):
with open('../data/top10querylog.txt', 'a') as f:
f.write('TOP10 QUERY IS BEING IN PROCESS !!!\n\n')
'''
self.embs = []
self.labels = []
cursors = self.mongo.mongodb_dashinfo.find({})
unique_labels = [cursor['represent_image_id'] for cursor in cursors]
cursors = self.mongo.mongodb_faceinfo.find({'image_id': {'$in': unique_labels}})
for cursor in cursors:
self.embs.append(np.array(cursor['embedding']))
self.labels.append(cursor['image_id'])
self.nof_registered_image_ids = len(self.labels)
'''
self.labels, self.embs = self.database.get_labels_and_embs_dashboard()
self.matcher = FaissMatcher()
self.matcher.fit(self.embs, self.labels)
with open('../data/top10querylog.txt', 'a') as f:
f.write('MATCHER BUILT!!!\n\n')
def run(self):
while not self.event.is_set():
# first update check for new faces in dashboard
'''
if self.nof_registered_image_ids < self.mongo.mongodb_dashinfo.find({}).count():
self.nof_registered_image_ids = self.mongo.mongodb_dashinfo.find({}).count()
print('[Query TOP10] Update new registered image_id ...')
cursors = self.mongo.mongodb_dashinfo.find({'represent_image_id': {'$nin': self.labels}})
unique_labels = [cursor['represent_image_id'] for cursor in cursors]
cursors = self.mongo.mongodb_faceinfo.find({'image_id': {'$in': unique_labels}})
adding_embs = []
adding_labels = []
for cursor in cursors:
adding_embs.append(np.array(cursor['embedding']))
adding_labels.append(cursor['image_id'])
'''
adding_labels, adding_embs = self.database.get_labels_and_embs_dashboard(
self.labels)
if len(adding_labels) > 0:
old_embs = self.embs
old_labels = self.labels
self.embs = old_embs + adding_embs
self.labels = old_labels + adding_labels
print('Find similar', len(adding_labels))
self.matcher.update(adding_embs, adding_labels)
# get new query from from queue, why not just trigger
action_msg = self.rabbit_mq.receive_str(Config.Queues.ACTION)
if action_msg is not None:
return_dict = json.loads(action_msg)
print('Receive: {}'.format(return_dict))
if return_dict['actionType'] == 'getNearest':
data = return_dict['data']
results = {}
session_id = data['sessionId']
image_id = data['imageId']
print('[Query TOP10] image_id: ' + image_id)
with open('../data/top10querylog.txt', 'a') as f:
f.write('image_id: ' + image_id + '\n')
cursors = self.database.mongodb_faceinfo.find(
{'image_id': image_id})
if cursors.count() == 0:
print(
'[Query TOP10] THIS QUERY IMAGE ID HAS YET TO REGISTER'
)
with open('../data/top10querylog.txt', 'a') as f:
f.write(
'THIS QUERY IMAGE ID HAS YET TO REGISTER\n')
face_id = self.database.mongodb_dashinfo.find(
{'represent_image_id': image_id})[0]['face_id']
unique_labels = [
cursor['represent_image_id']
for cursor in self.database.mongodb_dashinfo.find(
{'face_id': face_id})
]
for label in unique_labels:
results[label] = '0'
else:
query_emb = cursors[0]['embedding']
embs = np.array(query_emb).astype('float32').reshape(
(-1, 128))
dists, inds = self.matcher._classifier.search(embs,
k=15)
dists = np.squeeze(dists)
inds = np.squeeze(inds)
top_match_ids = [self.labels[idx] for idx in inds]
for i, top_match_id in enumerate(top_match_ids):
if i < 11 and top_match_id != image_id:
results[top_match_id] = str(dists[i])
msg_results = {
'actionType': 'getNearest',
'sessionId': session_id,
'data': {
'images': results
}
}
with open('../data/top10querylog.txt', 'a') as f:
f.write('Result: \n{}\n\n'.format(results))
print('[Query TOP10] Result: \n{}'.format(results))
self.rabbit_mq.send_with_exchange(
Config.Queues.ACTION_RESULT, session_id,
json.dumps(msg_results))
else:
time.sleep(1)