def find_id_card(candis, cap, values):
id_card_index_candidate = []
face_scores = []
rect_args = load_model()
for candi in candis:
idxs = pick_up_candi(candi, 10, values)
for idx in idxs:
cap.set(cv.CAP_PROP_POS_FRAMES, START_FRAME + idx)
res, frame = cap.read()
frame = frame[250:-250, 500:-500]
print(START_FRAME + idx)
croped_img, rect_score = id_card_detect(frame, rect_args)
if not croped_img.width:
continue
else:
croped_img = cv.cvtColor(np.asarray(croped_img),
cv.COLOR_RGB2BGR)
half = croped_img[int(croped_img.shape[0] *
0.1):int(croped_img.shape[0] * 0.7),
int(croped_img.shape[1] *
0.55):int(croped_img.shape[1] * 0.95)]
has_face, score, _ = retina_face_distinguish(half)
if has_face:
id_card_index_candidate.append(idx)
face_scores.append(score)
best_face_score_idx = np.argmax(np.array(face_scores))
best_idx = id_card_index_candidate[best_face_score_idx]
return best_idx, id_card_index_candidate
def get_hSv_value_and_extract_face(cap):
num_frames = cap.get(7)
pix_value_average = []
interval = 15
face_idx_candis = []
face_scores = []
front_face_score = []
lm_test = []
i = START_FRAME
cap.set(cv.CAP_PROP_POS_FRAMES, i)
while cap.isOpened():
ret, frame = cap.read()
if ret:
frame_roi = frame[int(frame.shape[0] * 0.1): int(frame.shape[0] * 0.9),
int(frame.shape[1] * 0.1): int(frame.shape[1] * 0.9)]
hsv_img = cv.cvtColor(frame_roi, cv.COLOR_BGR2HSV)
pix_value_average.append(hsv_img[:, :, 1].mean())
# visualization
imgzi = put_text(hsv_img, str(pix_value_average[-1]))
cv.imshow('2', hsv_img)
cv.waitKey(10)
# end
if i % interval == 0:
has_face, score, lm = retina_face_distinguish(frame)
# if has_face and score.shape[0] == 2:
if has_face:
face_idx_candis.append(i)
face_scores.append(np.min(score))
front_face_score.append((lm[:, 1] - lm[:, 0]).mean())
lm_test.append(lm)
if i >= num_frames - 30: # end point
break
print(i)
else:
print('video cap ends at' + str(i) + 'th frame')
break
i += 1
value_np = np.stack(pix_value_average, 0)
cv.destroyAllWindows()
if len(face_scores):
front_face_score_np = np.array(front_face_score) + 0.01
front_face_score_np = (front_face_score_np - front_face_score_np.min()) / (
front_face_score_np.max() - front_face_score_np.min())
front_face_score_np = front_face_score_np * 0.1 + 0.9
face_scores_np = np.array(face_scores)
access_score_np = front_face_score_np + face_scores_np
best_face_score_idx = np.argmax(access_score_np)
best_idx = face_idx_candis[best_face_score_idx]
return value_np, best_idx
else:
return value_np, None
def find_best_face(frames_num, cap):
interval = 15
face_idx_candis = []
face_scores = []
face_scores_ori = []
front_face_score = []
lm_test = []
for cur_frame_idx in range(0, frames_num, interval):
cap.set(cv.CAP_PROP_POS_FRAMES, cur_frame_idx + START_FRAME)
print(cur_frame_idx + START_FRAME)
s_f, frame = cap.read()
if s_f:
has_face, score, lm = retina_face_distinguish(frame[250: -250, 500:-500])
if has_face and score.shape[0] == 2:
face_idx_candis.append(cur_frame_idx)
face_scores.append(np.min(score))
face_scores_ori.append(score)
front_face_score.append((lm[:, 1] - lm[:, 0]).mean())
lm_test.append(lm)
front_face_score_np = np.array(front_face_score)
front_face_score_np = (front_face_score_np - front_face_score_np.min()) / (front_face_score_np.max() - front_face_score_np.min())
front_face_score_np = front_face_score_np * 0.1 + 0.9
face_scores_np = np.array(face_scores)
access_score_np = front_face_score_np + face_scores_np
best_face_score_idx = np.argmax(access_score_np)
best_idx = face_idx_candis[best_face_score_idx]
# for i in range(len(face_scores)):
# print(face_idx_candis[i], face_scores[i])
# print(best_idx)
return best_idx
def detect_id_card(img):
"""
looking for an id card from a frame, return the confidence score
"""
croped_img, rect_score = id_card_detect(img, rect_args)
if not croped_img.width:
return 0
else:
# looking for a face in the right-middle part of the cropped area
croped_img = cv.cvtColor(np.asarray(croped_img), cv.COLOR_RGB2BGR)
half = croped_img[int(croped_img.shape[0] *
0.1):int(croped_img.shape[0] * 0.7),
int(croped_img.shape[1] *
0.55):int(croped_img.shape[1] * 0.95)]
has_face, score, _ = retina_face_distinguish(half)
# cv.imshow('1', croped_img)
# cv.waitKey(1)
if has_face:
return score
else:
return 0
def find_id_card(candis, cap, values):
id_card_index_candidate = []
face_scores = []
rect_args = load_model()
for candi in candis:
idxs = pick_up_candi(candi, 10, values)
for idx in idxs:
cap.set(cv.CAP_PROP_POS_FRAMES, START_FRAME + idx)
res, frame = cap.read()
frame = frame
print('find id ', START_FRAME + idx)
croped_img, rect_score = id_card_detect(frame, rect_args)
if not croped_img.width:
continue
else:
croped_img = cv.cvtColor(np.asarray(croped_img), cv.COLOR_RGB2BGR)
half = croped_img[int(croped_img.shape[0] * 0.1): int(croped_img.shape[0] * 0.7),
int(croped_img.shape[1] * 0.55): int(croped_img.shape[1] * 0.95)]
has_face, score, _ = retina_face_distinguish(half)
cv.imshow(f'{idx} finding id-{has_face}', croped_img)
cv.waitKey(5)
if has_face:
id_card_index_candidate.append(idx)
face_scores.append(score)
# cv.waitKey(0)
if len(face_scores) == 0:
raise ValueError('id_card_candidates wrong')
best_face_score_idx = np.argmax(np.array(face_scores))
best_idx = id_card_index_candidate[best_face_score_idx]
return best_idx, id_card_index_candidate
def get_hSv_value_and_extract_face(cap, box0, box1, box2, box3):
"""
Args:
cap: cv.VideoCapture
box0: Area of Interest, x-left
box1: Area of Interest, x-right
box2: Area of Interest, y-left
box3: Area of Interest, y-right
Returns:
value_np: [np.ndarray] pixel saturation average
best_face_idx: [int] predicted best frame containing faces
best_id_idx: [int] predicted best frame containing id cards
"""
from config import INTERVAL, START_FRAME, N_FACE
num_frames = cap.get(7)
pix_value_average = []
face_idx_candis = []
face_scores = []
front_face_score = []
id_card_scores = []
id_card_idxes = []
# lm_test = [] # retina landmarks
i = START_FRAME
cap.set(cv.CAP_PROP_POS_FRAMES, i)
while cap.isOpened():
ret, frame = cap.read()
# crop original frame
frame = frame[box0:box1, box2:box3]
if ret:
# shrink down the region of interest
frame_roi = frame[int(frame.shape[0] * 0.1):int(frame.shape[0] *
0.9),
int(frame.shape[1] * 0.1):int(frame.shape[1] *
0.9)]
# get saturation (s-value) and save its mean value
hsv_img = cv.cvtColor(frame_roi, cv.COLOR_BGR2HSV)
pix_value_average.append(hsv_img[:, :, 1].mean())
# detect using CNN
vis_title = 'ori_pic'
if i % INTERVAL == 0:
# expected n faces. if detected, save the frame as a candidate
has_face, score, lm = retina_face_distinguish(frame)
if has_face and score.shape[0] == N_FACE:
vis_title += "-" + str(score.shape[0]) + " faces"
face_idx_candis.append(i)
face_scores.append(np.mean(score))
front_face_score.append((lm[:, 1] - lm[:, 0]).mean())
# lm_test.append(lm)
# detect_id_card
id_card_score = detect_id_card(frame)
if id_card_score:
vis_title += "-with card"
id_card_scores.append(id_card_score)
id_card_idxes.append(i)
print("Detecting ", i)
# visualization
# imgzi = put_text(hsv_img, str(pix_value_average[-1]))
cv.imshow(vis_title, frame)
cv.waitKey(1)
# end
if i >= num_frames - 30: # end point
break
else:
print('video cap ends at' + str(i) + 'th frame')
break
i += 1
# from list to numpy
value_np = np.stack(pix_value_average, 0)
cv.destroyAllWindows()
# find best faces and best id card, return their frame index
if not len(face_scores) * len(id_card_scores):
zero = 'FACE' if not len(face_scores) else 'IDCARD'
raise ValueError(
f'NO {zero} DETECTED. Modify the configurations and try again (see config.py).'
)
else:
front_face_score_np = np.array(front_face_score) + 0.01
front_face_score_np = (front_face_score_np - front_face_score_np.min()
) / (front_face_score_np.max() -
front_face_score_np.min())
front_face_score_np = front_face_score_np * 0.1 + 0.3
face_scores_np = np.array(face_scores)
access_score_np = front_face_score_np + face_scores_np * 0.01
best_face_score_idx = int(np.argmax(access_score_np))
best_face_idx = face_idx_candis[best_face_score_idx]
best_id_card_score_idx = int(np.argmax(id_card_scores))
best_id_idx = id_card_idxes[best_id_card_score_idx]
return value_np, best_face_idx, best_id_idx