def predictor(vstop, predQ, faceQ):
window_size = 14
N_FEATURES = 512
faceDet = MTCNNFaceDet()
featureExtractor = MobileNetCustomFeatureExtract(N_FEATURES)
model = load_model('mobilecus_' + str(N_FEATURES) + '_train.h5')
tracker = cv2.TrackerMedianFlow_create()
frames = 0
fin = np.empty(shape=(window_size, N_FEATURES))
vin = cv2.VideoCapture(0)
fea_logs = [] # feature extraction time logs
pred_logs = [] # predition time logs
while True:
ret, frame = vin.read()
if not ret:
break
if frames == 0:
faceQ.put(frame)
(startX, startY, endX, endY, landmarks, face_time) = predQ.get()
if endX == 0 or endY == 0:
continue
tracker.clear()
tracker = cv2.TrackerMedianFlow_create()
tracker.init(frame, (startX, startY, endX - startX, endY - startY))
else:
if frames == 1:
faceQ.put(frame)
try:
(startX, startY, endX, endY, landmarks,
face_time) = predQ.get(False)
faceQ.put(frame)
tracker.clear()
tracker = cv2.TrackerMedianFlow_create()
tracker.init(frame,
(startX, startY, endX - startX, endY - startY))
except:
(success, box) = tracker.update(frame)
if success:
(startX, startY, w, h) = [int(v) for v in box]
endX = startX + w
endY = startY + h
# Extract features by good model
#print(startX, startY, endX, endY, landmarks)
startX = int(startX)
startY = int(startY)
endX = int(endX)
endY = int(endY)
bw = endX - startX
bh = endY - startY
sx, sy, ex, ey = faceDet.landmark2mouth(landmarks, bw, bh)
face_img = frame[sy:ey, sx:ex]
stime = time.time()
features = featureExtractor.feature_extract(face_img)
fea_logs.append(time.time() - stime)
fea_time = 0
for i in range(len(fea_logs)):
fea_time += fea_logs[i]
fea_time = fea_time / len(fea_logs)
if len(fea_logs) >= TIME_PERIOD:
fea_logs.pop()
# prediction
fin[0:window_size - 1] = fin[1:window_size]
fin[window_size - 1] = features
frames += 1
try:
text = 'face%.3f' % (1.0 / face_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 30), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 255), 2)
try:
text = 'fea%.3f' % (1.0 / fea_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 60), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 255), 2)
if frames >= window_size:
stime = time.time()
pred_in = fin[np.newaxis, :]
#pred_in = fin
#print(pred_in.shape)
pred = model.predict(pred_in)
pred_logs.append(time.time() - stime)
pred_time = 0
for i in range(len(pred_logs)):
pred_time += pred_logs[i]
pred_time = pred_time / len(pred_logs)
if len(pred_logs) >= TIME_PERIOD:
pred_logs.pop()
try:
text = 'pred%.3f' % (1 / pred_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 90), cv2.FONT_HERSHEY_SIMPLEX, 1,
(0, 0, 255), 2)
cv2.putText(frame, 'yawn: %d' % (int(pred + 0.5)), (20, 120),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 0, 255),
2)
cv2.rectangle(frame, (sx, sy), (ex, ey), (255, 0, 0), 2)
# idx = frames - 1
# cv2.rectangle(frame, (cord['sx'][idx], cord['sy'][idx]),
# (cord['ex'][idx], cord['ey'][idx]), (0, 255, 0), 2)
cv2.imshow('', frame)
key = cv2.waitKey(10) & 0xFF
if key == ord("q"):
break
vstop.value = 1
vin.release()
cv2.destroyAllWindows()
from dense121_fea_extract import Dense121FeatureExtract from mobilenet_fea_extract import MobileNetFeatureExtract from mobilenet_custom_fea_extract import MobileNetCustomFeatureExtract N_FEATURES = 512 video_path = '/home/jovyan/at072-group04/aiaDDD/videos/' #extractor = 'dense121' #featureExtracter = Dense121FeatureExtract(N_FEATURES) #extractor = 'mobilenet' #featureExtracter = MobileNetFeatureExtract(N_FEATURES) extractor = 'mobilecus' featureExtracter = MobileNetCustomFeatureExtract(N_FEATURES)
import pandas as pd
import os
import re
from keras.models import load_model
import multiprocessing as mp
from mobilenet_custom_fea_extract import MobileNetCustomFeatureExtract
def isMale(name):
return re.match('[0-9]{1,2}-Male.+', name) != None
window_size = 14
N_FEATURES = 512
featureExtractor = MobileNetCustomFeatureExtract(N_FEATURES)
model = load_model('mobilecus_' + str(N_FEATURES) + '_train.h5')
video_path = '/projectdata/driver/YawDD/'
#marker_path = '../../YawDD/'
cord_prefix = '../YawDD/ssd_face/bbox/'
#check_set = 'yawn_valid'
for check_set in ['yawn_test', 'yawn_valid']:
files = pd.read_csv('../YawDD/' + check_set + '.csv')
file_list = files['Name'].values
cord_path = cord_prefix + check_set + '/'
for fname in file_list:
frames = 0
fin = np.empty(shape=(window_size, N_FEATURES))
src_path = video_path
def predictor(vstop, predQ, faceQ):
window_size = 14
N_FEATURES = 512
#featureExtractor = Dense121FeatureExtract(N_FEATURES)
#featureExtractor = MobileNetFeatureExtract(N_FEATURES)
featureExtractor = MobileNetCustomFeatureExtract(N_FEATURES)
model = load_model('mobilecus_'+str(N_FEATURES)+'_train.h5')
tracker = cv2.TrackerMedianFlow_create()
frames = 0
fin = np.empty(shape=(window_size,N_FEATURES))
#vin = cv2.VideoCapture('VID_20180804_152934.avi')
vin = cv2.VideoCapture(0)
#fea = np.load('VID_20180804_152934.npy')
#cord = pd.read_csv('VID_20180804_152934_bbox.csv')
while True:
ret, frame = vin.read()
if not ret:
break
stime = time.time()
if frames == 0:
faceQ.put(frame)
(startX, startY, endX, endY) = predQ.get()
tracker.clear()
tracker = cv2.TrackerMedianFlow_create()
tracker.init(frame, (startX, startY, endX-startX, endY-startY))
else:
if frames == 1:
faceQ.put(frame)
try:
(startX, startY, endX, endY) = predQ.get(False)
faceQ.put(frame)
tracker.clear()
tracker = cv2.TrackerMedianFlow_create()
tracker.init(frame, (startX, startY, endX-startX, endY-startY))
except:
(success, box) = tracker.update(frame)
if success:
(startX, startY, w, h) = [int(v) for v in box]
endX = startX + w
endY = startY + h
face_time = time.time() - stime
#cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 0, 255), 2)
# Extract features by good model
startX = int(startX)
startY = int(startY)
endX = int(endX)
endY = int(endY)
# y: 6/12 -> 11/12, use 5/12 region size
yoff = int((endY-startY)*6/12)
xoff = int((endX-startX)/4)
ybot = int((endY-startY)*1/12)
face_img = frame[startY+yoff:endY-ybot, startX+xoff:endX]
fea_time = time.time()
features = featureExtractor.feature_extract(face_img)
fea_time = time.time() - fea_time
# prediction
fin[0:window_size-1] = fin[1:window_size]
fin[window_size-1] = features
frames += 1
try:
text='face%.3f'%(1.0/face_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 30),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
try:
text='fea%.3f'%(1.0/fea_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 60),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
if frames >= window_size:
pred_time = time.time()
pred_in = fin[np.newaxis, :]
#pred_in = fin
#print(pred_in.shape)
pred = model.predict(pred_in)
pred_time = time.time() - pred_time
try:
text='pred%.3f'%(1/pred_time)
except ZeroDivisionError:
text = 'Inf'
cv2.putText(frame, text, (20, 90),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
cv2.putText(frame, 'yawn: %d'%(int(pred+0.5)), (20, 120),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), 2)
cv2.rectangle(frame, (startX, startY), (endX, endY), (0, 0, 255), 2)
cv2.rectangle(frame, (startX+xoff, startY+yoff), (endX, endY-ybot), (255, 0, 0), 2)
# idx = frames - 1
# cv2.rectangle(frame, (cord['sx'][idx], cord['sy'][idx]),
# (cord['ex'][idx], cord['ey'][idx]), (0, 255, 0), 2)
cv2.imshow('', frame)
key = cv2.waitKey(10) & 0xFF
if key == ord("q"):
break
vstop.value = 1
vin.release()
cv2.destroyAllWindows()