def Eye_tracking(yolo): import cv2 cam = cv2.VideoCapture(0) cam.set(3, 1000) cam.set(4, 600) gaze = GazeTracking() while True: _, frame = cam.read() image = Image.fromarray(frame) res, Faces = yolo.detect_image_with_coord(image) output = np.array(res) for face in Faces: x1, y1, x2, y2 = face #Face_img = frame[y1:y2,(x1-10):(x2+10)] gaze.refresh(output) Ox1, Oy1, Ox2, Oy2 = gaze.annotated_frame(x1, y1) color = (0, 0, 255) cv2.line(output, (Ox1 - 5, Oy1), (Ox1 + 5, Oy1), color) cv2.line(output, (Ox1, Oy1 - 5), (Ox1, Oy1 + 5), color) cv2.line(output, (Ox2 - 5, Oy2), (Ox2 + 5, Oy2), color) cv2.line(output, (Ox2, Oy2 - 5), (Ox2, Oy2 + 5), color) # 60 130 165 text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" elif gaze.is_left(): text = "Looking left" elif gaze.is_center(): text = "Looking center" left_pupil = gaze.pupil_left_relative_coords() right_pupil = gaze.pupil_right_relative_coords() cv2.putText(output, text, (x1, y1 + 20), cv2.FONT_HERSHEY_DUPLEX, 0.8, (147, 58, 31), 2) cv2.putText(output, "Left pupil: " + str(left_pupil), (x1, y1 + 50), cv2.FONT_HERSHEY_DUPLEX, 0.4, (147, 58, 31), 1) cv2.putText(output, "Right pupil: " + str(right_pupil), (x1, y1 + 65), cv2.FONT_HERSHEY_DUPLEX, 0.4, (147, 58, 31), 1) cv2.imshow("Camera", output) if cv2.waitKey(5) == 27: break yolo.close_session()
def imagecov(photoname, relative_eye_size=1.5): global count ''' Keep the image in the folder source_image and put in the name of image in photoname ''' photoname = photoname sourcename = DIRNAME + '/source_img/' + photoname finalname = DIRNAME + '/static/' + str(count) + ".jpg" ''' You can change the relative eye size to optimize the image further ''' # relative_eye_size = 1.5 gaze = GazeTracking() frame = cv2.imread(sourcename) # cv2.imshow("Demo1", frame) gaze.refresh(frame) frame = gaze.annotated_frame() left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() try: distance = (left_pupil[0] - right_pupil[0]) * ( left_pupil[0] - right_pupil[0] ) + (left_pupil[1] - right_pupil[1]) * (left_pupil[1] - right_pupil[1]) except: return False distance = np.sqrt(distance) print(distance) face_image = Image.open(sourcename) eye_image = Image.open(DIRNAME + '/source_img/redeye.png') eye_image = eye_image.resize((int(distance * 2 * relative_eye_size), int(distance * relative_eye_size))) eye_image = eye_image.rotate(15) Image.Image.paste(face_image, eye_image, (left_pupil[0] - int(distance * relative_eye_size), left_pupil[1] - int(distance * relative_eye_size / 2)), eye_image) Image.Image.paste(face_image, eye_image, (right_pupil[0] - int(distance * relative_eye_size), right_pupil[1] - int(distance * relative_eye_size / 2)), eye_image) count += 1 # face_image.show() face_image.save(finalname) # eye_image.show() return True
class GazeExtractor: def __init__(self): self.cv_bridge = CvBridge() self.gaze = GazeTracking() self.publish_annotated_frame = rospy.get_param( "~publish_annotated_frame", True) if self.publish_annotated_frame: self.annotated_frame_publisher = rospy.Publisher( 'image_annotated_raw', Image, queue_size=10) self.gaze_publisher = rospy.Publisher('gaze_state', GazeState, queue_size=10) def extract_from_image(self, img_msg): #convert image to opencv type try: cv_image = self.cv_bridge.imgmsg_to_cv2(img_msg, "bgr8") except CvBridgeError as e: print(e) return #run gaze detection self.gaze.refresh(cv_image) #if desired, publish annotated frame if self.publish_annotated_frame: annotated_image_msg = self.cv_bridge.cv2_to_imgmsg( self.gaze.annotated_frame(), "bgr8") self.annotated_frame_publisher.publish(annotated_image_msg) #if no pupils detected, stop here if not self.gaze.pupils_located: return #pack gaze tracking result into a GazeState message and publish result_msg = GazeState() result_msg.header = img_msg.header result_msg.is_left = self.gaze.is_left() result_msg.is_right = self.gaze.is_right() result_msg.is_center = self.gaze.is_center() result_msg.is_blinking = self.gaze.is_blinking() result_msg.pupil_left_coords.x = self.gaze.pupil_left_coords()[0] result_msg.pupil_left_coords.y = self.gaze.pupil_left_coords()[1] result_msg.pupil_right_coords.x = self.gaze.pupil_right_coords()[0] result_msg.pupil_right_coords.y = self.gaze.pupil_right_coords()[1] result_msg.horizontal_ratio = self.gaze.horizontal_ratio() result_msg.vertical_ratio = self.gaze.vertical_ratio() self.gaze_publisher.publish(result_msg)
def adhdRes(): #app3 code # p = multiprocessing.Process(target = ValuePredictor, name="valuePredictor", args=()) # p.start() # time.sleep(10) # p.terminate() sns.set(style="ticks", context="talk") plt.style.use("dark_background") gaze = GazeTracking() webcam = cv2.VideoCapture(0) while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) if (left_pupil == (0, 0) or right_pupil == (0, 0)): pass else: plt.plot(left_pupil, right_pupil) cv2.imshow("Demo", frame) if cv2.waitKey(1) == ord(' '): break plt.savefig('2.png') img = open_image('./2.png') result = ValuePredictor(img) if result == 'ASD': prediction = "ADHD" else: prediction = "No ADHD" return render_template("./test/adhd.html", prediction=prediction)
def gaze(): gaze = GazeTracking() webcam = cv2.VideoCapture(0) face_detect = 1 while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) if face_detect: phonix_tracking.detect_video() frame = gaze.annotated_frame() text = "" # if gaze.is_blinking(): # text = "Blinking" # print("blinking") if gaze.is_right(): text = "Looking right" print("right") elif gaze.is_left(): text = "Looking left" print("left") # elif gaze.is_center(): # text = "Looking center" # print("center") # cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() # cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) # cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) # cv2.imshow("Demo", frame) # esc(27번키) 를 누르면 프로그램 종료 if cv2.waitKey(1) == 27: break
def eye_tracking(image_path): gaze = GazeTracking() frame = cv2.imread(image_path) gaze.refresh(frame) frame = gaze.annotated_frame() if gaze.is_right(): value = 0.5 elif gaze.is_left(): value = 0.5 elif gaze.is_center(): value = 1 else: value = 0 return value
def run_gazetracker(seconds): gaze = GazeTracking() webcam = cv2.VideoCapture(0) current_time = time.time() gaze_matrix = [] while time.time() - current_time <= 60: # We get a new frame from the webcam success, frame = webcam.read() if not success: print('NOT SUCCESSFUL') break else: # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" elif gaze.is_left(): text = "Looking left" elif gaze.is_center(): text = "Looking center" cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.imshow("Demo", frame) gaze_matrix.append([time.time(), left_pupil, right_pupil]) if cv2.waitKey(1) == 27: break print(left_pupil) print(right_pupil) print(gaze_matrix)
def get_frame(self): '''success, image = self.video.read() image=cv2.resize(image,None,fx=ds_factor,fy=ds_factor,interpolation=cv2.INTER_AREA) gray=cv2.cvtColor(image,cv2.COLOR_BGR2GRAY) face_rects=face_cascade.detectMultiScale(gray,1.3,5) for (x,y,w,h) in face_rects: cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,0),2) break ret, jpeg = cv2.imencode('.jpg', image) return jpeg.tobytes()''' gaze = GazeTracking() webcam = cv2.VideoCapture(0) while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right,please look at the screen" elif gaze.is_left(): text = "Looking left,please look at the screen" elif gaze.is_center(): text = "Looking center" cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) '''cv2.imshow("Demo", frame) if cv2.waitKey(1) == 27: break''' #comment lower part and decomment upper part for unique tab ret, jpeg = cv2.imencode('.jpg', frame) return jpeg.tobytes()
def getEyeResults(): gaze = GazeTracking() frame = cv2.imread("./images/analysis/proctor.png") gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" elif gaze.is_left(): text = "Looking left" elif gaze.is_center(): text = "Looking center" print(text) return text
def GazeYourEye(video, student): gaze = GazeTracking() webcam = cv2.VideoCapture(video) result = [] while True: value, frame = webcam.read() if value == False: break gaze.refresh(frame) frame = gaze.annotated_frame() if gaze.is_blinking(): result.append('B') elif gaze.is_right(): result.append('R') elif gaze.is_left(): result.append('L') elif gaze.is_center(): result.append('C') if cv2.waitKey(1) == 27: break whole = len(result) ret = [ round(result.count('C') / whole * 100, 2), round(result.count('B') / whole * 100, 2), round(result.count('L') / whole * 100, 2), round(result.count('R') / whole * 100, 2) ] student = Students.query.filter( Students.student_number == student.student_number) student.update({ 'eye_ratio_center': ret[0], 'eye_ratio_blink': ret[1], 'eye_ratio_left': ret[2], 'eye_ratio_right': ret[3] }) data = np.array([[ret[0], ret[1], ret[2], ret[3]]]) [result] = load_model.predict(data) student.update({'eye_result': bool(result)}) db.session.commit()
def startCam(): import cv2 from gaze_tracking import GazeTracking import time gaze = GazeTracking() webcam = cv2.VideoCapture(0) startTime = time.time() totalFrames = 0 framesDistracted = 0 framesFocused = 0 while True: _, frame = webcam.read() totalFrames += 1 gaze.refresh(frame) frame = gaze.annotated_frame() if gaze.is_blinking(): framesDistracted += 1 elif gaze.is_right(): framesDistracted += 1 elif gaze.is_left(): framesDistracted += 1 elif gaze.is_center(): framesFocused += 1 else: framesDistracted += 1 cv2.imshow("Camera", frame) if cv2.waitKey(1) == ord('q'): break webcam.release() cv2.destroyAllWindows() totalTime = truncate(time.time() - startTime, 2) percentFocused = truncate((framesFocused / totalFrames) * 100, 2) percentDistracted = truncate((framesDistracted / totalFrames) * 100, 2) return totalTime, percentFocused, percentDistracted
def eyeTrack(self): gaze = GazeTracking() blinkCount = 0 while True: # Grab a single frame of video ret, frame = self.video_capture.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Goz Kirpildi" blinkCount += 1 elif gaze.is_right(): text = "Saga Bakildi" elif gaze.is_left(): text = "Sola Bakildi" elif gaze.is_center(): text = "Merkeze Bakildi" cv2.putText(frame, text, (0, 30), cv2.FONT_HERSHEY_DUPLEX, 1, (147, 58, 31), 2) # Display the resulting image cv2.imshow('Video', frame) print("Goz Kırpma: " + str(blinkCount)) if blinkCount >= 3: return 1 if cv2.waitKey(1) & 0xFF == ord('q'): break
def main(args): filename = args["input_file"] faceCascade = cv2.CascadeClassifier( 'models/haarcascade_frontalface_default.xml') model = load_model('models/facenet_keras.h5') if filename is None: isVideo = False webcam = cv2.VideoCapture(0) webcam.set(3, args['wh'][0]) webcam.set(4, args['wh'][1]) else: isVideo = True webcam = cv2.VideoCapture(filename) fps = webcam.get(cv2.webcam_PROP_FPS) width = int(webcam.get(cv2.webcam_PROP_FRAME_WIDTH)) height = int(webcam.get(cv2.webcam_PROP_FRAME_HEIGHT)) fourcc = cv2.VideoWriter_fourcc(*'XVID') name, ext = osp.splitext(filename) out = cv2.VideoWriter(args["output_file"], fourcc, fps, (width, height)) # Variable Setting hpd = headpose.HeadposeDetection( args["landmark_type"], args["landmark_predictor"]) #import headpose gaze = GazeTracking() # import gazetracking yellocard = 0 redcard = 0 tempval = 0 timee = int( input("시험 시간을 입력하세요(Minute): ")) # Input time for limit test time max_time_end = time.time() + (60 * timee) # Infinity Loop for Detect Cheating for Online test while (webcam.isOpened()): ret, frame = webcam.read() # Read wabcam gaze.refresh(frame) frame = gaze.annotated_frame() # Mark pupil for frame gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = faceCascade.detectMultiScale( gray, scaleFactor=1.1, minNeighbors=3, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) # face structure # Get point from pupil if gaze.is_blinking(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_right(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_left(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_center(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) else: yellocard = yellocard + 2 # Get redcard optiom if yellocard > 50: yellocard = 0 tempval = tempval + 1 redcard = redcard + 1 # if get 1redcard, then give Aural and Text Warning(Loop) if tempval == 1: text1 = "WARNING" cv2.putText(frame, text1, (10, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (0, 0, 255), 2) my_thread = threading.Thread(target=Sound) my_thread.start() tempval = 0 # if you are not GPU environment, Do not run this code by # -------------- # if get 2redcard, then give Picture Warning(Once) if redcard == 2: warn_img = cv2.imread("Warning/warning.png", cv2.IMREAD_COLOR) cv2.imshow('Warning', warn_img) cv2.waitKey(1) redcard = 2.1 # ----------------------------------------------------------------------- # Get log consistently print("<< *의심수준:", yellocard, " || ", "*경고횟수:", redcard, " >>") #Detect head position if isVideo: frame, angles = hpd.process_image(frame) if frame is None: break else: out.write(frame) else: frame, angles = hpd.process_image(frame) if angles is None: pass else: #angles = [x,y,z] , get point from headposition if angles[0] > 15 or angles[0] < -15 or angles[ 1] > 15 or angles[1] < -15 or angles[2] > 15 or angles[ 2] < -15: yellocard = yellocard + 2 else: yellocard = yellocard - 1 yellocard = notnegative(yellocard) yellocard = yellocard + hpd.yello(frame) if yellocard < 0: yellocard = notnegative(yellocard) # Draw a rectangle around the faces and predict the face name for (x, y, w, h) in faces: cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) # take the face pixels from the frame crop_frame = frame[y:y + h, x:x + w] # turn the face pixels back into an image new_crop = Image.fromarray( crop_frame ) # resize the image to meet the size requirment of facenet new_crop = new_crop.resize( (160, 160)) # turn the image back into a tensor crop_frame = np.asarray( new_crop) # get the face embedding using the face net model face_embed = get_embedding( model, crop_frame ) # it is a 1d array need to reshape it as a 2d tensor for svm face_embed = face_embed.reshape( -1, face_embed.shape[0]) # predict using our SVM model pred = svm.predict(face_embed) # get the prediction probabiltiy pred_prob = svm.predict_proba( face_embed) # pred_prob has probabilities of each class # get name class_index = pred[0] class_probability = pred_prob[0, class_index] * 100 predict_names = out_encoder.inverse_transform(pred) text = 'Predicted: %s (%.3f%%)' % (predict_names[0], class_probability) #add the name to frame but only if the pred is above a certain threshold if (class_probability > 70): cv2.putText(frame, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) # Display the resulting frame cv2.imshow('POCAS', frame) if cv2.waitKey(1) & 0xFF == ord('q'): print("관리자에 의해 시험이 강제 종료 되었습니다") PrintResult(yellocard, redcard) Fail(timee, redcard) break elif time.time() > max_time_end: print(timee, "분의 시험이 종료되었습니다.") PrintResult(yellocard, redcard) Fail(timee, redcard) break # When everything done, release the webcam webcam.release() if isVideo: out.release() cv2.destroyAllWindows()
def main(): """MAIN""" # Video source from webcam or video file. video_src = args.cam if args.cam is not None else args.video if video_src is None: engine.say( "Warning: video source not assigned, default webcam will be used") engine.runAndWait() video_src = 0 cap = cv2.VideoCapture(video_src) if video_src == 0: cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) _, sample_frame = cap.read() # Introduce mark_detector to detect landmarks. mark_detector = MarkDetector() # Setup process and queues for multiprocessing. img_queue = Queue() box_queue = Queue() img_queue.put(sample_frame) box_process = Process(target=get_face, args=( mark_detector, img_queue, box_queue, )) box_process.start() gaze = GazeTracking() # Introduce pose estimator to solve pose. Get one frame to setup the # estimator according to the image size. height, width = sample_frame.shape[:2] pose_estimator = PoseEstimator(img_size=(height, width)) # Introduce scalar stabilizers for pose. pose_stabilizers = [ Stabilizer(state_num=2, measure_num=1, cov_process=0.1, cov_measure=0.1) for _ in range(6) ] tm = cv2.TickMeter() head_flag = 0 gaze_flag = 0 while True: # Read frame, crop it, flip it, suits your needs. frame_got, frame = cap.read() if frame_got is False: break # Crop it if frame is larger than expected. # frame = frame[0:480, 300:940] #audio_record(AUDIO_OUTPUT, 3) #sphinx_recog(AUDIO_OUTPUT) # If frame comes from webcam, flip it so it looks like a mirror. if video_src == 0: frame = cv2.flip(frame, 2) # Pose estimation by 3 steps: # 1. detect face; # 2. detect landmarks; # 3. estimate pose # Feed frame to image queue. img_queue.put(frame) # Get face from box queue. facebox = box_queue.get() gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if facebox is not None: # Detect landmarks from image of 128x128. face_img = frame[facebox[1]:facebox[3], facebox[0]:facebox[2]] face_img = cv2.resize(face_img, (CNN_INPUT_SIZE, CNN_INPUT_SIZE)) gray = cv2.cvtColor(face_img, cv2.COLOR_BGR2GRAY) face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB) rects = detector(gray, 0) tm.start() marks = mark_detector.detect_marks([face_img]) tm.stop() # for rect in rects: # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array shape = predictor(gray, rect) shape = face_utils.shape_to_np( shape) #converting to NumPy Array矩阵运算 mouth = shape[Start:End] leftEye = shape[lStart:lEnd] rightEye = shape[rStart:rEnd] leftEAR = eye_aspect_ratio(leftEye) #眼睛长宽比 rightEAR = eye_aspect_ratio(rightEye) ear = (leftEAR + rightEAR) / 2.0 #长宽比平均值 lipdistance = lip_distance(shape) if (lipdistance > YAWN_THRESH): #print(lipdistance) flag0 += 1 print("yawning time: ", flag0) if flag0 >= 40: cv2.putText(frame, "Yawn Alert", (10, 150), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) cv2.putText(frame, "Yawn Alert", (220, 150), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) engine.say("don't yawn") engine.runAndWait() flag0 = 0 else: flag0 = 0 if (ear < thresh): flag += 1 print("eyes closing time: ", flag) if flag >= frame_check: cv2.putText(frame, "****************ALERT!****************", (10, 30), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) cv2.putText(frame, "****************ALERT!****************", (10, 250), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) engine.say("open your eyes") engine.runAndWait() flag = 0 else: flag = 0 if gaze.is_right(): print("Looking right") text = "Looking right" elif gaze.is_left(): print("Looking left") text = "Looking left" elif gaze.is_up(): text = "Looking up" else: text = "Looking center" if text is not "Looking center": gaze_flag += 1 if gaze_flag >= 20: engine.say("look forward") engine.runAndWait() gaze_flag = 0 else: gaze_flag = 0 marks *= (facebox[2] - facebox[0]) marks[:, 0] += facebox[0] marks[:, 1] += facebox[1] # Uncomment following line to show raw marks. # mark_detector.draw_marks( # frame, marks, color=(0, 255, 0)) # Uncomment following line to show facebox. # mark_detector.draw_box(frame, [facebox]) # Try pose estimation with 68 points. pose = pose_estimator.solve_pose_by_68_points(marks) # get angles angles = pose_estimator.get_angles(pose[0], pose[1]) if ((-8 > angles[0] or angles[0] > 8) or (-8 > angles[1] or angles[1] > 8)): head_flag += 1 if head_flag >= 40: print(angles[0]) engine.say("please look ahead") engine.runAndWait() else: head_flag = 0 # pose_estimator.draw_info(frame, angles) # Stabilize the pose. steady_pose = [] pose_np = np.array(pose).flatten() for value, ps_stb in zip(pose_np, pose_stabilizers): ps_stb.update([value]) steady_pose.append(ps_stb.state[0]) steady_pose = np.reshape(steady_pose, (-1, 3)) # Uncomment following line to draw pose annotation on frame. pose_estimator.draw_annotation_box(frame, pose[0], pose[1], color=(255, 128, 128)) # Uncomment following line to draw stabile pose annotation on frame. pose_estimator.draw_annotation_box(frame, steady_pose[0], steady_pose[1], color=(128, 255, 128)) # Uncomment following line to draw head axes on frame. pose_estimator.draw_axes(frame, steady_pose[0], steady_pose[1]) #pose_estimator.show_3d_model # Show preview. cv2.imshow("Preview", frame) if cv2.waitKey(1) & 0xFF == ord("q"): break # Clean up the multiprocessing process. box_process.terminate() box_process.join()
def eyeGaze(): gaze = GazeTracking() parser = argparse.ArgumentParser() parser.add_argument( "-i", help= 'Path to input image or video file. Skip this argument to capture frames from a camera.' ) detector = dlib.get_frontal_face_detector() predictor = dlib.shape_predictor( "pretrained_model/shape_predictor_68_face_landmarks.dat") args = parser.parse_args() cap = cv2.VideoCapture(args.i if args.i else 0) while cv2.waitKey(1) < 0: t = time.time() ret, frame = cap.read() if not ret: cv2.waitKey() break gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = detector(gray) if faces is not None: i = np.zeros(shape=(frame.shape), dtype=np.uint8) for face in faces: # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" elif gaze.is_left(): text = "Looking left" elif gaze.is_center(): text = "Looking center" left = face.left() top = face.top() right = face.right() bottom = face.bottom() cv2.rectangle(frame, (left, top), (right, bottom), (147, 58, 31), 2) cv2.rectangle(frame, (left, bottom - 10), (right, bottom), (147, 58, 31), cv2.FILLED) cv2.putText(frame, text, (left + 2, bottom - 2), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 0), 1) #cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1, (147, 58, 31), 2) '''left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.7, (147, 58, 31), 1)''' cv2.imshow("Demo", frame) print("Time : {:.3f}".format(time.time() - t)) if cv2.waitKey(1) & 0xFF == ord('q'): break print('[INFO] Stopping System') cap.release() cv2.destroyAllWindows()
(150.0, -150.0, -125.0) # Right mouth corner ]) # Camera internals focal_length = size[1] center = (size[1] / 2, size[0] / 2) camera_matrix = np.array( [[focal_length, 0, center[0]], [0, focal_length, center[1]], [0, 0, 1]], dtype="double") while True: ret, img = cap.read() _, frame = cap.read() #--------- gaze.refresh(frame) #------- img = gaze.annotated_frame() #----- text = "" #------ msgg = "" #----- if ret == True: faces = find_faces(img, face_model) for face in faces: marks = detect_marks(img, landmark_model, face) draw_marks(img, marks, color=(0, 255, 0)) image_points = np.array( [ marks[30], # Nose tip marks[8], # Chin marks[36], # Left eye left corner marks[45], # Right eye right corne marks[48], # Left Mouth corner
import cv2 import serial from gaze_tracking import GazeTracking gaze = GazeTracking() webcam = cv2.VideoCapture(0) #ser = serial.Serial('COM3', 9600) while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Down" #ser.write(b'B') elif gaze.is_right(): text = "Right" #ser.write(b'R') elif gaze.is_left(): text = "Left" #ser.write(b'L') elif gaze.is_up(): text = "Up" ser.write(b'F') elif gaze.is_center():
def run(models): #카메라 열기 cap = cv2.VideoCapture(0) gaze = GazeTracking() cheat_cnt = 0 right_cnt = 0 left_cnt = 0 up_cnt = 0 down_cnt = 0 name = "" while True: #카메라로 부터 사진 한장 읽기 ret, frame = cap.read() gaze.refresh(frame) # 얼굴 검출 시도 image, face = face_detector(frame) try: min_score = 999 #가장 낮은 점수로 예측된 사람의 점수 min_score_name = "" #가장 높은 점수로 예측된 사람의 이름 #검출된 사진을 흑백으로 변환 face = cv2.cvtColor(face, cv2.COLOR_BGR2GRAY) #위에서 학습한 모델로 예측시도 for key, model in models.items(): result = model.predict(face) if min_score > result[1]: min_score = result[1] min_score_name = key #min_score 신뢰도이고 0에 가까울수록 자신과 같다는 뜻이다. if min_score < 500: confidence = int(100 * (1 - (min_score) / 300)) # 유사도 화면에 표시 display_string = str( confidence) + '% Confidence it is ' + min_score_name name = min_score_name cv2.putText(image, display_string, (90, 170), cv2.FONT_HERSHEY_COMPLEX, 1, (250, 120, 255), 2) #75 보다 크면 동일 인물로 간주해 Match! if confidence > 75: cv2.putText(image, "Match : " + min_score_name, (250, 450), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 255, 0), 2) cv2.imshow('Face Detection', image) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" right_cnt += 1 elif gaze.is_left(): text = "Looking left" left_cnt += 1 elif gaze.is_up(): text = "Looking up" up_cnt += 1 elif gaze.is_down(): text = "Looking down" down_cnt += 1 elif gaze.is_center(): text = "Looking center" cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 100), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 135), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.imshow("Face Detection", frame) if left_pupil is None and right_pupil is None: cheat_cnt += 1 elif right_cnt > 5: cheat_cnt += 1 print('Too much looking right') elif left_cnt > 5: cheat_cnt += 1 print('Too much looking left') elif up_cnt > 5: cheat_cnt += 1 print('Too much looking up') elif down_cnt > 5: cheat_cnt += 1 print('Too much looking down') else: #75 이하면 Unmatch cv2.putText(image, "Unmatch", (250, 450), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2) cv2.imshow('Face Detection', image) except: #얼굴 검출 안됨 cv2.putText(image, "Face Not Found", (250, 450), cv2.FONT_HERSHEY_COMPLEX, 1, (255, 0, 0), 2) cv2.imshow('Face Detection', image) pass if cheat_cnt > 100: print(name + " Cheating Probability is high") break if cv2.waitKey(1) == 13: break cap.release() cv2.destroyAllWindows()
def main(): gaze = GazeTracking() webcam = cv2.VideoCapture(0) face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml') eye_cascade = cv2.CascadeClassifier('haarcascade_eye.xml') horizontal_ratio = 0 vertical_ratio = 0 hr_filtered = 0 vr_filtered = 0 cursor_x = 0 cursor_y = 0 while True: # We get a new frame from the webcam _, frame = webcam.read() # face_frame = detect_faces(frame, face_cascade) # if face_frame is not None: # eyes = detect_eyes(face_frame, eye_cascade) # # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" # if gaze.is_blinking(): # text = "Blinking" # elif gaze.is_right(): # text = "Looking right" # elif gaze.is_left(): # text = "Looking left" # elif gaze.is_center(): # text = "Looking center" # cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) h, v = gaze.position() # h = h*2-1 # v = v*2-1 # horizontal_ratio = clamp(h,-1,1) # vertical_ratio = clamp(v,-1,1) if h != -1: horizontal_ratio = clamp(h, 0, 1) vertical_ratio = clamp(v, 0, 1) alpha = 0.1 hr_filtered = lowpass_filter(horizontal_ratio, hr_filtered, alpha) vr_filtered = lowpass_filter(vertical_ratio, vr_filtered, alpha) print(hr_filtered, vr_filtered) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) width = 1000 height = 500 # blank_image = np.zeros((height, width)) blank_image = np.zeros(shape=[height, width, 3], dtype=np.uint8) cv2.imshow("blank", blank_image) cursor_x = int(width * vr_filtered) cursor_y = int(height * hr_filtered) cursor_x = clamp(cursor_x, 0, width) cursor_y = clamp(cursor_y, 0, height) # cursor_x = int(width*0.5) # cursor_y = int(height*0.5) radius = 10 cv2.circle(blank_image, (cursor_x, cursor_y), radius, [255, 255, 255], 2) cv2.imshow("blank", blank_image) frame = cv2.flip(frame, 1) cv2.imshow("Demo", frame) if cv2.waitKey(1) == 27: break
class face_detector(): def __init__(self): # Load the parameters self.conf = config() # initialize dlib's face detector (HOG-based) and then create the # facial landmark predictor print("[INFO] loading facial landmark predictor...") self.detector = dlib.get_frontal_face_detector() self.predictor = dlib.shape_predictor(self.conf.shape_predictor_path) # grab the indexes of the facial landmarks for the left and # right eye, respectively (self.lStart, self.lEnd) = face_utils.FACIAL_LANDMARKS_IDXS["left_eye"] (self.rStart, self.rEnd) = face_utils.FACIAL_LANDMARKS_IDXS["right_eye"] # initialize the video stream and sleep for a bit, allowing the # camera sensor to warm up self.cap = cv2.VideoCapture(0) if self.conf.vedio_path == 0: self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640) _, sample_frame = self.cap.read() # Introduce mark_detector to detect landmarks. self.mark_detector = MarkDetector() # Setup process and queues for multiprocessing. self.img_queue = Queue() self.box_queue = Queue() self.img_queue.put(sample_frame) self.box_process = Process(target=get_face, args=( self.mark_detector, self.img_queue, self.box_queue,)) self.box_process.start() # Introduce pose estimator to solve pose. Get one frame to setup the # estimator according to the image size. self.height, self.width = sample_frame.shape[:2] self.pose_estimator = PoseEstimator(img_size=(self.height, self.width)) # Introduce scalar stabilizers for pose. self.pose_stabilizers = [Stabilizer( state_num=2, measure_num=1, cov_process=0.1, cov_measure=0.1) for _ in range(6)] self.tm = cv2.TickMeter() # Gaze tracking self.gaze = GazeTracking() def detect(self): # loop over the frames from the video stream temp_steady_pose = 0 while True: # grab the frame from the threaded video stream, resize it to # have a maximum width of 400 pixels, and convert it to # grayscale frame_got, frame = self.cap.read() # Empty frame frame_empty = np.zeros(frame.shape) # frame = imutils.rotate(frame, 90) frame = imutils.resize(frame, width=400) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # detect faces in the grayscale frame rects = self.detector(gray, 0) # initialize the frame counters and the total number of blinks TOTAL = 0 COUNTER = 0 # loop over the face detections for (i, rect) in enumerate(rects): # determine the facial landmarks for the face region, then # convert the facial landmark (x, y)-coordinates to a NumPy # array self.shape = self.predictor(gray, rect) self.shape = face_utils.shape_to_np(self.shape) # ******************************** # Blink detection # extract the left and right eye coordinates, then use the # coordinates to compute the eye aspect ratio for both eyes self.leftEye = self.shape[self.lStart:self.lEnd] self.rightEye = self.shape[self.rStart:self.rEnd] self.leftEAR = eye_aspect_ratio(self.leftEye) self.rightEAR = eye_aspect_ratio(self.rightEye) # average the eye aspect ratio together for both eyes ear = (self.leftEAR + self.rightEAR) / 2.0 # check to see if the eye aspect ratio is below the blink # threshold, and if so, increment the blink frame counter if ear < self.conf.EYE_AR_THRESH: COUNTER += 1 # otherwise, the eye aspect ratio is not below the blink # threshold else: # if the eyes were closed for a sufficient number of # then increment the total number of blinks if COUNTER >= self.conf.EYE_AR_CONSEC_FRAMES: TOTAL += 1 # reset the eye frame counter COUNTER = 0 # Frame empty cv2.putText(frame_empty, "Blinks: {}".format(TOTAL), (30, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2) cv2.putText(frame_empty, "EAR: {:.2f}".format(ear), (30, 90), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2) # ******************************** # convert dlib's rectangle to a OpenCV-style bounding box # [i.e., (x, y, w, h)], then draw the face bounding box (x, y, w, h) = face_utils.rect_to_bb(rect) self.bounding_box = (x, y, w, h) # cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) # Frame empty cv2.rectangle(frame_empty, (x, y), (x + w, y + h), (0, 255, 0), 2) # show the face number cv2.putText(frame_empty, "Face #{}".format(i + 1), (30, 120), cv2.FONT_HERSHEY_SIMPLEX, 0.3, (0, 0, 255), 2) # loop over the (x, y)-coordinates for the facial landmarks # and draw them on the image for (x, y) in self.shape: # cv2.circle(frame, (x, y), 1, (0, 255, 255), -1) cv2.circle(frame_empty, (x, y), 1, (0, 255, 255), -1) # ********************************************************** if frame_got is False: break # If frame comes from webcam, flip it so it looks like a mirror. if self.conf.vedio_path == 0: frame = cv2.flip(frame, 2) # Pose estimation by 3 steps: # 1. detect face; # 2. detect landmarks; # 3. estimate pose # Feed frame to image queue. self.img_queue.put(frame) # Get face from box queue. self.facebox = self.box_queue.get() if self.facebox is not None: # Detect landmarks from image of 128x128. face_img = frame[self.facebox[1]: self.facebox[3], self.facebox[0]: self.facebox[2]] face_img = cv2.resize(face_img, (self.conf.CNN_INPUT_SIZE, self.conf.CNN_INPUT_SIZE)) face_img = cv2.cvtColor(face_img, cv2.COLOR_BGR2RGB) self.tm.start() # marks = self.mark_detector.detect_marks([face_img]) self.tm.stop() # Convert the marks locations from local CNN to global image. self.shape *= (self.facebox[2] - self.facebox[0]) self.shape[:, 0] += self.facebox[0] self.shape[:, 1] += self.facebox[1] # Uncomment following line to show raw marks. # mark_detector.draw_marks( # frame, marks, color=(0, 255, 0)) # Uncomment following line to show facebox. # mark_detector.draw_box(frame, [facebox]) # Try pose estimation with 68 points. self.pose = self.pose_estimator.solve_pose_by_68_points(self.shape) # Stabilize the pose. self.steady_pose = [] pose_np = np.array(self.pose).flatten() for value, ps_stb in zip(pose_np, self.pose_stabilizers): ps_stb.update([value]) self.steady_pose.append(ps_stb.state[0]) self.steady_pose = np.reshape(self.steady_pose, (-1, 3)) # Uncomment following line to draw pose annotation on frame. # pose_estimator.draw_annotation_box( # frame, pose[0], pose[1], color=(255, 128, 128)) # Uncomment following line to draw stabile pose annotation on frame. # pose_estimator.draw_annotation_box(frame, steady_pose[0], steady_pose[1], color=(128, 255, 128)) # Uncomment following line to draw head axes on frame. # pose_estimator.draw_axes(frame, steady_pose[0], steady_pose[1]) self.pose_estimator.draw_axes(frame_empty, self.steady_pose[0], self.steady_pose[1]) print('steady pose vector: {}'.format(self.steady_pose[0], self.steady_pose[1])) else: # cv2.putText(frame, "Signal loss", (200, 200), # cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) cv2.putText(frame_empty, "Signal loss", (200, 200), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 0), 2) # ****************************************************************** # We send this frame to GazeTracking to analyze it self.gaze.refresh(frame) frame = self.gaze.annotated_frame() text = "" if self.gaze.is_blinking(): text = "Blinking" elif self.gaze.is_right(): text = "Looking right" elif self.gaze.is_left(): text = "Looking left" elif self.gaze.is_center(): text = "Looking center" cv2.putText(frame_empty, text, (250, 250), cv2.FONT_HERSHEY_DUPLEX, 0.5, (147, 58, 31), 2) left_pupil = self.gaze.pupil_left_coords() right_pupil = self.gaze.pupil_right_coords() cv2.putText(frame_empty, "Left pupil: " + str(left_pupil), (250, 280), cv2.FONT_HERSHEY_DUPLEX, 0.5, (147, 58, 31), 1) cv2.putText(frame_empty, "Right pupil: " + str(right_pupil), (250, 310), cv2.FONT_HERSHEY_DUPLEX, 0.5, (147, 58, 31), 1) # ******************************************************************** # show the frame # cv2.imshow("Frame", frame) cv2.imshow("Frame", frame_empty) key = cv2.waitKey(1) & 0xFF self.pass_variable = np.array(1) try: self._listener(self.pass_variable) except: pass # if the `q` key was pressed, break from the loop if key == ord("q"): break # do a bit of cleanup cv2.destroyAllWindows() # self.cap.stop() def set_listener(self, listener): self._listener = listener
def main(): ### SET OPTIONS """ show_frame: True or False. Set to true if you want to see the video/webcam. video_path: String. Path to video file to play. source_path: String. Path to video to run emotion detector on, or just 'webcam' to use webcam. save_output: Save output of detected emotions video to a file called "output". save_log: String. Save log file of emotions to a text file. Specify log file name or leave empty string for none. """ show_frame = True video_path = 'test_video.mov' source_path = 'webcam' save_output = True save_log = "" ### count = 0 # frame counter # load the model model = tf.keras.models.load_model('tl-weights-improvement-08-0.69.hdf5', custom_objects=None, compile=False) if source_path != "webcam": webcam = cv2.VideoCapture(source_path) else: webcam = cv2.VideoCapture(0) video = cv2.VideoCapture(video_path) if save_output: # Used for writing video file frame_width = int(webcam.get(3)) frame_height = int(webcam.get(4)) out = cv2.VideoWriter('output.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10, (1000, 600)) # initialize face detector detector = mtcnn.MTCNN() # initialize gazer tracker gaze = GazeTracking() while True: ret, frame = webcam.read( ) # captures frame and returns boolean value and captured image vret, vframe = video.read() # if either of the videos end, if not ret or not vret: break print("Frame:", count) count += 1 gray_img = frame #cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) gaze.refresh(frame) frame = gaze.annotated_frame() where_looking = is_distracted(gaze, gray_img) if where_looking == "Distracted": cv2.putText(frame, "Distracted", (int(90), int(60)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) resized_img = cv2.resize(frame, (1000, 600)) if show_frame: resized_vframe = cv2.resize(vframe, (500, 300)) cv2.imshow(video_path, resized_vframe) cv2.imshow('Facial emotion analysis', resized_img) if save_output: out.write(resized_img) if cv2.waitKey(10) == ord('q'): #wait until 'q' key is pressed break continue # write eye location details cv2.putText(frame, where_looking, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) # use MTCNN to detect location of faces in image results = detector.detect_faces(gray_img) if len(results) != 0: x1, y1, width, height = results[0]['box'] faces_detected = [(x1, y1, width, height)] for (x, y, w, h) in faces_detected: cv2.rectangle(frame, (x, y), (x + w, y + h), (255, 0, 0), thickness=7) roi_gray = gray_img[ y:y + w, x:x + h] #cropping region of interest i.e. face area from image roi_gray = cv2.resize(roi_gray, (64, 64)) img_pixels = keras.preprocessing.image.img_to_array(roi_gray) img_pixels = np.expand_dims(img_pixels, axis=0) img_pixels /= 255 predictions = model.predict(img_pixels) #find max indexed array max_index = np.argmax(predictions[0]) emotions = ('angry', 'disgust', 'fear', 'happy', 'neutral', 'sad', 'surprise') predicted_emotion = emotions[max_index] cv2.putText(frame, predicted_emotion, (int(x), int(y)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2) resized_img = cv2.resize(frame, (1000, 600)) if show_frame: resized_vframe = cv2.resize(vframe, (500, 300)) cv2.imshow(video_path, resized_vframe) cv2.imshow('Facial emotion analysis', resized_img) if save_output: out.write(resized_img) if cv2.waitKey(10) == ord('q'): #wait until 'q' key is pressed break webcam.release() if save_output: out.release() cv2.destroyAllWindows
def getAttention(temp_photo): beta = 0.9 frame_count = 0 blink_counter = 0 sleep_frames_counter = 0 blink_th = 3 gaze = GazeTracking() yawns = 0 yawn_status = False smooth_attentiveness = -1 # 对图片进行处理 frame = temp_photo # 检测人脸 face = detector(frame) if len(face)==0 or face is None: return "no face" # We send this frame to GazeTracking to analyze it gaze.refresh(frame) landmarks = np.matrix([[p.x, p.y] for p in gaze.landmarks.parts()]) # print(np.matrix([[p.x, p.y] for p in landmarks.parts()])) frame = gaze.annotated_frame() text = "" attentiveness = 100 if gaze.is_blinking(): blink_counter += 1 elif is_right(gaze) or is_left(gaze): # or is_up(gaze) or is_down(gaze): 不看中间专注度基数就为0 attentiveness = 0 blink_counter = np.maximum(blink_counter - 1, 0) elif is_center(gaze): attentiveness = 100 blink_counter = np.maximum(blink_counter - 1, 0) # if blink_counter >= blink_th: # attentiveness = 0 # 瞳孔定位判断,用于显示 if gaze.pupils_located: if gaze.is_blinking(): gaze_text = "blinking" elif is_right(gaze): gaze_text = "looking right" elif is_left(gaze): gaze_text = "looking left" elif is_center(gaze): gaze_text = "looking center" elif is_down(gaze): gaze_text = "looking down" elif is_up(gaze): gaze_text = "looking up" else: gaze_text = "unsure" # 面部朝向判断 direction = check_face_direction(landmarks) if direction == 'Center': attentiveness = 90 else: print("人脸朝向") print(direction) attentiveness = 0.5 * attentiveness # 0.5 * attentiveness print("降低后的专注度") print(attentiveness) print("瞳孔方向") print(gaze_text) sleep_chance = check_for_sleep(landmarks) if sleep_chance: if direction == 'Down': sleep_frames_counter += 1 else: sleep_frames_counter = np.maximum(sleep_frames_counter - 1, 0) # 如果超过60次检测疲惫并且注意力变为为0,就设置为睡觉,这里要更改 if sleep_frames_counter > 60: attentiveness = 0 cv2.putText(frame, "sleep detected", (30, 300), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) # calculate smooth attentiveness score if smooth_attentiveness == -1: smooth_attentiveness = attentiveness average_attentivenss = attentiveness else: smooth_attentiveness = np.round((1.0 - beta) * attentiveness + beta * smooth_attentiveness, 1) #average_attentivenss = np.round( # (1. / (frame_count)) * attentiveness + (1. - 1. / (frame_count)) * average_attentivenss, 1) update_dt = datetime.now() #print("attentiveness is {}, smooth attentiveness is {} and average attentiveness is {}" \ # .format(attentiveness, smooth_attentiveness, average_attentivenss)) print(attentiveness) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() if gaze.pupils_located: v_ratio = np.round(gaze.vertical_ratio(), 2) h_ratio = np.round(gaze.horizontal_ratio(), 2) else: v_ratio, h_ratio = 0, 0 image_landmarks, lip_distance = mouth_open(frame, landmarks) #cv2.imshow('Live Landmarks', image_landmarks) cv2.putText(frame, "Attentiveness: " + str(attentiveness), (15, 95), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) # cv2.putText(frame, "Left pupil: " + str(left_pupil), (30, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) # cv2.putText(frame, "Right pupil: " + str(right_pupil), (30, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Gaze direction: " + str(gaze_text), (15, 130), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) cv2.putText(frame, "Head orientation: " + str(direction), (15, 165), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) # cv2.putText(frame, "v & h ratios: " + str(v_ratio) + " "+ str(h_ratio), (30, 200), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) # cv2.putText(frame, "Sleep counter: " + str(sleep_frames_counter), (15, 200), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 255), 2) cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1, (0, 0, 235), 2) prev_yawn_status = yawn_status if lip_distance > 20: yawn_status = True # 可以增加一个时间控制判断 cv2.putText(frame, "Subject is Yawning", (50, 450), cv2.FONT_HERSHEY_COMPLEX, 1, (0, 0, 255), 2) # output_text = " Yawn Count: " + str(yawns + 1) # cv2.putText(frame, output_text, (50,50), cv2.FONT_HERSHEY_COMPLEX, 1,(0,255,127),2) else: yawn_status = False if prev_yawn_status == True and yawn_status == False: yawns += 1 overlay = frame.copy() if smooth_attentiveness < 30: text_tpl = ' Low - {:04.1f}%' bkg_color = (0, 0, 255) # Red elif smooth_attentiveness < 60: text_tpl = 'Medium - {:04.1f}%' bkg_color = (0, 255, 255) # Yellow else: text_tpl = ' High - {:04.1f}%' bkg_color = (0, 255, 0) # Green # noise = frame_count % 10 / 10. # print(noise) # if smooth_attentiveness + noise >= 100: # noise -= 1 # text = text_tpl.format(smooth_attentiveness + noise) # text += ' - {:04.1f}%'.format(attentiveness) #cv2.rectangle(overlay, rect_org, rect_end, bkg_color, -1); # cv2.putText(overlay,text, # org=text_org, # fontFace=font, # fontScale=fontScale, # color=(0,0,0), # thickness=thickness, # lineType=cv2.LINE_AA) #cv2.addWeighted(overlay, alpha, frame, 1 - alpha, 0, frame) #out.write(frame) # cv2.imshow('Yawn Detection', frame ) #cv2.imshow("Demo", frame) #cv2.waitKey(0) #return attentiveness #print(str(direction)) #print(yawn_status) #print(sleep_chance) result = str(attentiveness)+"\n"+str(direction)+"\n"+str(yawn_status)+"\n"+str(sleep_chance) print("ddd"+result) return result
webcam2 = cv2.VideoCapture(4) webcam2.set(cv2.CAP_PROP_FRAME_WIDTH, 1280) webcam2.set(cv2.CAP_PROP_FRAME_HEIGHT, 720) detector = cv2.QRCodeDetector() cv2.namedWindow("d", cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty("d", cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) cv2.imshow('d', raw_show) cv2.waitKey(1) while True: # We get a new frame from the webcam _, frame1 = webcam1.read() _, frame2 = webcam2.read() raw1 = frame1.copy() raw2 = frame2.copy() gaze1.refresh(frame1) frame1 = gaze1.annotated_frame() left_pupil1 = gaze1.pupil_left_coords() right_pupil1 = gaze1.pupil_right_coords() # cv2.imshow('1', frame1) gaze2.refresh(frame2) frame2 = gaze2.annotated_frame() left_pupil2 = gaze2.pupil_left_coords() right_pupil2 = gaze2.pupil_right_coords() # cv2.imshow('2', frame2) if left_pupil1 is not None and left_pupil2 is not None: xtest = [[ gaze1.eye_left.center[0], gaze1.eye_left.center[1], gaze1.eye_right.center[0],
cali_flag = False left_screen_x_model = fitting.cali_fitting(cali_settings.left_cali, cali_settings.left_inner, cali_settings.mouse_pos,0) right_screen_x_model = fitting.cali_fitting(cali_settings.right_cali, cali_settings.right_inner, cali_settings.mouse_pos, 0) left_screen_y_model = fitting.cali_fitting(cali_settings.left_cali, cali_settings.left_inner, cali_settings.mouse_pos, 1) right_screen_y_model = fitting.cali_fitting(cali_settings.right_cali, cali_settings.right_inner, cali_settings.mouse_pos, 1) while True: _, frame = webcam.read() gaze.refresh(frame) frame, (left_x, left_y), (right_x, right_y) = gaze.annotated_frame() if left_x and left_y and right_x and right_y and gaze.eye_left.inner and gaze.eye_right.inner : left_mat = fitting.get_matrix(left_x,left_y,gaze.eye_left.inner[0],gaze.eye_left.inner[1]) right_mat = fitting.get_matrix(right_x,right_y,gaze.eye_right.inner[0],gaze.eye_right.inner[1]) left_screen_x = left_screen_x_model.predict([left_mat]) left_screen_y = left_screen_y_model.predict([left_mat]) right_screen_x = right_screen_x_model.predict([right_mat]) right_screen_y = right_screen_y_model.predict([right_mat]) draw_x = fitting.get_mid(left_screen_x, right_screen_x) draw_y = fitting.get_mid(left_screen_y, right_screen_y) pygame.draw.circle(screen, cali_settings.black, (draw_x, draw_y), 5, 0) for event in pygame.event.get(): if event.type == pygame.KEYDOWN: if event.key == pygame.K_ESCAPE: sys.exit() pygame.display.flip()
def get_data(): l = [] center_left = [] center_right = [] l_x = [] l_y = [] r_x = [] r_y = [] d = {'Time': [], 'Left eye': [], 'Right eye': []} gaze = GazeTracking() webcam = cv2.VideoCapture(0) while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) frame = gaze.annotated_frame() text = "" if gaze.is_blinking(): text = "Blinking" l.append(datetime.datetime.now()) elif gaze.is_left(): text = "Looking left" elif gaze.is_center(): text = "Looking right" cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() try: l_x.append(gaze.pupil_left_coords()[0]) l_y.append(gaze.pupil_left_coords()[1]) r_x.append(gaze.pupil_right_coords()[0]) r_y.append(gaze.pupil_right_coords()[1]) except: l_x.append(0) l_y.append(0) r_x.append(0) r_y.append(0) cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) #print((left_pupil,right_pupil)) try: d['Left eye'].append((left_pupil[0], left_pupil[1])) d['Right eye'].append((right_pupil[0], right_pupil[1])) d['Time'].append(datetime.datetime.now()) except: d['Left eye'].append(0) d['Right eye'].append(0) d['Time'].append(datetime.datetime.now()) cv2.imshow("Frame", frame) if cv2.waitKey(1) == 27: break eye_coordinates = pd.DataFrame(d) eye_coordinates.columns = ['Time', 'Left eye', 'Right eye'] eye_blinking = pd.Series(l) return eye_coordinates #,eye_blinking,center_right,center_left,l_x,l_y,r_x,r_y
def calculate_cog_load(): gaze = GazeTracking() webcam = cv2.VideoCapture(0) pupil_position = Pupil_position('center', 0) t = dt.datetime.now() start_time = dt.datetime.now() blink_count = 0 saccades = 0 pupil_dilation_x = [] pupil_dilation_y = [] fixations = [0] minute = 0 blink_rate = 0 saccades_rate = 0 pup_dil_x = 0 pup_dil_y = 0 fixation_avg = 0 cogload = 0 while True: # We get a new frame from the webcam _, frame = webcam.read() # We send this frame to GazeTracking to analyze it gaze.refresh(frame) response = requests.get( "https://api.fitbit.com/1/user/7QCRW3/activities/heart/date/today/today.json", headers=header).json() frame = gaze.annotated_frame() text = "" pupil_dilation_x.append(gaze.horizontal_ratio()) pupil_dilation_y.append(gaze.vertical_ratio()) horizontal_ratio = gaze.horizontal_ratio() vertical_ratio = gaze.vertical_ratio() if horizontal_ratio is not None: pupil_dilation_x.append(horizontal_ratio) if vertical_ratio is not None: pupil_dilation_y.append(vertical_ratio) if gaze.is_blinking(): text = "Blinking" blink_count = blink_count + 1 elif gaze.is_right(): delta = dt.datetime.now() - t position = Pupil_position('right', delta.seconds) if position.position != pupil_position.position: diff = delta.seconds - pupil_position.time fixations.append(diff) pupil_position = position saccades = saccades + 1 text = "Looking right" elif gaze.is_left(): delta = dt.datetime.now() - t position = Pupil_position('left', delta.seconds) if position.position != pupil_position.position: diff = delta.seconds - pupil_position.time fixations.append(diff) pupil_position = position saccades = saccades + 1 text = "Looking left" elif gaze.is_center(): delta = dt.datetime.now() - t position = Pupil_position('center', delta.seconds) if position.position != pupil_position.position: diff = delta.seconds - pupil_position.time fixations.append(diff) pupil_position = position saccades = saccades + 1 text = "Looking center" cv2.putText(frame, text, (90, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (147, 58, 31), 2) left_pupil = gaze.pupil_left_coords() right_pupil = gaze.pupil_right_coords() cv2.putText(frame, "Left pupil: " + str(left_pupil), (90, 130), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Right pupil: " + str(right_pupil), (90, 165), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Blink Rate: " + str(blink_rate), (90, 195), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Saccades Rate: " + str(saccades_rate), (90, 225), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Pupil dilation x: " + str(pup_dil_x), (90, 255), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Pupil dilation y: " + str(pup_dil_y), (90, 285), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Fixation: " + str(fixation_avg), (90, 315), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.putText(frame, "Cognitive Load: " + str(cogload), (90, 345), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) delta = dt.datetime.now() - t elapsed_time = dt.datetime.now() - start_time elapsed_time_second = elapsed_time.seconds cv2.putText(frame, "Elapsed Time: " + str(elapsed_time_second), (90, 375), cv2.FONT_HERSHEY_DUPLEX, 0.9, (147, 58, 31), 1) cv2.imshow("Demo", frame) if delta.seconds >= 10: minute = minute + 1 blink_rate = blink_count / 10 saccades_rate = saccades / 10 Not_none_values_x = filter(None.__ne__, pupil_dilation_x) Not_none_values_y = filter(None.__ne__, pupil_dilation_y) pupil_dilation_x = list(Not_none_values_x) pupil_dilation_y = list(Not_none_values_y) pup_dil_x = sum(pupil_dilation_x) / len(pupil_dilation_x) pup_dil_y = sum(pupil_dilation_y) / len(pupil_dilation_y) fixation_avg = sum(fixations) / len(fixations) blink_count = 0 saccades = 0 pupil_position = Pupil_position('center', 0) t = dt.datetime.now() pupil_dilation_x = [] pupil_dilation_y = [] fixations = [0] print( response['activities-heart-intraday']['dataset'][-1]['value']) cogload = blink_rate \ + math.sqrt(pup_dil_x * pup_dil_x + pup_dil_y * pup_dil_y) \ + saccades_rate \ - fixation_avg print(blink_rate) print(pup_dil_x) print(pup_dil_y) print(saccades_rate) print(fixation_avg) print(cogload) write_csv('data.csv', minute, blink_rate, pup_dil_x, pup_dil_y, fixation_avg, saccades_rate, cogload) if cv2.waitKey(33) == 27: break time.sleep(0.25)
class Program: def __init__(self): user32 = ctypes.windll.user32 self.img_width = user32.GetSystemMetrics(0) // 5 self.img_height = self.img_width // 2 self.act = "" self.acts = [] self.act_time = 0 self.act_started = False self.act_ended = True self.act_start_time = time.time() self.act_stop_time = time.time() self.current_gesture = "" self.gesture_text = "" self.end_of_gesture = False self.gesture_end_start_time = time.time() self.gesture_end_stop_time = time.time() self.gests = dict() self.detection_of_end = False self.list_of_acts = {"Closed left": 0, "Closed right": 0, "Neither": 0} self.counter = 0 self.time_of_output = 0 self.end_of_display_image = True self.current_act = "" self.read_settings() self.gaze = GazeTracking() self.webcam = cv2.VideoCapture(self.webcam_number) self.webcam.set(3, self.webcam_width) self.webcam.set(4, self.webcam_height) self.screen_width = int(self.webcam.get(3)) self.screen_height = int(self.webcam.get(4)) window_name = "Aurelium" cv2.namedWindow(window_name, cv2.WND_PROP_FULLSCREEN) cv2.setWindowProperty(window_name, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN) self.dim = (self.img_width, self.img_height) self.load_graphics() self.run() cv2.destroyAllWindows() def load_graphics(self): self.both_closed = cv2.imread("graphics/both_closed.png", -1) self.both_closed = cv2.resize(self.both_closed, self.dim) self.both_closed_long = cv2.imread("graphics/both_closed_long.png", -1) self.both_closed_long = cv2.resize(self.both_closed_long, self.dim) self.left_closed = cv2.imread("graphics/left_closed.png", -1) self.left_closed = cv2.resize(self.left_closed, self.dim) self.left_closed_long = cv2.imread("graphics/left_closed_long.png", -1) self.left_closed_long = cv2.resize(self.left_closed_long, self.dim) self.right_closed = cv2.imread("graphics/right_closed.png", -1) self.right_closed = cv2.resize(self.right_closed, self.dim) self.right_closed_long = cv2.imread("graphics/right_closed_long.png", -1) self.right_closed_long = cv2.resize(self.right_closed_long, self.dim) self.both_open = cv2.imread("graphics/both_open.png", -1) self.both_open = cv2.resize(self.both_open, self.dim) self.both_open_long = cv2.imread("graphics/both_open_long.png", -1) self.both_open_long = cv2.resize(self.both_open_long, self.dim) def read_settings(self): with open('configuration.txt', 'r') as file: self.webcam_number = int(file.readline()) self.webcam_width = int(file.readline()) self.webcam_height = int(file.readline()) self.gesture_end_duration = float(file.readline()) / 1000 self.short_act_duration = float(file.readline()) / 1000 self.long_act_duration = float(file.readline()) / 1000 self.result_display_duration = float(file.readline()) / 1000 for row in file: row = row.split('"') clear = [] for i in row: if i != '': clear.append(i) gesture = clear[2].strip("\n") self.gests[gesture] = clear[1] print(self.gests) def start_act(self): if self.act_started is False: self.act_started = True print('act started') self.act_start_time = time.time() def detect_act(self): self.act_stop_time = time.time() if self.act_stop_time - self.act_start_time > self.short_act_duration: count_l = self.acts.count('l') count_r = self.acts.count('r') count_b = self.acts.count('b') count_n = self.acts.count('_') max_act = max(count_l, count_r, count_b, count_n) if self.act_stop_time - self.act_start_time > self.long_act_duration: if max_act == count_l: max_act = 'L' elif max_act == count_r: max_act = 'R' elif max_act == count_b: max_act = 'B' elif max_act == count_n: max_act = '_' self.current_gesture += max_act else: if max_act == count_l: max_act = 'l' elif max_act == count_r: max_act = 'r' elif max_act == count_b: max_act = 'b' elif max_act == count_n: max_act = '_' self.current_gesture += max_act print("act: " + max_act) self.act = max_act self.act_ended = True def detect_end(self): if time.time() > self.gesture_end_stop_time: self.detection_of_end = False print("gesture: " + self.current_gesture) for i in self.gests: if i == self.current_gesture: self.gesture_text = self.gests[i] self.end_of_gesture = True print(self.gesture_text) self.current_gesture = "" def run(self): while True: '''get a new frame from webcam''' _, self.frame = self.webcam.read() self.frame = cv2.flip(self.frame, 1) '''GazeTracking analyzes the frame''' try: self.gaze.refresh(self.frame) except Exception: pass self.frame = self.gaze.annotated_frame() if self.gaze.image_too_dark(): cv2.putText(self.frame, "Nedostatok svetla", (10, self.screen_height - 10), cv2.FONT_HERSHEY_DUPLEX, 1.6, (50, 50, 200), 2) if self.gaze.face_recognition is False: cv2.putText(self.frame, "Neviem najst tvar", (10, self.screen_height - 70), cv2.FONT_HERSHEY_DUPLEX, 1.6, (50, 50, 200), 2) if self.gaze.face_too_small is True: cv2.putText(self.frame, "Priblizte sa", (10, self.screen_height - 130), cv2.FONT_HERSHEY_DUPLEX, 1.6, (50, 50, 200), 2) if self.current_gesture != "": cv2.putText( self.frame, self.current_gesture, (self.screen_width - self.img_width, self.img_height), cv2.FONT_HERSHEY_DUPLEX, 1.6, (75, 75, 75), 2) self.which_eye_is_closed() self.which_act() if (time.time() - self.time_of_output) < 5: cv2.putText(self.frame, self.gesture_text, (10, 50), cv2.FONT_HERSHEY_DUPLEX, 1.6, (75, 75, 75), 2) else: self.gesture_text = "" self.display_act() cv2.imshow("Aurelium", self.frame) if cv2.waitKey(1) & 0xFF == ord("q"): break def which_eye_is_closed(self): right_is_closed, right_value = self.gaze.is_closed_right() left_is_closed, left_value = self.gaze.is_closed_left() neither = True if left_is_closed: self.list_of_acts["Closed left"] += 1 if (right_value is not None and (self.gaze.left_eye_threshold + self.gaze.shift) >= right_value >= (self.gaze.left_eye_threshold - self.gaze.shift)): self.gaze.add_to_threshold("L", left_value) neither = False if right_is_closed: self.list_of_acts["Closed right"] += 1 if (left_value is not None and (self.gaze.right_eye_threshold + self.gaze.shift) >= left_value >= (self.gaze.right_eye_threshold - self.gaze.shift)): self.gaze.add_to_threshold("R", right_value) neither = False if neither is True: self.list_of_acts["Neither"] += 1 def which_act(self): self.counter += 1 if self.counter == 3: if self.list_of_acts["Closed right"] < self.list_of_acts[ "Neither"] > self.list_of_acts["Closed left"]: self.current_act = "" if self.act_started is True: self.detect_act() if self.act_ended is True: self.act_time = time.time() self.acts = [] self.act_started = False self.detection_of_end = True self.gesture_end_start_time = time.time() self.gesture_end_stop_time = self.gesture_end_start_time + self.gesture_end_duration elif self.list_of_acts["Closed right"] == self.list_of_acts[ "Closed left"]: self.current_act = "Closed both" self.detection_of_end = False self.start_act() self.acts.append("b") elif self.list_of_acts["Closed right"] < self.list_of_acts[ "Closed left"]: self.current_act = "Closed left" self.detection_of_end = False self.acts.append("l") elif self.list_of_acts["Closed right"] > self.list_of_acts[ "Closed left"]: self.current_act = "Closed right" self.detection_of_end = False self.acts.append("r") if self.detection_of_end is True: self.detect_end() if self.end_of_gesture is True: self.end_of_gesture = False self.time_of_output = time.time() self.list_of_acts = { "Closed left": 0, "Closed right": 0, "Neither": 0 } self.counter = 0 def display_act(self): no_image = True if self.end_of_display_image is True: no_image = False if self.current_act == "Closed left": self.act = "" self.act_ended = False added_image = self.display_image(self.left_closed) elif self.act_ended and self.act == "L": self.act = "" self.act_ended = False added_image = self.display_image(self.left_closed_long) elif self.current_act == "Closed right": self.act = "" self.act_ended = False added_image = self.display_image(self.right_closed) elif self.act_ended and self.act == "R": self.act = "" self.act_ended = False added_image = self.display_image(self.right_closed_long) elif self.current_act == "Closed both": self.act = "" self.act_ended = False added_image = self.display_image(self.both_closed) elif self.act_ended and self.act == "B": self.act = "" self.act_ended = False added_image = self.display_image(self.both_closed_long) elif self.gaze.pupils_located: added_image = self.display_image(self.both_open) else: no_image = True if (time.time() - self.act_time) < 1: self.end_of_display_image = False else: self.end_of_display_image = True if no_image is False: self.frame[0:self.img_height, self.screen_width - self.img_width:self.screen_width] = added_image def display_image(self, image): b, g, r, a = cv2.split(image) overlay_color = cv2.merge((b, g, r)) mask = cv2.medianBlur(a, 5) h, w, _ = overlay_color.shape roi = self.frame[0:self.img_height, self.screen_width - self.img_width:self.screen_width] img1_bg = cv2.bitwise_and(roi.copy(), roi.copy(), mask=cv2.bitwise_not(mask)) img2_fg = cv2.bitwise_and(overlay_color, overlay_color, mask=mask) added_image = cv2.add(img1_bg, img2_fg) return added_image
config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 30) pipeline.start(config) save_time = datetime.time(datetime.now()) data_to_file = [] while True: frames = pipeline.wait_for_frames() color_frame = frames.get_color_frame() bgr_image = np.asanyarray(color_frame.get_data()) gaze.refresh(bgr_image) bgr_image = gaze.annotated_frame() text = "" if gaze.is_right(): text = "Looking mono chino de pelo morado" elif gaze.is_left(): text = "Looking mona china" elif gaze.is_up(): text = "Looking mono chino rubio intenso" elif gaze.is_down(): text = "Looking logo" elif gaze.is_center(): text = "Looking mono chino de pelo verde" cv2.putText(bgr_image, text, (30, 30), cv2.FONT_HERSHEY_DUPLEX, 1, (147, 58, 31), 2)
def main(args): filename = args["input_file"] #<<<<<<< HEAD faceCascade = cv2.CascadeClassifier( 'C:/Capstone/models/haarcascade_frontalface_default.xml') model = load_model( 'C:\Capstone\poscas\POSCO_AIProject_OnlineTestCheatingDetectionAiSystem-master\models/facenet_keras.h5' ) #C:\Capstone\poscas\POSCO_AIProject_OnlineTestCheatingDetectionAiSystem-master\models #======= faceCascade = cv2.CascadeClassifier( 'models/haarcascade_frontalface_default.xml') model = load_model('models/facenet_keras.h5') #>>>>>>> 757e3559f2acad057224670a45fca1fc2d17309e if filename is None: isVideo = False #url='http://192.168.0.06:8091/?action=stream' #webcam = cv2.VideoCapture(url) webcam = cv2.VideoCapture(0) # 캠으로 이미지 받아오는 코드 webcam.set(3, args['wh'][0]) webcam.set(4, args['wh'][1]) else: isVideo = True webcam = cv2.VideoCapture(filename) fps = webcam.get(cv2.webcam_PROP_FPS) width = int(webcam.get(cv2.webcam_PROP_FRAME_WIDTH)) height = int(webcam.get(cv2.webcam_PROP_FRAME_HEIGHT)) fourcc = cv2.VideoWriter_fourcc(*'XVID') name, ext = osp.splitext(filename) out = cv2.VideoWriter(args["output_file"], fourcc, fps, (width, height)) ############################################################################################## # 이름 제대로 들어가는지 테스트 -> 이름 제대로 들어감 # print('main 문 ') # print(name1) # print(time1) # ############################## UserName = name1 f.write(UserName + " ") checktime = 1 start_check = time.time() + (10 * checktime) checktime_end = time.time() + (60 * checktime) #1분 동안 체크 (60) while (webcam.isOpened() ): # Infinity Loop for Detect Cheating for Online test ret, frame = webcam.read() # Read wabcam gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = faceCascade.detectMultiScale( gray, scaleFactor=1.1, minNeighbors=3, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) # face structure for (x, y, w, h) in faces: # take the face pixels from the frame cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2) # turn the face pixels back into an image crop_frame = frame[y:y + h, x:x + w] # resize the image to meet the size requirment of facenet new_crop = Image.fromarray(crop_frame) # turn the image back into a tensor new_crop = new_crop.resize((160, 160)) # get the face embedding using the face net model crop_frame = np.asarray(new_crop) # it is a 1d array need to reshape it as a 2d tensor for svm face_embed = get_embedding(model, crop_frame) # predict using our SVM model face_embed = face_embed.reshape(-1, face_embed.shape[0]) pred = svm.predict(face_embed) # get the prediction probabiltiy # pred_prob has probabilities of each class pred_prob = svm.predict_proba(face_embed) # get name class_index = pred[0] class_probability = pred_prob[0, class_index] * 100 predict_names = out_encoder.inverse_transform(pred) text = '%s (%.3f%%)' % (predict_names[0], class_probability) cv2.putText(frame, text, (x, y), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2) cv2.imshow('capstone', frame) #-> 정상 출력 되는 코드 #label.after(20,frame) #-> window에 비디오 창이 열리게 끔 하고 싶었으나 안됨 cv2.waitKey(1) if (time.time() > start_check and predict_names[0] == UserName and class_probability > 80): #print("얼굴이 일치합니다. 시험을 시작하겠습니다.") messagebox.showinfo("얼굴 확인", "얼굴이 일치합니다. 시험을 시작합니다") break if time.time() > checktime_end: #print("얼굴이 일치하지 않아 시험에 응시하지 못합니다.") messagebox.showerror("얼굴 확인", "얼굴이 일치하지 않아 시험에 응시하지 못합니다.") f.write(" 얼굴 불일치" + '\n') TxtOpen() # 작동 함 quit() window.destroy() # UI 화면 닫기 break ################################################################################## ######################################################################### # Variable Setting hpd = headpose.HeadposeDetection( args["landmark_type"], args["landmark_predictor"]) # import headpose gaze = GazeTracking() # import gazetracking yellocard = 0 redcard = 0 tempval = 0 # Input time for limit test time timee = int(time1) #timee = int(input("시험 시간을 입력하세요(Minute): ")) max_time_end = time.time() + (60 * timee) check_angle = time.time() + (10 * checktime) while (webcam.isOpened() ): # Infinity Loop for Detect Cheating for Online test ret, frame = webcam.read() # Read wabcam gaze.refresh(frame) frame = gaze.annotated_frame() # Mark pupil for frame gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) faces = faceCascade.detectMultiScale( gray, scaleFactor=1.1, minNeighbors=3, minSize=(30, 30), flags=cv2.CASCADE_SCALE_IMAGE) # face structure # Get point from pupil if gaze.is_blinking(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_right(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_left(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) elif gaze.is_center(): yellocard = yellocard - 1 yellocard = notnegative(yellocard) else: yellocard = yellocard + 2 # Get redcard optiom if yellocard > 50: yellocard = 0 tempval = tempval + 1 redcard = redcard + 1 # if get 1redcard, then give Aural and Text Warning(Loop) if tempval == 1: text1 = "WARNING" cv2.putText(frame, text1, (10, 60), cv2.FONT_HERSHEY_DUPLEX, 1.6, (0, 0, 255), 2) my_thread = threading.Thread(target=Sound) my_thread.start() tempval = 0 # if you are not GPU environment, Do not run this code by # -------------- # if get 2redcard, then give Picture Warning(Once) if redcard == 2: warn_img = cv2.imread("Warning/warning.png", cv2.IMREAD_COLOR) cv2.imshow('Warning', warn_img) cv2.waitKey(1) redcard = 2.1 # -----------------------------s------------------------------------------ # Get log consistently print("<< *의심수준:", yellocard, " || ", "*경고횟수:", redcard, " >>") #cv2.destroyWindow('Warning') # Detect head position if isVideo: frame, angles = hpd.process_image(frame) if frame is None: break else: out.write(frame) else: frame, angles = hpd.process_image(frame) if angles is None: #print("경고! 응시자가 사라졌습니다") #messagebox.showwarning("경고","경고! 응시자가 사라졌습니다") if time.time() > check_angle: redcard = timee / 3 + redcard #print("지속적으로 카메라 앵글 밖으로 나갔으므로, 시험을 강제종료합니다.") #messagebox.showinfo("확인","얼굴이 일치합니다. 시험을 시작합니다") messagebox.showerror( "경고", "지속적으로 카메라 앵글 밖으로 나갔으므로, 시험을 강제종료합니다.") PrintResult(yellocard, redcard) Fail(timee, redcard) TxtOpen() window.destroy() quit() else: pass else: # angles = [x,y,z] , get point from headposition if angles[0] > 15 or angles[0] < -15 or angles[ 1] > 15 or angles[1] < -15 or angles[2] > 15 or angles[ 2] < -15: yellocard = yellocard + 2 else: yellocard = yellocard - 1 yellocard = notnegative(yellocard) yellocard = yellocard + hpd.yello(frame) if yellocard < 0: yellocard = notnegative(yellocard) # Display the resulting frame cv2.imshow('capstone', frame) if cv2.waitKey(1) & 0xFF == ord('q'): #print("관리자에 의해 시험이 강제 종료 되었습니다") f.write("경고 횟수 : %s" % (redcard)) f.write(" -> 강제 종료 \n") messagebox.showerror("경고", "관리자에 의해 시험이 강제 종료 되었습니다") PrintResult(yellocard, redcard) Fail(timee, redcard) # messagebox.showinfo("결과출력","결과가 출력됩니다.") # #messagebox("결과출력","결과가 출력됩니다.") # data = open('C:/Capstone/result_data.txt', 'r') # contents = data.read() # messagebox.showinfo("결과 출력",contents) TxtOpen() window.destroy() # f.close() break elif time.time() > max_time_end: #print(timee, "분의 시험이 종료되었습니다.") f.write(" -> 정상 종료 \n") PrintResult(yellocard, redcard) Fail(timee, redcard) messagebox.showinfo("시험 종료", "시험이 종료되었습니다.") #messagebox.showinfo("결과출력","결과가 출력됩니다.") # data = open('C:/Capstone/result_data.txt', 'r') # contents = data.read() # messagebox.showinfo("결과 출력",contents) # f.close() TxtOpen() window.destroy() break # When everything done, release the webcam webcam.release() cv2.destroyAllWindows() #TxtOpen() -> 새로운 UI 창이 뜸.. 여기에 넣으면 안될 듯 quit() window.destroy() if isVideo: out.release() cv2.destroyAllWindows()
def TrackImages(): check_haarcascadefile() assure_path_exists("Attendance/") assure_path_exists("StudentDetails/") for k in tv.get_children(): tv.delete(k) msg = '' i = 0 j = 0 recognizer = cv2.face.LBPHFaceRecognizer_create( ) # cv2.createLBPHFaceRecognizer() exists3 = os.path.isfile("TrainingImageLabel\Trainner.yml") if exists3: recognizer.read("TrainingImageLabel\Trainner.yml") else: mess._show(title='Data Missing', message='Please click on Save Profile to reset data!!') return harcascadePath = "haarcascade_frontalface_default.xml" faceCascade = cv2.CascadeClassifier(harcascadePath) cam = cv2.VideoCapture(0) font = cv2.FONT_HERSHEY_SIMPLEX col_names = ['Id', '', 'Name', '', 'Date', '', 'Time'] exists1 = os.path.isfile("StudentDetails\StudentDetails.csv") if exists1: df = pd.read_csv("StudentDetails\StudentDetails.csv") else: mess._show(title='Details Missing', message='Students details are missing, please check!') cam.release() cv2.destroyAllWindows() window.destroy() gaze = GazeTracking() l = 0 r = 0 l_done = 0 r_done = 0 ok = 0 all_ok = 0 LARGE_FONT = ("Verdana", 12) NORM_FONT = ("Helvetica", 10) SMALL_FONT = ("Helvetica", 8) while True: ret, im = cam.read() _, frame = cam.read() #--------- gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY) faces = faceCascade.detectMultiScale(gray, 1.2, 5) gaze.refresh(frame) #------- im = gaze.annotated_frame() #----- text = "" #------ msgg = "" #----- if gaze.is_blinking(): text = "Blinking" elif gaze.is_right(): text = "Looking right" r_done = 1 elif gaze.is_left(): text = "Looking left" l_done = 1 elif gaze.is_center(): text = "Looking center" if (l_done == 0 and r_done == 0): msgg = "Please Trun Your Eye In Left Side " elif (l_done == 1 and r_done == 0): msgg = "Please Trun Your Eye In Right Side" elif (l_done == 0 and r_done == 1): msgg = "Please Trun Your Eye In Left Side" elif (l_done == 1 and r_done == 1): msgg = "Eye Movement Done" ok = 1 for (x, y, w, h) in faces: cv2.rectangle(im, (x, y), (x + w, y + h), (225, 0, 0), 2) serial, conf = recognizer.predict(gray[y:y + h, x:x + w]) if (conf < 50): ts = time.time() date = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y') timeStamp = datetime.datetime.fromtimestamp(ts).strftime( '%H:%M:%S') aa = df.loc[df['SERIAL NO.'] == serial]['NAME'].values ID = df.loc[df['SERIAL NO.'] == serial]['ID'].values ID = str(ID) ID = ID[1:-1] bb = str(aa) bb = bb[2:-2] attendance = [ str(ID), '', bb, '', str(date), '', str(timeStamp) ] if (ok == 1): cv2.putText(im, "{} Press 'Q' for save and exit !".format(bb), (50, 30), font, 0.7, (0, 0, 255), 2) all_ok = 1 else: cv2.putText( im, "{} Follow The Eye Movement Instructions !".format(bb), (30, 30), font, 0.7, (0, 0, 255), 2) else: Id = 'Unknown' bb = str(Id) cv2.putText(im, "You are unregistered !", (30, 30), font, 0.7, (0, 0, 255), 2) cv2.putText(im, str(bb), (x, y + h), font, 1, (255, 255, 255), 2) cv2.putText(im, "Eye position {}".format(text), (2, 70), font, 0.5, (95, 106, 106), 2) #------- cv2.putText(im, "{}".format(msgg), (150, 450), font, 0.6, (0, 0, 255), 2) #------- cv2.imshow('Taking Attendance', im) if (cv2.waitKey(1) == ord('q')): break if (all_ok == 1): ts = time.time() date = datetime.datetime.fromtimestamp(ts).strftime('%d-%m-%Y') exists = os.path.isfile("Attendance\Attendance_" + date + ".csv") if exists: with open("Attendance\Attendance_" + date + ".csv", 'a+') as csvFile1: writer = csv.writer(csvFile1) writer.writerow(attendance) csvFile1.close() else: with open("Attendance\Attendance_" + date + ".csv", 'a+') as csvFile1: writer = csv.writer(csvFile1) writer.writerow(col_names) writer.writerow(attendance) csvFile1.close() with open("Attendance\Attendance_" + date + ".csv", 'r') as csvFile1: reader1 = csv.reader(csvFile1) for lines in reader1: i = i + 1 if (i > 1): if (i % 2 != 0): iidd = str(lines[0]) + ' ' tv.insert('', 0, text=iidd, values=(str(lines[2]), str(lines[4]), str(lines[6]))) csvFile1.close() cam.release() cv2.destroyAllWindows() else: popup = tk.Tk() popup.wm_title("!") label = ttk.Label(popup, text="Something Wrong !", font=NORM_FONT) label.pack(side="top", fill="x", pady=10) B1 = ttk.Button(popup, text="Okay", command=popup.destroy) B1.pack() #popup.mainloop() cam.release() cv2.destroyAllWindows()