class ImageProcessor:
def __init__(self):
gopigo.set_speed(50)
gopigo.stop
self._face_detector = FaceDetector('/home/pi/opencv/data/haarcascades/haarcascade_frontalface_default.xml')
self.sizes_calculated = False
self.image_height = None
self.image_width = None
self.segment_detector = None
def faces(self, image):
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
return self._face_detector.detect(gray, scaleFactor = 1.1, minNeighbors = 5, minSize = (30, 30))
def calculate_sizes(self, image):
(self.height, self.width) = image.shape[:2]
self.segment_detector = SegmentDetector(self.width)
self.sizes_calculated = True
def process(self, stream):
start_time = time.time()
image = cv2.imdecode(np.fromstring(stream.getvalue(), dtype=np.uint8), 1)
if(self.sizes_calculated == False):
self.calculate_sizes(image)
faceRects = self.faces(image)
for (x, y, w, h) in faceRects:
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
if (len(faceRects) > 0):
(x, y, w, h) = faceRects[0]
self.move(x + (w/2))
else:
print("no faces found")
gopigo.stop()
print(time.time() - start_time)
#pdb.set_trace()
cv2.line(image, (self.segment_detector.left_cutoff, 0), (self.segment_detector.left_cutoff, self.height), (255, 0, 0), 1)
cv2.line(image, (self.segment_detector.right_cutoff, 0), (self.segment_detector.right_cutoff, self.height), (255, 0, 0), 1)
return cv2.imencode('.jpg', image)[1].tostring()
def move(self, horiz_x):
segment = self.segment_detector.segment(horiz_x)
print(segment)
if(segment == 'left'):
gopigo.set_speed(10)
gopigo.right_rot()
elif(segment == 'right'):
gopigo.set_speed(10)
gopigo.left_rot()
elif(segment == 'centre'):
gopigo.set_speed(50)
gopigo.fwd()
def main():
# original frame size is (720, 960)
W = 320
H = 240
image_cx = W // 2
image_cy = H // 2
num_skip_frames = 300
drone = tellopy.Tello()
controller = Controller(drone, image_cx, image_cy)
face_detector = FaceDetector()
renderer = Renderer()
display = PygameDisplay(W, H)
try:
drone.connect()
drone.wait_for_connection(60.0)
drone.subscribe(drone.EVENT_FLIGHT_DATA, flight_data_handler)
container = av.open(drone.get_video_stream())
drone.takeoff()
while True:
for frame in container.decode(video=0):
if num_skip_frames > 0:
num_skip_frames = num_skip_frames - 1
continue
start_time = time.time()
image = np.array(frame.to_image())
image = cv2.resize(image, (W, H))
face = face_detector.detect(image)
controller.control(face)
renderer.render(image, drone_state, face)
display.paint(image)
time_base = max(1.0 / 60, frame.time_base)
processing_time = time.time() - start_time
num_skip_frames = int(processing_time / time_base)
#print('Video steam %d FPS, frame time base=%f' % (1/frame.time_base, frame.time_base))
#print('Processing FPS=%d, time=%f ms, skip frames=%d' % (1/processing_time, 1000 * processing_time, num_skip_frames))
except Exception as ex:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
print(ex)
finally:
drone.land()
drone.quit()
display.dispose()
def main():
W = 432
H = 240
image_cx = W // 2
image_cy = H // 2
face_detector = FaceDetector()
renderer = Renderer()
#display = Cv2Display2D()
display = PygameDisplay(W, H)
try:
container = av.open('video/ball_tracking_example.mp4')
num_skip_frames = 0
while True:
for frame in container.decode(video=0):
if num_skip_frames > 0:
num_skip_frames = num_skip_frames - 1
continue
start_time = time.monotonic()
image = np.array(frame.to_image())
image = cv2.resize(image, (W, H))
face = face_detector.detect(image)
renderer.render(image, drone_state, face)
if face is not None:
offset_x = face.cx - image_cx
offset_y = face.cy - image_cy
print(offset_x, offset_y)
display.paint(image)
time_base = max(1 / 60, frame.time_base)
processing_time = time.monotonic() - start_time
num_skip_frames = int(processing_time / time_base)
print('Video steam %d FPS, frame time base=%f' %
(1 / frame.time_base, frame.time_base))
print('Processing FPS=%d, time=%f ms, skip frames=%d' %
(1 / processing_time, 1000 * processing_time,
num_skip_frames))
except Exception as ex:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback)
print(ex)
finally:
display.dispose()
# Load the cascade
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--face", required = True, help = "path to where the face cascade resides")
args = vars(ap.parse_args())
fd = FaceDetector(args["face"])
camera = cv2.VideoCapture(0)
while True:
(grabbed, frame) = camera.read()
if not grabbed:
break
frame = imutils.resize(frame, width = 300)
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
faceRects = fd.detect(gray, scaleFactor = 1.1, minNeighbors = 5, minSize = (30,30))
frameClone = frame.copy()
for (fX, fY, fW, fH) in faceRects:
cv2.rectangle(frameClone, (fX, fY), (fX + fW, fY + fH), (0, 255, 0), 2)
cv2.imshow("Face", frameClone)
if cv2.waitKey(1) & 0xFF == ord("q"):
break
camera.release()
cv2.destroyAllWindows()
from face_detector import FaceDetector
import argparse
import cv2
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--face", required = True, help = "Face cascade pathname")
ap.add_argument("-i", "--image", required = True, help = "Image pathname")
args = vars(ap.parse_args())
image = cv2.imread(args["image"])
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
fd = FaceDetector(args["face"])
faceRects = fd.detect(gray, scaleFactor = 1.2)
print "I found %d face(s)" % len(faceRects)
for (x, y, w, h) in faceRects:
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
cv2.imshow("Faces", image)
cv2.waitKey(0)
from face_detector import FaceDetector
import argparse
import cv2
ap = argparse.ArgumentParser()
ap.add_argument("-f", "--face", required=True, help="Face cascade pathname")
ap.add_argument("-i", "--image", required=True, help="Image pathname")
args = vars(ap.parse_args())
image = cv2.imread(args["image"])
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
fd = FaceDetector(args["face"])
faceRects = fd.detect(gray, scaleFactor=1.2)
print "I found %d face(s)" % len(faceRects)
for (x, y, w, h) in faceRects:
cv2.rectangle(image, (x, y), (x + w, y + h), (0, 255, 0), 2)
cv2.imshow("Faces", image)
cv2.waitKey(0)
# grab frame
frame = frame_grabber.read()
frame_show = frame.copy()
frame_roi = frame[up_offsetY:down_offsetY, left_offsetX:right_offsetX]
frame_gray = cv2.cvtColor(frame_roi, cv2.COLOR_BGR2GRAY)
frame_gray = cv2.GaussianBlur(frame_gray, (21, 21), 0)
motion_locs = motion_detector.update(frame_gray)
# form a nice average before motion detection
if num_frame_read < 15:
num_frame_read += 1
continue
if len(motion_locs) > 0:
# @ZC may consider to process in every other frame to accelerate
face_locs = face_detector.detect(frame_roi, motion_locs)
if len(face_locs) > 0:
# Save image with faces detected
timestamp = datetime.datetime.now()
ts = timestamp.strftime("%Y-%m-%d_%H:%M:%S_%f")
print("[INFO] " + str(len(face_locs)) + " face found." + ts)
image_save_path = "images/" + ts + ".jpg"
cv2.imwrite(image_save_path, frame_roi)
for top, right, bottom, left in face_locs:
# Scale back up face locations
top *= 1
right *= 1
bottom *= 1
left *= 1
# draw the bounding box for faces
class Observer:
def __init__(self, base_dir):
self.publisher = Publisher()
self.publisher.declare_queue('hello')
self.base_dir = base_dir
self.csv_filename = 'screenshot_list.csv'
self.face_detector = FaceDetector()
# self.cap0 = cv2.VideoCapture()
# self.cap0.open(0)
self.cap = cv2.VideoCapture(1) # default is 0
self.eyegaze_process = None
def __del__(self):
# self.cap0.release()
# if self.eyegaze_process is not None:
# self.eyegaze_process.shutdown()
self.cap.release()
cv2.destroyAllWindows()
def one_screenshot(self):
'''
:param base_dir:
:return:
'''
timestamp = str(time.time())
# face_img_path = self.base_dir + '/' + timestamp + ".jpeg"
face_img_name = timestamp + ".jpeg"
face_img_path = os.path.join(self.base_dir, face_img_name)
ret0, mycapture = self.cap.read()
# mycapture = pyautogui.screenshot()
# mycapture = cv2.cvtColor(np.array(mycapture), cv2.COLOR_RGB2BGR)
if mycapture is None:
print("capture null")
return
# cv2.imwrite(path, img)
print('saved to ' + face_img_path + " at " + timestamp)
# detect face
face_flag = self.face_detector.detect(mycapture, face_img_path)
if face_flag:
print("write face_img to csv!")
# write to csv
self._write_to_csv(self.csv_filename, face_img_name,
timestamp) # face_img_path
# publish message
self.publisher.publish(face_img_path)
# return [path]
def _write_to_csv(self, csv_filename, path, timestamp):
if not os.path.exists(csv_filename):
# file = open(csv_filename, 'w')
# file.close()
df = pd.DataFrame(columns=['img_name', 'time'])
df.to_csv(csv_filename, index=False)
with open(csv_filename, 'a') as fd:
fd.write(path + ',' + timestamp + '\n')
def screenshots(self, time_control: TimeControl):
for i in range(time_control.batch_num):
batch_start_time = time.time()
for j in range(time_control.unit_num):
# self.one_screenshot()
unit_start_time = time.time()
# threading.Thread(target=self.one_screenshot(), args=(), daemon=True)
p1 = multiprocessing.Process(target=self.one_screenshot(),
args=(),
daemon=True)
p1.start()
unit_end_time = time.time()
time_to_sleep = max(
0, time_control.unit_interval - unit_end_time +
unit_start_time)
time.sleep(time_to_sleep)
batch_end_time = time.time()
time_to_sleep = max(
0, time_control.batch_interval - batch_end_time +
batch_start_time)
time.sleep(time_to_sleep)
@staticmethod
def run_eyegaze():
HOST = '127.0.0.1'
PORT = 4242
base_dir = os.path.join(os.getcwd()) # , "data"
print(base_dir)
eyegaze = Eyegaze(HOST, PORT, base_dir)
eyegaze.run_gazepoint()
def run(self, time_control: TimeControl, use_eyegaze=False):
# open a new process to run the eyegaze program
if use_eyegaze is True:
self.eyegaze_process = multiprocessing.Process(
target=self.run_eyegaze, args=(), daemon=True)
self.eyegaze_process.start()
print(self.eyegaze_process.pid)
self.screenshots(time_control)
def tagMask(outputPath=None):
print("[INFO] Loading model")
fd_model = FaceDetector(
prototype='./checkpoints/deploy.prototxt.txt',
model='./checkpoints/res10_300x300_ssd_iter_140000.caffemodel',
)
model = MaskDetectorTrainer()
model.load_state_dict(
torch.load('./checkpoints/model_w.ckpt',
map_location=torch.device('cpu'))['state_dict'],
strict=False)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = model.to(device)
model.eval()
print("[INFO] Starting stream")
vs = VideoStream(src=0).start()
time.sleep(2)
transformations = Compose([
ToPILImage(),
Resize((100, 100)),
ToTensor(),
])
font = cv2.FONT_HERSHEY_SIMPLEX
labels = ['No Mask', 'Mask']
labelColor = [(255, 0, 9), (10, 255, 0)]
boxColor = [(255, 0, 9), (10, 255, 0)]
while True:
frame = vs.read()
frame = imutils.resize(frame, width=400)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
faces = fd_model.detect(frame)
for face in faces:
startX, startY, w, h = face
startX, startY = max(startX, 0), max(startY, 0)
faceImg = frame[startY:startY + h, startX:startX + w]
output = model(transformations(faceImg).unsqueeze(0).to(device))
_, predicted = torch.max(output.data, 1)
cv2.rectangle(frame, (startX, startY), (startX + w, startY + h),
boxColor[predicted], 2)
textSize = cv2.getTextSize(labels[predicted], font, 1, 2)[0]
textX = startX + w // 2 - textSize[0] // 2
cv2.putText(frame, labels[predicted], (textX, startY - 20), font,
1, labelColor[predicted], 2)
frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)
cv2.imshow('main', frame)
key = cv2.waitKey(1) & 0xFF
if key == ord("q"):
break
cv2.destroyAllWindows()
vs.stop()
model_path = args["model"] # Model file
# TODO: Check if file exists
frame_class = ModelMode(model_path)
# Face detection is needed in all modes
face_detector = FaceDetector("haarcascade_frontalface_default.xml")
cap = cv2.VideoCapture(0)
while True: # Main loop
ret, frame = cap.read()
frame = cv2.resize(frame, (0, 0), fx=0.5, fy=0.5)
ROIs = [] # Array for segmeted faces
ROIs_coordinates = [] # Array for x,y coordinates of segmented faces to draw bounding boxes
faces = face_detector.detect(frame)
for (x, y, w, h) in faces:
ROIs.append(frame[y : y + h, x : x + w])
ROIs_coordinates.append((x, y))
# cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 0, 255))
frame = frame_class.consume(frame, ROIs, ROIs_coordinates, faces)
if frame_class.EXIT_FLAG:
break
cv2.imshow("frame", frame)
cap.release()
cv2.destroyAllWindows()
class FaceRecognizer:
def __init__(self):
self.face_detector = FaceDetector()
# https://github.com/davisking/dlib-models
self.sp = dlib.shape_predictor(
'data/shape_predictor_5_face_landmarks.dat')
self.facerec = dlib.face_recognition_model_v1(
'data/dlib_face_recognition_resnet_model_v1.dat')
def _get_face_feat(self, img_x, box):
'''
获取脸部的128维特征向量
:param img_x:
:param box:
:return:
'''
rec = dlib.rectangle(*box)
shape = self.sp(img_x, rec)
feat = self.facerec.compute_face_descriptor(img_x, shape)
return feat
def _face_distance(self, face_encodings, face_to_compare):
"""
Given a list of face encodings, compare them to a known face encoding and get a euclidean distance
for each comparison face. The distance tells you how similar the faces are.
:param faces: List of face encodings to compare
:param face_to_compare: A face encoding to compare against
:return: A numpy ndarray with the distance for each face in the same order as the 'faces' array
"""
if len(face_encodings) == 0:
return np.empty((0))
face_encodings = np.asarray(face_encodings)
return np.linalg.norm(face_encodings - face_to_compare, axis=1)
def _get_img_face_encoding(self, fpath):
'''
获取路径图片的脸部特征编码
:param fpath: 图片路径
:return: 128位的向量
'''
img_x = cv2.imread(fpath)
img_x = cv2.cvtColor(img_x, cv2.COLOR_BGR2RGB)
item = self.face_detector.detect(img_x)
assert item is not None, 'can not find the face box,please check %s' % fpath
box, _ = item
return self._get_face_feat(img_x, box)
def create_known_faces(self, root):
'''
# 构建目标库,在程序启动时，或有新员工添加时执行
:param root:目标图片存放路径
:return: [[id..] [feat..]]
'''
self._known_faces = []
self._know_name = []
for i, fname in enumerate(os.listdir(root)):
fpath = os.path.join(root, fname)
self._known_faces.append(self._get_img_face_encoding(fpath))
self._know_name.append(fname.split('.')[0])
def recognize(self, image, score_threshold=0.6):
'''
识别人脸
:param image: 图片路径
:param score_threshold: 人脸识别得分阈值
:return: (know_name[ix], face_locations, cls) (人员id,box坐标(left,top,right,bottom),是否戴了口罩(0,1))
'''
if isinstance(image, str):
image = cv2.imread(image)
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
item = self.face_detector.detect(image)
if item:
box, cls = item
face_feat = self._get_face_feat(image, box)
scores = 1 - self._face_distance(self._known_faces, face_feat)
ix = np.argmax(scores).item()
if scores[ix] > score_threshold:
# 1 for mask
return self._know_name[ix], box, 1 - int(cls), scores[ix]
return None
def test_100x(self):
'''
测试使用，扩大目标库*128
:return:
'''
print('or_length', len(self._know_name))
for i in range(7):
self._know_name.extend(self._know_name)
self._known_faces.extend(self._known_faces)
print('10x_length', len(self._know_name))
class ImageProcessingThread(QThread):
signal_send_image = pyqtSignal(QImage)
def __init__(self, parent):
super().__init__(parent)
self._is_join_requested = False
self._is_joined = False
self._frame_source = None
self._mask = None
self._mask_next = None
self._face_detector = FaceDetector()
self._shape_predictor = ShapePredictor(
model_path=SHAPE_PREDICTOR_MODEL_PATH)
self.mask = FaceMaskPassthrough()
self.frame_source = CameraFrameSource(0)
def run(self):
while not self._is_join_requested:
self.update_mask()
frame_color = self._frame_source.get_frame()
frame_gray = cv2.cvtColor(frame_color, cv2.COLOR_BGR2GRAY)
faces = self._face_detector.detect(frame_gray)
for face in faces:
face_points = self._shape_predictor.predict_remapped(
frame_gray, face)
self._mask.apply(frame_color, face_points)
height, width, channel_count = frame_color.shape
bytes_per_line = channel_count * width
image_qt = QImage(
frame_color.data,
width,
height,
bytes_per_line,
QImage.Format_BGR888,
)
self.signal_send_image.emit(image_qt)
self._is_joined = True
def join(self):
self._is_join_requested = True
while not self._is_joined:
continue
@property
def frame_source(self) -> Type[FrameSource]:
return self._frame_source
@frame_source.setter
def frame_source(self, new_frame_source: Type[FrameSource]):
""" Sets the current frame source to the specified `new_frame_source`.
If the current frame source is an instance of `ThreadedFrameSource`, then
it is stopped before replacement.
If `new_frame_source` if an instance of `ThreadedFrameSource`, then it is
started after writing it to the `_frame_source` field.
Parameters
----------
new_frame_source : FrameSource
New frame source.
"""
if isinstance(self._frame_source, ThreadedFrameSource):
# Stop current threaded frame source
self._frame_source.join()
self._frame_source = new_frame_source
if isinstance(self._frame_source, ThreadedFrameSource):
# Start new threaded frame source
self._frame_source.start()
@property
def mask(self) -> FaceMask:
return self._mask
@mask.setter
def mask(self, mask_new: Type[FaceMask]):
self._mask = mask_new
def update_mask(self):
""" Updates current mask with the next mask, if available. """
if self._mask_next:
self.mask = self._mask_next
self._mask_next = None
def handle_mask_receive(self, mask_next: Type[FaceMask]):
""" Handles signal with the next mask to use.
Parameters
----------
mask_next : FaceMask
Next mask to use.
"""
self._mask_next = mask_next
print("[INFO] loading model...")
# initialize the video stream and allow the camera sensor to warmup
print("[INFO] starting video stream...")
vs = VideoStream(src=0).start()
#vs = VideoStream(usePiCamera=True).start()
time.sleep(2.0)
# loop over the frames from the video stream
while True:
# grab the frame from the threaded video stream and resize it
# to have a maximum width of 400 pixels
frame = vs.read()
frame = imutils.resize(frame, width=600)
faces = face_detector.detect(frame)
# loop over the detections
for face in faces:
# extract the confidence (i.e., probability) associated with the
# prediction
xStart, yStart, width, height = face
# clamp coordinates that are outside of the image
xStart, yStart = max(xStart, 0), max(yStart, 0)
# predict mask label on extracted face
faceImg = frame[yStart:yStart + height, xStart:xStart + width]
output = model(transformations(faceImg).unsqueeze(0).to(device))
_, predicted = torch.max(output.data, 1)
# draw face frame
# a command line argument
if args.get("video") and not grabbed:
break
# Resize the frame to have a width of only
# 600px solely for performance reasons
# associated with real-time face detection
# using a nice little helper function
frame = image_utils.resize(frame, width=600)
# Convert the frame to grayscale prior to analysis
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Attempt to detect faces in the grayscale
# version of the current video frame
faceRects = fd.detect(gray, scaleFactor=1.2,
minNeighbors=5, minSize=(30, 30))
# Before modifying the current frame, we'll clone
# it just in case we decide we want to do some
# additional processing on the original
frameClone = frame.copy()
# We'll draw a green bounding box around each of
# the faces that we detected within our video
# frame by modifying the cloned copy
for (fX, fY, fW, fH) in faceRects:
cv2.rectangle(frameClone, (fX, fY), (fX + fW, fY + fH),
(0, 255, 0), 2)
# We'll display the output of the cloned video
# frame immediately after we've detected faces
"""SELECT file.id as file_id, file.infinite_drop_user_id as user_id, filename, hash, dbx_user_id, dbx_access_token, infinite_drop_user.face_id FROM file, dbx_account, infinite_drop_user WHERE infinite_drop_user.id=file.infinite_drop_user_id AND file.dbx_account_id=dbx_account.id and file.id not in (SELECT file_id from processed_photos) and filename like '%%jpg'"""
)
rows = cur.fetchall()
for row in rows:
ifile_id = row[0]
idrop_user_id = row[1]
filename = row[2]
ext = filename[-4:]
hash = row[3]
dbx_user_id = row[4]
dbx_access_token = row[5]
face_id = row[6]
dbx = dropbox.Dropbox(dbx_access_token)
dbx.files_download_to_file(hash + ext, "/" + hash)
img = cv2.imread(hash + ext)
print(img.shape)
faces = fd.detect(img)
#faces = fd.detect_file(hash+ext)
for face in faces:
predicted_user_id, confidence = fr.recognize(face, face_id)
print(predicted_user_id)
if predicted_user_id:
print("INSERTED")
cur.execute(
"""INSERT INTO processed_photos (file_id, user_id) VALUES (%s, %s)""",
[ifile_id, predicted_user_id])
conn.commit()
time.sleep(.5)