def run(self, num_frames, preview_alpha, image_format, image_folder):
logger.info('Starting...')
leds = Leds()
player = Player(gpio=22, bpm=10)
photographer = Photographer(image_format, image_folder)
animator = Animator(leds, self._done)
try:
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
with PiCamera(sensor_mode=4,
resolution=(1640, 1232)) as camera, PrivacyLed(leds):
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
# Blend the preview layer with the alpha value from the flags.
if preview_alpha > 0:
logger.info('Starting preview with alpha %d',
preview_alpha)
camera.start_preview(alpha=preview_alpha)
else:
logger.info('Not starting preview, alpha 0')
button = Button(23)
button.when_pressed = take_photo
joy_score_moving_average = MovingAverage(10)
prev_joy_score = 0.0
with CameraInference(face_detection.model()) as inference:
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
for i, result in enumerate(inference.run()):
faces = face_detection.get_faces(result)
photographer.update_faces(faces)
joy_score = joy_score_moving_average.next(
average_joy_score(faces))
animator.update_joy_score(joy_score)
if joy_score > JOY_SCORE_PEAK > prev_joy_score:
player.play(JOY_SOUND)
elif joy_score < JOY_SCORE_MIN < prev_joy_score:
player.play(SAD_SOUND)
prev_joy_score = joy_score
if self._done.is_set() or i == num_frames:
break
finally:
player.stop()
photographer.stop()
player.join()
photographer.join()
animator.join()
def run(self, num_frames, preview_alpha, image_format, image_folder, enable_streaming):
logger.info('Starting...')
leds = Leds()
with contextlib.ExitStack() as stack:
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(StreamingServer(camera))
server.run()
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
button = Button(BUTTON_GPIO)
button.when_pressed = take_photo
joy_score_moving_average = MovingAverage(10)
prev_joy_score = 0.0
with CameraInference(face_detection.model()) as inference:
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
for i, result in enumerate(inference.run()):
faces = face_detection.get_faces(result)
photographer.update_faces(faces)
joy_score = joy_score_moving_average.next(average_joy_score(faces))
animator.update_joy_score(joy_score)
if server:
data = server_inference_data(result.width, result.height, faces, joy_score)
server.send_inference_data(data)
if joy_score > JOY_SCORE_PEAK > prev_joy_score:
player.play(JOY_SOUND)
elif joy_score < JOY_SCORE_MIN < prev_joy_score:
player.play(SAD_SOUND)
prev_joy_score = joy_score
if self._done.is_set() or i == num_frames:
break
def _run(self):
logger.info('Starting...')
leds = Leds()
with contextlib.ExitStack() as stack:
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(self.args.image_format, self.args.image_folder))
animator = stack.enter_context(Animator(leds))
stack.enter_context(PrivacyLed(leds))
server = None
if self.args.enable_streaming:
server = stack.enter_context(StreamingServer(self.camera))
server.run()
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(self.camera)
button = Button(BUTTON_GPIO)
button.when_pressed = take_photo
joy_score_moving_average = MovingAverage(10)
prev_joy_score = 0.0
with CameraInference(face_detection.model()) as inference:
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
for i, result in enumerate(inference.run()):
faces = face_detection.get_faces(result)
photographer.update_faces(faces)
avg_joy_score = average_joy_score(faces)
joy_score = joy_score_moving_average.next(avg_joy_score)
animator.update_joy_score(joy_score)
if server:
data = server_inference_data(result.width,
result.height, faces,
joy_score)
server.send_inference_data(data)
if avg_joy_score > JOY_SCORE_MIN:
photographer.shoot(self.camera)
# if joy_score > JOY_SCORE_PEAK > prev_joy_score:
# player.play(JOY_SOUND)
# elif joy_score < JOY_SCORE_MIN < prev_joy_score:
# player.play(SAD_SOUND)
prev_joy_score = joy_score
if self._done.is_set() or i == self.args.num_frames:
break
def __enter__(self):
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
with stopwatch('initialize camera'):
self._camera = self._stack.enter_context(
picamera.PiCamera(sensor_mode=4,
resolution=CAPTURE_RESOLUTION))
with stopwatch('initialize inference'):
self._inference = self._stack.enter_context(
_initialize_inference())
leds = self._stack.enter_context(Leds())
self._stack.enter_context(PrivacyLed(leds))
return self
print('RGB: Decrease RED brightness for 3.2 seconds')
for i in reversed(range(32)):
leds.update(Leds.rgb_on((8 * i, 0, 0)))
time.sleep(0.1)
print('RGB: Blend between GREEN and BLUE for 3.2 seconds')
for i in range(32):
leds.update(Leds.rgb_on(blend(BLUE, GREEN, i / 32)))
time.sleep(0.1)
print('RGB: Off for 1 second')
leds.update(Leds.rgb_off())
time.sleep(1)
print('Privacy: On for 2 seconds')
with PrivacyLed(leds):
time.sleep(2)
print('RGB: Solid GREEN for 2 seconds')
with RgbLeds(leds, Leds.rgb_on(GREEN)):
time.sleep(2)
print('Custom configuration for 5 seconds')
leds.update({
1: Leds.Channel(Leds.Channel.PATTERN, 128), # Red channel
2: Leds.Channel(Leds.Channel.OFF, 0), # Green channel
3: Leds.Channel(Leds.Channel.ON, 128), # Blue channel
4: Leds.Channel(Leds.Channel.PATTERN, 64), # Privacy channel
})
time.sleep(5)
def joy_detector(num_frames, preview_alpha, image_format, image_folder,
enable_streaming, streaming_bitrate, mdns_name):
done = threading.Event()
def stop():
logger.info('Stopping...')
done.set()
signal.signal(signal.SIGINT, lambda signum, frame: stop())
signal.signal(signal.SIGTERM, lambda signum, frame: stop())
logger.info('Starting...')
with contextlib.ExitStack() as stack:
leds = stack.enter_context(Leds())
board = stack.enter_context(Board())
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(
StreamingServer(camera,
bitrate=streaming_bitrate,
mdns_name=mdns_name))
def model_loaded():
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
board.button.when_pressed = take_photo
joy_moving_average = moving_average(10)
joy_moving_average.send(None) # Initialize.
joy_threshold_detector = threshold_detector(JOY_SCORE_LOW,
JOY_SCORE_HIGH)
joy_threshold_detector.send(None) # Initialize.
for faces, frame_size in run_inference(num_frames, model_loaded):
photographer.update_faces((faces, frame_size))
joy_score = joy_moving_average.send(average_joy_score(faces))
animator.update_joy_score(joy_score)
event = joy_threshold_detector.send(joy_score)
if event == 'high':
logger.info('High joy detected.')
player.play(JOY_SOUND)
elif event == 'low':
logger.info('Low joy detected.')
player.play(SAD_SOUND)
if server:
server.send_overlay(svg_overlay(faces, frame_size, joy_score))
if done.is_set():
break
def main():
with Leds() as leds:
print('RGB: Solid RED for 1 second')
leds.update(Leds.rgb_on(Color.RED))
time.sleep(1)
print('RGB: Solid GREEN for 1 second')
leds.update(Leds.rgb_on(Color.GREEN))
time.sleep(1)
print('RGB: Solid YELLOW for 1 second')
leds.update(Leds.rgb_on(Color.YELLOW))
time.sleep(1)
print('RGB: Solid BLUE for 1 second')
leds.update(Leds.rgb_on(Color.BLUE))
time.sleep(1)
print('RGB: Solid PURPLE for 1 second')
leds.update(Leds.rgb_on(Color.PURPLE))
time.sleep(1)
print('RGB: Solid CYAN for 1 second')
leds.update(Leds.rgb_on(Color.CYAN))
time.sleep(1)
print('RGB: Solid WHITE for 1 second')
leds.update(Leds.rgb_on(Color.WHITE))
time.sleep(1)
print('RGB: Off for 1 second')
leds.update(Leds.rgb_off())
time.sleep(1)
for _ in range(3):
print('Privacy: On (brightness=default)')
leds.update(Leds.privacy_on())
time.sleep(1)
print('Privacy: Off')
leds.update(Leds.privacy_off())
time.sleep(1)
for _ in range(3):
print('Privacy: On (brightness=5)')
leds.update(Leds.privacy_on(5))
time.sleep(1)
print('Privacy: Off')
leds.update(Leds.privacy_off())
time.sleep(1)
print('Set blink pattern: period=500ms (2Hz)')
leds.pattern = Pattern.blink(500)
print('RGB: Blink RED for 5 seconds')
leds.update(Leds.rgb_pattern(Color.RED))
time.sleep(5)
print('RGB: Blink GREEN for 5 seconds')
leds.update(Leds.rgb_pattern(Color.GREEN))
time.sleep(5)
print('RGB: Blink BLUE for 5 seconds')
leds.update(Leds.rgb_pattern(Color.BLUE))
time.sleep(5)
print('Set breathe pattern: period=1000ms (1Hz)')
leds.pattern = Pattern.breathe(1000)
print('RGB: Breathe RED for 5 seconds')
leds.update(Leds.rgb_pattern(Color.RED))
time.sleep(5)
print('RGB: Breathe GREEN for 5 seconds')
leds.update(Leds.rgb_pattern(Color.GREEN))
time.sleep(5)
print('RGB: Breathe BLUE for 5 seconds')
leds.update(Leds.rgb_pattern(Color.BLUE))
time.sleep(5)
print('RGB: Increase RED brightness for 3.2 seconds')
for i in range(32):
leds.update(Leds.rgb_on((8 * i, 0, 0)))
time.sleep(0.1)
print('RGB: Decrease RED brightness for 3.2 seconds')
for i in reversed(range(32)):
leds.update(Leds.rgb_on((8 * i, 0, 0)))
time.sleep(0.1)
print('RGB: Blend between GREEN and BLUE for 3.2 seconds')
for i in range(32):
color = Color.blend(Color.BLUE, Color.GREEN, i / 32)
leds.update(Leds.rgb_on(color))
time.sleep(0.1)
print('RGB: Off for 1 second')
leds.update(Leds.rgb_off())
time.sleep(1)
print('Privacy: On for 2 seconds')
with PrivacyLed(leds):
time.sleep(2)
print('RGB: Solid GREEN for 2 seconds')
with RgbLeds(leds, Leds.rgb_on(Color.GREEN)):
time.sleep(2)
print('Custom configuration for 5 seconds')
leds.update({
1: Leds.Channel(Leds.Channel.PATTERN, 128), # Red channel
2: Leds.Channel(Leds.Channel.OFF, 0), # Green channel
3: Leds.Channel(Leds.Channel.ON, 128), # Blue channel
4: Leds.Channel(Leds.Channel.PATTERN, 64), # Privacy channel
})
time.sleep(5)
print('Done')
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--num_frames',
'-f',
type=int,
dest='num_frames',
default=-1,
help='Sets the number of frames to run for, otherwise runs forever.')
parser.add_argument(
'--num_pics',
'-p',
type=int,
dest='num_pics',
default=-1,
help='Sets the max number of pictures to take, otherwise runs forever.'
)
args = parser.parse_args()
with PiCamera() as camera, PrivacyLed(Leds()):
# See the Raspicam documentation for mode and framerate limits:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# Set to the highest resolution possible at 16:9 aspect ratio
camera.sensor_mode = 5
camera.resolution = (1640, 922)
camera.start_preview(fullscreen=True)
with CameraInference(object_detection.model()) as inference:
print("Camera inference started")
player.play(*MODEL_LOAD_SOUND)
last_time = time()
pics = 0
save_pic = False
for f, result in enumerate(inference.run()):
for i, obj in enumerate(
object_detection.get_objects(result, score_threshold)):
print('%s Object #%d: %s' %
(strftime("%Y-%m-%d-%H:%M:%S"), i, str(obj)))
x, y, width, height = obj.bounding_box
# if obj.label == 'CAT':
if obj.label == '2m':
save_pic = True
player.play(*BEEP_SOUND)
# save the image if there was 1 or more cats detected
if save_pic:
# save the clean image
camera.capture("images/image_%s.jpg" %
strftime("%Y%m%d-%H%M%S"))
pics += 1
save_pic = False
if f == args.num_frames or pics == args.num_pics:
break
now = time()
duration = (now - last_time)
# The Movidius chip runs at 35 ms per image.
# Then there is some additional overhead for the object detector to
# interpret the result and to save the image. If total process time is
# running slower than 50 ms it could be a sign the CPU is geting overrun
if duration > 0.50:
print(
"Total process time: %s seconds. Bonnet inference time: %s ms "
% (duration, result.duration_ms))
last_time = now
camera.stop_preview()
def run_inference(run_event, model="face", framerate=15, cammode=5, hres=1640, vres=922, stats=True):
# See the Raspicam documentation for mode and framerate limits:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# Default to the highest resolution possible at 16:9 aspect ratio
global socket_connected, time_log
leds = Leds()
with PiCamera() as camera, PrivacyLed(leds):
camera.sensor_mode = cammode
camera.resolution = (hres, vres)
camera.framerate = framerate
camera.video_stabilization = True
camera.start_preview() # fullscreen=True)
def model_selector(argument):
options = {
"object": object_detection.model(),
"face": face_detection.model(),
"class": image_classification.model()
}
return options.get(argument, "nothing")
tf_model = model_selector(model)
# this is not needed because the function defaults to "face"
if tf_model == "nothing":
print("No tensorflow model or invalid model specified - exiting..")
camera.stop_preview()
os._exit(0)
return
with CameraInference(tf_model) as inference:
print("%s model loaded" % model)
last_time = time() # measure inference time
for result in inference.run():
# exit on shutdown
if not run_event.is_set():
camera.stop_preview()
return
output = ApiObject()
# handler for the AIY Vision object detection model
if model == "object":
output.threshold = 0.3
objects = object_detection.get_objects(result, output.threshold)
for obj in objects:
# print(object)
item = {
'name': 'object',
'class_name': obj._LABELS[obj.kind],
'score': obj.score,
'x': obj.bounding_box[0] / capture_width,
'y': obj.bounding_box[1] / capture_height,
'width': obj.bounding_box[2] / capture_width,
'height': obj.bounding_box[3] / capture_height
}
output.numObjects += 1
output.objects.append(item)
# handler for the AIY Vision face detection model
elif model == "face":
faces = face_detection.get_faces(result)
for face in faces:
# print(face)
item = {
'name': 'face',
'score': face.face_score,
'joy': face.joy_score,
'x': face.bounding_box[0] / capture_width,
'y': face.bounding_box[1] / capture_height,
'width': face.bounding_box[2] / capture_width,
'height': face.bounding_box[3] / capture_height,
}
output.numObjects += 1
output.objects.append(item)
elif model == "class":
output.threshold = 0.3
classes = image_classification.get_classes(result)
s = ""
for (obj, prob) in classes:
if prob > output.threshold:
s += '%s=%1.2f\t|\t' % (obj, prob)
item = {
'name': 'class',
'class_name': obj,
'score': prob
}
output.numObjects += 1
output.objects.append(item)
# print('%s\r' % s)
now = time()
output.timeStamp = now
output.inferenceTime = (now - last_time)
last_time = now
# No need to do anything else if there are no objects
if output.numObjects > 0:
output_json = output.to_json()
print(output_json)
# Send the json object if there is a socket connection
if socket_connected is True:
q.put(output_json)
# Additional data to measure inference time
if stats is True:
time_log.append(output.inferenceTime)
time_log = time_log[-10:] # just keep the last 10 times
print("Avg inference time: %s" % (sum(time_log)/len(time_log)))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--num_frames',
'-f',
type=int,
dest='num_frames',
default=-1,
help='Sets the number of frames to run for, otherwise runs forever.')
parser.add_argument(
'--num_pics',
'-p',
type=int,
dest='num_pics',
default=-1,
help='Sets the max number of pictures to take, otherwise runs forever.'
)
#cnx = mysql.connector.connect(user='******', password='******',
# host='34.65.17.107',
# database='lorecdb')
#cursor = cnx.cursor()
args = parser.parse_args()
with PiCamera() as camera, PrivacyLed(Leds()):
# See the Raspicam documentation for mode and framerate limits:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# Set to the highest resolution possible at 16:9 aspect ratio
camera.sensor_mode = 4
camera.resolution = (1640, 1232)
camera.start_preview(fullscreen=True)
def facedet():
with CameraInference(FaceDetection.model()) as inference:
for result in inference.run(args.num_frames):
faces = FaceDetection.get_faces(result)
annotator.clear()
for face in faces:
annotator.bounding_box(transform(face.bounding_box),
fill=0)
annotator.update()
print(
'#%05d (%5.2f fps): num_faces=%d, avg_joy_score=%.2f' %
(inference.count, inference.rate, len(faces),
avg_joy_score(faces)))
def objdet():
with CameraInference(ObjectDetection.model()) as inference:
print("Camera inference started")
player.play(*MODEL_LOAD_SOUND)
last_time = time()
pics = 0
save_pic = False
enable_label = True
# Annotator renders in software so use a smaller size and scale results
# for increased performace.
annotator = Annotator(camera, dimensions=(320, 240))
scale_x = 320 / 1640
scale_y = 240 / 1232
# Incoming boxes are of the form (x, y, width, height). Scale and
# transform to the form (x1, y1, x2, y2).
def transform(bounding_box):
x, y, width, height = bounding_box
return (scale_x * x, scale_y * y, scale_x * (x + width),
scale_y * (y + height))
def leftCorner(bounding_box):
x, y, width, height = bounding_box
return (scale_x * x, scale_y * y)
def truncateFloat(value):
return '%.3f' % (value)
for f, result in enumerate(inference.run()):
annotator.clear()
detections = enumerate(
ObjectDetection.get_objects(result, 0.3))
for i, obj in detections:
print('%s', obj.label)
annotator.bounding_box(transform(obj.bounding_box),
fill=0)
if enable_label:
annotator.text(
leftCorner(obj.bounding_box), obj.label +
" - " + str(truncateFloat(obj.score)))
print('%s Object #%d: %s' %
(strftime("%Y-%m-%d-%H:%M:%S"), i, str(obj)))
x, y, width, height = obj.bounding_box
if obj.label == 'chair':
#dt = datetime.datetime.now()
#os.system("ffplay -nodisp -autoexit /home/pi/AIY-projects-python/src/LorecObjectSoundFiles/insan.mp3")
#query = ("INSERT INTO Log (Time, Location, GlassNameDbid, ModulDbid, Screenshot, Tag, Distance) VALUES ('"+ dt+"', 'Ankara', '1', '2', 'No Screenshot', 'Insan', '150')")
#save_pic = True
player.play(*BEEP_SOUND)
#elif obj.label == 'tvmonitor':
#os.system("ffplay -nodisp -autoexit /home/pi/AIY-projects-python/src/LorecObjectSoundFiles/Ekran.mp3")
# save the image
if save_pic:
# save the clean image
camera.capture("images/image_%s.jpg" %
strftime("%Y%m%d-%H%M%S"))
pics += 1
save_pic = False
if f == args.num_frames or pics == args.num_pics:
break
annotator.update()
now = time()
duration = (now - last_time)
# The Movidius chip runs at 35 ms per image.
# Then there is some additional overhead for the object detector to
# interpret the result and to save the image. If total process time is
# running slower than 50 ms it could be a sign the CPU is geting overrun
if duration > 0.50:
print(
"Total process time: %s seconds. Bonnet inference time: %s ms "
% (duration, result.duration_ms))
last_time = now
#threading.Thread(target=facedet).start()
objdet()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--num_frames',
'-f',
type=int,
dest='num_frames',
default=-1,
help='Sets the number of frames to run for, otherwise runs forever.')
parser.add_argument(
'--num_pics',
'-p',
type=int,
dest='num_pics',
default=-1,
help='Sets the max number of pictures to take, otherwise runs forever.'
)
args = parser.parse_args()
with PiCamera() as camera, PrivacyLed(Leds()):
# See the Raspicam documentation for mode and framerate limits:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# Set to the highest resolution possible at 16:9 aspect ratio
camera.sensor_mode = 4
camera.resolution = (1640, 1232)
camera.start_preview(fullscreen=True)
with CameraInference(object_detection_custom.model()) as inference:
print("Camera inference started")
player.play(*MODEL_LOAD_SOUND)
last_time = time()
pics = 0
save_pic = False
enable_label = True
# Annotator renders in software so use a smaller size and scale results
# for increased performace.
annotator = Annotator(camera, dimensions=(320, 240))
scale_x = 320 / 1640
scale_y = 240 / 1232
# Incoming boxes are of the form (x, y, width, height). Scale and
# transform to the form (x1, y1, x2, y2).
def transform(bounding_box):
x, y, width, height = bounding_box
return (scale_x * x, scale_y * y, scale_x * (x + width),
scale_y * (y + height))
def leftCorner(bounding_box):
x, y, width, height = bounding_box
return (scale_x * x, scale_y * y)
def truncateFloat(value):
return '%.3f' % (value)
for f, result in enumerate(inference.run()):
annotator.clear()
detections = enumerate(
object_detection_custom.get_objects(result, 0.3))
for i, obj in detections:
print('%s', obj.label)
annotator.bounding_box(transform(obj.bounding_box), fill=0)
if enable_label:
annotator.text(
leftCorner(obj.bounding_box),
obj.label + " - " + str(truncateFloat(obj.score)))
print('%s Object #%d: %s' %
(strftime("%Y-%m-%d-%H:%M:%S"), i, str(obj)))
x, y, width, height = obj.bounding_box
if obj.label == 'person':
#save_pic = True
player.play(*BEEP_SOUND)
# save the image if there was 1 or more cats detected
if save_pic:
# save the clean image
camera.capture("images/image_%s.jpg" %
strftime("%Y%m%d-%H%M%S"))
pics += 1
save_pic = False
if f == args.num_frames or pics == args.num_pics:
break
annotator.update()
now = time()
duration = (now - last_time)
# The Movidius chip runs at 35 ms per image.
# Then there is some additional overhead for the object detector to
# interpret the result and to save the image. If total process time is
# running slower than 50 ms it could be a sign the CPU is geting overrun
if duration > 0.50:
print(
"Total process time: %s seconds. Bonnet inference time: %s ms "
% (duration, result.duration_ms))
last_time = now
camera.stop_preview()
def monitor_run(num_frames, preview_alpha, image_format, image_folder,
enable_streaming, streaming_bitrate, mdns_name, width, height,
fps, region, enter_side, use_annotator, url, uname, pw,
image_dir, dev):
# Sign the device in and get an access and a refresh token, if a password and username provided.
access_token = None
refresh_token = None
tokens = None
start_token_timer = timer()
if uname is not None and pw is not None:
try:
tokens = connect_to_server(url, uname, pw)
access_token = tokens['access']
refresh_token = tokens['refresh']
print(access_token)
print(refresh_token)
except:
print("Could not get tokens from the server.")
pass
# location where we want to send the faces + status for classification on web server.
classification_path = url + "/" + image_dir
done = threading.Event()
def stop():
logger.info('Stopping...')
done.set()
# Get the region center point and two corners
r_center = (region[0] + region[2] / 2, region[1] + region[3] / 2)
r_corners = (region[0], region[0] + region[2], region[1],
region[1] + region[3])
signal.signal(signal.SIGINT, lambda signum, frame: stop())
signal.signal(signal.SIGTERM, lambda signum, frame: stop())
logger.info('Starting...')
with contextlib.ExitStack() as stack:
leds = stack.enter_context(Leds())
board = stack.enter_context(Board())
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, framerate=fps, resolution=(width, height)))
stack.enter_context(PrivacyLed(leds))
# Annotator renders in software so use a smaller size and scale results
# for increased performace.
annotator = None
if use_annotator:
annotator = Annotator(camera, dimensions=(320, 240))
scale_x = 320 / width
scale_y = 240 / height
server = None
if enable_streaming:
server = stack.enter_context(
StreamingServer(camera,
bitrate=streaming_bitrate,
mdns_name=mdns_name))
def model_loaded():
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
board.button.when_pressed = take_photo
joy_moving_average = moving_average(10)
joy_moving_average.send(None) # Initialize.
joy_threshold_detector = threshold_detector(JOY_SCORE_LOW,
JOY_SCORE_HIGH)
joy_threshold_detector.send(None) # Initialize.
previous_faces3 = []
previous_faces2 = []
previous_faces = []
num_faces = 0
for faces, frame_size in run_inference(num_frames, model_loaded):
# If 4 mins have passed since access token obtained, refresh the token.
end_token_timer = timer() # time in seconds
if refresh_token is not None and end_token_timer - start_token_timer >= 240:
tokens = refresh_access_token(url, refresh_token)
access_token = tokens["access"]
photographer.update_faces((faces, frame_size))
joy_score = joy_moving_average.send(average_joy_score(faces))
animator.update_joy_score(joy_score)
event = joy_threshold_detector.send(joy_score)
if event == 'high':
logger.info('High joy detected.')
player.play(JOY_SOUND)
elif event == 'low':
logger.info('Low joy detected.')
player.play(SAD_SOUND)
num_previous_faces = num_faces
if use_annotator:
annotator.clear()
annotator.bounding_box(transform(region, scale_x, scale_y),
fill=0)
num_faces = 0
tmp_arr = []
faces_in_region = []
photo_taken = False
image = None
for face in faces:
face_center = (face.bounding_box[0] + face.bounding_box[2] / 2,
face.bounding_box[1] + face.bounding_box[3] / 2)
# check if the center of the face is in our region of interest:
if r_corners[0] <= face_center[0] <= r_corners[1] and \
r_corners[2] <= face_center[1] <= r_corners[3]:
if not photo_taken:
stream = io.BytesIO()
with stopwatch('Taking photo'):
camera.capture(stream,
format=image_format,
use_video_port=True)
stream.seek(0)
image = Image.open(stream)
photo_taken = True
num_faces = num_faces + 1
faces_in_region.append(face)
# creates a tuple ( image of the face, entering/exiting status)
tmp_arr.append([
crop_face(image, image_format, image_folder,
face.bounding_box),
get_status(face.bounding_box, r_center, enter_side)
])
if use_annotator:
annotator.bounding_box(
transform(face.bounding_box, scale_x, scale_y),
fill=0) # draw a box around the face
if server:
server.send_overlay(
svg_overlay(faces_in_region, frame_size, region,
joy_score))
if use_annotator:
annotator.update()
if num_faces < num_previous_faces:
# loop through previous faces: send face data, image and status
print(" A face left the region: send previous face data")
#if not use_annotator:
#take_photo()
faces_to_use = previous_faces
if previous_faces2:
faces_to_use = previous_faces2
if previous_faces3:
faces_to_use = previous_faces3
for face in faces_to_use:
print(classification_path, face, access_token)
if access_token is not None:
print("sent face with access token")
send_face(classification_path, face, access_token, dev)
previous_faces3 = previous_faces2
previous_faces2 = previous_faces
previous_faces = tmp_arr
if done.is_set():
break
def run(self, num_frames, preview_alpha, image_format, image_folder,
enable_streaming):
logger.info('Starting...')
leds = Leds()
with contextlib.ExitStack() as stack:
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(StreamingServer(camera))
server.run()
def take_photo():
logger.info('Button pressed.')
player.play(BEEP_SOUND)
photographer.shoot(camera)
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
button = Button(BUTTON_GPIO)
button.when_pressed = take_photo
joy_score_moving_average = MovingAverage(5) #Changed it from 10
prev_joy_score = 0.0
with CameraInference(face_detection.model()) as inference:
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
for i, result in enumerate(inference.run()):
faces = face_detection.get_faces(result)
photographer.update_faces(faces)
joy_score = joy_score_moving_average.next(
average_joy_score(faces))
animator.update_joy_score(joy_score)
if server:
data = server_inference_data(result.width,
result.height, faces,
joy_score)
server.send_inference_data(data)
if joy_score > JOY_SCORE_PEAK > prev_joy_score:
player.play(JOY_SOUND)
## picoSpeakNow(list_happy[np.random.randint(0,N_HAPPY)])
## os.system('pico2wave -w test.wav "keep smiling. I feed off of smile energy... do not let the smile die down." && aplay test.wav')
## time.sleep(3)
espeak_happy = 'espeak -s160 -g6 -ven+f3 ' + '"' + if_happy_list[
np.random.randint(0, N_HAPPY)] + '"'
os.system(espeak_happy)
elif joy_score < 0.35 < prev_joy_score:
player.play(SAD_SOUND)
espeak_sad = 'espeak -s160 -g6 -ven+f3 ' + '"' + if_sad[
0] + '"'
os.system(espeak_sad)
## picoSpeakNow(list_sad[np.random.randint(0,N_SAD)])
## time.sleep(3)
## os.system('espeak "Keep smiling. I feed off of smile energy... do not let the smile die down"')
## os.system('pico2wave -w test.wav "start smiling. I feed off of smile energy... do not let the smile die down." && aplay test.wav')
prev_joy_score = joy_score
if self._done.is_set() or i == num_frames:
break
def joy_detector(num_frames, preview_alpha, image_format, image_folder,
enable_streaming, streaming_bitrate, mdns_name):
readings = []
averages = []
num_reading = 0
num_average = 0
done = threading.Event()
def stop():
logger.info('Stopping...')
done.set()
signal.signal(signal.SIGINT, lambda signum, frame: stop())
signal.signal(signal.SIGTERM, lambda signum, frame: stop())
logger.info('Starting...')
with contextlib.ExitStack() as stack:
leds = stack.enter_context(Leds())
board = stack.enter_context(Board())
player = stack.enter_context(Player(gpio=BUZZER_GPIO, bpm=10))
photographer = stack.enter_context(
Photographer(image_format, image_folder))
animator = stack.enter_context(Animator(leds))
# Forced sensor mode, 1640x1232, full FoV. See:
# https://picamera.readthedocs.io/en/release-1.13/fov.html#sensor-modes
# This is the resolution inference run on.
# Use half of that for video streaming (820x616).
camera = stack.enter_context(
PiCamera(sensor_mode=4, resolution=(820, 616)))
stack.enter_context(PrivacyLed(leds))
server = None
if enable_streaming:
server = stack.enter_context(
StreamingServer(camera,
bitrate=streaming_bitrate,
mdns_name=mdns_name))
def model_loaded():
logger.info('Model loaded.')
player.play(MODEL_LOAD_SOUND)
def stop_playing():
client.loop_start()
client.subscribe("music")
client.publish("music", "stop")
client.loop_stop()
print("Sent stopping signal.")
if preview_alpha > 0:
camera.start_preview(alpha=preview_alpha)
board.button.when_pressed = stop_playing
joy_moving_average = moving_average(10)
joy_moving_average.send(None) # Initialize.
joy_threshold_detector = threshold_detector(JOY_SCORE_LOW,
JOY_SCORE_HIGH)
joy_threshold_detector.send(None) # Initialize.
for faces, frame_size in run_inference(num_frames, model_loaded):
photographer.update_faces((faces, frame_size))
joy_score = joy_moving_average.send(average_joy_score(faces))
# ----------------------------------
if len(readings) < 10:
readings.append(joy_score)
else:
x = readings[0]
readings.remove(x)
readings.append(joy_score)
num_reading += 1
time.sleep(0.2)
if num_reading % 10 == 0:
total_x = 0
for item_x in readings:
total_x += item_x
average = total_x / 10
message = str(average)
client.publish(topic, message)
print("published")
animator.update_joy_score(joy_score)
event = joy_threshold_detector.send(joy_score)
if event == 'high':
logger.info('High joy detected.')
player.play(JOY_SOUND)
elif event == 'low':
logger.info('Low joy detected.')
player.play(SAD_SOUND)
if server:
server.send_overlay(svg_overlay(faces, frame_size, joy_score))
if done.is_set():
break