def __init__(self, resolution, rate, callbacks):
# type: (CameraResolution, int, List[Callable[[AbstractImage], None]]) -> None
# Extract Image Dimensions from CameraResolution
self._resolution = resolution
self._width = self._resolution.value[1]
self._height = self._resolution.value[0]
self._shape = np.array([self.height, self.width, self.channels])
# Store Camera Rate and Callbacks
self._rate = rate
self._callbacks = callbacks
# Variables to do some performance statistics
self._dt_buffer = deque([], maxlen=10)
self._true_rate = rate
self._t0 = time()
# Create Mailbox and Image Processor:
# Each time an image is captured it is put in the mailbox, overriding whatever there might currently be.
# In a separate thread, the _processor worker takes an image and calls all registered callbacks.
# This way the processing of images does not block the acquisition of new images,
# while at the same new images don't build up a queue, but are discarded when the _processor is too busy.
self._mailbox = Mailbox()
self._processor_scheduler = Scheduler(self._processor,
name="CameraThread")
self._processor_scheduler.start()
# Default behaviour is to not run by default. Calling AbstractApplication.run() will activate the camera
self._running = False
self._log = logger.getChild(self.__class__.__name__)
def __init__(self, resolution, rate, callbacks=[], index=0):
"""
System Camera
Parameters
----------
resolution: pepper.framework.CameraResolution
rate: int
callbacks: list of callable
index: int
"""
super(SystemCamera, self).__init__(resolution, rate, callbacks)
# Get Camera and request resolution
self._camera = cv2.VideoCapture(index)
if not self.resolution == CameraResolution.NATIVE:
self._camera.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
self._camera.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
# Check if camera is working
if not self._camera.isOpened():
raise RuntimeError("{} could not be opened".format(
self.__class__.__name__))
# Run Image acquisition in Thread
self._scheduler = Scheduler(self._run, name="SystemCameraThread")
self._scheduler.start()
self._log.debug("Booted")
def __init__(self, rate, channels, callbacks):
# type: (int, int, List[Callable[[np.ndarray], None]]) -> None
self._rate = rate
self._channels = channels
self._callbacks = callbacks
# Variables to do some performance statistics
self._dt_buffer = deque([], maxlen=32)
self._true_rate = rate
self._t0 = time()
# Create Queue and Sound Processor:
# Each time audio samples are captured it is put in the audio processing queue
# In a separate thread, the _processor worker takes these samples and calls all registered callbacks.
# This way, samples are not accidentally skipped (NAOqi has some very strict timings)
self._queue = Queue()
self._processor_scheduler = Scheduler(self._processor,
0,
name="MicrophoneThread")
self._processor_scheduler.start()
# Default behaviour is to not run by default. Calling AbstractApplication.run() will activate the microphone
self._running = False
self._log = logger.getChild(self.__class__.__name__)
class SystemCamera(AbstractCamera):
"""
System Camera
Parameters
----------
resolution: pepper.framework.CameraResolution
rate: int
callbacks: list of callable
index: int
"""
def __init__(self, resolution, rate, callbacks=[], index=0):
# type: (CameraResolution, int, List[Callable[[AbstractImage], None]], int) -> None
super(SystemCamera, self).__init__(resolution, rate, callbacks)
# Get Camera and request resolution
self._camera = cv2.VideoCapture(index)
if not self.resolution == CameraResolution.NATIVE:
self._camera.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
self._camera.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
# Check if camera is working
if not self._camera.isOpened():
raise RuntimeError("{} could not be opened".format(
self.__class__.__name__))
# Run Image acquisition in Thread
self._scheduler = Scheduler(self._run, name="SystemCameraThread")
self._scheduler.start()
self._log.debug("Booted")
def _run(self):
t0 = time()
# Get frame from camera
status, image = self._camera.read()
if status:
if self._running:
# Resize Image and Convert to RGB
image = cv2.resize(image, (self.width, self.height))
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
# Call On Image Event
self.on_image(
SystemImage(
image,
Bounds(-0.55, -0.41 + np.pi / 2, 0.55,
0.41 + np.pi / 2)))
else:
self._camera.release()
raise RuntimeError("{} could not fetch image".format(
self.__class__.__name__))
# Maintain frame rate
sleep(max(0, 1. / self.rate - (time() - t0)))
def __init__(self, language):
self._language = language
self._queue = Queue()
self._talking_jobs = 0
self._scheduler = Scheduler(self._worker, name="TextToSpeechThread")
self._scheduler.start()
self._log = logger.getChild(self.__class__.__name__)
def __init__(self, backend):
super(StatisticsComponent, self).__init__(backend)
# Require Speech Recognition Component and Get Information from it
speech_recognition = self.require(
StatisticsComponent,
SpeechRecognitionComponent) # type: SpeechRecognitionComponent
vad, asr = speech_recognition.vad, speech_recognition.asr
def worker():
# Create Voice Activation Bar
activation = int(vad.activation * 10)
activation_print = "|" * activation + "." * (10 - activation)
voice_print = ("<{:10s}>" if vad._voice else
"[{:10s}]").format(activation_print)
empty_voice_print = "[ ]"
# Get Microphone Related Information
mic_running = self.backend.microphone.running
mic_rate = self.backend.microphone.rate
mic_rate_true = self.backend.microphone.true_rate
# Get Camera Related Information
cam_rate = self.backend.camera.rate
cam_rate_true = self.backend.camera.true_rate
# If Camera and/or Microphone are not running as fast as expected -> show stderr message instead of stdout
error = (
cam_rate_true < cam_rate * self.PERFORMANCE_ERROR_THRESHOLD
or mic_rate_true <
float(mic_rate) * self.PERFORMANCE_ERROR_THRESHOLD)
# Show Speech to Text Transcript 'live' as it happens
if asr.live:
self.LIVE_SPEECH = asr.live
self.LIVE_SPEECH_TIME = time()
elif time() - self.LIVE_SPEECH_TIME > self.LIVE_SPEECH_TIMEOUT:
self.LIVE_SPEECH = ""
# Display Statistics
print(
"\rThreads {:2d} | Cam {:4.1f} Hz | Mic {:4.1f} kHz | TTS {:12s} >>> {}"
.format(threading.active_count(), cam_rate_true,
mic_rate_true / 1000.0,
voice_print if mic_running else empty_voice_print,
self.LIVE_SPEECH),
end="",
file=(stderr if error else stdout))
# Run 10 times a second
# TODO: Bit Much?
schedule = Scheduler(worker, 0.1)
schedule.start()
def __init__(self, backend):
super(TextToSpeechComponent, self).__init__(backend)
self._microphone_lock = Lock()
def worker():
with self._microphone_lock:
# If talking is over & microphone is not yet running -> Start Microphone
if not self.backend.text_to_speech.talking and not self.backend.microphone.running:
self.backend.microphone.start()
schedule = Scheduler(worker, name="TextToSpeechComponentThread")
schedule.start()
def __init__(self, backend):
super(TextToSpeechComponent, self).__init__(backend)
# Prevent Racing Conditions
self._microphone_lock = Lock()
def worker():
# type: () -> None
"""Make sure Microphone is not listening when Text to Speech is Live"""
# Acquire Microphone Lock
with self._microphone_lock:
# If robot is not talking & microphone is not yet running -> Start Microphone
if not self.backend.text_to_speech.talking and not self.backend.microphone.running:
self.backend.microphone.start()
schedule = Scheduler(worker, name="TextToSpeechComponentThread")
schedule.start()
def __init__(self, rate, channels, callbacks):
self._rate = rate
self._channels = channels
self._callbacks = callbacks
self._dt_threshold_multiplier = 1.5
self._dt_buffer = deque([], maxlen=32)
self._true_rate = rate
self._t0 = time()
self._queue = Queue()
self._processor_scheduler = Scheduler(self._processor,
0,
name="MicrophoneThread")
self._processor_scheduler.start()
self._log = logger.getChild(self.__class__.__name__)
self._running = False
def __init__(self, resolution, rate, callbacks=[], index=0):
# type: (CameraResolution, int, List[Callable[[AbstractImage], None]], int) -> None
super(SystemCamera, self).__init__(resolution, rate, callbacks)
# Get Camera and request resolution
self._camera = cv2.VideoCapture(index)
if not self.resolution == CameraResolution.NATIVE:
self._camera.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
self._camera.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
# Check if camera is working
if not self._camera.isOpened():
raise RuntimeError("{} could not be opened".format(
self.__class__.__name__))
# Run Image acquisition in Thread
self._scheduler = Scheduler(self._run, name="SystemCameraThread")
self._scheduler.start()
self._log.debug("Booted")
def __init__(self, backend):
"""
Construct Statistics Component
Parameters
----------
backend: Backend
"""
super(StatisticsComponent, self).__init__(backend)
speech_recognition = self.require(StatisticsComponent, SpeechRecognitionComponent) # type: SpeechRecognitionComponent
vad = speech_recognition.vad
def worker():
# Create Activation Bar
activation = int(vad.activation * 10)
activation_print = "|" * activation + "." * (10 - activation)
voice_print = ("<{:10s}>" if vad._utterance else "[{:10s}]").format(activation_print)
empty_voice_print = "[ ]"
mic_running = self.backend.microphone.running
mic_rate = self.backend.microphone.rate
mic_rate_true = self.backend.microphone.true_rate
cam_rate = self.backend.camera.rate
cam_rate_true = self.backend.camera.true_rate
error = (cam_rate_true < cam_rate * self.PERFORMANCE_ERROR_THRESHOLD or
mic_rate_true < float(mic_rate) * self.PERFORMANCE_ERROR_THRESHOLD)
print("\rMicrophone {:3.1f} kHz | Camera {:4.1f} Hz | Voice {:12s} {:4.0%}".format(
mic_rate_true / 1000.0, cam_rate_true,
voice_print if mic_running else empty_voice_print,
vad.activation if mic_running else 0),
end="", file=(stderr if error else stdout))
schedule = Scheduler(worker, 0.1)
schedule.start()
def __init__(self, resolution, rate, callbacks):
self._resolution = resolution
self._width = self._resolution.value[1]
self._height = self._resolution.value[0]
self._rate = rate
self._callbacks = callbacks
self._shape = np.array([self.height, self.width, self.channels])
self._dt_buffer = deque([], maxlen=10)
self._true_rate = rate
self._t0 = time()
self._mailbox = Mailbox()
self._processor_scheduler = Scheduler(self._processor,
name="CameraThread")
self._processor_scheduler.start()
self._running = False
self._log = logger.getChild(self.__class__.__name__)
def __init__(self, backend):
super(ObjectDetectionComponent, self).__init__(backend)
# Public List of On Object Callbacks:
# Allowing other Components to Subscribe to it
self.on_object_callbacks = []
# Create Object Detection Client and a Mailbox per Target
# Make sure the corresponding server @ pepper_tensorflow is actually running
clients = [ObjectDetectionClient(target) for target in ObjectDetectionComponent.TARGETS]
mailboxes = {client: Mailbox() for client in clients} # type: Dict[ObjectDetectionClient, Mailbox]
def on_image(image):
# type: (AbstractImage) -> None
"""
Raw On Image Event. Called every time the camera yields a frame.
Parameters
----------
image: AbstractImage
"""
for client in clients:
mailboxes[client].put(image)
def worker(client):
# type: (ObjectDetectionClient) -> None
"""Object Detection Worker"""
# Get Image from Mailbox Corresponding with Client
image = mailboxes[client].get()
# Classify Objects in this Image using Client
objects = [obj for obj in client.classify(image) if obj.confidence > config.OBJECT_RECOGNITION_THRESHOLD]
if objects:
# Call on_object Callback Functions
for callback in self.on_object_callbacks:
callback(objects)
# Call on_object Event Function
self.on_object(objects)
# Initialize & Start Object Workers
schedule = [Scheduler(worker, args=(client,), name="{}Thread".format(client.target.name)) for client in clients]
for s in schedule:
s.start()
# Add on_image to Camera Callbacks
self.backend.camera.callbacks += [on_image]
def __init__(self, backend):
"""
Construct Face Detection Component
Parameters
----------
backend: AbstractBackend
"""
super(FaceDetectionComponent, self).__init__(backend)
self.on_face_callbacks = []
self.on_face_known_callbacks = []
self.on_face_new_callbacks = []
# Initialize OpenFace
open_face = OpenFace()
# Import Face Data
people = FaceClassifier.load_directory(config.PEOPLE_FRIENDS_ROOT)
people.update(FaceClassifier.load_directory(config.PEOPLE_NEW_ROOT))
# Initialize Face Classifier
self.face_classifier = FaceClassifier(people)
queue = Queue()
def on_image(image):
"""
Raw On Image Event. Called every time the camera yields a frame.
Parameters
----------
image: np.ndarray
"""
queue.put([
self.face_classifier.classify(r, b, image)
for r, b in open_face.represent(image)
])
def worker():
on_face = queue.get()
on_face_known = []
on_face_new = []
for face in on_face:
if face.confidence > config.FACE_RECOGNITION_THRESHOLD:
(on_face_new if face.name == FaceClassifier.NEW else
on_face_known).append(face)
if on_face:
for callback in self.on_face_callbacks:
callback(on_face)
self.on_face(on_face)
if on_face_known:
for callback in self.on_face_known_callbacks:
callback(on_face_known)
self.on_face_known(on_face_known)
if on_face_new:
for callback in self.on_face_new_callbacks:
callback(on_face_new)
self.on_face_new(on_face_new)
# Initialize Queue & Worker
schedule = Scheduler(worker, name="FaceDetectionComponentThread")
schedule.start()
# Add on_image to Camera Callbacks
self.backend.camera.callbacks += [on_image]
class AbstractMicrophone(object):
"""
Abstract Microphone
Parameters
----------
rate: int
channels: int
callbacks: list of callable
"""
def __init__(self, rate, channels, callbacks):
self._rate = rate
self._channels = channels
self._callbacks = callbacks
self._dt_threshold_multiplier = 1.5
self._dt_buffer = deque([], maxlen=32)
self._true_rate = rate
self._t0 = time()
self._queue = Queue()
self._processor_scheduler = Scheduler(self._processor,
0,
name="MicrophoneThread")
self._processor_scheduler.start()
self._log = logger.getChild(self.__class__.__name__)
self._running = False
@property
def rate(self):
"""
Audio bit rate
Returns
-------
rate: int
Audio bit rate
"""
return self._rate
@property
def true_rate(self):
"""
Actual Audio bit rate
Audio bit rate after accounting for latency & performance realities
Returns
-------
true_rate:
Actual Audio bit rate
"""
return self._true_rate
@property
def channels(self):
"""
Audio channels
Returns
-------
channels: int
Audio channels
"""
return self._channels
@property
def callbacks(self):
"""
Get/Set :func:`~AbstractCamera.on_audio` Callbacks
Returns
-------
callbacks: list of callable
"""
return self._callbacks
@callbacks.setter
def callbacks(self, value):
"""
Get/Set :func:`~AbstractCamera.on_audio` Callbacks
Parameters
----------
value: list of callable
"""
self._callbacks = value
@property
def running(self):
"""
Returns whether Microphone is Running
Returns
-------
running: bool
"""
return self._running
def on_audio(self, audio):
"""
On Audio Event, Called for every frame of audio captured by Microphone
Microphone Modules should call this function for every frame of audio acquired by Microphone
Parameters
----------
audio: np.ndarray
"""
self._queue.put(audio)
def start(self):
"""Start Microphone Stream"""
self._running = True
def stop(self):
"""Stop Microphone Stream"""
self._running = False
def _processor(self):
"""
Audio Processor
Calls each callback for each audio frame, threaded, for higher audio throughput
"""
audio = self._queue.get()
if self._running:
for callback in self.callbacks:
callback(audio)
self._update_dt(len(audio))
def _update_dt(self, n_bytes):
t1 = time()
self._dt_buffer.append((t1 - self._t0))
self._t0 = t1
self._true_rate = n_bytes / np.mean(self._dt_buffer)
def __init__(self, backend):
super(FaceRecognitionComponent, self).__init__(backend)
# Public Lists of Callbacks:
# Allowing other Components to Subscribe to them
self.on_face_callbacks = []
self.on_face_known_callbacks = []
self.on_face_new_callbacks = []
# Initialize OpenFace
open_face = OpenFace()
# Import Face Data (Friends & New)
people = FaceClassifier.load_directory(config.PEOPLE_FRIENDS_ROOT)
people.update(FaceClassifier.load_directory(config.PEOPLE_NEW_ROOT))
# Initialize Face Classifier
self.face_classifier = FaceClassifier(people)
# Initialize Image Mailbox
mailbox = Mailbox()
def on_image(image):
# type: (AbstractImage) -> None
"""
Private On Image Event. Called every time the camera yields a frame.
Parameters
----------
image: AbstractImage
"""
mailbox.put(image)
def worker():
# type: () -> None
"""Find and Classify Faces in Images"""
# Get latest Image from Mailbox
image = mailbox.get()
# Get All Face Representations from OpenFace & Initialize Known/New Face Categories
on_face = [
self.face_classifier.classify(r, b, image)
for r, b in open_face.represent(image.image)
]
on_face_known = []
on_face_new = []
# Distribute Faces over Known & New (Keeping them in the general on_face)
for face in on_face:
if face.name == config.HUMAN_UNKNOWN:
if face.confidence >= 1.0:
on_face_new.append(face)
elif face.confidence > config.FACE_RECOGNITION_THRESHOLD:
on_face_known.append(face)
# Call Appropriate Callbacks
if on_face:
for callback in self.on_face_callbacks:
callback(on_face)
self.on_face(on_face)
if on_face_known:
for callback in self.on_face_known_callbacks:
callback(on_face_known)
self.on_face_known(on_face_known)
if on_face_new:
for callback in self.on_face_new_callbacks:
callback(on_face_new)
self.on_face_new(on_face_new)
# Initialize Worker
schedule = Scheduler(worker, name="FaceDetectionComponentThread")
schedule.start()
# Add on_image to Camera Callbacks
self.backend.camera.callbacks += [on_image]
class AbstractTextToSpeech(object):
"""
Abstract Text To Speech
Parameters
----------
language: str
`Language Code <https://cloud.google.com/speech/docs/languages>`_
"""
def __init__(self, language):
# type: (str) -> None
self._language = language
self._queue = Queue()
self._talking_jobs = 0
self._scheduler = Scheduler(self._worker, name="TextToSpeechThread")
self._scheduler.start()
self._log = logger.getChild(self.__class__.__name__)
@property
def language(self):
# type: () -> str
"""
`Language Code <https://cloud.google.com/speech/docs/languages>`_
Returns
-------
language: str
`Language Code <https://cloud.google.com/speech/docs/languages>`_
"""
return self._language
@property
def talking(self):
# type: () -> bool
"""
Returns whether system is currently producing speech
Returns
-------
talking: bool
Whether system is currently producing speech
"""
return self._talking_jobs >= 1
def say(self, text, animation=None, block=False):
# type: (Union[str, unicode], Optional[str], bool) -> None
"""
Say Text (with optional Animation) through Text-to-Speech
Parameters
----------
text: str
Text to say through Text-to-Speech
animation: str or None
(Naoqi) Animation to play
block: bool
Whether this function should block or immediately return after calling
"""
# self._log.info(text.replace('\n', ' '))
self._talking_jobs += 1
self._queue.put((text, animation))
while block and self.talking:
sleep(1E-3)
def on_text_to_speech(self, text, animation=None):
# type: (Union[str, unicode], Optional[str]) -> None
"""
Say something through Text to Speech (Implementation)
Text To Speech Backends should implement this function
This function should block while speech is being produced
Parameters
----------
text: str
animation: str
"""
raise NotImplementedError()
def _worker(self):
self.on_text_to_speech(*self._queue.get())
self._talking_jobs -= 1
class AbstractMicrophone(object):
"""
Abstract Microphone
Parameters
----------
rate: int
Samples per Second
channels: int
Number of Channels
callbacks: list of callable
Functions to call each time some audio samples are captured
"""
def __init__(self, rate, channels, callbacks):
# type: (int, int, List[Callable[[np.ndarray], None]]) -> None
self._rate = rate
self._channels = channels
self._callbacks = callbacks
# Variables to do some performance statistics
self._dt_buffer = deque([], maxlen=32)
self._true_rate = rate
self._t0 = time()
# Create Queue and Sound Processor:
# Each time audio samples are captured it is put in the audio processing queue
# In a separate thread, the _processor worker takes these samples and calls all registered callbacks.
# This way, samples are not accidentally skipped (NAOqi has some very strict timings)
self._queue = Queue()
self._processor_scheduler = Scheduler(self._processor,
0,
name="MicrophoneThread")
self._processor_scheduler.start()
# Default behaviour is to not run by default. Calling AbstractApplication.run() will activate the microphone
self._running = False
self._log = logger.getChild(self.__class__.__name__)
@property
def rate(self):
# type: () -> int
"""
Audio bit rate
Returns
-------
rate: int
Audio bit rate
"""
return self._rate
@property
def true_rate(self):
# type: () -> float
"""
Actual Audio bit rate
Audio bit rate after accounting for latency & performance realities
Returns
-------
true_rate:
Actual Audio bit rate
"""
return self._true_rate
@property
def channels(self):
# type: () -> int
"""
Audio channels
Returns
-------
channels: int
Audio channels
"""
return self._channels
@property
def callbacks(self):
# type: () -> List[Callable[[np.ndarray], None]]
"""
Get/Set :func:`~AbstractCamera.on_audio` Callbacks
Returns
-------
callbacks: list of callable
"""
return self._callbacks
@callbacks.setter
def callbacks(self, value):
# type: (List[Callable[[np.ndarray], None]]) -> None
"""
Get/Set :func:`~AbstractCamera.on_audio` Callbacks
Parameters
----------
value: list of callable
"""
self._callbacks = value
@property
def running(self):
# type: () -> bool
"""
Returns whether Microphone is Running
Returns
-------
running: bool
"""
return self._running
def on_audio(self, audio):
# type: (np.ndarray) -> None
"""
On Audio Event, Called for every frame of audio captured by Microphone
Microphone Modules should call this function for every frame of audio acquired by Microphone
Parameters
----------
audio: np.ndarray
"""
self._queue.put(audio)
def start(self):
"""Start Microphone Stream"""
self._running = True
def stop(self):
"""Stop Microphone Stream"""
self._running = False
def _processor(self):
"""
Audio Processor
Calls each callback for each audio frame, threaded, for higher audio throughput
"""
# Get Audio Samples from Buffer
audio = self._queue.get()
# Call each regisered Callback with Samples
if self._running:
for callback in self.callbacks:
callback(audio)
# Update Statistics
self._update_dt(len(audio))
def _update_dt(self, n_bytes):
t1 = time()
self._dt_buffer.append((t1 - self._t0))
self._t0 = t1
self._true_rate = n_bytes / np.mean(self._dt_buffer)
class AbstractCamera(object):
"""
Abstract Camera
Parameters
----------
resolution: CameraResolution
:class:`~pepper.config.CameraResolution`
rate: int
callbacks: list of callable
"""
def __init__(self, resolution, rate, callbacks):
self._resolution = resolution
self._width = self._resolution.value[1]
self._height = self._resolution.value[0]
self._rate = rate
self._callbacks = callbacks
self._shape = np.array([self.height, self.width, self.channels])
self._dt_buffer = deque([], maxlen=10)
self._true_rate = rate
self._t0 = time()
self._mailbox = Mailbox()
self._processor_scheduler = Scheduler(self._processor,
name="CameraThread")
self._processor_scheduler.start()
self._running = False
self._log = logger.getChild(self.__class__.__name__)
@property
def resolution(self):
"""
Returns :class:`~pepper.config.CameraResolution`
Returns
-------
resolution: CameraResolution
"""
return self._resolution
@property
def width(self):
"""
Image Width
Returns
-------
width: int
Image width
"""
return self._width
@property
def height(self):
"""
Image Height
Returns
-------
height: int
Image height
"""
return self._height
@property
def channels(self):
"""
Image (Color) Channels
Returns
-------
channels: int
Image (Color) channels
"""
return 3
@property
def rate(self):
"""
Image Rate
Returns
-------
rate: int
Image rate
"""
return self._rate
@property
def true_rate(self):
"""
Actual Image Rate
Image rate after accounting for latency & performance realities
Returns
-------
true_rate: float
Actual Image Rate
"""
return self._true_rate
@property
def shape(self):
"""
Image Shape
Returns
-------
shape: np.ndarray
Image Shape
"""
return self._shape
@property
def callbacks(self):
"""
Get/Set :func:`~AbstractCamera.on_image` Callbacks
Returns
-------
callbacks: list of callable
on_image callbacks
"""
return self._callbacks
@callbacks.setter
def callbacks(self, value):
"""
Get/Set :func:`~AbstractCamera.on_image` Callbacks
Parameters
----------
value: list of callable
"""
self._callbacks = value
@property
def running(self):
"""
Returns whether Camera is Running
Returns
-------
running: bool
"""
return self._running
def on_image(self, image):
"""
On Image Event, Called for every Image captured by Camera
Camera Modules should call this function for every frame acquired by the Camera
Parameters
----------
image: np.ndarray
"""
self._mailbox.put(image)
def start(self):
"""Start Streaming Images from Camera"""
self._running = True
def stop(self):
"""Stop Streaming Images from Camera"""
self._running = False
def _processor(self):
"""
Image Processor
Calls each callback for each image, threaded, for higher image throughput
"""
image = self._mailbox.get()
if self._running:
for callback in self.callbacks:
callback(image)
self._update_dt()
def _update_dt(self):
t1 = time()
self._dt_buffer.append((t1 - self._t0))
self._t0 = t1
self._true_rate = 1 / np.mean(self._dt_buffer)
class AbstractCamera(object):
"""
Abstract Camera
Parameters
----------
resolution: CameraResolution
:class:`~pepper.config.CameraResolution`
rate: int
Camera Frames per Second
callbacks: List[Callable[[AbstractImage], None]]
Functions to call each time an AbstractImage is captured
"""
def __init__(self, resolution, rate, callbacks):
# type: (CameraResolution, int, List[Callable[[AbstractImage], None]]) -> None
# Extract Image Dimensions from CameraResolution
self._resolution = resolution
self._width = self._resolution.value[1]
self._height = self._resolution.value[0]
self._shape = np.array([self.height, self.width, self.channels])
# Store Camera Rate and Callbacks
self._rate = rate
self._callbacks = callbacks
# Variables to do some performance statistics
self._dt_buffer = deque([], maxlen=10)
self._true_rate = rate
self._t0 = time()
# Create Mailbox and Image Processor:
# Each time an image is captured it is put in the mailbox, overriding whatever there might currently be.
# In a separate thread, the _processor worker takes an image and calls all registered callbacks.
# This way the processing of images does not block the acquisition of new images,
# while at the same new images don't build up a queue, but are discarded when the _processor is too busy.
self._mailbox = Mailbox()
self._processor_scheduler = Scheduler(self._processor,
name="CameraThread")
self._processor_scheduler.start()
# Default behaviour is to not run by default. Calling AbstractApplication.run() will activate the camera
self._running = False
self._log = logger.getChild(self.__class__.__name__)
@property
def resolution(self):
# type: () -> CameraResolution
"""
Returns :class:`~pepper.config.CameraResolution`
Returns
-------
resolution: CameraResolution
"""
return self._resolution
@property
def width(self):
# type: () -> int
"""
Image Width
Returns
-------
width: int
Image width
"""
return self._width
@property
def height(self):
# type: () -> int
"""
Image Height
Returns
-------
height: int
Image height
"""
return self._height
@property
def channels(self):
# type: () -> int
"""
Number of Image (Color) Channels
Returns
-------
channels: int
Number of Image (Color) channels
"""
return 3
@property
def rate(self):
# type: () -> int
"""
Image Rate (Frames per Second)
Returns
-------
rate: int
Image rate (Frames per Second)
"""
return self._rate
@property
def true_rate(self):
# type: () -> float
"""
Actual Image Rate (Frames per Second)
Image rate after accounting for latency & performance realities
Returns
-------
true_rate: float
Actual Image Rate (Frames per Second)
"""
return self._true_rate
@property
def shape(self):
# type: () -> np.ndarray
"""
Image Shape (height, width, channels)
Returns
-------
shape: np.ndarray
Image Shape (height, width, channels)
"""
return self._shape
@property
def callbacks(self):
# type: () -> List[Callable[[AbstractImage], None]]
"""
Get/Set :func:`~AbstractCamera.on_image` Callbacks
Returns
-------
callbacks: list of callable
on_image callbacks
"""
return self._callbacks
@callbacks.setter
def callbacks(self, value):
# type: (List[Callable[[AbstractImage], None]]) -> None
"""
Get/Set :func:`~AbstractCamera.on_image` Callbacks
Parameters
----------
value: list of callable
"""
self._callbacks = value
@property
def running(self):
# type: () -> bool
"""
Returns whether Camera is Running
Returns
-------
running: bool
"""
return self._running
def on_image(self, image):
# type: (AbstractImage) -> None
"""
On Image Event, Called for every Image captured by Camera
Custom Camera Backends should call this function for every frame acquired by the Camera
Parameters
----------
image: AbstractImage
"""
self._mailbox.put(image)
def start(self):
"""Start Streaming Images from Camera"""
self._running = True
def stop(self):
"""Stop Streaming Images from Camera"""
self._running = False
def _processor(self):
"""
Image Processor
Calls each callback for each image, threaded, for higher image throughput and less image latency
"""
# Get latest image from Mailbox
image = self._mailbox.get()
# Call Every Registered Callback
if self._running:
for callback in self.callbacks:
callback(image)
# Update Statistics
self._update_dt()
def _update_dt(self):
t1 = time()
self._dt_buffer.append((t1 - self._t0))
self._t0 = t1
self._true_rate = 1 / np.mean(self._dt_buffer)
def __init__(self, backend):
"""
Construct Object Detection Component
Parameters
----------
backend: AbstractBackend
target: ObjectDetectionTarget
"""
super(ObjectDetectionComponent, self).__init__(backend)
# Callbacks
self.on_object_callbacks = []
clients = [
ObjectDetectionClient(target)
for target in ObjectDetectionComponent.TARGETS
]
mailboxes = {client: Mailbox()
for client in clients
} # type: Dict[ObjectDetectionClient, Mailbox]
def on_image(image):
"""
Raw On Image Event. Called every time the camera yields a frame.
Parameters
----------
image: np.ndarray
"""
for client in clients:
mailboxes[client].put(image)
def worker(client):
# type: (ObjectDetectionClient) -> None
"""Object Detection Event Worker"""
image = mailboxes[client].get()
objects = [
obj for obj in client.classify(image)
if obj.confidence > config.OBJECT_RECOGNITION_THRESHOLD
]
if objects:
# Call on_object Callback Functions
for callback in self.on_object_callbacks:
callback(image, objects)
# Call on_object Event Function
self.on_object(image, objects)
# Initialize Object Queue & Worker
schedule = [
Scheduler(worker,
args=(client, ),
name="{}Thread".format(client.target.name))
for client in clients
]
for s in schedule:
s.start()
# Add on_image to Camera Callbacks
self.backend.camera.callbacks += [on_image]
class AbstractTextToSpeech(object):
"""
Abstract Text To Speech
Parameters
----------
language: str
Language Code, See: https://cloud.google.com/speech/docs/languages
"""
def __init__(self, language):
self._language = language
self._queue = Queue()
self._talking_jobs = 0
self._scheduler = Scheduler(self._worker, name="TextToSpeechThread")
self._scheduler.start()
self._log = logger.getChild(self.__class__.__name__)
@property
def language(self):
"""
Language Code, See: https://cloud.google.com/speech/docs/languages
Returns
-------
language: str
Language Code, See: https://cloud.google.com/speech/docs/languages
"""
return self._language
@property
def talking(self):
"""
Returns whether system is currently producing speech
Returns
-------
talking: bool
Whether system is currently producing speech
"""
return self._talking_jobs >= 1
def say(self, text, animation=None, block=False):
"""
Say something through Text to Speech (Interface)
Parameters
----------
text: str
animation: str
"""
# self._log.info(text.replace('\n', ' '))
self._talking_jobs += 1
self._queue.put((text, animation))
while block and self.talking:
sleep(1E-3)
def on_text_to_speech(self, text, animation=None):
"""
Say something through Text to Speech (Implementation)
Text To Speech Modules should implement this function
This function should block while speech is being produced
Parameters
----------
text: str
animation: str
"""
pass
def _worker(self):
self.on_text_to_speech(*self._queue.get())
self._talking_jobs -= 1
