class PreciseConnection: """Represents a socket connection routed to a precise process""" def __init__(self, connection, address): self.address = address self.connection = connection # type: socket.socket self.stream = ReadWriteStream() self.runner = PreciseRunner( ListenerEngine(Listener(MODEL_NAME, CHUNK_SIZE), CHUNK_SIZE), 1, stream=self.stream, on_activation=self.on_activation, on_prediction=self.on_prediction ) self.runner.start() def on_activation(self): print(' --- ACTIVATION from:', self.address, '--- ') def on_prediction(self, conf): print('!' if conf > 0.5 else '.', end='', flush=True) def update(self): """Return whether connection still alive""" data = self.connection.recv(CHUNK_SIZE) self.stream.write(data) return bool(data) def close(self): print('Closing connection from', self.address) self.runner.stop() self.connection.close()
class Precise(): def __init__(self): # engine data: https://github.com/mycroftai/precise-data/tree/dist # precise models: https://github.com/MycroftAI/precise-data/tree/models?files=1 sensitivity = float(0.5) trigger_level = int(3) model_path = "models/hey-mycroft.pb" engine_path = "precise-engine/precise-engine" engine = PreciseEngine( engine_path, model_path) self.stream = ReadWriteStream() self.runner = PreciseRunner( engine, sensitivity=sensitivity, trigger_level=trigger_level, on_activation=self.on_activation ) print("Starting precise") self.runner.start() while True: data = self.stream.read() self.update_runner(data) def update_runner(self, data): self.stream.write(data) def on_activation(self): print("Activation!")
def detect(): event = threading.Event() runner = PreciseRunner(engine, on_activation=lambda: event.set()) runner.start() event.wait() wake_statuses[system] = 'detected' runner.stop()
def main(): args = create_parser(usage).parse_args() print('chunk_size: ', args.chunk_size) def on_activation(): activate_notify() if args.save_dir: global chunk_num nm = join(args.save_dir, args.save_prefix + session_id + '.' + str(chunk_num) + '.wav') save_audio(nm, audio_buffer) print() print('Saved to ' + nm + '.') chunk_num += 1 def on_prediction(conf): print('!' if conf > 0.8 else '.', end='', flush=True) listener = Listener(args.model, args.chunk_size) audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float) def get_prediction(chunk): nonlocal audio_buffer audio = buffer_to_audio(chunk) audio_buffer = np.concatenate((audio_buffer[len(audio):], audio)) return listener.update(chunk) engine = ListenerEngine(listener, args.chunk_size) engine.get_prediction = get_prediction runner = PreciseRunner(engine, args.threshold, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
def async_listen(call): nonlocal runner, detected_event hass.states.async_set(OBJECT_DECODER, STATE_LISTENING, state_attrs) engine = PreciseEngine('precise-engine', model) runner = PreciseRunner(engine, sensitivity=sensitivity, trigger_level=trigger_level, on_activation=lambda: detected_event.set()) # Runs in a separate thread detected_event.clear() runner.start() yield from asyncio.get_event_loop().run_in_executor(None, detected_event.wait) if not terminated: runner.stop() runner = None hass.states.async_set(OBJECT_DECODER, STATE_IDLE, state_attrs) # Fire detected event hass.bus.async_fire(EVENT_HOTWORD_DETECTED, { 'name': name, # name of the component 'model': model # model used })
def run(self): def on_activation(): activate_notify() def on_prediction(conf): print('!' if conf > 0.5 else '.', end='', flush=True) args = self.args runner = PreciseRunner(ListenerEngine( PocketsphinxListener(args.key_phrase, args.dict_file, args.hmm_folder, args.threshold, args.chunk_size)), 3, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
def main(): args = create_parser(usage).parse_args() def on_activation(): Popen(['aplay', '-q', 'data/activate.wav']) def on_prediction(conf): print('!' if conf > 0.5 else '.', end='', flush=True) runner = PreciseRunner(ListenerEngine( PocketsphinxListener(args.key_phrase, args.dict_file, args.hmm_folder, args.threshold, args.chunk_size)), 3, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
def main(): args = create_parser(usage).parse_args() sensitivity = 0.5 def on_activation(): activate_notify() if args.save_dir: global chunk_num nm = join( args.save_dir, args.save_prefix + session_id + '.' + str(chunk_num) + '.wav') save_audio(nm, audio_buffer) print() print('Saved to ' + nm + '.') chunk_num += 1 def on_prediction(conf): if args.light_mode: print('!' if conf > 0.7 else '.', end='', flush=True) else: max_width = 80 width = min(get_terminal_size()[0], max_width) units = int(round(conf * width)) bar = 'X' * units + '-' * (width - units) cutoff = round((1.0 - sensitivity) * width) print(bar[:cutoff] + bar[cutoff:].replace('X', 'x')) listener = Listener(args.model, args.chunk_size) audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float) def get_prediction(chunk): nonlocal audio_buffer audio = buffer_to_audio(chunk) audio_buffer = np.concatenate((audio_buffer[len(audio):], audio)) return listener.update(chunk) engine = ListenerEngine(listener, args.chunk_size) engine.get_prediction = get_prediction runner = PreciseRunner(engine, args.threshold, sensitivity=sensitivity, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
class MycroftScript(BaseScript): usage = Usage(__doc__) def __init__(self, args): super().__init__(args) if args.model == 'hey-mycroft': args.model = None self.engine = PreciseEngine(exe_file=None, model_file=args.model, chunk_size=args.chunk_size) self.runner = PreciseRunner(self.engine, args.trigger_level, sensitivity=args.sensitivity, on_activation=self.on_activation, on_prediction=self.on_prediction) self.session_id, self.chunk_num = '%09d' % randint(0, 999999999), 0 def on_activation(self): activate_notify() if self.args.save_dir: nm = join( self.args.save_dir, self.args.save_prefix + self.session_id + '.' + str(self.chunk_num) + '.wav') save_audio(nm, self.audio_buffer) print() print('Saved to ' + nm + '.') self.chunk_num += 1 def on_prediction(self, conf): if self.args.basic_mode: print('!' if conf > 0.7 else '.', end='', flush=True) else: max_width = 80 width = min(get_terminal_size()[0], max_width) units = int(round(conf * width)) bar = 'X' * units + '-' * (width - units) cutoff = round((1.0 - self.args.sensitivity) * width) print(bar[:cutoff] + bar[cutoff:].replace('X', 'x')) def run(self): self.runner.start() Event().wait() # Wait forever
class PreciseEngine(WakeWordEnginePlugin): program_url = ( 'https://raw.githubusercontent.com/MycroftAI/' 'precise-data/dist/{arch}/precise-engine.tar.gz' ) model_url = ( 'https://raw.githubusercontent.com/MycroftAI/' 'precise-data/models/{model_name}.tar.gz' ) def __init__(self, rt, on_activation: Callable): super().__init__(rt, on_activation) exe_file = which('precise-engine') precise_folder = join(self.rt.paths.user_config, 'precise') if not exe_file: exe_file = join(precise_folder, 'precise-engine', 'precise-engine') download_extract_tar( self.program_url.format(arch=platform.machine()), precise_folder, check_md5=False, subdir='precise-engine', on_update=lambda: self.rt.interfaces.faceplate.text('Updating listener...'), on_complete=lambda: self.rt.interfaces.faceplate.reset() ) log.debug('Using precise executable: ' + exe_file) model_folder = join(precise_folder, 'models', self.wake_word) model_file = join(model_folder, self.wake_word + '.pb') model_url = self.model_url.format(model_name=self.wake_word) download_extract_tar(model_url, model_folder, check_md5=True) from precise_runner import PreciseRunner, PreciseEngine engine = PreciseEngine(exe_file, model_file, chunk_size=1024) self.runner = PreciseRunner(engine, on_activation=on_activation) def startup(self): self.runner.start() def shutdown(self): self.runner.stop() def continue_listening(self): self.runner.play() def pause_listening(self): self.runner.pause()
def main(): args = create_parser(usage).parse_args() def on_activation(): activate_notify() if args.save_dir: global chunk_num nm = join( args.save_dir, args.save_prefix + session_id + '.' + str(chunk_num) + '.wav') save_audio(nm, audio_buffer) print() print('Saved to ' + nm + '.') chunk_num += 1 def on_prediction(conf): global detecting_flag # print('!' if conf > 0.5 else '.', end='', flush=True) if conf > 0.5: detecting_flag = True if conf < 0.5 and detecting_flag: print(colored("Yeah! I'm Here.", 'green')) detecting_flag = False sunshine_model = './ok-sunshine.net' listener = Listener(sunshine_model, args.chunk_size) audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float) def get_prediction(chunk): nonlocal audio_buffer audio = buffer_to_audio(chunk) audio_buffer = np.concatenate((audio_buffer[len(audio):], audio)) return listener.update(chunk) engine = ListenerEngine(listener, args.chunk_size) engine.get_prediction = get_prediction runner = PreciseRunner(engine, args.threshold, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
def main(): rospy.init_node('wake_word_detection_node') print("node is up") def on_activation(): print("activate") playsound(res_path + "/attention.wav") try: requests.get('http://www.google.com') try: response = stop_speech_perception_service(True) print(response) except rospy.ServiceException as exc: print("Service did not process request: " + str(exc)) except requests.ConnectionError: print("no internet") speak_pub.publish( "I'm sorry. I am not connected to the internet now and cannot answer" ) set_emotion_service(state="SADNESS", timeout=5500, restore=True) def on_prediction(conf): print(".") listener = Listener(res_path + "/stevie_10_06.pb", chunk_size) audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float) def get_prediction(chunk): nonlocal audio_buffer audio = buffer_to_audio(chunk) audio_buffer = np.concatenate((audio_buffer[len(audio):], audio)) return listener.update(chunk) engine = ListenerEngine(listener, chunk_size) engine.get_prediction = get_prediction runner = PreciseRunner(engine, trigger_level=3, sensitivity=0.5, on_activation=on_activation, on_prediction=on_prediction) runner.start() print("spinning") rospy.spin()
def main(): args = create_parser(usage).parse_args() def on_activation(): activate_notify() # TODO: trigger VMSE def on_prediction(conf): if args.basic_mode: print('!' if conf > 0.7 else '.', end='', flush=True) else: max_width = 80 width = min(get_terminal_size()[0], max_width) units = int(round(conf * width)) bar = 'X' * units + '-' * (width - units) cutoff = round((1.0 - args.sensitivity) * width) print(bar[:cutoff] + bar[cutoff:].replace('X', 'x')) listener = Listener(args.model, args.chunk_size) audio_buffer = np.zeros(listener.pr.buffer_samples, dtype=float) def get_prediction(chunk): nonlocal audio_buffer audio = buffer_to_audio(chunk) audio_buffer = np.concatenate((audio_buffer[len(audio):], audio)) return listener.update(chunk) engine = ListenerEngine(listener, args.chunk_size) engine.get_prediction = get_prediction runner = PreciseRunner(engine, args.trigger_level, sensitivity=args.sensitivity, on_activation=on_activation, on_prediction=on_prediction) runner.start() Event().wait() # Wait forever
class PreciseWakeListener(RhasspyActor): """Listens for a wake word using Mycroft Precise.""" def __init__(self) -> None: # pylint: disable=E0401 from precise_runner import ReadWriteStream RhasspyActor.__init__(self) self.audio_buffer: bytes = bytes() self.audio_info: Dict[Any, Any] = {} self.chunk_delay = 0 self.chunk_size = 2048 self.detected: bool = False self.engine = None self.engine_path = "" self.model_name = "" self.model_path = "" self.prediction_sem = threading.Semaphore() self.preload = False self.receivers: List[RhasspyActor] = [] self.recorder: Optional[RhasspyActor] = None self.runner = None self.send_not_detected = False self.stream: Optional[ReadWriteStream] = None def to_started(self, from_state: str) -> None: """Transition to started state.""" self.recorder = self.config["recorder"] self.preload = self.config.get("preload", False) self.send_not_detected = self.config.get("not_detected", False) self.chunk_size = self.profile.get("wake.precise.chunk_size", 2048) self.chunk_delay = self.profile.get("wake.precise.chunk_delay", 0) if self.preload: try: self.load_runner() except Exception: pass self.transition("loaded") def in_loaded(self, message: Any, sender: RhasspyActor) -> None: """Handle messages in loaded state.""" if isinstance(message, ListenForWakeWord): try: self.load_runner() self.receivers.append(message.receiver or sender) self.transition("listening") if message.record: self.send(self.recorder, StartStreaming(self.myAddress)) except Exception: self._logger.exception("in_loaded") def in_listening(self, message: Any, sender: RhasspyActor) -> None: """Handle messages in listening state.""" try: if isinstance(message, AudioData): self.audio_info = message.info self.detected = False self.audio_buffer += message.data num_chunks = len(self.audio_buffer) // self.chunk_size if num_chunks > 0: assert self.stream is not None self.prediction_sem = threading.Semaphore() for _ in range(num_chunks): chunk = self.audio_buffer[:self.chunk_size] self.stream.write(chunk) self.audio_buffer = self.audio_buffer[self.chunk_size:] if self.send_not_detected: # Wait for all chunks to finish processing for _ in range(num_chunks): self.prediction_sem.acquire(timeout=0.1) # Wait a little bit for the precise engine to finish processing time.sleep(self.chunk_delay) if not self.detected: # Not detected not_detected_event = WakeWordNotDetected( self.model_name, audio_data_info=message.info) for receiver in self.receivers: self.send(receiver, not_detected_event) elif isinstance(message, StopListeningForWakeWord): self.receivers.remove(message.receiver or sender) if len(self.receivers) == 0: if message.record: self.send(self.recorder, StopStreaming(self.myAddress)) self.transition("loaded") elif isinstance(message, str): # Detected self._logger.debug("Hotword detected (%s)", self.model_name) detected_event = WakeWordDetected( self.model_name, audio_data_info=self.audio_info) for receiver in self.receivers: self.send(receiver, detected_event) except Exception: self._logger.exception("in_listening") def to_stopped(self, from_state: str) -> None: """Transition to stopped state.""" self.stream = None if self.runner is not None: self.runner.stop() # ------------------------------------------------------------------------- def load_runner(self) -> None: """Load precise runner.""" if self.engine is None: # pylint: disable=E0401 from precise_runner import PreciseEngine self.model_name = self.profile.get("wake.precise.model", "hey-mycroft-2.pb") self.model_path = self.profile.read_path(self.model_name) self.engine_path = os.path.expandvars( self.profile.get("wake.precise.engine_path", "precise-engine")) self._logger.debug("Loading Precise engine at %s", self.engine_path) self.engine = PreciseEngine(self.engine_path, self.model_path, chunk_size=self.chunk_size) if self.runner is None: # pylint: disable=E0401 from precise_runner import PreciseRunner, ReadWriteStream self.stream = ReadWriteStream() sensitivity = float( self.profile.get("wake.precise.sensitivity", 0.5)) trigger_level = int( self.profile.get("wake.precise.trigger_level", 3)) def on_prediction(prob: float) -> None: self.prediction_sem.release() def on_activation() -> None: self.detected = True self.send(self.myAddress, "activated") self.runner = PreciseRunner( self.engine, stream=self.stream, sensitivity=sensitivity, trigger_level=trigger_level, on_activation=on_activation, on_prediction=on_prediction, ) assert self.runner is not None self.runner.start() self._logger.debug( "Loaded Mycroft Precise (model=%s, sensitivity=%s, trigger_level=%s)", self.model_path, sensitivity, trigger_level, ) # ------------------------------------------------------------------------- def get_problems(self) -> Dict[str, Any]: """Get problems at startup.""" problems: Dict[str, Any] = {} try: # pylint: disable=E0401,W0611 from precise_runner import PreciseRunner, ReadWriteStream # noqa: F401 except Exception: problems[ "precise_runner not installed"] = "The precise_runner Python library is not installed. Try pip3 install precise_runner" engine_path = os.path.expandvars( self.profile.get("wake.precise.engine_path", "precise-engine")) if not os.path.exists(engine_path) and not shutil.which(engine_path): problems[ "Missing precise-engine"] = 'The Mycroft Precise engine is not installed. Follow the <a href="https://github.com/MycroftAI/mycroft-precise#binary-install">binary install instructions</a>.' model_name = self.profile.get("wake.precise.model", "hey-mycroft-2.pb") model_path = self.profile.read_path(model_name) if not os.path.exists(model_path): problems[ "Missing model"] = f"Your Mycroft Precise model could not be loaded from {model_path}" return problems
class PreciseHotword(HotWordEngine): """Precice is the default wakeword engine for mycroft. Precise is developed by Mycroft AI and produces quite good wake word spotting when trained on a decent dataset. """ def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"): super().__init__(key_phrase, config, lang) from precise_runner import ( PreciseRunner, PreciseEngine, ReadWriteStream ) local_conf = LocalConf(USER_CONFIG) if (local_conf.get('precise', {}).get('dist_url') == 'http://bootstrap.mycroft.ai/artifacts/static/daily/'): del local_conf['precise']['dist_url'] local_conf.store() Configuration.updated(None) self.download_complete = True self.show_download_progress = Timer(0, lambda: None) precise_config = Configuration.get()['precise'] precise_exe = self.update_precise(precise_config) local_model = self.config.get('local_model_file') if local_model: self.precise_model = expanduser(local_model) else: self.precise_model = self.install_model( precise_config['model_url'], key_phrase.replace(' ', '-') ).replace('.tar.gz', '.pb') self.has_found = False self.stream = ReadWriteStream() def on_activation(): self.has_found = True trigger_level = self.config.get('trigger_level', 3) sensitivity = self.config.get('sensitivity', 0.5) self.runner = PreciseRunner( PreciseEngine(precise_exe, self.precise_model), trigger_level, sensitivity, stream=self.stream, on_activation=on_activation, ) self.runner.start() def update_precise(self, precise_config): """Continously try to download precise until successful""" precise_exe = None while not precise_exe: try: precise_exe = self.install_exe(precise_config['dist_url']) except TriggerReload: raise except Exception as e: LOG.error( 'Precise could not be downloaded({})'.format(repr(e))) if exists(self.install_destination): precise_exe = self.install_destination else: # Wait one minute before retrying sleep(60) return precise_exe @property def folder(self): return join(expanduser('~'), '.mycroft', 'precise') @property def install_destination(self): return join(self.folder, 'precise-engine', 'precise-engine') def install_exe(self, url: str) -> str: url = url.format(arch=platform.machine()) if not url.endswith('.tar.gz'): url = requests.get(url).text.strip() if install_package( url, self.folder, on_download=self.on_download, on_complete=self.on_complete ): raise TriggerReload return self.install_destination def install_model(self, url: str, wake_word: str) -> str: model_url = url.format(wake_word=wake_word) model_file = join(self.folder, posixpath.basename(model_url)) try: install_package( model_url, self.folder, on_download=lambda: LOG.info('Updated precise model') ) except (HTTPError, ValueError): if isfile(model_file): LOG.info("Couldn't find remote model. Using local file") else: raise NoModelAvailable('Failed to download model:', model_url) return model_file @staticmethod def _snd_msg(cmd): with suppress(OSError): with open('/dev/ttyAMA0', 'w') as f: print(cmd, file=f) def on_download(self): LOG.info('Downloading Precise executable...') if isdir(join(self.folder, 'precise-stream')): rmtree(join(self.folder, 'precise-stream')) for old_package in glob(join(self.folder, 'precise-engine_*.tar.gz')): os.remove(old_package) self.download_complete = False self.show_download_progress = Timer( 5, self.during_download, args=[True] ) self.show_download_progress.start() def during_download(self, first_run=False): LOG.info('Still downloading executable...') if first_run: # TODO: Localize self._snd_msg('mouth.text=Updating listener...') if not self.download_complete: self.show_download_progress = Timer(30, self.during_download) self.show_download_progress.start() def on_complete(self): LOG.info('Precise download complete!') self.download_complete = True self.show_download_progress.cancel() self._snd_msg('mouth.reset') def update(self, chunk): self.stream.write(chunk) def found_wake_word(self, frame_data): if self.has_found: self.has_found = False return True return False def stop(self): if self.runner: self.runner.stop()
class WebsocketAudioListener(Thread): def __init__(self, factory, client, queue, sample_rate=16000): super(WebsocketAudioListener, self).__init__() self.client = client self.factory = factory self.sample_rate = sample_rate self.vad = webrtcvad.Vad(1) self.queue = queue self.hotword_found = False self.hotword_stream = ReadWriteStream() def on_activation(): self.hotword_found = True trigger_level = 1 sensitivity = 0.5 self.hotword_runner = PreciseRunner( PreciseEngine('/opt/backend/precise-engine/precise-engine', '/opt/backend/hey-mycroft.pb'), trigger_level, sensitivity, stream=self.hotword_stream, on_activation=on_activation, ) self.hotword_runner.start() BLOCKS_PER_SECOND = 50 self.block_size = int(self.sample_rate / float(BLOCKS_PER_SECOND)) # 320 padding_ms = 600 block_duration_ms = 1000 * \ self.block_size // self.sample_rate # 20 num_padding_blocks = padding_ms // block_duration_ms # 30 self.ratio = 0.75 self.ring_buffer = deque(maxlen=num_padding_blocks) self.triggered = False self.running = True def run(self): while self.keep_running(): self.wait_for_hotword() if not self.keep_running(): break results = CALL_STT(self.vad_generator( )) # pseudo-code, change with method that will receive audio bytes try: for r in results: try: if r.chunks[0].final: self.emit_utterance( r.chunks[0].alternatives[0].text) break except LookupError: LOG.debug('No available chunks') except grpc._channel._Rendezvous as err: LOG.error('Error code %s, message: %s' % (err._state.code, err._state.details)) self.stop() def keep_running(self): return self.running and self.factory.clients.get(self.client.peer) def queue_generator(self): audio_data = bytearray() while self.keep_running(): if len(audio_data) < self.block_size: try: audio_from_queue = self.queue.get(timeout=1) audio_data.extend(audio_from_queue) except Empty: pass else: audio_block = audio_data[:self.block_size] audio_data = audio_data[self.block_size:] yield audio_block def wait_for_hotword(self): client_config = self.factory.clients.get(self.client.peer) if not client_config or not client_config.get('use_hotword'): LOG.debug('Skip hotword detection') return LOG.debug(client_config) buffered_audio = bytearray() LOG.debug('Start hotword detection') self.factory.emit_hotword_message_to_ona("start", self.client) for audio_block in self.queue_generator(): buffered_audio.extend(audio_block) if len(buffered_audio) > 2048: self.hotword_stream.write(bytes(buffered_audio[:2048])) buffered_audio = buffered_audio[2048:] if not client_config.get('use_hotword'): break if self.hotword_found: self.hotword_found = False LOG.debug('Hotword detected') self.factory.emit_hotword_message_to_ona( "detected", self.client) break def vad_generator(self): for audio_block in self.queue_generator(): try: is_speech = self.vad.is_speech(audio_block, self.sample_rate) except: is_speech = False if not self.triggered: self.ring_buffer.append((audio_block, is_speech)) num_voiced = len( [f for f, speech in self.ring_buffer if speech]) if num_voiced > self.ratio * self.ring_buffer.maxlen: self.triggered = True for f, s in self.ring_buffer: yield f self.ring_buffer.clear() else: yield audio_block self.ring_buffer.append((audio_block, is_speech)) num_unvoiced = len( [f for f, speech in self.ring_buffer if not speech]) if num_unvoiced > self.ratio * self.ring_buffer.maxlen: self.triggered = False self.ring_buffer.clear() break def emit_utterance(self, utterance): if len(utterance) > 0: self.factory.emit_utterance_to_bus(self.client, utterance) def stop(self): self.hotword_runner.stop() self.running = False
class PreciseHotword(HotWordEngine): def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"): super(PreciseHotword, self).__init__(key_phrase, config, lang) from precise_runner import ( PreciseRunner, PreciseEngine, ReadWriteStream ) local_conf = LocalConf(USER_CONFIG) if local_conf.get('precise', {}).get('dist_url') == \ 'http://bootstrap.mycroft.ai/artifacts/static/daily/': del local_conf['precise']['dist_url'] local_conf.store() Configuration.updated(None) self.download_complete = True self.show_download_progress = Timer(0, lambda: None) precise_config = Configuration.get()['precise'] precise_exe = self.install_exe(precise_config['dist_url']) local_model = self.config.get('local_model_file') if local_model: self.precise_model = expanduser(local_model) else: self.precise_model = self.install_model( precise_config['model_url'], key_phrase.replace(' ', '-') ).replace('.tar.gz', '.pb') self.has_found = False self.stream = ReadWriteStream() def on_activation(): self.has_found = True self.runner = PreciseRunner( PreciseEngine(precise_exe, self.precise_model), stream=self.stream, on_activation=on_activation ) self.runner.start() @property def folder(self): return join(expanduser('~'), '.mycroft', 'precise') def install_exe(self, url: str) -> str: url = url.format(arch=platform.machine()) if not url.endswith('.tar.gz'): url = requests.get(url).text.strip() if install_package( url, self.folder, on_download=self.on_download, on_complete=self.on_complete ): raise TriggerReload return join(self.folder, 'precise-engine', 'precise-engine') def install_model(self, url: str, wake_word: str) -> str: model_url = url.format(wake_word=wake_word) model_file = join(self.folder, posixpath.basename(model_url)) try: install_package( model_url, self.folder, on_download=lambda: LOG.info('Updated precise model') ) except (HTTPError, ValueError): if isfile(model_file): LOG.info("Couldn't find remote model. Using local file") else: raise NoModelAvailable('Failed to download model:', model_url) return model_file @staticmethod def _snd_msg(cmd): with suppress(OSError): with open('/dev/ttyAMA0', 'w') as f: print(cmd, file=f) def on_download(self): LOG.info('Downloading Precise executable...') if isdir(join(self.folder, 'precise-stream')): rmtree(join(self.folder, 'precise-stream')) for old_package in glob(join(self.folder, 'precise-engine_*.tar.gz')): os.remove(old_package) self.download_complete = False self.show_download_progress = Timer( 5, self.during_download, args=[True] ) self.show_download_progress.start() def during_download(self, first_run=False): LOG.info('Still downloading executable...') if first_run: # TODO: Localize self._snd_msg('mouth.text=Updating listener...') if not self.download_complete: self.show_download_progress = Timer(30, self.during_download) self.show_download_progress.start() def on_complete(self): LOG.info('Precise download complete!') self.download_complete = True self.show_download_progress.cancel() self._snd_msg('mouth.reset') def update(self, chunk): self.stream.write(chunk) def found_wake_word(self, frame_data): if self.has_found: self.has_found = False return True return False def stop(self): if self.runner: self.runner.stop()
time.sleep(3) print('hotword detection restart') ros.pub_hotword.publish(False) def start(): print('hotword node start') while not rospy.is_shutdown(): pass print('hotword node stop') if __name__ == "__main__": #ros ros = HotWordRos() rospy.init_node('Hotword', anonymous=True) rate = rospy.Rate(10) # 10hz # engine = PreciseEngine('/home/rastech/catkin_ws/src/venv/bin/precise-engine', # '/home/rastech/catkin_ws/src/fero_speaker/script/precise-data-models/new_ifero_191021.pb') engine = PreciseEngine(precise_engine, wake_word_model) runner = PreciseRunner(engine, sensitivity=0.5, on_activation=lambda: detectd()) runner.start() try: start() except rospy.ROSInterruptException: pass
def main(): # Parse arguments parser = argparse.ArgumentParser(description='mycroft-precise') parser.add_argument('--host', help='MQTT host (default=localhost)', type=str, default='localhost') parser.add_argument('--port', help='MQTT port (default=1883)', type=int, default=1883) parser.add_argument('--site-id', help='Hermes siteId (default=default)', type=str, default='default') parser.add_argument('--wakeword-id', help='Hermes wakewordId (default=default)', type=str, default='default') parser.add_argument('--model', type=str, required=True, help='Path to model file (.pb)') parser.add_argument('--sensitivity', help='Model sensitivity (default=0.5)', type=float, default=0.5) parser.add_argument('--trigger-level', help='Number of trigger events required (default=3)', type=int, default=3) parser.add_argument('--feedback', help='Show printed feedback', action='store_true') args = parser.parse_args() topic_audio_frame = 'hermes/audioServer/%s/audioFrame' % args.site_id topic_hotword_detected = 'hermes/hotword/%s/detected' % args.wakeword_id # Create runner engine = PreciseEngine('precise-engine', args.model) stream = ByteStream() client = mqtt.Client() first_frame = True def on_activation(): nonlocal first_frame if args.feedback: print('!', end='', flush=True) logging.debug('Hotword detected!') payload = json.dumps({ 'siteId': args.site_id, 'modelId': args.model, 'modelVersion': '', 'modelType': 'personal', 'currentSensitivity': args.sensitivity }).encode() client.publish(topic_hotword_detected, payload) first_frame = True runner = PreciseRunner(engine, stream=stream, sensitivity=args.sensitivity, trigger_level=args.trigger_level, on_activation=on_activation) # Set up MQTT def on_connect(client, userdata, flags, rc): client.subscribe(topic_audio_frame) logging.debug('Connected to %s' % args.host) def on_message(client, userdata, message): nonlocal first_frame try: if message.topic == topic_audio_frame: if first_frame: logging.debug('Receiving audio data') first_frame = False if args.feedback: print('.', end='', flush=True) # Extract audio data with io.BytesIO(message.payload) as wav_buffer: with wave.open(wav_buffer, mode='rb') as wav_file: audio_data = wav_file.readframes(wav_file.getnframes()) stream.write(audio_data) except Exception as e: logging.exception('on_message') client.on_connect = on_connect client.on_message = on_message client.connect(args.host, args.port) runner.start() try: logging.info('Listening') client.loop_forever() except KeyboardInterrupt: pass try: stream.close() runner.stop() except: pass
def start_listening(self): import gc, queue, json, threading, alsaaudio from vosk import Model, KaldiRecognizer from precise_runner import PreciseEngine, PreciseRunner, ReadWriteStream # Note: Seemingly precise-engine wants N samples, not N bytes (as documented) COMMAND_DURATON = 64 class Stream: QUEUE_DEPTH = 64 CHUNK_SIZE = 2048 def __init__(self): self.queue = queue.Queue(self.QUEUE_DEPTH) def write(self, chunk): assert (len(chunk) == self.CHUNK_SIZE) self.queue.put(chunk) def read(self, size): assert (size * 2 == self.CHUNK_SIZE) chunk = self.queue.get() return bytes(chunk) def clear(self): try: while True: self.queue.get(block=False) except queue.Empty: pass wake_stream = Stream() vosk_stream = Stream() vosk_restart = False vosk_model = Model( "/home/pi/Applications/Vosk/vosk-model-small-en-us-0.4") precise_engine = '/home/pi/Applications/Mycroft/precise-engine/precise-engine' precise_model = '/home/pi/Applications/Mycroft/hey-mycroft.pb' command_queue = queue.Queue(6) timeout = 0 def patch_precise( ): # Fix a couple of bugs in the precise_runner module def get_prediction(self, chunk): if len(chunk) != self.chunk_size * 2: raise ValueError('Invalid chunk size: ' + str(len(chunk))) self.proc.stdin.write(chunk) self.proc.stdin.flush() line = self.proc.stdout.readline() try: return float(line) except ValueError: return 0.0 def stop(self): if self.thread: self.running = False if isinstance(self.stream, ReadWriteStream): self.stream.write(b'\0' * self.chunk_size * 2) self.thread.join() self.thread = None self.engine.stop() if self.pa: self.pa.terminate() self.stream.stop_stream() self.stream = self.pa = None PreciseEngine.get_prediction = get_prediction PreciseRunner.stop = stop patch_precise() capture = alsaaudio.PCM( alsaaudio.PCM_CAPTURE, channels=1, rate=16000, format=alsaaudio.PCM_FORMAT_S16_LE, periodsize=2048 # Doesn't actually work ) def vosk_thread_function( ): # TODO: We should periodically rebuild the recognizer even if AcceptWaveform never returns true nonlocal vosk_restart try: recognizer = None def rebuild_recognizer(): nonlocal recognizer if recognizer is not None: del recognizer recognizer = KaldiRecognizer(vosk_model, 16000) count = 0 rebuild_recognizer() while True: data = vosk_stream.read(Stream.CHUNK_SIZE // 2) if vosk_restart: vosk_restart = False recognizer.Result( ) # Causes the recognizer to disregard previous input if recognizer.AcceptWaveform(data): result = json.loads(recognizer.Result()) command_queue.put(result['text']) count += 1 if count > 4: recognizer.FinalResult( ) # Causes the recognizer to free some memory rebuild_recognizer() gc.collect() count = 0 except (SystemExit, KeyboardInterrupt): return vosk_thread = threading.Thread(target=vosk_thread_function, daemon=True) vosk_thread.start() def stream_thread_function(): nonlocal timeout try: bytes = bytearray() while True: length, data = capture.read() if length > 0: bytes.extend(data) if len(bytes) >= Stream.CHUNK_SIZE: head = bytes[:Stream.CHUNK_SIZE] bytes = bytes[Stream.CHUNK_SIZE:] wake_stream.write(head) if timeout > 0: vosk_stream.write(head) timeout = timeout - 1 except (SystemExit, KeyboardInterrupt): return stream_thread = threading.Thread(target=stream_thread_function, daemon=True) stream_thread.start() def accept_command(): nonlocal vosk_restart, timeout speaker.say('yes') vosk_stream.clear() vosk_restart = True timeout = COMMAND_DURATON engine = PreciseEngine(precise_engine, precise_model, chunk_size=Stream.CHUNK_SIZE // 2) runner = PreciseRunner(engine, stream=wake_stream, on_activation=lambda: accept_command()) runner.start() return command_queue
class ListenScript(BaseScript): usage = Usage(''' Run a model on microphone audio input :model str Either Keras (.net) or TensorFlow (.pb) model to run :-c --chunk-size int 2048 Samples between inferences :-l --trigger-level int 3 Number of activated chunks to cause an activation :-s --sensitivity float 0.5 Network output required to be considered activated :-b --basic-mode Report using . or ! rather than a visual representation :-d --save-dir str - Folder to save false positives :-p --save-prefix str - Prefix for saved filenames ''') def __init__(self, args): super().__init__(args) self.listener = Listener(args.model, args.chunk_size) self.audio_buffer = np.zeros(self.listener.pr.buffer_samples, dtype=float) self.engine = ListenerEngine(self.listener, args.chunk_size) self.engine.get_prediction = self.get_prediction self.runner = PreciseRunner(self.engine, args.trigger_level, sensitivity=args.sensitivity, on_activation=self.on_activation, on_prediction=self.on_prediction) self.session_id, self.chunk_num = '%09d' % randint(0, 999999999), 0 def on_activation(self): activate_notify() if self.args.save_dir: nm = join( self.args.save_dir, self.args.save_prefix + self.session_id + '.' + str(self.chunk_num) + '.wav') save_audio(nm, self.audio_buffer) print() print('Saved to ' + nm + '.') self.chunk_num += 1 def on_prediction(self, conf): if self.args.basic_mode: print('!' if conf > 0.7 else '.', end='', flush=True) else: max_width = 80 width = min(get_terminal_size()[0], max_width) units = int(round(conf * width)) bar = 'X' * units + '-' * (width - units) cutoff = round((1.0 - self.args.sensitivity) * width) print(bar[:cutoff] + bar[cutoff:].replace('X', 'x')) def get_prediction(self, chunk): audio = buffer_to_audio(chunk) self.audio_buffer = np.concatenate( (self.audio_buffer[len(audio):], audio)) return self.listener.update(chunk) def run(self): self.runner.start() Event().wait() # Wait forever
class PreciseWakeword(WakewordEngine): NAME = 'Precise' DEPENDENCIES = {'system': [], 'pip': {'mycroft-precise==0.3.0'}} def __init__(self): super().__init__() self._hotwordThread = None try: self._stream = ReadWriteStream() self._handler = PreciseRunner(PreciseEngine( exe_file=f'{self.Commons.rootDir()}/venv/bin/precise-engine', model_file= f'{self.Commons.rootDir()}/trained/hotwords/mycroft-precise/athena.pb' ), sensitivity=self.ConfigManager. getAliceConfigByName( 'wakewordSensitivity'), stream=self._stream, on_activation=self.hotwordSpotted) except: self._enabled = False def onBooted(self): super().onBooted() if self._enabled: if not self._handler: self.logWarning('Hotword engine failed to init') else: self._handler.start() def onStop(self): super().onStop() if self._handler: self._handler.stop() def hotwordSpotted(self): self.logDebug('Detected wakeword') self._handler.pause() self.MqttManager.publish( topic=constants.TOPIC_HOTWORD_DETECTED.format('default'), payload={ 'siteId': self.ConfigManager.getAliceConfigByName('uuid'), 'modelId': f'precise_athena', 'modelVersion': '0.3.0', 'modelType': 'universal', 'currentSensitivity': self.ConfigManager.getAliceConfigByName('wakewordSensitivity') }) def onHotwordToggleOn(self, siteId: str, session: DialogSession): if self._enabled and self._handler: self._handler.start() def onAudioFrame(self, message: MQTTMessage, siteId: str): if not self.enabled or not self._handler or self._handler.is_paused or self._stream is None: return with io.BytesIO(message.payload) as buffer: try: with wave.open(buffer, 'rb') as wav: frame = wav.readframes(self.AudioServer.FRAMES_PER_BUFFER) while frame: self._stream.write(frame) frame = wav.readframes( self.AudioServer.FRAMES_PER_BUFFER) except Exception as e: self.logError(f'Error recording audio frame: {e}')
class PreciseHotword(HotWordEngine): def __init__(self, key_phrase="hey mycroft", config=None, lang="en-us"): super(PreciseHotword, self).__init__(key_phrase, config, lang) from precise_runner import (PreciseRunner, PreciseEngine, ReadWriteStream) local_conf = LocalConf(USER_CONFIG) if local_conf.get('precise', {}).get('dist_url') == \ 'http://bootstrap.mycroft.ai/artifacts/static/daily/': del local_conf['precise']['dist_url'] local_conf.store() Configuration.updated(None) self.download_complete = True self.show_download_progress = Timer(0, lambda: None) precise_config = Configuration.get()['precise'] precise_exe = self.install_exe(precise_config['dist_url']) self.precise_model = self.install_model( precise_config['model_url'], key_phrase.replace(' ', '-')).replace('.tar.gz', '.pb') self.has_found = False self.stream = ReadWriteStream() def on_activation(): self.has_found = True self.runner = PreciseRunner(PreciseEngine(precise_exe, self.precise_model), stream=self.stream, on_activation=on_activation) self.runner.start() @property def folder(self): return join(expanduser('~'), '.mycroft', 'precise') def install_exe(self, url: str) -> str: url = url.format(arch=platform.machine()) if not url.endswith('.tar.gz'): url = requests.get(url).text.strip() if install_package(url, self.folder, on_download=self.on_download, on_complete=self.on_complete): raise TriggerReload return join(self.folder, 'precise-engine', 'precise-engine') def install_model(self, url: str, wake_word: str) -> str: model_url = url.format(wake_word=wake_word) model_file = join(self.folder, posixpath.basename(model_url)) try: install_package( model_url, self.folder, on_download=lambda: LOG.info('Updated precise model')) except HTTPError: if isfile(model_file): LOG.info("Couldn't find remote model. Using local file") else: raise RuntimeError('Failed to download model:', model_url) return model_file @staticmethod def _snd_msg(cmd): with suppress(OSError): with open('/dev/ttyAMA0', 'w') as f: print(cmd, file=f) def on_download(self): LOG.info('Downloading Precise executable...') if isdir(join(self.folder, 'precise-stream')): rmtree(join(self.folder, 'precise-stream')) for old_package in glob(join(self.folder, 'precise-engine_*.tar.gz')): os.remove(old_package) self.download_complete = False self.show_download_progress = Timer(5, self.during_download, args=[True]) self.show_download_progress.start() def during_download(self, first_run=False): LOG.info('Still downloading executable...') if first_run: # TODO: Localize self._snd_msg('mouth.text=Updating listener...') if not self.download_complete: self.show_download_progress = Timer(30, self.during_download) self.show_download_progress.start() def on_complete(self): LOG.info('Precise download complete!') self.download_complete = True self.show_download_progress.cancel() self._snd_msg('mouth.reset') def update(self, chunk): self.stream.write(chunk) def found_wake_word(self, frame_data): if self.has_found: self.has_found = False return True return False
class HotwordDetector(Thread): """ Precise decoder to detect whether a keyword specified by `decoder_model` exists in a microphone input stream. :param decoder_model: decoder model file path, a string or a list of strings :param sensitivity: decoder sensitivity, a float of a list of floats. The bigger the value, the more senstive the decoder. If an empty list is provided, then the default sensitivity in the model will be used. """ def __init__(self, keyword=None, sensitivity=None, detected_callback=None ): super(HotwordDetector, self).__init__() sl = SettingLoader() self.settings = sl.settings self.paused_loop = False self.detected_callback = detected_callback self.sensitivity = sensitivity trigger_level = 3 self.keyword = keyword self.found_keyword = False if not os.path.exists(RESOURCE_FILE): if self.downloadPreciseEngine(): Utils.print_info("[Precise] Download complete") else: raise PreciseEngineNotFound("Error downloading precise engine, check your internet connection or try again later.") engine = PreciseEngine(RESOURCE_FILE, self.keyword) self.stream = ReadWriteStream() self.runner = PreciseRunner(engine, sensitivity=float(self.sensitivity), trigger_level=trigger_level, on_activation=self.activation ) self.runner.start() self.pause() # To avoid that precise starts detecting without beeing ready, we pause it right after start if self.settings.machine.startswith("arm"): # Because importing tensorflow takes up to 10 seconds, we sleep a while Utils.print_info("Starting precise trigger") time.sleep(10) def run(self): logger.debug("detecting...") while True: if not self.paused_loop: data = self.stream.read() if len(data) > 0: self.stream.write(data) if self.found_keyword: self.pause() # We start pausing it here, to avoid double activations message = "[Precise] Keyword detected" Utils.print_info(message) logger.debug(message) self.detected_callback() time.sleep(0.01) logger.debug("finished") def activation(self): self.found_keyword = True def pause(self): self.runner.pause() self.paused_loop = True def unpause(self): self.runner.play() self.paused_loop = False self.found_keyword = False def downloadPreciseEngine(self): import json import requests import tarfile Utils.print_info("[Preicse] Precise engine not present, starting download now") url = "https://api.github.com/repos/MycroftAI/mycroft-precise/releases/latest" response = requests.get(url) if response.status_code == 200: download_url = None arch = self.settings.machine for asset in response.json()["assets"]: if arch in asset.get("name"): if asset.get("name").startswith("precise-engine") and asset.get("name").endswith(".tar.gz"): download_name = asset.get("name") download_url = asset.get('browser_download_url') filepath = os.path.join(TOP_DIR, download_name) if download_url: Utils.print_info("[Precise] Downloading %s this can take a moment" % download_name) file = requests.get(download_url) if file.status_code == 200: with open(filepath, 'wb') as f: f.write(file.content) with tarfile.open(filepath) as tar: tar.extractall(path=TOP_DIR) os.remove(filepath) return True return False
class PreciseWakeListener(RhasspyActor): """Listens for a wake word using Mycroft Precise.""" def __init__(self) -> None: RhasspyActor.__init__(self) self.receivers: List[RhasspyActor] = [] self.stream: Optional[ByteStream] = None self.engine = None self.runner = None self.prediction_event = threading.Event() self.detected: bool = False def to_started(self, from_state: str) -> None: self.recorder = self.config["recorder"] self.preload = self.config.get("preload", False) self.not_detected: bool = self.config.get("not_detected", False) self.chunk_size: int = self.profile.get("wake.precise.chunk_size", 2048) self.chunk_delay: float = self.profile.get("wake.precise.chunk_delay", 0) if self.preload: self.load_runner() self.transition("loaded") def in_loaded(self, message: Any, sender: RhasspyActor) -> None: if isinstance(message, ListenForWakeWord): self.load_runner() self.receivers.append(message.receiver or sender) self.transition("listening") if message.record: self.send(self.recorder, StartStreaming(self.myAddress)) def in_listening(self, message: Any, sender: RhasspyActor) -> None: if isinstance(message, AudioData): audio_data = message.data chunk = audio_data[: self.chunk_size] detected = False while len(chunk) > 0: self.process_data(chunk) time.sleep(self.chunk_delay) if self.detected: break audio_data = audio_data[self.chunk_size :] chunk = audio_data[: self.chunk_size] if self.detected: # Detected self._logger.debug("Hotword detected (%s)" % self.model_name) detected_event = WakeWordDetected( self.model_name, audio_data_info=message.info ) for receiver in self.receivers: self.send(receiver, detected_event) self.detected = False # reset elif self.not_detected: # Not detected not_detected_event = WakeWordNotDetected( self.model_name, audio_data_info=message.info ) for receiver in self.receivers: self.send(receiver, not_detected_event) elif isinstance(message, StopListeningForWakeWord): self.receivers.remove(message.receiver or sender) if len(self.receivers) == 0: if message.record: self.send(self.recorder, StopStreaming(self.myAddress)) self.transition("loaded") def to_stopped(self, from_state: str) -> None: if self.stream is not None: self.stream.close() if self.runner is not None: self.runner.stop() # ------------------------------------------------------------------------- def process_data(self, data: bytes) -> None: assert self.stream is not None self.stream.write(data) self.prediction_event.wait() # ------------------------------------------------------------------------- def load_runner(self) -> None: if self.engine is None: from precise_runner import PreciseEngine self.model_name = self.profile.get("wake.precise.model") self.model_path = self.profile.read_path(self.model_name) self.engine_path = self.profile.get("wake.precise.engine_path") self._logger.debug(f"Loading Precise engine at {self.engine_path}") self.engine = PreciseEngine( self.engine_path, self.model_path, chunk_size=self.chunk_size ) if self.runner is None: from precise_runner import PreciseRunner self.stream = ByteStream() sensitivity = float(self.profile.get("wake.precise.sensitivity", 0.5)) trigger_level = int(self.profile.get("wake.precise.trigger_level", 3)) def on_prediction(prob: float) -> None: self.prediction_event.set() def on_activation() -> None: self.detected = True self.runner = PreciseRunner( self.engine, stream=self.stream, sensitivity=sensitivity, trigger_level=trigger_level, on_prediction=on_prediction, on_activation=on_activation, ) assert self.runner is not None self.runner.start() self._logger.debug( "Loaded Mycroft Precise (model=%s, sensitivity=%s, trigger_level=%s)" % (self.model_path, sensitivity, trigger_level) )