def dict_make_equal_keys(dct_to_change: MutableMapping, keys_dct: MutableMapping, max_depth: int = 1, cur_depth: int = 0) -> MutableMapping: """ Adds and removes keys from dct_to_change such that it has the same keys as keys_dct. Values from dct_to_change are preserved with any added keys using default values from keys_dct. Args: dct_to_change: Dict of user preferences to modify and return keys_dct: Dict containing all keys and default values max_depth: Int depth to recurse (0-indexed) cur_depth: Current depth relative to top-level config (0-indexed) Returns: dct_to_change with any keys not in keys_dct removed and any new keys added with default values """ if not isinstance(dct_to_change, MutableMapping) or not isinstance(keys_dct, MutableMapping): raise AttributeError("merge_recursive_dicts expects two dict objects as args") for key in list(dct_to_change.keys()): if isinstance(keys_dct.get(key), dict) and isinstance(dct_to_change[key], MutableMapping): if max_depth > cur_depth and key not in ("tts", "stt"): dct_to_change[key] = dict_make_equal_keys(dct_to_change[key], keys_dct[key], max_depth, cur_depth + 1) elif key not in keys_dct.keys(): dct_to_change.pop(key) LOG.warning(f"Removing '{key}' from dict!") # del dct_to_change[key] for key, value in keys_dct.items(): if key not in dct_to_change.keys(): dct_to_change[key] = value return dct_to_change
def get_neon_lang_config() -> dict: """ Get a language config for language utilities Returns: dict of config params used by Language Detector and Translator modules """ core_config = get_neon_local_config() language_config = deepcopy(get_neon_user_config().content.get( "speech", {})) language_config["internal"] = language_config.get( "internal", "en-us") # TODO: This is core, not user DM language_config["user"] = language_config.get("stt_language", "en-us") language_config["boost"] = False language_config["detection_module"] = core_config.get( "stt", {}).get("detection_module") language_config["translation_module"] = core_config.get( "stt", {}).get("translation_module") merged_language = { **_safe_mycroft_config().get("language", {}), **language_config } if merged_language.keys() != language_config.keys(): LOG.warning(f"Keys missing from Neon config! {merged_language.keys()}") return merged_language
def get_config_dir(): """ Get a default directory in which to find configuration files Returns: Path to configuration or else default """ site = sysconfig.get_paths()['platlib'] if exists(join(site, 'NGI')): return join(site, "NGI") for p in [path for path in sys.path if path != ""]: if exists(join(p, "NGI")): return join(p, "NGI") if re.match(".*/lib/python.*/site-packages", p): clean_path = "/".join(p.split("/")[0:-4]) if exists(join(clean_path, "NGI")): LOG.warning( f"Depreciated core structure found at {clean_path}") return join(clean_path, "NGI") elif exists(join(clean_path, "neon_core")): # Dev Environment return clean_path elif exists(join(clean_path, "mycroft")): LOG.info(f"Mycroft core structure found at {clean_path}") return clean_path elif exists(join(clean_path, ".venv")): # Localized Production Environment (Servers) return clean_path default_path = expanduser("~/.local/share/neon") # LOG.info(f"System packaged core found! Using default configuration at {default_path}") return default_path
def _move_config_sections(user_config, local_config): """ Temporary method to handle one-time migration of user_config params to local_config Args: user_config (NGIConfig): user configuration object local_config (NGIConfig): local configuration object """ depreciated_user_configs = ("interface", "listener", "skills", "session", "tts", "stt", "logs", "device") if any([d in user_config.content for d in depreciated_user_configs]): LOG.warning( "Depreciated keys found in user config! Adding them to local config" ) if "wake_words_enabled" in user_config.content.get( "interface", dict()): user_config["interface"]["wake_word_enabled"] = user_config[ "interface"].pop("wake_words_enabled") config_to_move = { "interface": user_config.content.pop("interface", {}), "listener": user_config.content.pop("listener", {}), "skills": user_config.content.pop("skills", {}), "session": user_config.content.pop("session", {}), "tts": user_config.content.pop("tts", {}), "stt": user_config.content.pop("stt", {}), "logs": user_config.content.pop("logs", {}), "device": user_config.content.pop("device", {}) } local_config.update_keys(config_to_move)
def synchronize(self): """ Upload namespaces, pages and data to the last connected. """ namespace_pos = 0 enclosure = self.application.enclosure for namespace, pages in enclosure.loaded: LOG.info('Sync {}'.format(namespace)) # Insert namespace self.send({"type": "mycroft.session.list.insert", "namespace": "mycroft.system.active_skills", "position": namespace_pos, "data": [{"skill_id": namespace}] }) # Insert pages self.send({"type": "mycroft.gui.list.insert", "namespace": namespace, "position": 0, "data": [{"url": p} for p in pages] }) # Insert data data = enclosure.datastore.get(namespace, {}) for key in data: self.send({"type": "mycroft.session.set", "namespace": namespace, "data": {key: data[key]} }) namespace_pos += 1
def send(self, msg_dict): """ Send to all registered GUIs. """ for connection in GUIWebsocketHandler.clients: try: connection.send(msg_dict) except Exception as e: LOG.exception(repr(e))
def __insert_new_namespace(self, namespace, pages): """ Insert new namespace and pages. This first sends a message adding a new namespace at the highest priority (position 0 in the namespace stack) Args: namespace (str): The skill namespace to create pages (str): Pages to insert (name matches QML) """ LOG.debug("Inserting new namespace") self.send({"type": "mycroft.session.list.insert", "namespace": "mycroft.system.active_skills", "position": 0, "data": [{"skill_id": namespace}] }) # Load any already stored Data data = self.datastore.get(namespace, {}) for key in data: msg = {"type": "mycroft.session.set", "namespace": namespace, "data": {key: data[key]}} self.send(msg) LOG.debug("Inserting new page") self.send({"type": "mycroft.gui.list.insert", "namespace": namespace, "position": 0, "data": [{"url": p} for p in pages] }) # Make sure the local copy is updated self.loaded.insert(0, Namespace(namespace, pages))
def on_gui_show_page(self, message): try: page, namespace, index = _get_page_data(message) # Pass the request to the GUI(s) to pull up a page template with namespace_lock: self.show(namespace, page, index) except Exception as e: LOG.exception(repr(e))
def on_gui_delete_namespace(self, message): """ Bus handler for removing namespace. """ try: namespace = message.data['__from'] with namespace_lock: self.remove_namespace(namespace) except Exception as e: LOG.exception(repr(e))
def on_gui_delete_page(self, message): """ Bus handler for removing pages. """ page, namespace, _ = _get_page_data(message) try: with namespace_lock: self.remove_pages(namespace, page) except Exception as e: LOG.exception(repr(e))
def tcp_client(self): """Simple tcp socket client emitting just one message""" with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.connect(TEST_SOCKET_ADDRESS) s.sendall(TEST_DICT_B64) LOG.info('Client sent data') data = get_packet_data(socket=s, sequentially=True) self.assertEqual(data, TEST_DICT_B64)
def get_neon_client_config() -> dict: core_config = get_neon_local_config() server_addr = core_config.get("remoteVars", {}).get("remoteHost", "167.172.112.7") if server_addr == "64.34.186.92": LOG.warning(f"Depreciated call to host: {server_addr}") server_addr = "167.172.112.7" return {"server_addr": server_addr, "devVars": core_config["devVars"], "remoteVars": core_config["remoteVars"]}
def populate(self, content, check_existing=False): if not check_existing: self.__add__(content) return old_content = deepcopy(self._content) self._content = dict_merge(content, self._content) # to_change, one_with_all_keys if old_content == self._content: LOG.warning(f"Update called with no change: {self.file_path}") return self._write_yaml_file()
def on_gui_send_event(self, message): """ Send an event to the GUIs. """ try: data = {'type': 'mycroft.events.triggered', 'namespace': message.data.get('__from'), 'event_name': message.data.get('event_name'), 'params': message.data.get('params')} self.send(data) except Exception as e: LOG.error('Could not send event ({})'.format(repr(e)))
def file_path(self): """ Returns the path to the yml file associated with this configuration Returns: path to this configuration yml """ file_path = join(self.path, self.name + ".yml") if not isfile(file_path): create_file(file_path) LOG.debug(f"New YAML created: {file_path}") return file_path
def on_gui_set_value(self, message): data = message.data namespace = data.get("__from", "") # Pass these values on to the GUI renderers for key in data: if key not in RESERVED_KEYS: try: self.set(namespace, key, data[key]) except Exception as e: LOG.exception(repr(e))
def test_03_get_packet_data(self): with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.bind(TEST_SOCKET_ADDRESS) s.listen() threading.Thread(target=self.tcp_client).start() conn, addr = s.accept() with conn: LOG.info(f'Connected by {addr}') data = get_packet_data(conn, sequentially=False) self.assertEqual(data, TEST_DICT_B64) conn.sendall(data)
def get_neon_bus_config() -> dict: """ Get a configuration dict for the messagebus. Merge any values from Mycroft config if missing from Neon. Returns: dict of config params used for a messagebus client """ mycroft = _safe_mycroft_config().get("websocket", {}) neon = get_neon_local_config().get("websocket", {}) merged = {**mycroft, **neon} if merged.keys() != neon.keys(): LOG.warning(f"Keys missing from Neon config! {merged.keys()}") return merged
def create_gui_service(enclosure, config): import tornado.options LOG.info('Starting message bus for GUI...') # Disable all tornado logging so mycroft loglevel isn't overridden tornado.options.parse_command_line(['--logging=None']) routes = [(config['route'], GUIWebsocketHandler)] application = web.Application(routes, debug=True) application.enclosure = enclosure application.listen(config['base_port'], config['host']) create_daemon(ioloop.IOLoop.instance().start) LOG.info('GUI Message bus started!') return application
def get_coordinates(gps_loc: dict) -> (float, float): """ Gets the latitude and longitude for the passed location :param gps_loc: dict of "city", "state", "country" :return: lat, lng float values """ coordinates = Nominatim(user_agent="neon-ai") try: location = coordinates.geocode(gps_loc) LOG.debug(f"{location}") return location.latitude, location.longitude except Exception as x: LOG.error(x) return -1, -1
def get_neon_api_config() -> dict: """ Get a configuration dict for the api module. Merge any values from Mycroft config if missing from Neon. Returns: dict of config params used for the Mycroft API module """ core_config = get_neon_local_config() api_config = deepcopy(core_config.get("api")) api_config["metrics"] = core_config["prefFlags"].get("metrics", False) mycroft = _safe_mycroft_config().get("server", {}) merged = {**mycroft, **api_config} if merged.keys() != api_config.keys(): LOG.warning(f"Keys missing from Neon config! {merged.keys()}") return merged
def update_keys(self, other): """ Adds keys to this config such that it has all keys in 'other'. Configuration values are preserved with any added keys using default values from 'other'. Args: other: dict of keys and default values this should be added to this configuration """ old_content = deepcopy(self._content) self._content = dict_update_keys(self._content, other) # to_change, one_with_all_keys if old_content == self._content: LOG.warning(f"Update called with no change: {self.file_path}") return self._write_yaml_file()
def get_neon_auth_config(path: Optional[str] = None) -> NGIConfig: """ Returns a dict authentication configuration and handles populating values from key files Args: path: optional path to yml configuration files Returns: NGIConfig object with authentication config """ auth_config = NGIConfig("ngi_auth_vars", path) if not auth_config.content: LOG.info("Populating empty auth configuration") auth_config._content = build_new_auth_config(path) LOG.info(f"Loaded auth config from {auth_config.file_path}") return auth_config
def encode_file_to_base64_string(path: str) -> str: """ Encodes a file to a base64 string (useful for passing file data over a messagebus) :param path: Path to file to be encoded :return: encoded string """ if not isinstance(path, str): raise TypeError path = os.path.expanduser(path) if not os.path.isfile(path): LOG.error(f"File Not Found: {path}") raise FileNotFoundError with open(path, "rb") as file_in: encoded = base64.b64encode(file_in.read()).decode("utf-8") return encoded
def get_neon_audio_config() -> dict: """ Get a configuration dict for the audio module. Merge any values from Mycroft config if missing from Neon. Returns: dict of config params used for the Audio module """ mycroft = _safe_mycroft_config() local_config = get_neon_local_config() neon_audio = local_config.get("audioService", {}) merged_audio = {**mycroft.get("Audio", {}), **neon_audio} if merged_audio.keys() != neon_audio.keys(): LOG.warning(f"Keys missing from Neon config! {merged_audio.keys()}") return {"Audio": merged_audio, "tts": get_neon_tts_config(), "language": get_neon_lang_config()}
def _load_yaml_file(self) -> dict: """ Loads and parses the YAML file at a given filepath into the Python dictionary object. :return: dictionary, containing all keys and values from the most current selected YAML. """ try: self._loaded = os.path.getmtime(self.file_path) with self.lock: with open(self.file_path, 'r') as f: return self.parser.load(f) or dict() except FileNotFoundError as x: LOG.error(f"Configuration file not found error: {x}") except Exception as c: LOG.error(f"{self.file_path} Configuration file error: {c}") return dict()
def listen(self, source, stream): """Listens for chunks of audio that Mycroft should perform STT on. This will listen continuously for a wake-up-word, then return the audio chunk containing the spoken phrase that comes immediately afterwards. Args: source (AudioSource): Source producing the audio chunks stream (AudioStreamHandler): Stream target that will receive chunks of the utterance audio while it is being recorded Returns: (AudioData, lang): audio with the user's utterance (minus the wake-up-word), stt_lang """ assert isinstance(source, AudioSource), "Source must be an AudioSource" # If skipping wake words, just pass audio to our streaming STT # TODO: Check config updates? if self.loop.stt.can_stream and not self.use_wake_word: lang = self.loop.stt.lang self.loop.emit("recognizer_loop:record_begin") self.loop.stt.stream.stream_start() frame_data = get_silence(source.SAMPLE_WIDTH) LOG.debug("Stream starting!") # event set in OPM while not self.loop.stt.transcript_ready.is_set(): # Pass audio until STT tells us to stop (this is called again immediately) chunk = self.record_sound_chunk(source) if not is_speaking(): # Filter out Neon speech self.loop.stt.stream.stream_chunk(chunk) frame_data += chunk LOG.debug("stream ended!") audio_data = self._create_audio_data(frame_data, source) self.loop.emit("recognizer_loop:record_end") # If using wake words, wait until the wake_word is detected and then record the following phrase else: audio_data, lang = super().listen(source, stream) # one of the default plugins saves the speech to file and adds "filename" to context audio_data, context = self.audio_consumers.transform(audio_data) context["lang"] = lang return audio_data, context
def create(config=None, results_event: Event = None): if config and not config.get( "module" ): # No module, try getting stt config from passed config config = config.get("stt") if not config: # No config, go get it config = get_neon_speech_config().get("stt", {}) LOG.info(f"Create STT with config: {config}") clazz = OVOSSTTFactory.get_class(config) if not clazz: LOG.warning("plugin not found, falling back to Chromium STT") config["module"] = "google" # TODO configurable fallback plugin clazz = OVOSSTTFactory.get_class(config) if not clazz: raise ValueError("fallback plugin not found") return WrappedSTT(clazz, config=config, results_event=results_event)
def decode_base64_string_to_file(encoded_string: str, output_path: str) -> str: """ Writes out a base64 string to a file object at the specified path :param encoded_string: Base64 encoded string :param output_path: Path to file to write (throws exception if file exists) :return: Path to output file """ if not isinstance(output_path, str): raise TypeError output_path = os.path.expanduser(output_path) if os.path.isfile(output_path): LOG.error(f"File already exists: {output_path}") raise FileExistsError ensure_directory_exists(os.path.dirname(output_path)) with open(output_path, "wb+") as file_out: byte_data = base64.b64decode(encoded_string.encode("utf-8")) file_out.write(byte_data) return output_path
def get_neon_cli_config() -> dict: """ Get a configuration dict for the neon_cli Returns: dict of config params used by the neon_cli """ local_config = NGIConfig("ngi_local_conf").content wake_words_enabled = local_config.get("interface", {}).get("wake_word_enabled", True) try: neon_core_version = os.path.basename(glob(local_config['dirVars']['ngiDir'] + '/*.release')[0]).split('.release')[0] except Exception as e: LOG.error(e) neon_core_version = "Unknown" log_dir = local_config.get("dirVars", {}).get("logsDir", "/var/log/mycroft") return {"neon_core_version": neon_core_version, "wake_words_enabled": wake_words_enabled, "log_dir": log_dir}