class PadatiousFileIntent(IntentPlugin): """Interface for Padatious intent engine""" def __init__(self, rt): super().__init__(rt) self.container = IntentContainer( join(rt.paths.user_config, 'intent_cache')) def register(self, intent: Any, skill_name: str, intent_id: str): file_name = join(self.rt.paths.skill_locale(skill_name), intent + '.intent') self.container.load_intent(name=intent_id, file_name=file_name) def register_entity(self, entity: Any, entity_id: str, skill_name: str): file_name = join(self.rt.paths.skill_locale(skill_name), entity + '.intent') self.container.load_intent(name=entity_id, file_name=file_name) def unregister(self, intent_id: str): self.container.remove_intent(intent_id) def unregister_entity(self, entity_id: str): self.container.remove_entity(entity_id) def compile(self): log.info('Training...') self.container.train() log.info('Training complete!') def calc_intents(self, query): return [ IntentMatch(intent_id=data.name, confidence=data.conf, matches=data.matches, query=query) for data in self.container.calc_intents(query) ]
class PadatiousService(FallbackSkill): instance = None fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match def __init__(self, bus, service): FallbackSkill.__init__(self, use_settings=False) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) self.bus.on('intent.service.padatious.get', self.handle_get_padatious) self.bus.on('intent.service.padatious.manifest.get', self.handle_manifest) self.bus.on('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] self.registered_entities = [] def make_active(self): """Override the make active since this is not a real fallback skill.""" pass def train(self, message=None): padatious_single_thread = Configuration.get( )['padatious']['single_thread'] if message is None: single_thread = padatious_single_thread else: single_thread = message.data.get('single_thread', padatious_single_thread) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() if not self.finished_initial_train: LOG.info("Mycroft is all loaded and ready to roll!") self.bus.emit(Message('mycroft.ready')) self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def __detach_intent(self, intent_name): """ Remove an intent if it has been registered. Arguments: intent_name (str): intent identifier """ if intent_name in self.registered_intents: self.registered_intents.remove(intent_name) self.container.remove_intent(intent_name) def handle_detach_intent(self, message): self.__detach_intent(message.data.get('intent_name')) def handle_detach_skill(self, message): skill_id = message.data['skill_id'] remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.__detach_intent(i) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self.registered_intents.append(message.data['name']) self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self.registered_entities.append(message.data) self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message, threshold=0.8): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance', '') LOG.debug("Padatious fallback attempt: " + utt) intent = self.calc_intent(utt) if not intent or intent.conf < threshold: # Attempt to use normalized() version norm = message.data.get('norm_utt', utt) if norm != utt: LOG.debug(" alt attempt: " + norm) intent = self.calc_intent(norm) utt = norm if not intent or intent.conf < threshold: return False intent.matches['utterance'] = utt self.service.add_active_skill(intent.name.split(':')[0]) self.bus.emit(message.forward(intent.name, data=intent.matches)) return True def handle_fallback_last_chance(self, message): return self.handle_fallback(message, 0.5) def handle_get_padatious(self, message): utterance = message.data["utterance"] norm = message.data.get('norm_utt', utterance) intent = self.calc_intent(utterance) if not intent and norm != utterance: intent = PadatiousService.instance.calc_intent(norm) if intent: intent = intent.__dict__ self.bus.emit( message.reply("intent.service.padatious.reply", {"intent": intent})) def handle_manifest(self, message): self.bus.emit( message.reply("intent.service.padatious.manifest", {"intents": self.registered_intents})) def handle_entity_manifest(self, message): self.bus.emit( message.reply("intent.service.padatious.entities.manifest", {"entities": self.registered_entities})) # NOTE: This cache will keep a reference to this calss (PadatiousService), # but we can live with that since it is used as a singleton. @lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache def calc_intent(self, utt): return self.container.calc_intent(utt)
class PadatiousService(FallbackSkill): instance = None def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self.bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('owo.skills.initialized', self.train) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay def train(self, message=None): if message is None: single_thread = False else: single_thread = message.data.get('single_thread', False) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def handle_detach_intent(self, message): intent_name = message.data.get('intent_name') self.container.remove_intent(intent_name) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance') LOG.debug("Padatious fallback attempt: " + utt) data = self.calc_intent(utt) if data.conf < 0.5: return False data.matches['utterance'] = utt self.service.add_active_skill(data.name.split(':')[0]) self.bus.emit(message.reply(data.name, data=data.matches)) return True def calc_intent(self, utt): return self.container.calc_intent(utt)
class PadatiousExtractor(IntentExtractor): keyword_based = False def __init__(self, cache_dir=None, *args, **kwargs): super().__init__(*args, **kwargs) # TODO xdg data_dir data_dir = expanduser(self.config.get("data_dir", "~/.padatious")) cache_dir = cache_dir or join(data_dir, "padatious") self.lock = Lock() self.container = IntentContainer(cache_dir) self.registered_intents = [] def detach_intent(self, intent_name): if intent_name in self.registered_intents: LOG.debug("Detaching padatious intent: " + intent_name) with self.lock: self.container.remove_intent(intent_name) self.registered_intents.remove(intent_name) def detach_skill(self, skill_id): LOG.debug("Detaching padatious skill: " + str(skill_id)) remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.detach_intent(i) def register_entity(self, entity_name, samples=None, reload_cache=True): samples = samples or [entity_name] with self.lock: self.container.add_entity(entity_name, samples, reload_cache=reload_cache) def register_intent(self, intent_name, samples=None, reload_cache=True): samples = samples or [intent_name] if intent_name not in self._intent_samples: self._intent_samples[intent_name] = samples else: self._intent_samples[intent_name] += samples with self.lock: self.container.add_intent(intent_name, samples, reload_cache=reload_cache) self.registered_intents.append(intent_name) def register_entity_from_file(self, entity_name, file_name, reload_cache=True): with self.lock: self.container.load_entity(entity_name, file_name, reload_cache=reload_cache) def register_intent_from_file(self, intent_name, file_name, single_thread=True, timeout=120, reload_cache=True, force_training=True): try: with self.lock: self.container.load_intent(intent_name, file_name, reload_cache=reload_cache) self.registered_intents.append(intent_name) success = self._train(single_thread=single_thread, timeout=timeout, force_training=force_training) if success: LOG.debug(file_name + " trained successfully") else: LOG.error(file_name + " FAILED TO TRAIN") except Exception as e: LOG.exception(e) def _get_remainder(self, intent, utterance): if intent["name"] in self.intent_samples: return get_utterance_remainder( utterance, samples=self.intent_samples[intent["name"]]) return utterance def calc_intent(self, utterance, min_conf=None): min_conf = min_conf or self.config.get("padatious_min_conf", 0.65) utterance = utterance.strip().lower() with self.lock: intent = self.container.calc_intent(utterance).__dict__ if intent["conf"] < min_conf: return { "intent_type": "unknown", "entities": {}, "conf": 0, "intent_engine": "padatious", "utterance": utterance, "utterance_remainder": utterance } intent["utterance_remainder"] = self._get_remainder(intent, utterance) intent["entities"] = intent.pop("matches") intent["intent_engine"] = "padatious" intent["intent_type"] = intent.pop("name") intent["utterance"] = intent.pop("sent") if isinstance(intent["utterance"], list): intent["utterance"] = " ".join(intent["utterance"]) return intent def intent_scores(self, utterance): utterance = utterance.strip().lower() intents = [i.__dict__ for i in self.container.calc_intents(utterance)] for idx, intent in enumerate(intents): intent["utterance_remainder"] = self._get_remainder( intent, utterance) intents[idx]["entities"] = intents[idx].pop("matches") intents[idx]["intent_type"] = intents[idx].pop("name") intent["intent_engine"] = "padatious" intent["utterance"] = intent.pop("sent") if isinstance(intents[idx]["utterance"], list): intents[idx]["utterance"] = " ".join(intents[idx]["utterance"]) return intents def calc_intents(self, utterance, min_conf=None): min_conf = min_conf or self.config.get("padatious_min_conf", 0.65) utterance = utterance.strip().lower() bucket = {} for ut in self.segmenter.segment(utterance): intent = self.calc_intent(ut) if intent["conf"] < min_conf: bucket[ut] = None else: bucket[ut] = intent return bucket def calc_intents_list(self, utterance): utterance = utterance.strip().lower() bucket = {} for ut in self.segmenter.segment(utterance): bucket[ut] = self.filter_intents(ut) return bucket def manifest(self): # TODO vocab, skill ids, intent_data return {"intent_names": self.registered_intents} def _train(self, single_thread=True, timeout=120, force_training=True): with self.lock: return self.container.train(single_thread=single_thread, timeout=timeout, force=force_training, debug=True)
class PadatiousService(FallbackSkill): instance = None fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] def train(self, message=None): if message is None: single_thread = False else: single_thread = message.data.get('single_thread', False) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() if not self.finished_initial_train: LOG.info("Mycroft is all loaded and ready to roll!") self.bus.emit(Message('mycroft.ready')) self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def __detach_intent(self, intent_name): self.registered_intents.remove(intent_name) self.container.remove_intent(intent_name) def handle_detach_intent(self, message): self.__detach_intent(message.data.get('intent_name')) def handle_detach_skill(self, message): skill_id = message.data['skill_id'] remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.__detach_intent(i) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self.registered_intents.append(message.data['name']) self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message, threshold=0.8): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance', '') LOG.debug("Padatious fallback attempt: " + utt) intent = self.calc_intent(utt) if not intent or intent.conf < threshold: # Attempt to use normalized() version norm = message.data.get('norm_utt', '') if norm != utt: LOG.debug(" alt attempt: " + norm) intent = self.calc_intent(norm) utt = norm if not intent or intent.conf < threshold: return False intent.matches['utterance'] = utt self.service.add_active_skill(intent.name.split(':')[0]) self.bus.emit(message.reply(intent.name, data=intent.matches)) return True def handle_fallback_last_chance(self, message): return self.handle_fallback(message, 0.5) # NOTE: This cache will keep a reference to this calss (PadatiousService), # but we can live with that since it is used as a singleton. @lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache def calc_intent(self, utt): return self.container.calc_intent(utt)
class PadatiousService: """Service class for padatious intent matching.""" def __init__(self, bus, config): self.padatious_config = config self.bus = bus intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] self.registered_entities = [] def train(self, message=None): """Perform padatious training. Arguments: message (Message): optional triggering message """ padatious_single_thread = Configuration.get()[ 'padatious']['single_thread'] if message is None: single_thread = padatious_single_thread else: single_thread = message.data.get('single_thread', padatious_single_thread) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() if not self.finished_initial_train: self.bus.emit(Message('mycroft.skills.trained')) self.finished_initial_train = True def wait_and_train(self): """Wait for minimum time between training and start training.""" if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def __detach_intent(self, intent_name): """ Remove an intent if it has been registered. Arguments: intent_name (str): intent identifier """ if intent_name in self.registered_intents: self.registered_intents.remove(intent_name) self.container.remove_intent(intent_name) def handle_detach_intent(self, message): """Messagebus handler for detaching padatious intent. Arguments: message (Message): message triggering action """ self.__detach_intent(message.data.get('intent_name')) def handle_detach_skill(self, message): """Messagebus handler for detaching all intents for skill. Arguments: message (Message): message triggering action """ skill_id = message.data['skill_id'] remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.__detach_intent(i) def _register_object(self, message, object_name, register_func): """Generic method for registering a padatious object. Arguments: message (Message): trigger for action object_name (str): type of entry to register register_func (callable): function to call for registration """ file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): """Messagebus handler for registering intents. Arguments: message (Message): message triggering action """ self.registered_intents.append(message.data['name']) self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): """Messagebus handler for registering entities. Arguments: message (Message): message triggering action """ self.registered_entities.append(message.data) self._register_object(message, 'entity', self.container.load_entity) def _match_level(self, utterances, limit): """Match intent and make sure a certain level of confidence is reached. Arguments: utterances (list of tuples): Utterances to parse, originals paired with optional normalized version. limit (float): required confidence level. """ padatious_intent = None LOG.debug('Padatious Matching confidence > {}'.format(limit)) for utt in utterances: for variant in utt: intent = self.calc_intent(variant) if intent: best = padatious_intent.conf if padatious_intent else 0.0 if best < intent.conf: padatious_intent = intent padatious_intent.matches['utterance'] = utt[0] if padatious_intent and padatious_intent.conf > limit: skill_id = padatious_intent.name.split(':')[0] ret = IntentMatch( 'Padatious', padatious_intent.name, padatious_intent.matches, skill_id ) else: ret = None return ret def match_high(self, utterances, _=None, __=None): """Intent matcher for high confidence. Arguments: utterances (list of tuples): Utterances to parse, originals paired with optional normalized version. """ return self._match_level(utterances, 0.95) def match_medium(self, utterances, _=None, __=None): """Intent matcher for medium confidence. Arguments: utterances (list of tuples): Utterances to parse, originals paired with optional normalized version. """ return self._match_level(utterances, 0.8) def match_low(self, utterances, _=None, __=None): """Intent matcher for low confidence. Arguments: utterances (list of tuples): Utterances to parse, originals paired with optional normalized version. """ return self._match_level(utterances, 0.5) @lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache def calc_intent(self, utt): """Cached version of container calc_intent. This improves speed when called multiple times for different confidence levels. NOTE: This cache will keep a reference to this class (PadatiousService), but we can live with that since it is used as a singleton. Arguments: utt (str): utterance to calculate best intent for """ return self.container.calc_intent(utt)