def initialize_intent_parser(self): self._intents_container = IntentContainer("%s_cache" % self._name) self._adapt_intent_engine = DomainIntentDeterminationEngine() self._adapt_intent_engine.register_domain(self._name) for intent_name, intent_file_path in self.get_intent_names(): #print ("###### IntentBuilder: %s, %s" % (intent_name, intent_file_path)) adapt_intent_builder = IntentBuilder(intent_name) for intent_name, intent_example_sentences_array in self.intent_training_file_content( intent_file_path, 'intent'): #print ("add intent %s, %s" % (intent_name, intent_example_sentences_array)) self._intents_container.add_intent( intent_name, intent_example_sentences_array) for entity_name, entities_array in self.intent_training_file_content( intent_file_path, 'entities'): #print ("add entity %s, %s " % (entity_name, entities_array)) self._intents_container.add_entity(entity_name, entities_array) # adapt if entity_name.endswith("_keyword"): for k in entities_array: #print ("add keyword %s to %s" % (k, intent_name)) self._adapt_intent_engine.register_entity( k, entity_name, domain=self._name) adapt_intent_builder.require(entity_name) adapt_intent = adapt_intent_builder.build() self._adapt_intent_engine.register_intent_parser(adapt_intent, domain=self._name) self._intents_container.train(debug=False)
def __init__(self): cache = expanduser("~/.simple_NER") if not isdir(cache): makedirs(cache) self._container = IntentContainer(join(cache, "rule_cache")) self._rules = {} self._examples = {}
def __init__(self, emitter, service): FallbackSkill.__init__(self) self.config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return ver = get_distribution('padatious').version if ver != PADATIOUS_VERSION: LOG.warning('Using Padatious v' + ver + '. Please re-run ' + 'dev_setup.sh to install ' + PADATIOUS_VERSION) self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.emitter.on('padatious:register_entity', self.register_entity) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train()
def __init__(self, emitter): FallbackSkill.__init__(self) self.config = ConfigurationManager.get()['padatious'] intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: logger.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train()
def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self.bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('owo.skills.initialized', self.train) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay
def __init__(self, bus, config): self.padatious_config = config self.bus = bus intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] self.registered_entities = []
def __init__(self, cache_dir=None, *args, **kwargs): super().__init__(*args, **kwargs) # TODO xdg data_dir data_dir = expanduser(self.config.get("data_dir", "~/.padatious")) cache_dir = cache_dir or join(data_dir, "padatious") self.lock = Lock() self.container = IntentContainer(cache_dir) self.registered_intents = []
def train(parser, args): if bool(args.input_files) == bool(args.data): parser.error( 'You must specify one of input_files or --data (but not both)') cont = IntentContainer(args.intent_cache) if args.data: cont.apply_training_args(args.data) else: for fn in args.input_files: obj_name, ext = splitext(basename(fn)) if ext == '.intent': cont.load_intent(obj_name, fn) elif ext == '.entity': cont.load_entity(obj_name, fn) else: parser.error('Unknown file extension: {}'.format(ext)) kwargs = inspect.signature(cont.train).bind(*(args.args or [])).arguments kwargs.update(args.kwargs or {}) kwargs.setdefault('debug', True) kwargs.setdefault('single_thread', args.single_thread) kwargs.setdefault('force', args.force) if cont.train(**kwargs): return 0 return 10 # timeout
def __init__(self, object_type="tnalagmes_object", adapt=None): self.adapt = adapt or IntentDeterminationEngine() self.context_manager = ContextManager(self.adapt) self.object_type = object_type self.container = IntentContainer(self.cache_dir) self.intents = {} self.register_default_intents() self.register_core_intents() self.container.train() self.waiting_for_user = False self._output = "" self.input = ""
def __init__(self, bus, service): FallbackSkill.__init__(self, use_settings=False) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) self.bus.on('intent.service.padatious.get', self.handle_get_padatious) self.bus.on('intent.service.padatious.manifest.get', self.handle_manifest) self.bus.on('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] self.registered_entities = []
def _init_padatious(self): # i want to check in converse method if some intent by this skill will trigger # however there is no mechanism to query the intent parser # PR incoming intent_cache = expanduser( self.config_core['padatious']['intent_cache']) self.container = IntentContainer(intent_cache) for intent in ["restore.intent", "play.intent", "credits.intent"]: name = str(self.skill_id) + ':' + intent filename = self.find_resource(intent, 'vocab') if filename is not None: with open(filename, "r") as f: self.container.add_intent(name, f.readlines()) self.container.train()
def __init__(self, emitter): FallbackSkill.__init__(self) self.config = ConfigurationManager.get()['padatious'] intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return ver = get_distribution('padatious').version if ver != PADATIOUS_VERSION: LOG.warning('Using Padatious v' + ver + '. Please re-run ' + 'dev_setup.sh to install ' + PADATIOUS_VERSION) self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.emitter.on('padatious:register_entity', self.register_entity) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train()
class NeuralQuestionParser(RegexQuestionParser): def __init__(self, lang="en-us"): self.container = IntentContainer(INTENT_CACHE_PATH) self._intents = [] self.lang = lang self.register_default_intents() self.container.train() def parse(self, utterance): data = BasicQuestionParser().parse(utterance) match = self.container.calc_intent(utterance) if match.name: data["QuestionIntent"] = match.name data.update(match.matches) data["conf"] = match.conf return data
def __init__(self): super().__init__() self.translator = google_translator() self.tx_cache = {} # avoid translating twice self.duck_cache = {} self.rake = Rake() # only english for now # for usage in tell me more / follow up questions self.idx = 0 self.results = [] self.image = None # subparser, intents just for this skill # not part of main intent service intent_cache = expanduser( self.config_core['padatious']['intent_cache']) self.intents = IntentContainer(intent_cache)
class CommandDetector: def __init__(self, ): self.container = IntentContainer('intent_cache') self.commands = Commands() def add_command(self, commands): for c in self.commands.family_list: for intent, filename in c.items(): self.container.load_file(intent, filename) def train_commands(self): for c in self.commands.family_list: self.add_command(c) self.container.train() def calc_intent(self, text): return self.container.calc_intent(text)
def create_container(): container = IntentContainer('intent_cache') dir = os.getenv('VOCAB_DIR', '/qabot/vocab/en-us/') for file in os.listdir(dir): print(file) if file.endswith(".intent"): container.load_intent(basename(file), join(dir, file)) elif file.endswith(".entity"): container.load_entity(basename(file), join(dir, file)) container.train() return container
def initialize(self): """ Loads all skills into memory for referencing as required. """ self.logger.debug("Initalizing") self.intentParser = IntentContainer('/tmp/intent_cache') if self.skill_folder is None: self.skill_folder = os.path.join(os.path.dirname(__file__), "skills") if self.skill_folder not in sys.path: sys.path.insert(0, str(pathlib.Path(self.skill_folder).absolute())) skillModules = [ os.path.join(self.skill_folder, o) for o in os.listdir(self.skill_folder) if os.path.isdir(os.path.join(self.skill_folder, o)) ] for f in skillModules: mySkillName = os.path.basename(f) self.logger.debug("Loading " + mySkillName) mySkillModule = __import__(mySkillName) mySkillClass = mySkillModule.create_skill() mySkillClass.brain = self.brain mySkillClass.initialize() self.logger.debug("Skills load is complete.") self.intentParser.train( False) # False = be quiet and don't print messages to stdout self.logger.debug("Training completed.") self.logger.info("Initialization completed.")
def __init__(self): super(CountriesSkill, self).__init__() if "map_style" not in self.settings: self.settings["map_style"] = "ortho" self.countries_data = {} self.country_codes = {} self.regions = [ u'Asia', u'Europe', u'Africa', u'Oceania', u'Americas', u'Polar' ] self.subregions = [ u'Southern Asia', u'Northern Europe', u'Southern Europe', u'Northern Africa', u'Polynesia', u'Middle Africa', u'Caribbean', u'South America', u'Western Asia', u'Australia and New Zealand', u'Western Europe', u'Eastern Europe', u'Central America', u'Western Africa', u'Northern America', u'Southern Africa', u'Eastern Africa', u'South-Eastern Asia', u'Eastern Asia', u'Melanesia', u'Micronesia', u'Central Asia' ] self.get_country_data() intent_cache = expanduser( self.config_core['padatious']['intent_cache']) self.intents = IntentContainer(intent_cache)
def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = []
def __init__(self): super(WebMusicControl, self).__init__() self.regexes = {} self.is_playing = True self.intent_container: IntentContainer = IntentContainer( 'play_intent_cache') self.add_intents_from_file("play.song.intent") self.add_intents_from_file("play.album.intent") self.add_intents_from_file("play.artist.intent") self.add_intents_from_file("play.playlist.intent") self.add_entity_from_file("song_name.entity") self.add_entity_from_file("playlist_name.entity") # self.add_intents_from_file("play.something.intent") # self.add_intents_from_file("continue.intent") self.log.info("Training play intent parser") self.intent_container.train() self.log.info("Done Training")
class NeuralNER(RuleNER): def __init__(self): cache = expanduser("~/.simple_NER") if not isdir(cache): makedirs(cache) self._container = IntentContainer(join(cache, "rule_cache")) self._rules = {} self._examples = {} def extract_entities(self, text, as_json=False): for rule in self._container.calc_intents(text): for e in rule.matches: if as_json: yield Entity(rule.matches[e], entity_type=e, source_text=text, confidence=rule.conf, rules=self._rules[rule.name]).as_json() else: yield Entity(rule.matches[e], entity_type=e, source_text=text, confidence=rule.conf, rules=self._rules[rule.name])
def __init__(self, lang="en-us"): self.container = IntentContainer(INTENT_CACHE_PATH) self._intents = [] self.lang = lang self.register_default_intents() self.container.train()
class PadatiousService(FallbackSkill): instance = None def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self.bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('owo.skills.initialized', self.train) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay def train(self, message=None): if message is None: single_thread = False else: single_thread = message.data.get('single_thread', False) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def handle_detach_intent(self, message): intent_name = message.data.get('intent_name') self.container.remove_intent(intent_name) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance') LOG.debug("Padatious fallback attempt: " + utt) data = self.calc_intent(utt) if data.conf < 0.5: return False data.matches['utterance'] = utt self.service.add_active_skill(data.name.split(':')[0]) self.bus.emit(message.reply(data.name, data=data.matches)) return True def calc_intent(self, utt): return self.container.calc_intent(utt)
def parseMsg(msg): container = IntentContainer('intent_cache') container.add_intent('greetings', ['hi.', 'hellow', '(waz|wazz|wad) up?', 'hey', 'yoo', 'buddy', '(you|hey) there?']) container.add_intent('bye', ['bye.', 'tata', 'thank you', 'adios', 'thanks', 'allah hafez', 'done']) # container.add_intent('goal', ['i want to go (|to) {goal}', '(destination|i wanna go) {goal}', # 'destination is {goal}']) # container.add_intent('start', ['from {start}', '(i am (in|at){start}' # ]) container.add_intent('search', ['I want to go from {start} to {goal}.', 'I want to (go to|go) {goal} from {start}.']) container.add_intent('ajaira', ['lala', 'kchdskfhsk', 'iwurhb', 'uerwyvdsvjjkc', 'sufgbsdjc']) container.train() result = container.calc_intent(msg) print(result) print(result.name) # print(str(result.matches['start'])) # print(result.matches['start']) # print(result.matches['goal']) return result
class DuckDuckGoSkill(CommonQuerySkill): def __init__(self): super().__init__() self.translator = google_translator() self.tx_cache = {} # avoid translating twice self.duck_cache = {} self.rake = Rake() # only english for now # for usage in tell me more / follow up questions self.idx = 0 self.results = [] self.image = None # subparser, intents just for this skill # not part of main intent service intent_cache = expanduser( self.config_core['padatious']['intent_cache']) self.intents = IntentContainer(intent_cache) def initialize(self): self.load_intents() # check for conflicting skills just in case # done after all skills loaded to ensure proper shutdown self.add_event("mycroft.skills.initialized", self.blacklist_default_skill) def load_intents(self): # TODO intents for other infobox fields for intent in ["who", "birthdate"]: path = self.find_resource(intent + '.intent', "locale") if path: self.intents.load_intent(intent, path) self.intents.train(single_thread=True) def get_intro_message(self): # blacklist conflicting skills on install self.blacklist_default_skill() def blacklist_default_skill(self): # load the current list of already blacklisted skills blacklist = self.config_core["skills"]["blacklisted_skills"] # check the folder name (skill_id) of the skill you want to replace skill_id = "mycroft-fallback-duck-duck-go.mycroftai" # add the skill to the blacklist if skill_id not in blacklist: self.log.debug("Blacklisting official mycroft skill") blacklist.append(skill_id) # load the user config file (~/.mycroft/mycroft.conf) conf = LocalConf(USER_CONFIG) if "skills" not in conf: conf["skills"] = {} # update the blacklist field conf["skills"]["blacklisted_skills"] = blacklist # save the user config file conf.store() # tell the intent service to unload the skill in case it was loaded already # this should avoid the need to restart self.bus.emit(Message("detach_skill", {"skill_id": skill_id})) def stop(self): self.gui.release() # intents @intent_handler("search_duck.intent") def handle_search(self, message): query = message.data["query"] summary = self.ask_the_duck(query) if summary: self.speak_result() else: answer, _, _ = self.parse_subintents(query) if answer: self.speakr(answer) else: self.speak_dialog("no_answer") @intent_handler( IntentBuilder("DuckMore").require("More").require("DuckKnows")) def handle_tell_more(self, message): """ Follow up query handler, "tell me more".""" query = message.data["DuckKnows"] data, related_queries = self.get_infobox(query) # TODO maybe do something with the infobox data ? self.speak_result() # common query def parse_subintents(self, utt): # Get response from intents, this is a subparser that will handle # queries about the infobox returned by duckduckgo # eg. when was {person} born match = self.intents.calc_intent(utt) level = CQSMatchLevel.CATEGORY data = match.matches intent = match.name score = match.conf data["intent"] = intent data["score"] = score query = utt if score > 0.8: level = CQSMatchLevel.EXACT elif score > 0.5: level = CQSMatchLevel.CATEGORY elif score > 0.3: level = CQSMatchLevel.GENERAL else: intent = None self.log.debug("DuckDuckGo Intent: " + str(intent)) if "person" in data: query = data["person"] summary = self.ask_the_duck(query) answer = summary if summary: answer = self.results[0] infobox, related_queries = self.get_infobox(query) self.log.debug("DuckDuckGo infobox: " + str(infobox)) data["infobox"] = infobox data["related_queries"] = related_queries if intent == "birthdate": answer = infobox.get("born") data["query"] = query data["answer"] = answer data["image"] = self.image if not answer: level = CQSMatchLevel.GENERAL return answer, level, data def CQS_match_query_phrase(self, utt): self.log.debug("DuckDuckGo query: " + utt) answer, match, data = self.parse_subintents(utt) if answer: self.idx += 1 return (utt, match, answer, data) # extract most relevant keyword utt = self.translate(utt, "en", self.lang) keywords = self.rake.extract_keywords(utt) self.log.debug("Extracted keywords: " + str(keywords)) # TODO better selection / merging of top keywords with same # confidence?? for kw in keywords: query = kw[0] self.log.debug("Selected keyword: " + query) summary = self.ask_the_duck(query, translate=False) if summary: self.idx += 1 return (utt, CQSMatchLevel.GENERAL, self.results[0], { 'query': query, 'answer': self.results[0], "keywords": keywords, "image": self.image }) def CQS_action(self, phrase, data): """ If selected show gui """ self.display_ddg(data["answer"], data["image"]) # duck duck go api def ask_the_duck(self, query, translate=True): if translate: # Automatic translation to English utt = self.translate(query, "en", self.lang) else: utt = query # cache so we dont hit the api twice for the same query if query not in self.duck_cache: self.duck_cache[query] = requests.get("https://api.duckduckgo.com", params={ "format": "json", "q": utt }).json() data = self.duck_cache[query] # GUI self.gui.clear() # clear previous answer just in case title = data.get("Heading") self.image = data.get("Image", "") # summary summary = data.get("AbstractText") if not summary: return None self.log.debug("DuckDuckGo answer: " + summary) # context for follow up questions # TODO intents for this, with this context intents can look up all data self.set_context("DuckKnows", query) self.idx = 0 self.results = summary.split(". ") return summary def display_ddg(self, summary, image): if image.startswith("/"): image = "https://duckduckgo.com" + image self.gui['summary'] = summary self.gui['imgLink'] = image self.gui.show_page("DuckDelegate.qml", override_idle=60) def speak_result(self): if self.idx + 1 > len(self.results): # TODO ask user if he wants to hear about related topics self.speak_dialog("thats all") self.remove_context("ddg") self.idx = 0 else: if self.image: self.display_ddg(self.results[self.idx], self.image) self.speak(self.results[self.idx]) self.idx += 1 def get_infobox(self, query): if query not in self.duck_cache: self.ask_the_duck(query) data = self.duck_cache[query] # info related_topics = [t.get("Text") for t in data.get("RelatedTopics", [])] infobox = {} infodict = data.get("Infobox") or {} for entry in infodict.get("content", []): k = entry["label"].lower().strip() infobox[k] = entry["value"] return infobox, related_topics def translate(self, utterance, lang_tgt=None, lang_src="en"): lang_tgt = lang_tgt or self.lang # if langs are the same do nothing if not lang_tgt.startswith(lang_src): if lang_tgt not in self.tx_cache: self.tx_cache[lang_tgt] = {} # if translated before, dont translate again if utterance in self.tx_cache[lang_tgt]: # get previous translated value translated_utt = self.tx_cache[lang_tgt][utterance] else: # translate this utterance translated_utt = self.translator.translate( utterance, lang_tgt=lang_tgt, lang_src=lang_src).strip() # save the translation if we need it again self.tx_cache[lang_tgt][utterance] = translated_utt self.log.debug("translated {src} -- {tgt}".format( src=utterance, tgt=translated_utt)) else: translated_utt = utterance.strip() return translated_utt
class PadatiousService(FallbackSkill): def __init__(self, emitter, service): FallbackSkill.__init__(self) self.config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return ver = get_distribution('padatious').version if ver != PADATIOUS_VERSION: LOG.warning('Using Padatious v' + ver + '. Please re-run ' + 'dev_setup.sh to install ' + PADATIOUS_VERSION) self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.emitter.on('padatious:register_entity', self.register_entity) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train() def wait_and_train(self): sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.finished_training_event.clear() LOG.info('Training...') self.container.train() LOG.info('Training complete.') self.finished_training_event.set() def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message): utt = message.data.get('utterance') LOG.debug("Padatious fallback attempt: " + utt) if not self.finished_training_event.is_set(): LOG.debug('Waiting for training to finish...') self.finished_training_event.wait() data = self.container.calc_intent(utt) if data.conf < 0.5: return False data.matches['utterance'] = utt self.service.add_active_skill(int(data.name.split(':')[0])) self.emitter.emit(Message(data.name, data=data.matches)) return True
class PadatiousExtractor(IntentExtractor): keyword_based = False def __init__(self, cache_dir=None, *args, **kwargs): super().__init__(*args, **kwargs) # TODO xdg data_dir data_dir = expanduser(self.config.get("data_dir", "~/.padatious")) cache_dir = cache_dir or join(data_dir, "padatious") self.lock = Lock() self.container = IntentContainer(cache_dir) self.registered_intents = [] def detach_intent(self, intent_name): if intent_name in self.registered_intents: LOG.debug("Detaching padatious intent: " + intent_name) with self.lock: self.container.remove_intent(intent_name) self.registered_intents.remove(intent_name) def detach_skill(self, skill_id): LOG.debug("Detaching padatious skill: " + str(skill_id)) remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.detach_intent(i) def register_entity(self, entity_name, samples=None, reload_cache=True): samples = samples or [entity_name] with self.lock: self.container.add_entity(entity_name, samples, reload_cache=reload_cache) def register_intent(self, intent_name, samples=None, reload_cache=True): samples = samples or [intent_name] if intent_name not in self._intent_samples: self._intent_samples[intent_name] = samples else: self._intent_samples[intent_name] += samples with self.lock: self.container.add_intent(intent_name, samples, reload_cache=reload_cache) self.registered_intents.append(intent_name) def register_entity_from_file(self, entity_name, file_name, reload_cache=True): with self.lock: self.container.load_entity(entity_name, file_name, reload_cache=reload_cache) def register_intent_from_file(self, intent_name, file_name, single_thread=True, timeout=120, reload_cache=True, force_training=True): try: with self.lock: self.container.load_intent(intent_name, file_name, reload_cache=reload_cache) self.registered_intents.append(intent_name) success = self._train(single_thread=single_thread, timeout=timeout, force_training=force_training) if success: LOG.debug(file_name + " trained successfully") else: LOG.error(file_name + " FAILED TO TRAIN") except Exception as e: LOG.exception(e) def _get_remainder(self, intent, utterance): if intent["name"] in self.intent_samples: return get_utterance_remainder( utterance, samples=self.intent_samples[intent["name"]]) return utterance def calc_intent(self, utterance, min_conf=None): min_conf = min_conf or self.config.get("padatious_min_conf", 0.65) utterance = utterance.strip().lower() with self.lock: intent = self.container.calc_intent(utterance).__dict__ if intent["conf"] < min_conf: return { "intent_type": "unknown", "entities": {}, "conf": 0, "intent_engine": "padatious", "utterance": utterance, "utterance_remainder": utterance } intent["utterance_remainder"] = self._get_remainder(intent, utterance) intent["entities"] = intent.pop("matches") intent["intent_engine"] = "padatious" intent["intent_type"] = intent.pop("name") intent["utterance"] = intent.pop("sent") if isinstance(intent["utterance"], list): intent["utterance"] = " ".join(intent["utterance"]) return intent def intent_scores(self, utterance): utterance = utterance.strip().lower() intents = [i.__dict__ for i in self.container.calc_intents(utterance)] for idx, intent in enumerate(intents): intent["utterance_remainder"] = self._get_remainder( intent, utterance) intents[idx]["entities"] = intents[idx].pop("matches") intents[idx]["intent_type"] = intents[idx].pop("name") intent["intent_engine"] = "padatious" intent["utterance"] = intent.pop("sent") if isinstance(intents[idx]["utterance"], list): intents[idx]["utterance"] = " ".join(intents[idx]["utterance"]) return intents def calc_intents(self, utterance, min_conf=None): min_conf = min_conf or self.config.get("padatious_min_conf", 0.65) utterance = utterance.strip().lower() bucket = {} for ut in self.segmenter.segment(utterance): intent = self.calc_intent(ut) if intent["conf"] < min_conf: bucket[ut] = None else: bucket[ut] = intent return bucket def calc_intents_list(self, utterance): utterance = utterance.strip().lower() bucket = {} for ut in self.segmenter.segment(utterance): bucket[ut] = self.filter_intents(ut) return bucket def manifest(self): # TODO vocab, skill ids, intent_data return {"intent_names": self.registered_intents} def _train(self, single_thread=True, timeout=120, force_training=True): with self.lock: return self.container.train(single_thread=single_thread, timeout=timeout, force=force_training, debug=True)
class PadatiousService(FallbackSkill): def __init__(self, emitter): FallbackSkill.__init__(self) self.config = ConfigurationManager.get()['padatious'] intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return ver = get_distribution('padatious').version LOG.warning('VERSION: ' + ver) if ver != PADATIOUS_VERSION: LOG.warning('Using Padatious v' + ver + '. Please re-run ' + 'dev_setup.sh to install ' + PADATIOUS_VERSION) self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train() def wait_and_train(self): sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.finished_training_event.clear() LOG.info('Training...') self.container.train(print_updates=False) LOG.info('Training complete.') self.finished_training_event.set() def register_intent(self, message): LOG.debug('Registering Padatious intent: ' + message.data['intent_name']) file_name = message.data['file_name'] intent_name = message.data['intent_name'] if not isfile(file_name): return self.container.load_file(intent_name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def handle_fallback(self, message): utt = message.data.get('utterance') LOG.debug("Padatious fallback attempt: " + utt) utt = normalize(utt, message.data.get('lang', 'en-us')) if not self.finished_training_event.is_set(): LOG.debug('Waiting for training to finish...') self.finished_training_event.wait() data = self.container.calc_intent(utt) if data.conf < 0.5: return False self.emitter.emit(Message(data.name, data=data.matches)) return True
class PadatiousService(FallbackSkill): instance = None fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match def __init__(self, bus, service): FallbackSkill.__init__(self, use_settings=False) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call([ 'notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again' ]) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) self.bus.on('intent.service.padatious.get', self.handle_get_padatious) self.bus.on('intent.service.padatious.manifest.get', self.handle_manifest) self.bus.on('intent.service.padatious.entities.manifest.get', self.handle_entity_manifest) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] self.registered_entities = [] def make_active(self): """Override the make active since this is not a real fallback skill.""" pass def train(self, message=None): padatious_single_thread = Configuration.get( )['padatious']['single_thread'] if message is None: single_thread = padatious_single_thread else: single_thread = message.data.get('single_thread', padatious_single_thread) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() if not self.finished_initial_train: LOG.info("Mycroft is all loaded and ready to roll!") self.bus.emit(Message('mycroft.ready')) self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def __detach_intent(self, intent_name): """ Remove an intent if it has been registered. Arguments: intent_name (str): intent identifier """ if intent_name in self.registered_intents: self.registered_intents.remove(intent_name) self.container.remove_intent(intent_name) def handle_detach_intent(self, message): self.__detach_intent(message.data.get('intent_name')) def handle_detach_skill(self, message): skill_id = message.data['skill_id'] remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.__detach_intent(i) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self.registered_intents.append(message.data['name']) self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self.registered_entities.append(message.data) self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message, threshold=0.8): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance', '') LOG.debug("Padatious fallback attempt: " + utt) intent = self.calc_intent(utt) if not intent or intent.conf < threshold: # Attempt to use normalized() version norm = message.data.get('norm_utt', utt) if norm != utt: LOG.debug(" alt attempt: " + norm) intent = self.calc_intent(norm) utt = norm if not intent or intent.conf < threshold: return False intent.matches['utterance'] = utt self.service.add_active_skill(intent.name.split(':')[0]) self.bus.emit(message.forward(intent.name, data=intent.matches)) return True def handle_fallback_last_chance(self, message): return self.handle_fallback(message, 0.5) def handle_get_padatious(self, message): utterance = message.data["utterance"] norm = message.data.get('norm_utt', utterance) intent = self.calc_intent(utterance) if not intent and norm != utterance: intent = PadatiousService.instance.calc_intent(norm) if intent: intent = intent.__dict__ self.bus.emit( message.reply("intent.service.padatious.reply", {"intent": intent})) def handle_manifest(self, message): self.bus.emit( message.reply("intent.service.padatious.manifest", {"intents": self.registered_intents})) def handle_entity_manifest(self, message): self.bus.emit( message.reply("intent.service.padatious.entities.manifest", {"entities": self.registered_entities})) # NOTE: This cache will keep a reference to this calss (PadatiousService), # but we can live with that since it is used as a singleton. @lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache def calc_intent(self, utt): return self.container.calc_intent(utt)
class PadatiousService(FallbackSkill): def __init__(self, emitter): FallbackSkill.__init__(self) self.config = ConfigurationManager.get()['padatious'] intent_cache = expanduser(self.config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return ver = get_distribution('padatious').version if ver != PADATIOUS_VERSION: LOG.warning('Using Padatious v' + ver + '. Please re-run ' + 'dev_setup.sh to install ' + PADATIOUS_VERSION) self.container = IntentContainer(intent_cache) self.emitter = emitter self.emitter.on('padatious:register_intent', self.register_intent) self.emitter.on('padatious:register_entity', self.register_entity) self.register_fallback(self.handle_fallback, 5) self.finished_training_event = Event() self.train_delay = self.config['train_delay'] self.train_time = get_time() + self.train_delay self.wait_and_train() def wait_and_train(self): sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.finished_training_event.clear() LOG.info('Training...') self.container.train() LOG.info('Training complete.') self.finished_training_event.set() def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message): utt = message.data.get('utterance') LOG.debug("Padatious fallback attempt: " + utt) if not self.finished_training_event.is_set(): LOG.debug('Waiting for training to finish...') self.finished_training_event.wait() data = self.container.calc_intent(utt) if data.conf < 0.5: return False data.matches['utterance'] = utt self.emitter.emit(Message(data.name, data=data.matches)) return True
class PadatiousService(FallbackSkill): instance = None fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match def __init__(self, bus, service): FallbackSkill.__init__(self) if not PadatiousService.instance: PadatiousService.instance = self self.padatious_config = Configuration.get()['padatious'] self.service = service intent_cache = expanduser(self.padatious_config['intent_cache']) try: from padatious import IntentContainer except ImportError: LOG.error('Padatious not installed. Please re-run dev_setup.sh') try: call(['notify-send', 'Padatious not installed', 'Please run build_host_setup and dev_setup again']) except OSError: pass return self.container = IntentContainer(intent_cache) self._bus = bus self.bus.on('padatious:register_intent', self.register_intent) self.bus.on('padatious:register_entity', self.register_entity) self.bus.on('detach_intent', self.handle_detach_intent) self.bus.on('detach_skill', self.handle_detach_skill) self.bus.on('mycroft.skills.initialized', self.train) # Call Padatious an an early fallback, looking for a high match intent self.register_fallback(self.handle_fallback, PadatiousService.fallback_tight_match) # Try loose Padatious intent match before going to fallback-unknown self.register_fallback(self.handle_fallback_last_chance, PadatiousService.fallback_loose_match) self.finished_training_event = Event() self.finished_initial_train = False self.train_delay = self.padatious_config['train_delay'] self.train_time = get_time() + self.train_delay self.registered_intents = [] def train(self, message=None): if message is None: single_thread = False else: single_thread = message.data.get('single_thread', False) self.finished_training_event.clear() LOG.info('Training... (single_thread={})'.format(single_thread)) self.container.train(single_thread=single_thread) LOG.info('Training complete.') self.finished_training_event.set() if not self.finished_initial_train: LOG.info("Mycroft is all loaded and ready to roll!") self.bus.emit(Message('mycroft.ready')) self.finished_initial_train = True def wait_and_train(self): if not self.finished_initial_train: return sleep(self.train_delay) if self.train_time < 0.0: return if self.train_time <= get_time() + 0.01: self.train_time = -1.0 self.train() def __detach_intent(self, intent_name): self.registered_intents.remove(intent_name) self.container.remove_intent(intent_name) def handle_detach_intent(self, message): self.__detach_intent(message.data.get('intent_name')) def handle_detach_skill(self, message): skill_id = message.data['skill_id'] remove_list = [i for i in self.registered_intents if skill_id in i] for i in remove_list: self.__detach_intent(i) def _register_object(self, message, object_name, register_func): file_name = message.data['file_name'] name = message.data['name'] LOG.debug('Registering Padatious ' + object_name + ': ' + name) if not isfile(file_name): LOG.warning('Could not find file ' + file_name) return register_func(name, file_name) self.train_time = get_time() + self.train_delay self.wait_and_train() def register_intent(self, message): self.registered_intents.append(message.data['name']) self._register_object(message, 'intent', self.container.load_intent) def register_entity(self, message): self._register_object(message, 'entity', self.container.load_entity) def handle_fallback(self, message, threshold=0.8): if not self.finished_training_event.is_set(): LOG.debug('Waiting for Padatious training to finish...') return False utt = message.data.get('utterance', '') LOG.debug("Padatious fallback attempt: " + utt) intent = self.calc_intent(utt) if not intent or intent.conf < threshold: # Attempt to use normalized() version norm = message.data.get('norm_utt', '') if norm != utt: LOG.debug(" alt attempt: " + norm) intent = self.calc_intent(norm) utt = norm if not intent or intent.conf < threshold: return False intent.matches['utterance'] = utt self.service.add_active_skill(intent.name.split(':')[0]) self.bus.emit(message.reply(intent.name, data=intent.matches)) return True def handle_fallback_last_chance(self, message): return self.handle_fallback(message, 0.5) # NOTE: This cache will keep a reference to this calss (PadatiousService), # but we can live with that since it is used as a singleton. @lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache def calc_intent(self, utt): return self.container.calc_intent(utt)
def determine_intent(msg): container = IntentContainer('intent_cache') container.add_intent('time', ['What is the time now.', 'Tell me the time']) container.add_intent( 'goodbye', ['See you!', 'Goodbye!', 'Bye', 'Bye Bye', 'Go to sleep']) container.add_intent('search', [ 'Search for {query} (using|on) (internet|web|google).', 'Lookup {query} (using|on) the (internet|web|google).', 'Search {addons} {query}.' ]) container.add_intent('device', [ 'Turn the {location} {device} {state}.', 'Turn {state} the {location} {device}.', 'Turn the {device} {state}.' ]) #container.add_intent('fans', ['Turn the {location} (fan|fans) {state}.', 'Turn {state} the {location} (fan|fans).']) container.add_intent('Wolfram', [ 'What (is|are|was|were) {query}.', 'Who (is|was|were|invented|did|scored|will be) {query}.', 'When (is|are|was|were) {query}.', 'How (is|are|was|were) {query}.' ]) container.add_intent( 'Creator', ['Who created you.', 'Who made you.', 'By whom were you created.']) container.add_intent( 'news', ['Tell me the news.', 'What is the news.', 'Get me the news update']) container.add_intent( 'weather', ['Tell me about the weather.', 'What is the weather like.']) container.add_intent('Joke', [ 'Tell me a joke.', 'Tell me a {type} joke.', 'Can you (say|tell) a joke.', 'Can you (say|tell) a {type} joke.' ]) container.add_intent('Me', ['Tell me about yourself.', '(Who|What) are you.'], 'What is your name.') container.add_intent( 'Praise', ['I am great.', 'I (did|completed) {task}.']) #Sarcasm container.train() data = container.calc_intent(msg) return data
def __init__(self, ): self.container = IntentContainer('intent_cache') self.commands = Commands()
#!/usr/bin/env python3 # Sample Padatious program used for testing import sys from glob import glob from os.path import basename from padatious import IntentContainer reload_cache = len(sys.argv) > 1 and sys.argv[1] == '-r' container = IntentContainer('intent_cache') for file_name in glob('data/*.intent'): name = basename(file_name).replace('.intent', '') container.load_file(name, file_name, reload_cache=reload_cache) for file_name in glob('data/*.entity'): name = basename(file_name).replace('.entity', '') container.load_entity(name, file_name, reload_cache=reload_cache) container.train() query = None while query != 'q': query = input('> ') data = container.calc_intent(query) print(data.name + ': ' + str(data.conf)) for key, val in data.matches.items(): print('\t' + key + ': ' + val)