class PadatiousFileIntent(IntentPlugin):
"""Interface for Padatious intent engine"""
def __init__(self, rt):
super().__init__(rt)
self.container = IntentContainer(
join(rt.paths.user_config, 'intent_cache'))
def register(self, intent: Any, skill_name: str, intent_id: str):
file_name = join(self.rt.paths.skill_locale(skill_name),
intent + '.intent')
self.container.load_intent(name=intent_id, file_name=file_name)
def register_entity(self, entity: Any, entity_id: str, skill_name: str):
file_name = join(self.rt.paths.skill_locale(skill_name),
entity + '.intent')
self.container.load_intent(name=entity_id, file_name=file_name)
def unregister(self, intent_id: str):
self.container.remove_intent(intent_id)
def unregister_entity(self, entity_id: str):
self.container.remove_entity(entity_id)
def compile(self):
log.info('Training...')
self.container.train()
log.info('Training complete!')
def calc_intents(self, query):
return [
IntentMatch(intent_id=data.name,
confidence=data.conf,
matches=data.matches,
query=query)
for data in self.container.calc_intents(query)
]
class PadatiousService(FallbackSkill):
instance = None
fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match
fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match
def __init__(self, bus, service):
FallbackSkill.__init__(self, use_settings=False)
if not PadatiousService.instance:
PadatiousService.instance = self
self.padatious_config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call([
'notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'
])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
self.bus.on('intent.service.padatious.get', self.handle_get_padatious)
self.bus.on('intent.service.padatious.manifest.get',
self.handle_manifest)
self.bus.on('intent.service.padatious.entities.manifest.get',
self.handle_entity_manifest)
# Call Padatious an an early fallback, looking for a high match intent
self.register_fallback(self.handle_fallback,
PadatiousService.fallback_tight_match)
# Try loose Padatious intent match before going to fallback-unknown
self.register_fallback(self.handle_fallback_last_chance,
PadatiousService.fallback_loose_match)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
self.registered_entities = []
def make_active(self):
"""Override the make active since this is not a real fallback skill."""
pass
def train(self, message=None):
padatious_single_thread = Configuration.get(
)['padatious']['single_thread']
if message is None:
single_thread = padatious_single_thread
else:
single_thread = message.data.get('single_thread',
padatious_single_thread)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
LOG.info("Mycroft is all loaded and ready to roll!")
self.bus.emit(Message('mycroft.ready'))
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
""" Remove an intent if it has been registered.
Arguments:
intent_name (str): intent identifier
"""
if intent_name in self.registered_intents:
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self.registered_entities.append(message.data)
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message, threshold=0.8):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance', '')
LOG.debug("Padatious fallback attempt: " + utt)
intent = self.calc_intent(utt)
if not intent or intent.conf < threshold:
# Attempt to use normalized() version
norm = message.data.get('norm_utt', utt)
if norm != utt:
LOG.debug(" alt attempt: " + norm)
intent = self.calc_intent(norm)
utt = norm
if not intent or intent.conf < threshold:
return False
intent.matches['utterance'] = utt
self.service.add_active_skill(intent.name.split(':')[0])
self.bus.emit(message.forward(intent.name, data=intent.matches))
return True
def handle_fallback_last_chance(self, message):
return self.handle_fallback(message, 0.5)
def handle_get_padatious(self, message):
utterance = message.data["utterance"]
norm = message.data.get('norm_utt', utterance)
intent = self.calc_intent(utterance)
if not intent and norm != utterance:
intent = PadatiousService.instance.calc_intent(norm)
if intent:
intent = intent.__dict__
self.bus.emit(
message.reply("intent.service.padatious.reply",
{"intent": intent}))
def handle_manifest(self, message):
self.bus.emit(
message.reply("intent.service.padatious.manifest",
{"intents": self.registered_intents}))
def handle_entity_manifest(self, message):
self.bus.emit(
message.reply("intent.service.padatious.entities.manifest",
{"entities": self.registered_entities}))
# NOTE: This cache will keep a reference to this calss (PadatiousService),
# but we can live with that since it is used as a singleton.
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class PadatiousService(FallbackSkill):
instance = None
def __init__(self, bus, service):
FallbackSkill.__init__(self)
if not PadatiousService.instance:
PadatiousService.instance = self
self.config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call([
'notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'
])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self.bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('owo.skills.initialized', self.train)
self.register_fallback(self.handle_fallback, 5)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.config['train_delay']
self.train_time = get_time() + self.train_delay
def train(self, message=None):
if message is None:
single_thread = False
else:
single_thread = message.data.get('single_thread', False)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
self.container.remove_intent(intent_name)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance')
LOG.debug("Padatious fallback attempt: " + utt)
data = self.calc_intent(utt)
if data.conf < 0.5:
return False
data.matches['utterance'] = utt
self.service.add_active_skill(data.name.split(':')[0])
self.bus.emit(message.reply(data.name, data=data.matches))
return True
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class PadatiousExtractor(IntentExtractor):
keyword_based = False
def __init__(self, cache_dir=None, *args, **kwargs):
super().__init__(*args, **kwargs)
# TODO xdg data_dir
data_dir = expanduser(self.config.get("data_dir", "~/.padatious"))
cache_dir = cache_dir or join(data_dir, "padatious")
self.lock = Lock()
self.container = IntentContainer(cache_dir)
self.registered_intents = []
def detach_intent(self, intent_name):
if intent_name in self.registered_intents:
LOG.debug("Detaching padatious intent: " + intent_name)
with self.lock:
self.container.remove_intent(intent_name)
self.registered_intents.remove(intent_name)
def detach_skill(self, skill_id):
LOG.debug("Detaching padatious skill: " + str(skill_id))
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.detach_intent(i)
def register_entity(self, entity_name, samples=None, reload_cache=True):
samples = samples or [entity_name]
with self.lock:
self.container.add_entity(entity_name,
samples,
reload_cache=reload_cache)
def register_intent(self, intent_name, samples=None, reload_cache=True):
samples = samples or [intent_name]
if intent_name not in self._intent_samples:
self._intent_samples[intent_name] = samples
else:
self._intent_samples[intent_name] += samples
with self.lock:
self.container.add_intent(intent_name,
samples,
reload_cache=reload_cache)
self.registered_intents.append(intent_name)
def register_entity_from_file(self,
entity_name,
file_name,
reload_cache=True):
with self.lock:
self.container.load_entity(entity_name,
file_name,
reload_cache=reload_cache)
def register_intent_from_file(self,
intent_name,
file_name,
single_thread=True,
timeout=120,
reload_cache=True,
force_training=True):
try:
with self.lock:
self.container.load_intent(intent_name,
file_name,
reload_cache=reload_cache)
self.registered_intents.append(intent_name)
success = self._train(single_thread=single_thread,
timeout=timeout,
force_training=force_training)
if success:
LOG.debug(file_name + " trained successfully")
else:
LOG.error(file_name + " FAILED TO TRAIN")
except Exception as e:
LOG.exception(e)
def _get_remainder(self, intent, utterance):
if intent["name"] in self.intent_samples:
return get_utterance_remainder(
utterance, samples=self.intent_samples[intent["name"]])
return utterance
def calc_intent(self, utterance, min_conf=None):
min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
utterance = utterance.strip().lower()
with self.lock:
intent = self.container.calc_intent(utterance).__dict__
if intent["conf"] < min_conf:
return {
"intent_type": "unknown",
"entities": {},
"conf": 0,
"intent_engine": "padatious",
"utterance": utterance,
"utterance_remainder": utterance
}
intent["utterance_remainder"] = self._get_remainder(intent, utterance)
intent["entities"] = intent.pop("matches")
intent["intent_engine"] = "padatious"
intent["intent_type"] = intent.pop("name")
intent["utterance"] = intent.pop("sent")
if isinstance(intent["utterance"], list):
intent["utterance"] = " ".join(intent["utterance"])
return intent
def intent_scores(self, utterance):
utterance = utterance.strip().lower()
intents = [i.__dict__ for i in self.container.calc_intents(utterance)]
for idx, intent in enumerate(intents):
intent["utterance_remainder"] = self._get_remainder(
intent, utterance)
intents[idx]["entities"] = intents[idx].pop("matches")
intents[idx]["intent_type"] = intents[idx].pop("name")
intent["intent_engine"] = "padatious"
intent["utterance"] = intent.pop("sent")
if isinstance(intents[idx]["utterance"], list):
intents[idx]["utterance"] = " ".join(intents[idx]["utterance"])
return intents
def calc_intents(self, utterance, min_conf=None):
min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
utterance = utterance.strip().lower()
bucket = {}
for ut in self.segmenter.segment(utterance):
intent = self.calc_intent(ut)
if intent["conf"] < min_conf:
bucket[ut] = None
else:
bucket[ut] = intent
return bucket
def calc_intents_list(self, utterance):
utterance = utterance.strip().lower()
bucket = {}
for ut in self.segmenter.segment(utterance):
bucket[ut] = self.filter_intents(ut)
return bucket
def manifest(self):
# TODO vocab, skill ids, intent_data
return {"intent_names": self.registered_intents}
def _train(self, single_thread=True, timeout=120, force_training=True):
with self.lock:
return self.container.train(single_thread=single_thread,
timeout=timeout,
force=force_training,
debug=True)
class PadatiousService(FallbackSkill):
instance = None
fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match
fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match
def __init__(self, bus, service):
FallbackSkill.__init__(self)
if not PadatiousService.instance:
PadatiousService.instance = self
self.padatious_config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
# Call Padatious an an early fallback, looking for a high match intent
self.register_fallback(self.handle_fallback,
PadatiousService.fallback_tight_match)
# Try loose Padatious intent match before going to fallback-unknown
self.register_fallback(self.handle_fallback_last_chance,
PadatiousService.fallback_loose_match)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
def train(self, message=None):
if message is None:
single_thread = False
else:
single_thread = message.data.get('single_thread', False)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
LOG.info("Mycroft is all loaded and ready to roll!")
self.bus.emit(Message('mycroft.ready'))
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message, threshold=0.8):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance', '')
LOG.debug("Padatious fallback attempt: " + utt)
intent = self.calc_intent(utt)
if not intent or intent.conf < threshold:
# Attempt to use normalized() version
norm = message.data.get('norm_utt', '')
if norm != utt:
LOG.debug(" alt attempt: " + norm)
intent = self.calc_intent(norm)
utt = norm
if not intent or intent.conf < threshold:
return False
intent.matches['utterance'] = utt
self.service.add_active_skill(intent.name.split(':')[0])
self.bus.emit(message.reply(intent.name, data=intent.matches))
return True
def handle_fallback_last_chance(self, message):
return self.handle_fallback(message, 0.5)
# NOTE: This cache will keep a reference to this calss (PadatiousService),
# but we can live with that since it is used as a singleton.
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class PadatiousService:
"""Service class for padatious intent matching."""
def __init__(self, bus, config):
self.padatious_config = config
self.bus = bus
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
self.registered_entities = []
def train(self, message=None):
"""Perform padatious training.
Arguments:
message (Message): optional triggering message
"""
padatious_single_thread = Configuration.get()[
'padatious']['single_thread']
if message is None:
single_thread = padatious_single_thread
else:
single_thread = message.data.get('single_thread',
padatious_single_thread)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
self.bus.emit(Message('mycroft.skills.trained'))
self.finished_initial_train = True
def wait_and_train(self):
"""Wait for minimum time between training and start training."""
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
""" Remove an intent if it has been registered.
Arguments:
intent_name (str): intent identifier
"""
if intent_name in self.registered_intents:
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
"""Messagebus handler for detaching padatious intent.
Arguments:
message (Message): message triggering action
"""
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
"""Messagebus handler for detaching all intents for skill.
Arguments:
message (Message): message triggering action
"""
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
"""Generic method for registering a padatious object.
Arguments:
message (Message): trigger for action
object_name (str): type of entry to register
register_func (callable): function to call for registration
"""
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
"""Messagebus handler for registering intents.
Arguments:
message (Message): message triggering action
"""
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
"""Messagebus handler for registering entities.
Arguments:
message (Message): message triggering action
"""
self.registered_entities.append(message.data)
self._register_object(message, 'entity', self.container.load_entity)
def _match_level(self, utterances, limit):
"""Match intent and make sure a certain level of confidence is reached.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
limit (float): required confidence level.
"""
padatious_intent = None
LOG.debug('Padatious Matching confidence > {}'.format(limit))
for utt in utterances:
for variant in utt:
intent = self.calc_intent(variant)
if intent:
best = padatious_intent.conf if padatious_intent else 0.0
if best < intent.conf:
padatious_intent = intent
padatious_intent.matches['utterance'] = utt[0]
if padatious_intent and padatious_intent.conf > limit:
skill_id = padatious_intent.name.split(':')[0]
ret = IntentMatch(
'Padatious', padatious_intent.name, padatious_intent.matches,
skill_id
)
else:
ret = None
return ret
def match_high(self, utterances, _=None, __=None):
"""Intent matcher for high confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.95)
def match_medium(self, utterances, _=None, __=None):
"""Intent matcher for medium confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.8)
def match_low(self, utterances, _=None, __=None):
"""Intent matcher for low confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.5)
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
"""Cached version of container calc_intent.
This improves speed when called multiple times for different confidence
levels.
NOTE: This cache will keep a reference to this class
(PadatiousService), but we can live with that since it is used as a
singleton.
Arguments:
utt (str): utterance to calculate best intent for
"""
return self.container.calc_intent(utt)