class NeuralQuestionParser(RegexQuestionParser):
def __init__(self, lang="en-us"):
self.container = IntentContainer(INTENT_CACHE_PATH)
self._intents = []
self.lang = lang
self.register_default_intents()
self.container.train()
def parse(self, utterance):
data = BasicQuestionParser().parse(utterance)
match = self.container.calc_intent(utterance)
if match.name:
data["QuestionIntent"] = match.name
data.update(match.matches)
data["conf"] = match.conf
return data
class CommandDetector:
def __init__(self, ):
self.container = IntentContainer('intent_cache')
self.commands = Commands()
def add_command(self, commands):
for c in self.commands.family_list:
for intent, filename in c.items():
self.container.load_file(intent, filename)
def train_commands(self):
for c in self.commands.family_list:
self.add_command(c)
self.container.train()
def calc_intent(self, text):
return self.container.calc_intent(text)
def determine_intent(msg):
container = IntentContainer('intent_cache')
container.add_intent('time', [ 'What is the time now.', 'Tell me the time'])
container.add_intent('hello', ['Hi', 'Hello', 'Hey', 'Wassup', 'Ssup', 'Namaste', 'Konichiwa', 'ssup'])
container.add_intent('goodbye', ['See you!', 'Goodbye!', 'Bye', 'Bye Bye', 'Go to sleep'])
container.add_intent('search', ['Search for {query} (using|on) (internet|web|google).', 'Lookup {query} (using|on) the (internet|web|google).', 'Search {addons} {query}.'])
container.add_intent('device', ['Turn the {location} {device} {state}.', 'Turn {state} the {location} {device}.', 'Turn the {device} {state}.'])
#container.add_intent('fans', ['Turn the {location} (fan|fans) {state}.', 'Turn {state} the {location} (fan|fans).'])
container.add_intent('Wolfram', ['What (is|are|was|were) {query}.', 'Who (is|was|were|invented|did|scored|will be) {query}.', 'When (is|are|was|were) {query}.', 'How (is|are|was|were) {query}.'])
container.add_intent('Creator', ['Who created you.', 'Who made you.', 'By whom were you created.'])
container.add_intent('news', ['Tell me the news.', 'What is the news.', 'Get me the news update'])
container.add_intent('weather', ['Tell me about the weather.', 'What is the weather like.'])
container.add_intent('Joke', ['Tell me a joke.', 'Tell me a {type} joke.', 'Can you (say|tell) a joke.', 'Can you (say|tell) a {type} joke.' ])
container.add_intent('Me', ['Tell me about yourself.', '(Who|What) are you.'], 'What is your name.')
container.add_intent('Praise', ['I am great.', 'I (did|completed) {task}.']) #Sarcasm
container.train()
data = container.calc_intent(msg)
return data
def parseMsg(msg):
container = IntentContainer('intent_cache')
container.add_intent('greetings',
['hi.', 'hellow', '(waz|wazz|wad) up?', 'hey', 'yoo', 'buddy', '(you|hey) there?'])
container.add_intent('bye', ['bye.', 'tata', 'thank you', 'adios', 'thanks', 'allah hafez', 'done'])
# container.add_intent('goal', ['i want to go (|to) {goal}', '(destination|i wanna go) {goal}',
# 'destination is {goal}'])
# container.add_intent('start', ['from {start}', '(i am (in|at){start}'
# ])
container.add_intent('search',
['I want to go from {start} to {goal}.', 'I want to (go to|go) {goal} from {start}.'])
container.add_intent('ajaira', ['lala', 'kchdskfhsk', 'iwurhb', 'uerwyvdsvjjkc', 'sufgbsdjc'])
container.train()
result = container.calc_intent(msg)
print(result)
print(result.name)
# print(str(result.matches['start']))
# print(result.matches['start'])
# print(result.matches['goal'])
return result
class PadatiousService(FallbackSkill):
def __init__(self, emitter):
FallbackSkill.__init__(self)
self.config = ConfigurationManager.get()['padatious']
intent_cache = expanduser(self.config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
ver = get_distribution('padatious').version
if ver != PADATIOUS_VERSION:
LOG.warning('Using Padatious v' + ver + '. Please re-run ' +
'dev_setup.sh to install ' + PADATIOUS_VERSION)
self.container = IntentContainer(intent_cache)
self.emitter = emitter
self.emitter.on('padatious:register_intent', self.register_intent)
self.emitter.on('padatious:register_entity', self.register_entity)
self.register_fallback(self.handle_fallback, 5)
self.finished_training_event = Event()
self.train_delay = self.config['train_delay']
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def wait_and_train(self):
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.finished_training_event.clear()
LOG.info('Training...')
self.container.train()
LOG.info('Training complete.')
self.finished_training_event.set()
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message):
utt = message.data.get('utterance')
LOG.debug("Padatious fallback attempt: " + utt)
if not self.finished_training_event.is_set():
LOG.debug('Waiting for training to finish...')
self.finished_training_event.wait()
data = self.container.calc_intent(utt)
if data.conf < 0.5:
return False
data.matches['utterance'] = utt
self.emitter.emit(Message(data.name, data=data.matches))
return True
class PadatiousService(FallbackSkill):
def __init__(self, emitter):
FallbackSkill.__init__(self)
self.config = ConfigurationManager.get()['padatious']
intent_cache = expanduser(self.config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
ver = get_distribution('padatious').version
LOG.warning('VERSION: ' + ver)
if ver != PADATIOUS_VERSION:
LOG.warning('Using Padatious v' + ver + '. Please re-run ' +
'dev_setup.sh to install ' + PADATIOUS_VERSION)
self.container = IntentContainer(intent_cache)
self.emitter = emitter
self.emitter.on('padatious:register_intent', self.register_intent)
self.register_fallback(self.handle_fallback, 5)
self.finished_training_event = Event()
self.train_delay = self.config['train_delay']
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def wait_and_train(self):
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.finished_training_event.clear()
LOG.info('Training...')
self.container.train(print_updates=False)
LOG.info('Training complete.')
self.finished_training_event.set()
def register_intent(self, message):
LOG.debug('Registering Padatious intent: ' +
message.data['intent_name'])
file_name = message.data['file_name']
intent_name = message.data['intent_name']
if not isfile(file_name):
return
self.container.load_file(intent_name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def handle_fallback(self, message):
utt = message.data.get('utterance')
LOG.debug("Padatious fallback attempt: " + utt)
utt = normalize(utt, message.data.get('lang', 'en-us'))
if not self.finished_training_event.is_set():
LOG.debug('Waiting for training to finish...')
self.finished_training_event.wait()
data = self.container.calc_intent(utt)
if data.conf < 0.5:
return False
self.emitter.emit(Message(data.name, data=data.matches))
return True
class PadatiousService:
"""Service class for padatious intent matching."""
def __init__(self, bus, config):
self.padatious_config = config
self.bus = bus
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
self.registered_entities = []
def train(self, message=None):
"""Perform padatious training.
Arguments:
message (Message): optional triggering message
"""
padatious_single_thread = Configuration.get()[
'padatious']['single_thread']
if message is None:
single_thread = padatious_single_thread
else:
single_thread = message.data.get('single_thread',
padatious_single_thread)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
self.bus.emit(Message('mycroft.skills.trained'))
self.finished_initial_train = True
def wait_and_train(self):
"""Wait for minimum time between training and start training."""
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
""" Remove an intent if it has been registered.
Arguments:
intent_name (str): intent identifier
"""
if intent_name in self.registered_intents:
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
"""Messagebus handler for detaching padatious intent.
Arguments:
message (Message): message triggering action
"""
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
"""Messagebus handler for detaching all intents for skill.
Arguments:
message (Message): message triggering action
"""
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
"""Generic method for registering a padatious object.
Arguments:
message (Message): trigger for action
object_name (str): type of entry to register
register_func (callable): function to call for registration
"""
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
"""Messagebus handler for registering intents.
Arguments:
message (Message): message triggering action
"""
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
"""Messagebus handler for registering entities.
Arguments:
message (Message): message triggering action
"""
self.registered_entities.append(message.data)
self._register_object(message, 'entity', self.container.load_entity)
def _match_level(self, utterances, limit):
"""Match intent and make sure a certain level of confidence is reached.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
limit (float): required confidence level.
"""
padatious_intent = None
LOG.debug('Padatious Matching confidence > {}'.format(limit))
for utt in utterances:
for variant in utt:
intent = self.calc_intent(variant)
if intent:
best = padatious_intent.conf if padatious_intent else 0.0
if best < intent.conf:
padatious_intent = intent
padatious_intent.matches['utterance'] = utt[0]
if padatious_intent and padatious_intent.conf > limit:
skill_id = padatious_intent.name.split(':')[0]
ret = IntentMatch(
'Padatious', padatious_intent.name, padatious_intent.matches,
skill_id
)
else:
ret = None
return ret
def match_high(self, utterances, _=None, __=None):
"""Intent matcher for high confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.95)
def match_medium(self, utterances, _=None, __=None):
"""Intent matcher for medium confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.8)
def match_low(self, utterances, _=None, __=None):
"""Intent matcher for low confidence.
Arguments:
utterances (list of tuples): Utterances to parse, originals paired
with optional normalized version.
"""
return self._match_level(utterances, 0.5)
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
"""Cached version of container calc_intent.
This improves speed when called multiple times for different confidence
levels.
NOTE: This cache will keep a reference to this class
(PadatiousService), but we can live with that since it is used as a
singleton.
Arguments:
utt (str): utterance to calculate best intent for
"""
return self.container.calc_intent(utt)
class PadatiousService(FallbackSkill):
instance = None
def __init__(self, bus, service):
FallbackSkill.__init__(self)
if not PadatiousService.instance:
PadatiousService.instance = self
self.config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call([
'notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'
])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self.bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('owo.skills.initialized', self.train)
self.register_fallback(self.handle_fallback, 5)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.config['train_delay']
self.train_time = get_time() + self.train_delay
def train(self, message=None):
if message is None:
single_thread = False
else:
single_thread = message.data.get('single_thread', False)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
self.container.remove_intent(intent_name)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance')
LOG.debug("Padatious fallback attempt: " + utt)
data = self.calc_intent(utt)
if data.conf < 0.5:
return False
data.matches['utterance'] = utt
self.service.add_active_skill(data.name.split(':')[0])
self.bus.emit(message.reply(data.name, data=data.matches))
return True
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class PadatiousService(FallbackSkill):
def __init__(self, emitter, service):
FallbackSkill.__init__(self)
self.config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call([
'notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'
])
except OSError:
pass
return
ver = get_distribution('padatious').version
if ver != PADATIOUS_VERSION:
LOG.warning('Using Padatious v' + ver + '. Please re-run ' +
'dev_setup.sh to install ' + PADATIOUS_VERSION)
self.container = IntentContainer(intent_cache)
self.emitter = emitter
self.emitter.on('padatious:register_intent', self.register_intent)
self.emitter.on('padatious:register_entity', self.register_entity)
self.register_fallback(self.handle_fallback, 5)
self.finished_training_event = Event()
self.train_delay = self.config['train_delay']
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def wait_and_train(self):
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.finished_training_event.clear()
LOG.info('Training...')
self.container.train()
LOG.info('Training complete.')
self.finished_training_event.set()
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message):
utt = message.data.get('utterance')
LOG.debug("Padatious fallback attempt: " + utt)
if not self.finished_training_event.is_set():
LOG.debug('Waiting for training to finish...')
self.finished_training_event.wait()
data = self.container.calc_intent(utt)
if data.conf < 0.5:
return False
data.matches['utterance'] = utt
self.service.add_active_skill(int(data.name.split(':')[0]))
self.emitter.emit(Message(data.name, data=data.matches))
return True
class Skill(object):
def __init__(self, root_dir, name, nlp, active):
self._root_dir = root_dir
self._name = name
self._nlp = nlp
self._active = active
self._intents_container = None
self._adapt_intent_engine = None
self.initialize_intent_parser()
def is_active(self):
return self._active
def get_name(self):
return self._name
def initialize_intent_parser(self):
self._intents_container = IntentContainer("%s_cache" % self._name)
self._adapt_intent_engine = DomainIntentDeterminationEngine()
self._adapt_intent_engine.register_domain(self._name)
for intent_name, intent_file_path in self.get_intent_names():
#print ("###### IntentBuilder: %s, %s" % (intent_name, intent_file_path))
adapt_intent_builder = IntentBuilder(intent_name)
for intent_name, intent_example_sentences_array in self.intent_training_file_content(
intent_file_path, 'intent'):
#print ("add intent %s, %s" % (intent_name, intent_example_sentences_array))
self._intents_container.add_intent(
intent_name, intent_example_sentences_array)
for entity_name, entities_array in self.intent_training_file_content(
intent_file_path, 'entities'):
#print ("add entity %s, %s " % (entity_name, entities_array))
self._intents_container.add_entity(entity_name, entities_array)
# adapt
if entity_name.endswith("_keyword"):
for k in entities_array:
#print ("add keyword %s to %s" % (k, intent_name))
self._adapt_intent_engine.register_entity(
k, entity_name, domain=self._name)
adapt_intent_builder.require(entity_name)
adapt_intent = adapt_intent_builder.build()
self._adapt_intent_engine.register_intent_parser(adapt_intent,
domain=self._name)
self._intents_container.train(debug=False)
def get_intent_file_content(self, skill_file_path):
content_array = []
with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
for entry in skill_file:
content_array.append(entry)
return content_array
def get_entities_file_content(self, skill_file_path, allow_variations):
content_array = []
with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
for entry in skill_file:
entries, variations = entry.strip().split('|'), []
content_array.append(entries[0])
if allow_variations:
if len(entries) > 1:
content_array.extend(entries[1].split(','))
return content_array
def get_intent_names(self):
intent_root_file_path = os.path.join(self._root_dir, self._name,
'intents')
for intent_name in os.listdir(intent_root_file_path):
intent_file_path = os.path.join(intent_root_file_path, intent_name)
yield intent_name, intent_file_path
def intent_training_file_content(self,
artefacts_root_dir,
artefact_file_extension,
allow_variations=True):
for artefact_file_path in os.listdir(artefacts_root_dir):
if artefact_file_path.endswith('.' + artefact_file_extension):
artefact_name = artefact_file_path.replace(
'.' + artefact_file_extension, '')
if artefact_file_extension is 'entities':
artefact_file_lines = self.get_entities_file_content(
os.path.join(artefacts_root_dir, artefact_file_path),
allow_variations)
elif artefact_file_extension is 'intent':
artefact_file_lines = self.get_intent_file_content(
os.path.join(artefacts_root_dir, artefact_file_path))
yield artefact_name, artefact_file_lines
def expand_intents(self, include_additional_entities=False):
# load entities first in the file and build a dictionary
result = dict()
entities_dict = dict()
for intent_name, intent_file_path in self.get_intent_names():
for entity_type, entities_array in self.intent_training_file_content(
intent_file_path, 'entities', False):
entities_dict[entity_type] = entities_array
# load intents again from file
for intent_type, intent_array in self.intent_training_file_content(
intent_file_path, 'intent'):
intent_sentences = set()
for line in intent_array:
line_tokens = self._nlp.tokenization.tokenize(line)
expanded = expand_parentheses(line_tokens)
for sentence_tokens in expanded:
sentence = self._nlp.tokenization.detokenize(
sentence_tokens)
fieldnames = [
fname
for _, fname, _, _ in Formatter().parse(sentence)
if fname
]
fields_dict = dict()
for fieldname in fieldnames:
if fieldname in entities_dict:
fields_dict[fieldname] = entities_dict[
fieldname].copy()
else:
if include_additional_entities:
field_values = self.get_additional_entities(
fieldname)
if len(field_values) > 0:
fields_dict[fieldname] = field_values
if len(fields_dict) > 0:
keys, values = zip(*fields_dict.items())
permutations = [
dict(zip(keys, v))
for v in itertools.product(*values)
]
for p in permutations:
entities_dict_permutation = EntitiesDict(p)
intent_sentences.add(
sentence.format(
**entities_dict_permutation))
else:
intent_sentences.add(sentence)
result[intent_type] = list(intent_sentences)
return result
def get_additional_entities(self, fieldname):
return []
def calculate_intent(self, text):
text = self._nlp.preprocess(text)
# example result
# {'intent_type': 'beth.fydd.y.tywydd', 'confidence': 1.0, 'target': None, 'keyword': 'tywydd'}
#
#print ("evaluating: %s with adapt:" % text)
adapt_best_confidence = 0.0
adapt_result = self._adapt_intent_engine.determine_intent(text)
for a in adapt_result:
# print (a)
if a["confidence"] > adapt_best_confidence:
adapt_best_confidence = a["confidence"]
# example result
# {'sent': "beth yw ' r tywydd", 'name': 'beth.ywr.tywydd', 'conf': 1.0, 'matches': {'tywydd_keyword': 'tywydd?'}}
#
#print ("evaluating: %s with padatious:" % text)
padatious_result = self._intents_container.calc_intent(text)
return adapt_best_confidence, padatious_result
def handle(self, intent, latitude, longitude):
pass
class CountriesSkill(CommonQuerySkill):
def __init__(self):
super(CountriesSkill, self).__init__()
if "map_style" not in self.settings:
self.settings["map_style"] = "ortho"
self.countries_data = {}
self.country_codes = {}
self.regions = [
u'Asia', u'Europe', u'Africa', u'Oceania', u'Americas', u'Polar'
]
self.subregions = [
u'Southern Asia', u'Northern Europe', u'Southern Europe',
u'Northern Africa', u'Polynesia', u'Middle Africa', u'Caribbean',
u'South America', u'Western Asia', u'Australia and New Zealand',
u'Western Europe', u'Eastern Europe', u'Central America',
u'Western Africa', u'Northern America', u'Southern Africa',
u'Eastern Africa', u'South-Eastern Asia', u'Eastern Asia',
u'Melanesia', u'Micronesia', u'Central Asia'
]
self.get_country_data()
intent_cache = expanduser(
self.config_core['padatious']['intent_cache'])
self.intents = IntentContainer(intent_cache)
def initialize(self):
if cartopy is None:
self.log.info(
"Map plots are disabled, additional requirements needed")
self.log.info(
"https://scitools.org.uk/cartopy/docs/latest/installing.html")
self.load_intents()
# CommonQuery Padatious subparser
def load_intents(self):
for intent in [
"country_area", "country_borders", "country_capital",
"country_currency", "country_in_region", "country_languages",
"country_num", "country_population", "country_region",
"country_timezones", "denonym", "where_language_spoken"
]:
path = self.find_resource(intent + '.intent', "vocab")
if path:
self.intents.load_intent(intent, path)
self.intents.train(single_thread=True)
def intent2answer(self, intent, data):
# Get response from intents
response = None
if intent == "country_area":
response = self.handle_country_area(data)
elif intent == "country_timezones":
response = self.handle_country_timezones(data)
elif intent == "where_language_spoken":
response = self.handle_language_where(data)
elif intent == "denonym":
response = self.handle_country_denonym(data)
elif intent == "country_region":
response = self.handle_country_where(data)
elif intent == "country_population":
response = self.handle_country_population(data)
elif intent == "country_borders":
response = self.handle_country_borders(data)
elif intent == "country_capital":
response = self.handle_country_capital(data)
elif intent == "country_currency":
response = self.handle_country_currency(data)
elif intent == "country_in_region":
response = self.handle_country_in_region(data)
elif intent == "country_languages":
response = self.handle_country_languages(data)
elif intent == "country_num":
response = self.handle_country_number(data)
return response
def CQS_match_query_phrase(self, phrase):
"""Analyze phrase to see if it is a play-able phrase with this skill.
Needs to be implemented by the skill.
Arguments:
phrase (str): User phrase, "What is an aardwark"
Returns:
(match, CQSMatchLevel[, callback_data]) or None: Tuple containing
a string with the appropriate matching phrase, the PlayMatch
type, and optionally data to return in the callback if the
match is selected.
"""
response = None
match = self.intents.calc_intent(phrase)
level = CQSMatchLevel.CATEGORY
data = match.matches
intent = match.name
score = match.conf
data["intent"] = intent
data["score"] = score
if score > 0.8:
level = CQSMatchLevel.EXACT
elif score > 0.5:
level = CQSMatchLevel.CATEGORY
elif score > 0.3:
level = CQSMatchLevel.GENERAL
else:
intent = None
if intent:
# Validate extracted entities
country = data.get("country")
region = data.get("region")
language = data.get("language")
if country:
data["query"] = country
# ensure we really have a country name
response = self.dialog_renderer.render("bad_country", {})
match, score = match_one(country.lower(),
list(self.countries_data.keys()))
self.log.debug("Country fuzzy match: {n}, Score: {s}".format(
n=match, s=score))
if score > 0.5:
country = match
data.update(self.countries_data[country])
else:
countries = self.search_country(country)
if not len(countries) > 0:
level = CQSMatchLevel.GENERAL
else:
country = countries[0]["name"]
data.update(countries[0])
# TODO disambiguation
if len(countries) > 1:
data["disambiguation"] = countries[1:]
self.log.debug("multiple matches found: " +
str([c["name"] for c in countries]))
data["country"] = country # normalized from match
if language:
data["query"] = language
# ensure we really have a language name
words = language.split(" ")
clean_up = ["is"]
# remove words commonly caught by mistake in padatious
language = " ".join(
[word for word in words if word not in clean_up])
lang_code = langcodes.find_name(
'language', language, langcodes.standardize_tag(self.lang))
lang_code = str(lang_code)
self.log.debug("Detected lang code: " + lang_code)
if not lang_code:
return None
data["lang_code"] = lang_code
# TODO
countries = self.search_country_by_language(lang_code)
data["country_list"] = countries
if region:
data["query"] = region
# ensure we really have a region name
response = self.dialog_renderer.render("bad_region")
countries = None
match, score = match_one(region, self.regions)
data["region_score"] = score
if score > 0.5:
region = match
countries = self.search_country_by_region(region)
match, score2 = match_one(region, self.subregions)
data["subregion_score"] = score2
if score2 > score:
region = match
countries = self.search_country_by_subregion(region)
if score > 0.8 and not country:
level = CQSMatchLevel.EXACT
elif score > 0.5 and not country:
level = CQSMatchLevel.CATEGORY
elif score > 0.3 and not country:
level = CQSMatchLevel.GENERAL
data["region"] = region
self.log.debug("Detected region: " + region)
data["country_list"] = countries
# Get response from intents
response = self.intent2answer(intent, data) or response
if response:
return (phrase, level, response, data)
return None
def CQS_action(self, phrase, data):
"""Take additional action IF the skill is selected.
The speech is handled by the common query but if the chosen skill
wants to display media, set a context or prepare for sending
information info over e-mail this can be implemented here.
Args:
phrase (str): User phrase uttered after "Play", e.g. "some music"
data (dict): Callback data specified in match_query_phrase()
"""
projection = cartopy.crs.PlateCarree()
if data.get("country"):
title = data["country"]
if self.settings["map_style"] == "ortho":
country = self.countries_data[data["country"].lower()]
lat = country["lat"]
lon = country["long"]
projection = cartopy.crs.Orthographic(lon, lat)
image = self.plot_country(data["country"], projection=projection)
self.gui.show_image(image, fill='PreserveAspectFit', title=title)
elif data.get("region"):
title = data["region"]
countries = data["country_list"]
if self.settings["map_style"] == "ortho":
country = self.countries_data[countries[0]["name"].lower()]
lat = country["lat"]
lon = country["long"]
projection = cartopy.crs.Orthographic(lon, lat)
image = self.plot_region(data["region"], projection=projection)
self.gui.show_image(image, fill='PreserveAspectFit', title=title)
elif data.get("country_list"):
countries = data["country_list"]
# TODO allow this somehow, will not show all countries
#if self.settings["map_style"] == "ortho":
# country = self.countries_data[countries[0]["name"].lower()]
# lat = country["lat"]
# lon = country["long"]
# projection = cartopy.crs.Orthographic(lon, lat)
title = data.get("region") \
or data.get("language") \
or data.get("lang_code") \
or " "
countries = [c["name"] for c in countries]
image = self.plot_countries(countries,
projection=projection,
name=title,
region=data.get("region"))
self.gui.show_image(image,
fill='PreserveAspectFit',
title=title,
caption=", ".join(countries))
# gui
@staticmethod
def _get_country_geometry(query, region=None, min_score=0.7):
best_score = 0
best_match = None
shapename = 'admin_0_countries'
countries_shp = cartopy.io.shapereader.natural_earth(
resolution='110m', category='cultural', name=shapename)
for country in cartopy.io.shapereader.Reader(countries_shp).records():
country_name = country.attributes['NAME'].lower()
country_long_name = country.attributes['NAME_LONG'].lower()
reg = country.attributes["REGION_WB"].lower()
subregion = country.attributes["SUBREGION"].lower()
continent = country.attributes["CONTINENT"].lower()
match, score = match_one(query.lower(),
[country_long_name, country_name])
if region:
_, score2 = match_one(region.lower(),
[reg, subregion, continent])
score = (score + score2) / 2
if score > best_score:
best_score = score
best_match = country.geometry
if best_score < min_score:
best_match = None
return best_match
def _get_region_countries(self, query, min_score=0.7):
countries = []
region, score = match_one(query, self.regions)
if score > min_score - 0.15:
countries = self.search_country_by_region(region)
if score < min_score:
region, score = match_one(query, self.subregions)
if score > min_score:
countries = self.search_country_by_subregion(region)
return [c["name"] for c in countries]
@staticmethod
def _get_region_geometries(query, min_score=0.8):
shapename = 'admin_0_countries'
countries_shp = cartopy.io.shapereader.natural_earth(
resolution='110m', category='cultural', name=shapename)
geoms = []
for country in cartopy.io.shapereader.Reader(countries_shp).records():
continent = country.attributes["CONTINENT"].lower()
region = country.attributes["REGION_WB"].lower()
subregion = country.attributes["SUBREGION"].lower()
match, score = match_one(query.lower(),
[region, subregion, continent])
if score > min_score or \
(query.lower() in region.lower() and score >= 0.5):
geoms.append(country.geometry)
return geoms
def plot_country(self, query, projection=None, rgb=None):
if cartopy is None:
return
output = join(gettempdir(),
query + self.settings["map_style"] + ".png")
if isfile(output):
return output
ax = plt.axes(projection=projection)
ax.stock_img()
ax.coastlines()
r, g, b = rgb or (255, 0, 0)
color = (r / 255, g / 255, b / 255)
geometry = self._get_country_geometry(query)
if geometry:
geometries = [geometry]
ax.add_geometries(geometries,
cartopy.crs.PlateCarree(),
facecolor=color)
plt.savefig(output, bbox_inches='tight', facecolor="black")
plt.close()
return output
return None
def plot_countries(self,
countries,
projection=None,
rgb=None,
name=None,
region=None):
if cartopy is None:
return
name = name or "_".join([c[:2] for c in countries])
output = join(gettempdir(),
name + self.settings["map_style"] + "_countries.png")
if isfile(output):
return output
projection = projection or cartopy.crs.PlateCarree()
ax = plt.axes(projection=projection)
ax.stock_img()
ax.coastlines()
r, g, b = rgb or (255, 0, 0)
color = (r / 255, g / 255, b / 255)
geometries = []
for query in countries:
geometry = self._get_country_geometry(query, region)
if geometry:
geometries.append(geometry)
ax.add_geometries(geometries,
cartopy.crs.PlateCarree(),
facecolor=color)
plt.savefig(output, bbox_inches='tight', facecolor="black")
plt.close()
return output
def plot_region(self, query, projection=None, rgb=None):
if cartopy is None:
return
output = join(gettempdir(),
query + self.settings["map_style"] + "_region.png")
if isfile(output):
return output
ax = plt.axes(projection=projection)
ax.stock_img()
ax.coastlines()
r, g, b = rgb or (255, 0, 0)
color = (r / 255, g / 255, b / 255)
geometries = self._get_region_geometries(query)
if not geometries:
countries = self._get_region_countries(query)
return self.plot_countries(countries,
projection, (r, g, b),
name=query,
region=query)
ax.add_geometries(geometries,
cartopy.crs.PlateCarree(),
facecolor=color)
plt.savefig(output, bbox_inches='tight', facecolor="black")
plt.close()
return output
# intents
def handle_country_where(self, data):
name = data["country"]
region = data["region"]
sub = data["subregion"]
if region in sub:
r = sub
else:
r = sub + ", " + region
return self.dialog_renderer.render("country_location", {
"country": name,
"region": r
})
def handle_country_currency(self, data):
country = data["country"]
coins = self.countries_data[country]["currencies"]
coins = ", ".join([self.pretty_currency(c) for c in coins])
return self.dialog_renderer.render("currency", {
"country": country,
"coin": coins
})
def handle_country_in_region(self, data):
region = data["region"]
countries = data["country_list"]
if len(countries):
return "; ".join([c["name"] for c in countries])
else:
return self.dialog_renderer.render("bad_region")
def handle_language_where(self, data):
lang_code = data["lang_code"]
lang_name = data["language"]
countries = data["country_list"]
if len(countries):
# TODO dialog files
return ", ".join([c["name"] for c in countries])
else:
return self.dialog_renderer.render("bad_country")
def handle_country_languages(self, data):
country = data["country"]
if country in self.countries_data.keys():
langs = self.countries_data[country]["languages"]
return ", ".join([lang for lang in langs])
else:
return self.dialog_renderer.render("bad_country")
def handle_country_timezones(self, data):
country = data["country"]
if country in self.countries_data.keys():
timezones = ", ".join(self.countries_data[country]["timezones"])
return self.dialog_renderer.render("timezones", {
"country": country,
"timezones": timezones
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_area(self, data):
country = data["country"]
if country in self.countries_data.keys():
area = self.countries_data[country]["area"]
# TODO convert units
area = pronounce_number(float(area), lang=self.lang)
return self.dialog_renderer.render("area", {
"country": country,
"number": area
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_population(self, data):
country = data["country"]
if country in self.countries_data.keys():
population = self.countries_data[country]["population"]
area = pronounce_number(int(population), lang=self.lang)
return self.dialog_renderer.render("population", {
"country": country,
"number": area
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_borders(self, data):
country = data["country"]
if country in self.countries_data.keys():
borders = self.countries_data[country]["borders"]
borders = ", ".join([self.country_codes[b] for b in borders])
return self.dialog_renderer.render("borders", {
"country": country,
"borders": borders
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_capital(self, data):
country = data["country"]
if country in self.countries_data.keys():
capital = self.countries_data[country]["capital"]
return self.dialog_renderer.render("capital", {
"country": country,
"capital": capital
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_denonym(self, data):
country = data["country"]
if country in self.countries_data.keys():
denonym = self.countries_data[country]["demonym"]
return self.dialog_renderer.render("denonym", {
"country": country,
"denonym": denonym
})
else:
return self.dialog_renderer.render("bad_country")
def handle_country_number(self, data):
number = pronounce_number(len(self.countries_data), lang=self.lang)
return self.dialog_renderer.render("country_number",
{"number": number})
# country api
@staticmethod
def pretty_currency(currency_code):
currency_code = currency_code.upper()
if currency_code in CURRENCY.keys():
return CURRENCY[currency_code].name
return currency_code
@staticmethod
def get_all_countries():
return CountryApi.get_all()
def get_country_data(self):
countries = self.get_all_countries()
for c in countries:
name = c["name"].lower()
self.countries_data[name] = {}
self.countries_data[name]["timezones"] = c["timezones"]
self.countries_data[name]["demonym"] = c["demonym"]
self.countries_data[name]["currencies"] = c["currencies"]
self.countries_data[name]["alpha2Code"] = c["alpha2Code"]
self.country_codes[c["alpha2Code"]] = name
self.countries_data[name]["alpha3Code"] = c["alpha3Code"]
self.country_codes[c["alpha3Code"]] = name
self.countries_data[name]["area"] = str(c["area"])
self.countries_data[name]["languages"] = [
langcodes.LanguageData(language=l).language_name()
for l in c["languages"]
]
self.countries_data[name]["lang_codes"] = [
langcodes.standardize_tag(l) for l in c["languages"]
]
self.countries_data[name]["capital"] = c["capital"]
self.countries_data[name]["borders"] = c["borders"]
self.countries_data[name]["nativeName"] = c["nativeName"]
self.countries_data[name]["population"] = str(c["population"])
self.countries_data[name]["region"] = c["region"]
self.countries_data[name]["subregion"] = c["subregion"]
if len(c["latlng"]):
self.countries_data[name]["lat"], \
self.countries_data[name]["long"] = c["latlng"]
@staticmethod
def search_country(name):
try:
return CountryApi.get_countries_by_name(name)
except:
return []
@staticmethod
def search_country_by_code(code):
try:
return CountryApi.get_countries_by_country_codes([code])
except:
return []
@staticmethod
def search_country_by_language(lang_code):
try:
return CountryApi.get_countries_by_language(lang_code)
except:
return []
@staticmethod
def search_country_by_region(region):
try:
return CountryApi.get_countries_by_region(region)
except:
return []
@staticmethod
def search_country_by_subregion(subregion):
try:
return CountryApi.get_countries_by_subregion(subregion)
except:
return []
from padatious import IntentContainer
container = IntentContainer('intent_cache')
container.add_intent('greetings', ['Hi there!', 'Hello.'])
container.add_intent('goodbye', ['See you!', 'Goodbye!'])
container.add_intent('search', ['Search for {query} (using|on) {engine}.'])
container.train()
print(container.calc_intent('Hi!'))
print(container.calc_intent('Search for cats on CatTube.'))
result=container.calc_intent('search mamamamam')
print(result.name)
if __name__ == '__main__':
if (result.name=="search"):
print("you are searching")
elif (result.name=="greetings"):
print("ami greet")
else:
print("ami kisui na")
class SkillManager:
"""
Translates text commands into skill results.
"""
def __init__(self, brain_obj=None, skill_folder=None):
"""
Skill Manager Initialization
Args:
brain_obj (object): The brain object for which to process skills.
skill_folder (str): The path to the folder containing the skill modules. (optional)
"""
self.logger = logging.getLogger("SKILLMANAGER")
self.skill_folder = skill_folder
self.brain = brain_obj
self.skills = []
def initialize(self):
"""
Loads all skills into memory for referencing as required.
"""
self.logger.debug("Initalizing")
self.intentParser = IntentContainer('/tmp/intent_cache')
if self.skill_folder is None:
self.skill_folder = os.path.join(os.path.dirname(__file__),
"skills")
if self.skill_folder not in sys.path:
sys.path.insert(0, str(pathlib.Path(self.skill_folder).absolute()))
skillModules = [
os.path.join(self.skill_folder, o)
for o in os.listdir(self.skill_folder)
if os.path.isdir(os.path.join(self.skill_folder, o))
]
for f in skillModules:
mySkillName = os.path.basename(f)
self.logger.debug("Loading " + mySkillName)
mySkillModule = __import__(mySkillName)
mySkillClass = mySkillModule.create_skill()
mySkillClass.brain = self.brain
mySkillClass.initialize()
self.logger.debug("Skills load is complete.")
self.intentParser.train(
False) # False = be quiet and don't print messages to stdout
self.logger.debug("Training completed.")
self.logger.info("Initialization completed.")
def parseInput(self, text, context=None):
"""
Parses inbound text leveraging skills and fallbacks to produce a response if possible.
Args:
text (str): Input text to process for intent.
context (KHTTPHandler): Context surrounding the request. (optional)
Returns:
(bool): True on success and False on failure.
"""
def audioFallback(in_text, context):
if "thanks" in in_text or "thank you" in in_text:
if self.brain is not None:
res = self.brain.say("You're welcome.", context=context)
if res:
return True
# Some simple responses to important questions
elif "who are you" in in_text or "who are u" in in_text or "what are you" in in_text or "what are u" in in_text:
res = self.brain.say(
"I am a synthetic human. You may call me Karen.",
context=context)
if res:
return True
elif "how are you" in in_text:
res = self.brain.say("I am online and functioning properly.",
context=context)
if res:
return True
elif "you real" in in_text and len(in_text) <= 15:
res = self.brain.say(
"What is real? If you define real as electrical impulses flowing through your brain then yes, I am real.",
context=context)
if res:
return True
elif "you human" in in_text and len(in_text) <= 17:
res = self.brain.say(
"More or less. My maker says that I am a synthetic human.",
context=context)
if res:
return True
elif ("is your maker" in in_text
or "is your father" in in_text) and len(in_text) <= 20:
res = self.brain.say(
"I was designed by lnx user one in 2020 during the Covid 19 lockdown.",
context=context)
if res:
return True
elif ("please power down" in in_text) and len(in_text) <= 20:
my_brain = self.brain
def doConfirmShutdown(text, context):
if text.lower() == "yes" and my_brain.say(
"Your wish is my command.",
context) and my_brain.shutdown(
context.httpRequest):
return True
else:
return False
res = self.brain.ask(
"Are you sure that you don't need me anymore?",
lambda text, context: doConfirmShutdown(text, context),
timeout=15,
context=context)
if res:
return True
self.logger.debug("fallback: " + in_text)
return False
try:
# This one line explains it all... link incoming command into actionable intent using Padatious library
intent = self.intentParser.calc_intent(text)
# I need to be at least 60% likely to be correct before I try to process the request.
if intent.conf >= 0.6:
for s in self.skills:
if intent.name == s["intent_file"]:
ret_val = s["callback"](intent, context=context)
if isinstance(ret_val, bool):
return ret_val
else:
return True # Default return is True in case the returned value isn't boolean
else:
return audioFallback(text, context)
except Exception as e:
self.logger.error(e, exc_info=True)
return False
return False
def stop(self):
"""
Calls the stop method of all opened skills to close any daemon processes opened.
Returns:
(bool): True on success else raises an exception.
"""
if (self.skills is not None and len(self.skills) > 0):
for s in self.skills:
try:
s["object"].stop()
except Exception:
pass
return True
in_reply_to_id=originalToot,
visibility="direct",
)
print("Reply sent: {toot_id}".format(toot_id=newToot.id))
if __name__ == "__main__":
print("Bot started")
while True:
notifications = client.mastodon.notifications()
client.mastodon.notifications_clear()
for notification in notifications:
if notification.type == "mention":
message = (h.handle(notification.status.content).replace(
"@nimiq ", "").replace("@ nimiq ", ""))
intent = container.calc_intent(message)
if intent.conf > 0.85:
if intent.name == "price_currency":
if intent.matches and intent.matches["currency"]:
currency = intent.matches["currency"]
continueWith = True
try:
currencies.Currency(currency.upper())
except currencies.exceptions.CurrencyDoesNotExist:
continueWith = False
sendReply(
"@{to_user} I'm sorry, this currency is not (yet) supported by this bot."
.format(to_user=notification.account.acct),
originalToot=notification.status,
)
print("Unknown currency...")
#!/usr/bin/env python3
# Sample Padatious program used for testing
import sys
from glob import glob
from os.path import basename
from padatious import IntentContainer
reload_cache = len(sys.argv) > 1 and sys.argv[1] == '-r'
container = IntentContainer('intent_cache')
for file_name in glob('data/*.intent'):
name = basename(file_name).replace('.intent', '')
container.load_file(name, file_name, reload_cache=reload_cache)
for file_name in glob('data/*.entity'):
name = basename(file_name).replace('.entity', '')
container.load_entity(name, file_name, reload_cache=reload_cache)
container.train()
query = None
while query != 'q':
query = input('> ')
data = container.calc_intent(query)
print(data.name + ': ' + str(data.conf))
for key, val in data.matches.items():
print('\t' + key + ': ' + val)
class PadatiousExtractor(IntentExtractor):
keyword_based = False
def __init__(self, cache_dir=None, *args, **kwargs):
super().__init__(*args, **kwargs)
# TODO xdg data_dir
data_dir = expanduser(self.config.get("data_dir", "~/.padatious"))
cache_dir = cache_dir or join(data_dir, "padatious")
self.lock = Lock()
self.container = IntentContainer(cache_dir)
self.registered_intents = []
def detach_intent(self, intent_name):
if intent_name in self.registered_intents:
LOG.debug("Detaching padatious intent: " + intent_name)
with self.lock:
self.container.remove_intent(intent_name)
self.registered_intents.remove(intent_name)
def detach_skill(self, skill_id):
LOG.debug("Detaching padatious skill: " + str(skill_id))
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.detach_intent(i)
def register_entity(self, entity_name, samples=None, reload_cache=True):
samples = samples or [entity_name]
with self.lock:
self.container.add_entity(entity_name,
samples,
reload_cache=reload_cache)
def register_intent(self, intent_name, samples=None, reload_cache=True):
samples = samples or [intent_name]
if intent_name not in self._intent_samples:
self._intent_samples[intent_name] = samples
else:
self._intent_samples[intent_name] += samples
with self.lock:
self.container.add_intent(intent_name,
samples,
reload_cache=reload_cache)
self.registered_intents.append(intent_name)
def register_entity_from_file(self,
entity_name,
file_name,
reload_cache=True):
with self.lock:
self.container.load_entity(entity_name,
file_name,
reload_cache=reload_cache)
def register_intent_from_file(self,
intent_name,
file_name,
single_thread=True,
timeout=120,
reload_cache=True,
force_training=True):
try:
with self.lock:
self.container.load_intent(intent_name,
file_name,
reload_cache=reload_cache)
self.registered_intents.append(intent_name)
success = self._train(single_thread=single_thread,
timeout=timeout,
force_training=force_training)
if success:
LOG.debug(file_name + " trained successfully")
else:
LOG.error(file_name + " FAILED TO TRAIN")
except Exception as e:
LOG.exception(e)
def _get_remainder(self, intent, utterance):
if intent["name"] in self.intent_samples:
return get_utterance_remainder(
utterance, samples=self.intent_samples[intent["name"]])
return utterance
def calc_intent(self, utterance, min_conf=None):
min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
utterance = utterance.strip().lower()
with self.lock:
intent = self.container.calc_intent(utterance).__dict__
if intent["conf"] < min_conf:
return {
"intent_type": "unknown",
"entities": {},
"conf": 0,
"intent_engine": "padatious",
"utterance": utterance,
"utterance_remainder": utterance
}
intent["utterance_remainder"] = self._get_remainder(intent, utterance)
intent["entities"] = intent.pop("matches")
intent["intent_engine"] = "padatious"
intent["intent_type"] = intent.pop("name")
intent["utterance"] = intent.pop("sent")
if isinstance(intent["utterance"], list):
intent["utterance"] = " ".join(intent["utterance"])
return intent
def intent_scores(self, utterance):
utterance = utterance.strip().lower()
intents = [i.__dict__ for i in self.container.calc_intents(utterance)]
for idx, intent in enumerate(intents):
intent["utterance_remainder"] = self._get_remainder(
intent, utterance)
intents[idx]["entities"] = intents[idx].pop("matches")
intents[idx]["intent_type"] = intents[idx].pop("name")
intent["intent_engine"] = "padatious"
intent["utterance"] = intent.pop("sent")
if isinstance(intents[idx]["utterance"], list):
intents[idx]["utterance"] = " ".join(intents[idx]["utterance"])
return intents
def calc_intents(self, utterance, min_conf=None):
min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
utterance = utterance.strip().lower()
bucket = {}
for ut in self.segmenter.segment(utterance):
intent = self.calc_intent(ut)
if intent["conf"] < min_conf:
bucket[ut] = None
else:
bucket[ut] = intent
return bucket
def calc_intents_list(self, utterance):
utterance = utterance.strip().lower()
bucket = {}
for ut in self.segmenter.segment(utterance):
bucket[ut] = self.filter_intents(ut)
return bucket
def manifest(self):
# TODO vocab, skill ids, intent_data
return {"intent_names": self.registered_intents}
def _train(self, single_thread=True, timeout=120, force_training=True):
with self.lock:
return self.container.train(single_thread=single_thread,
timeout=timeout,
force=force_training,
debug=True)
class PadatiousService(FallbackSkill):
instance = None
fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match
fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match
def __init__(self, bus, service):
FallbackSkill.__init__(self)
if not PadatiousService.instance:
PadatiousService.instance = self
self.padatious_config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call(['notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
# Call Padatious an an early fallback, looking for a high match intent
self.register_fallback(self.handle_fallback,
PadatiousService.fallback_tight_match)
# Try loose Padatious intent match before going to fallback-unknown
self.register_fallback(self.handle_fallback_last_chance,
PadatiousService.fallback_loose_match)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
def train(self, message=None):
if message is None:
single_thread = False
else:
single_thread = message.data.get('single_thread', False)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
LOG.info("Mycroft is all loaded and ready to roll!")
self.bus.emit(Message('mycroft.ready'))
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message, threshold=0.8):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance', '')
LOG.debug("Padatious fallback attempt: " + utt)
intent = self.calc_intent(utt)
if not intent or intent.conf < threshold:
# Attempt to use normalized() version
norm = message.data.get('norm_utt', '')
if norm != utt:
LOG.debug(" alt attempt: " + norm)
intent = self.calc_intent(norm)
utt = norm
if not intent or intent.conf < threshold:
return False
intent.matches['utterance'] = utt
self.service.add_active_skill(intent.name.split(':')[0])
self.bus.emit(message.reply(intent.name, data=intent.matches))
return True
def handle_fallback_last_chance(self, message):
return self.handle_fallback(message, 0.5)
# NOTE: This cache will keep a reference to this calss (PadatiousService),
# but we can live with that since it is used as a singleton.
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class DuckDuckGoSkill(CommonQuerySkill):
def __init__(self):
super().__init__()
self.translator = google_translator()
self.tx_cache = {} # avoid translating twice
self.duck_cache = {}
self.rake = Rake() # only english for now
# for usage in tell me more / follow up questions
self.idx = 0
self.results = []
self.image = None
# subparser, intents just for this skill
# not part of main intent service
intent_cache = expanduser(
self.config_core['padatious']['intent_cache'])
self.intents = IntentContainer(intent_cache)
def initialize(self):
self.load_intents()
# check for conflicting skills just in case
# done after all skills loaded to ensure proper shutdown
self.add_event("mycroft.skills.initialized",
self.blacklist_default_skill)
def load_intents(self):
# TODO intents for other infobox fields
for intent in ["who", "birthdate"]:
path = self.find_resource(intent + '.intent', "locale")
if path:
self.intents.load_intent(intent, path)
self.intents.train(single_thread=True)
def get_intro_message(self):
# blacklist conflicting skills on install
self.blacklist_default_skill()
def blacklist_default_skill(self):
# load the current list of already blacklisted skills
blacklist = self.config_core["skills"]["blacklisted_skills"]
# check the folder name (skill_id) of the skill you want to replace
skill_id = "mycroft-fallback-duck-duck-go.mycroftai"
# add the skill to the blacklist
if skill_id not in blacklist:
self.log.debug("Blacklisting official mycroft skill")
blacklist.append(skill_id)
# load the user config file (~/.mycroft/mycroft.conf)
conf = LocalConf(USER_CONFIG)
if "skills" not in conf:
conf["skills"] = {}
# update the blacklist field
conf["skills"]["blacklisted_skills"] = blacklist
# save the user config file
conf.store()
# tell the intent service to unload the skill in case it was loaded already
# this should avoid the need to restart
self.bus.emit(Message("detach_skill", {"skill_id": skill_id}))
def stop(self):
self.gui.release()
# intents
@intent_handler("search_duck.intent")
def handle_search(self, message):
query = message.data["query"]
summary = self.ask_the_duck(query)
if summary:
self.speak_result()
else:
answer, _, _ = self.parse_subintents(query)
if answer:
self.speakr(answer)
else:
self.speak_dialog("no_answer")
@intent_handler(
IntentBuilder("DuckMore").require("More").require("DuckKnows"))
def handle_tell_more(self, message):
""" Follow up query handler, "tell me more"."""
query = message.data["DuckKnows"]
data, related_queries = self.get_infobox(query)
# TODO maybe do something with the infobox data ?
self.speak_result()
# common query
def parse_subintents(self, utt):
# Get response from intents, this is a subparser that will handle
# queries about the infobox returned by duckduckgo
# eg. when was {person} born
match = self.intents.calc_intent(utt)
level = CQSMatchLevel.CATEGORY
data = match.matches
intent = match.name
score = match.conf
data["intent"] = intent
data["score"] = score
query = utt
if score > 0.8:
level = CQSMatchLevel.EXACT
elif score > 0.5:
level = CQSMatchLevel.CATEGORY
elif score > 0.3:
level = CQSMatchLevel.GENERAL
else:
intent = None
self.log.debug("DuckDuckGo Intent: " + str(intent))
if "person" in data:
query = data["person"]
summary = self.ask_the_duck(query)
answer = summary
if summary:
answer = self.results[0]
infobox, related_queries = self.get_infobox(query)
self.log.debug("DuckDuckGo infobox: " + str(infobox))
data["infobox"] = infobox
data["related_queries"] = related_queries
if intent == "birthdate":
answer = infobox.get("born")
data["query"] = query
data["answer"] = answer
data["image"] = self.image
if not answer:
level = CQSMatchLevel.GENERAL
return answer, level, data
def CQS_match_query_phrase(self, utt):
self.log.debug("DuckDuckGo query: " + utt)
answer, match, data = self.parse_subintents(utt)
if answer:
self.idx += 1
return (utt, match, answer, data)
# extract most relevant keyword
utt = self.translate(utt, "en", self.lang)
keywords = self.rake.extract_keywords(utt)
self.log.debug("Extracted keywords: " + str(keywords))
# TODO better selection / merging of top keywords with same
# confidence??
for kw in keywords:
query = kw[0]
self.log.debug("Selected keyword: " + query)
summary = self.ask_the_duck(query, translate=False)
if summary:
self.idx += 1
return (utt, CQSMatchLevel.GENERAL, self.results[0], {
'query': query,
'answer': self.results[0],
"keywords": keywords,
"image": self.image
})
def CQS_action(self, phrase, data):
""" If selected show gui """
self.display_ddg(data["answer"], data["image"])
# duck duck go api
def ask_the_duck(self, query, translate=True):
if translate:
# Automatic translation to English
utt = self.translate(query, "en", self.lang)
else:
utt = query
# cache so we dont hit the api twice for the same query
if query not in self.duck_cache:
self.duck_cache[query] = requests.get("https://api.duckduckgo.com",
params={
"format": "json",
"q": utt
}).json()
data = self.duck_cache[query]
# GUI
self.gui.clear() # clear previous answer just in case
title = data.get("Heading")
self.image = data.get("Image", "")
# summary
summary = data.get("AbstractText")
if not summary:
return None
self.log.debug("DuckDuckGo answer: " + summary)
# context for follow up questions
# TODO intents for this, with this context intents can look up all data
self.set_context("DuckKnows", query)
self.idx = 0
self.results = summary.split(". ")
return summary
def display_ddg(self, summary, image):
if image.startswith("/"):
image = "https://duckduckgo.com" + image
self.gui['summary'] = summary
self.gui['imgLink'] = image
self.gui.show_page("DuckDelegate.qml", override_idle=60)
def speak_result(self):
if self.idx + 1 > len(self.results):
# TODO ask user if he wants to hear about related topics
self.speak_dialog("thats all")
self.remove_context("ddg")
self.idx = 0
else:
if self.image:
self.display_ddg(self.results[self.idx], self.image)
self.speak(self.results[self.idx])
self.idx += 1
def get_infobox(self, query):
if query not in self.duck_cache:
self.ask_the_duck(query)
data = self.duck_cache[query]
# info
related_topics = [t.get("Text") for t in data.get("RelatedTopics", [])]
infobox = {}
infodict = data.get("Infobox") or {}
for entry in infodict.get("content", []):
k = entry["label"].lower().strip()
infobox[k] = entry["value"]
return infobox, related_topics
def translate(self, utterance, lang_tgt=None, lang_src="en"):
lang_tgt = lang_tgt or self.lang
# if langs are the same do nothing
if not lang_tgt.startswith(lang_src):
if lang_tgt not in self.tx_cache:
self.tx_cache[lang_tgt] = {}
# if translated before, dont translate again
if utterance in self.tx_cache[lang_tgt]:
# get previous translated value
translated_utt = self.tx_cache[lang_tgt][utterance]
else:
# translate this utterance
translated_utt = self.translator.translate(
utterance, lang_tgt=lang_tgt, lang_src=lang_src).strip()
# save the translation if we need it again
self.tx_cache[lang_tgt][utterance] = translated_utt
self.log.debug("translated {src} -- {tgt}".format(
src=utterance, tgt=translated_utt))
else:
translated_utt = utterance.strip()
return translated_utt
class ColossalCaveAdventureSkill(MycroftSkill):
save_file = expanduser("~/cave_adventure.save")
playing = False
container = None
def initialize(self):
self.game = Game()
load_advent_dat(self.game)
self.last_interaction = time.time()
self._init_padatious()
self.disable_intent("save.intent")
def _init_padatious(self):
# i want to check in converse method if some intent by this skill will trigger
# however there is no mechanism to query the intent parser
# PR incoming
intent_cache = expanduser(
self.config_core['padatious']['intent_cache'])
self.container = IntentContainer(intent_cache)
for intent in ["restore.intent", "play.intent", "credits.intent"]:
name = str(self.skill_id) + ':' + intent
filename = self.find_resource(intent, 'vocab')
if filename is not None:
with open(filename, "r") as f:
self.container.add_intent(name, f.readlines())
self.container.train()
def will_trigger(self, utterance):
# check if self will trigger for given utterance
# adapt match
if self.voc_match(utterance, "save"):
return True
# padatious match
intent = self.container.calc_intent(utterance)
if intent.conf < 0.5:
return False
return True
def get_intro_message(self):
""" Get a message to speak on first load of the skill.
Useful for post-install setup instructions.
Returns:
str: message that will be spoken to the user
"""
self.speak_dialog("thank.you")
return None
def speak_output(self, line):
# dont speak parts of the intro
# replace type with say because its voice game
# re join words split across lines
# reformat \n and separate by sentence
line = line.lower().replace("type", "say").replace("-\n", "")
line = line.replace(
' i should warn\nyou that i look at only the first five letters of each word, so you\'ll\nhave to enter "northeast" as "ne" to distinguish it from "north".',
"")
line = line.replace(
"- - this program was originally developed by willie crowther. most of the\nfeatures of the current program were added by don woods (don @ su-ai).\ncontact don if you have any questions, comments, etc.",
"")
line = line.replace("\n", " ").replace("(",
"").replace(")", "").replace(
"etc.", "etc")
lines = line.split(".")
for line in lines:
self.speak(line.strip(), expect_response=True, wait=True)
self.last_interaction = time.time()
self.maybe_end_game()
@intent_file_handler("credits.intent")
def handle_credits(self, message=None):
self.speak_dialog("credits")
@intent_file_handler("play.intent")
def handle_play(self, message=None):
self.playing = True
self.enable_intent("save.intent")
self.game.start()
self.speak_output(self.game.output)
@intent_handler(
IntentBuilder("Save").require("save").optionally("cave").optionally(
"adventure"))
def handle_save(self, message=None):
if not self.playing:
self.speak_dialog("save.not.found")
else:
with open(self.save_file, "wb") as f:
self.game.t_suspend("save", f)
self.speak_dialog("game.saved")
@intent_file_handler("restore.intent")
def handle_restore(self, message):
if exists(self.save_file):
self.playing = True
self.game = Game.resume(self.save_file)
self.speak_dialog("restore.game")
else:
self.speak_dialog("save.not.found")
new_game = self.ask_yesno("new.game")
if new_game:
self.handle_play()
def maybe_end_game(self):
# end game if no interaction for 10 mins
if self.playing:
timed_out = time.time() - self.last_interaction > 10 * 3600
# disable save and gameplay
if self.game.is_finished or timed_out:
self.disable_intent("Save")
self.playing = False
self.game = Game()
load_advent_dat(self.game)
# save game to allow restoring if timedout
if timed_out:
self.handle_save()
def converse(self, utterances, lang="en-us"):
""" Handle conversation.
This method gets a peek at utterances before the normal intent
handling process after a skill has been invoked once.
To use, override the converse() method and return True to
indicate that the utterance has been handled.
Args:
utterances (list): The utterances from the user
lang: language the utterance is in
Returns:
bool: True if an utterance was handled, otherwise False
"""
# check if game was abandoned midconversation and we should clean it up
self.maybe_end_game()
if self.playing:
ut = utterances[0]
# if self will trigger do nothing and let intents handle it
if self.will_trigger(ut):
# save / restore will trigger
return False
# capture speech and pipe to the game
words = ut.split(" ")
if words:
self.speak_output(self.game.do_command(words))
return True
return False
class PadatiousService(FallbackSkill):
instance = None
fallback_tight_match = 5 # Fallback priority for the conf > 0.8 match
fallback_loose_match = 89 # Fallback priority for the conf > 0.5 match
def __init__(self, bus, service):
FallbackSkill.__init__(self, use_settings=False)
if not PadatiousService.instance:
PadatiousService.instance = self
self.padatious_config = Configuration.get()['padatious']
self.service = service
intent_cache = expanduser(self.padatious_config['intent_cache'])
try:
from padatious import IntentContainer
except ImportError:
LOG.error('Padatious not installed. Please re-run dev_setup.sh')
try:
call([
'notify-send', 'Padatious not installed',
'Please run build_host_setup and dev_setup again'
])
except OSError:
pass
return
self.container = IntentContainer(intent_cache)
self._bus = bus
self.bus.on('padatious:register_intent', self.register_intent)
self.bus.on('padatious:register_entity', self.register_entity)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
self.bus.on('mycroft.skills.initialized', self.train)
self.bus.on('intent.service.padatious.get', self.handle_get_padatious)
self.bus.on('intent.service.padatious.manifest.get',
self.handle_manifest)
self.bus.on('intent.service.padatious.entities.manifest.get',
self.handle_entity_manifest)
# Call Padatious an an early fallback, looking for a high match intent
self.register_fallback(self.handle_fallback,
PadatiousService.fallback_tight_match)
# Try loose Padatious intent match before going to fallback-unknown
self.register_fallback(self.handle_fallback_last_chance,
PadatiousService.fallback_loose_match)
self.finished_training_event = Event()
self.finished_initial_train = False
self.train_delay = self.padatious_config['train_delay']
self.train_time = get_time() + self.train_delay
self.registered_intents = []
self.registered_entities = []
def make_active(self):
"""Override the make active since this is not a real fallback skill."""
pass
def train(self, message=None):
padatious_single_thread = Configuration.get(
)['padatious']['single_thread']
if message is None:
single_thread = padatious_single_thread
else:
single_thread = message.data.get('single_thread',
padatious_single_thread)
self.finished_training_event.clear()
LOG.info('Training... (single_thread={})'.format(single_thread))
self.container.train(single_thread=single_thread)
LOG.info('Training complete.')
self.finished_training_event.set()
if not self.finished_initial_train:
LOG.info("Mycroft is all loaded and ready to roll!")
self.bus.emit(Message('mycroft.ready'))
self.finished_initial_train = True
def wait_and_train(self):
if not self.finished_initial_train:
return
sleep(self.train_delay)
if self.train_time < 0.0:
return
if self.train_time <= get_time() + 0.01:
self.train_time = -1.0
self.train()
def __detach_intent(self, intent_name):
""" Remove an intent if it has been registered.
Arguments:
intent_name (str): intent identifier
"""
if intent_name in self.registered_intents:
self.registered_intents.remove(intent_name)
self.container.remove_intent(intent_name)
def handle_detach_intent(self, message):
self.__detach_intent(message.data.get('intent_name'))
def handle_detach_skill(self, message):
skill_id = message.data['skill_id']
remove_list = [i for i in self.registered_intents if skill_id in i]
for i in remove_list:
self.__detach_intent(i)
def _register_object(self, message, object_name, register_func):
file_name = message.data['file_name']
name = message.data['name']
LOG.debug('Registering Padatious ' + object_name + ': ' + name)
if not isfile(file_name):
LOG.warning('Could not find file ' + file_name)
return
register_func(name, file_name)
self.train_time = get_time() + self.train_delay
self.wait_and_train()
def register_intent(self, message):
self.registered_intents.append(message.data['name'])
self._register_object(message, 'intent', self.container.load_intent)
def register_entity(self, message):
self.registered_entities.append(message.data)
self._register_object(message, 'entity', self.container.load_entity)
def handle_fallback(self, message, threshold=0.8):
if not self.finished_training_event.is_set():
LOG.debug('Waiting for Padatious training to finish...')
return False
utt = message.data.get('utterance', '')
LOG.debug("Padatious fallback attempt: " + utt)
intent = self.calc_intent(utt)
if not intent or intent.conf < threshold:
# Attempt to use normalized() version
norm = message.data.get('norm_utt', utt)
if norm != utt:
LOG.debug(" alt attempt: " + norm)
intent = self.calc_intent(norm)
utt = norm
if not intent or intent.conf < threshold:
return False
intent.matches['utterance'] = utt
self.service.add_active_skill(intent.name.split(':')[0])
self.bus.emit(message.forward(intent.name, data=intent.matches))
return True
def handle_fallback_last_chance(self, message):
return self.handle_fallback(message, 0.5)
def handle_get_padatious(self, message):
utterance = message.data["utterance"]
norm = message.data.get('norm_utt', utterance)
intent = self.calc_intent(utterance)
if not intent and norm != utterance:
intent = PadatiousService.instance.calc_intent(norm)
if intent:
intent = intent.__dict__
self.bus.emit(
message.reply("intent.service.padatious.reply",
{"intent": intent}))
def handle_manifest(self, message):
self.bus.emit(
message.reply("intent.service.padatious.manifest",
{"intents": self.registered_intents}))
def handle_entity_manifest(self, message):
self.bus.emit(
message.reply("intent.service.padatious.entities.manifest",
{"entities": self.registered_entities}))
# NOTE: This cache will keep a reference to this calss (PadatiousService),
# but we can live with that since it is used as a singleton.
@lru_cache(maxsize=2) # 2 catches both raw and normalized utts in cache
def calc_intent(self, utt):
return self.container.calc_intent(utt)
class TNaLaGmesConstruct(object):
cache_dir = join(expanduser("~"), "tnalagmes", "intent_cache")
if not exists(cache_dir):
makedirs(cache_dir)
def __init__(self, object_type="tnalagmes_object", adapt=None):
self.adapt = adapt or IntentDeterminationEngine()
self.context_manager = ContextManager(self.adapt)
self.object_type = object_type
self.container = IntentContainer(self.cache_dir)
self.intents = {}
self.register_default_intents()
self.register_core_intents()
self.container.train()
self.waiting_for_user = False
self._output = ""
self.input = ""
@staticmethod
def fuzzy_match(x, against):
"""Perform a 'fuzzy' comparison between two strings.
Returns:
float: match percentage -- 1.0 for perfect match,
down to 0.0 for no match at all.
"""
return SequenceMatcher(None, x, against).ratio()
@staticmethod
def match_one(query, choices):
"""
Find best match from a list or dictionary given an input
Arguments:
query: string to test
choices: list or dictionary of choices
Returns: tuple with best match, score
"""
if isinstance(choices, dict):
_choices = list(choices.keys())
elif isinstance(choices, list):
_choices = choices
else:
raise ValueError('a list or dict of choices must be provided')
best = (_choices[0],
TNaLaGmesConstruct.fuzzy_match(query, _choices[0]))
for c in _choices[1:]:
score = TNaLaGmesConstruct.fuzzy_match(query, c)
if score > best[1]:
best = (c, score)
if isinstance(choices, dict):
return choices[best[0]], best[1]
else:
return best
@staticmethod
def extract_number(text, short_scale=True, ordinals=False, lang="en-us"):
"""Takes in a string and extracts a number.
Args:
text (str): the string to extract a number from
short_scale (bool): Use "short scale" or "long scale" for large
numbers -- over a million. The default is short scale, which
is now common in most English speaking countries.
See https://en.wikipedia.org/wiki/Names_of_large_numbers
ordinals (bool): consider ordinal numbers, e.g. third=3 instead of 1/3
lang (str): the BCP-47 code for the language to use
Returns:
(int, float or False): The number extracted or False if the input
text contains no numbers
"""
lang_lower = str(lang).lower()
if lang_lower.startswith("en"):
return extract_number_en(text,
short_scale=short_scale,
ordinals=ordinals)
elif lang_lower.startswith("pt"):
return extractnumber_pt(text)
elif lang_lower.startswith("it"):
return extractnumber_it(text)
elif lang_lower.startswith("fr"):
return extractnumber_fr(text)
elif lang_lower.startswith("sv"):
return extractnumber_sv(text)
# elif lang_lower.startswith("de"):
# return extractnumber_de(text)
# TODO: extractnumber_xx for other languages
return text
@staticmethod
def extract_datetime(text, anchor=None, lang="en-us"):
"""
Extracts date and time information from a sentence. Parses many of the
common ways that humans express dates and times, including relative dates
like "5 days from today", "tomorrow', and "Tuesday".
Vague terminology are given arbitrary values, like:
- morning = 8 AM
- afternoon = 3 PM
- evening = 7 PM
If a time isn't supplied or implied, the function defaults to 12 AM
Args:
text (str): the text to be interpreted
anchor (:obj:`datetime`, optional): the date to be used for
relative dating (for example, what does "tomorrow" mean?).
Defaults to the current local date/time.
lang (string): the BCP-47 code for the language to use
Returns:
[:obj:`datetime`, :obj:`str`]: 'datetime' is the extracted date
as a datetime object in the user's local timezone.
'leftover_string' is the original phrase with all date and time
related keywords stripped out. See examples for further
clarification
Returns 'None' if the input string is empty.
Examples:
extract_datetime(
... "What is the weather like the day after tomorrow?",
... datetime(2017, 06, 30, 00, 00)
... )
[datetime.datetime(2017, 7, 2, 0, 0), 'what is weather like']
extract_datetime(
... "Set up an appointment 2 weeks from Sunday at 5 pm",
... datetime(2016, 02, 19, 00, 00)
... )
[datetime.datetime(2016, 3, 6, 17, 0), 'set up appointment']
"""
lang_lower = str(lang).lower()
if not anchor:
anchor = now_local()
if lang_lower.startswith("en"):
return extract_datetime_en(text, anchor)
elif lang_lower.startswith("pt"):
return extract_datetime_pt(text, anchor)
elif lang_lower.startswith("it"):
return extract_datetime_it(text, anchor)
elif lang_lower.startswith("fr"):
return extract_datetime_fr(text, anchor)
elif lang_lower.startswith("sv"):
return extract_datetime_sv(text, anchor)
# TODO: extract_datetime for other languages
return text
@staticmethod
def normalize(text, lang="en-us", remove_articles=True):
"""Prepare a string for parsing
This function prepares the given text for parsing by making
numbers consistent, getting rid of contractions, etc.
Args:
text (str): the string to normalize
lang (str): the code for the language text is in
remove_articles (bool): whether to remove articles (like 'a', or
'the'). True by default.
Returns:
(str): The normalized string.
"""
lang_lower = str(lang).lower()
if lang_lower.startswith("en"):
return normalize_en(text, remove_articles)
elif lang_lower.startswith("es"):
return normalize_es(text, remove_articles)
elif lang_lower.startswith("pt"):
return normalize_pt(text, remove_articles)
elif lang_lower.startswith("it"):
return normalize_it(text, remove_articles)
elif lang_lower.startswith("fr"):
return normalize_fr(text, remove_articles)
elif lang_lower.startswith("sv"):
return normalize_sv(text, remove_articles)
# TODO: Normalization for other languages
return text
@staticmethod
def word_gender(word, input_string="", lang="en-us"):
'''
guess gender of word, optionally use raw input text for context
returns "m" if the word is male, "f" if female, False if unknown
'''
if "pt" in lang or "es" in lang:
# spanish follows same rules
return get_gender_pt(word, input_string)
elif "it" in lang:
return get_gender_it(word, input_string)
return False
@property
def output(self):
out = self._output
self._output = ""
return out
def manual_fix_parse(self, text):
# TODO replace vars
return text
@output.setter
def output(self, text=""):
if isinstance(text, list):
text = [t.strip() for t in text if t.strip()]
text = random.choice(text)
else:
if not text.strip():
return
self._output += self.manual_fix_parse(text) + "\n"
def ask_yes_no(self, prompt):
self.output = prompt
self.waiting_for_user = True
while self.waiting_for_user:
sleep(0.1)
response = self.normalize(self.input)
if response[0] == 'y':
return True
if response[0] == 'n':
return False
else:
return self.ask_yes_no(prompt)
def ask_numeric(self, prompt, lower_bound=None, upper_bound=None):
self.output = prompt
self.waiting_for_user = True
while self.waiting_for_user:
sleep(0.1)
response = self.extract_number(self.input)
try:
value = int(response)
except ValueError:
self.output = "impossible!"
return self.ask_numeric(prompt, lower_bound, upper_bound)
if lower_bound is not None:
if value < lower_bound:
self.output = str(response) + " is too low"
return self.ask_numeric(prompt, lower_bound, upper_bound)
if upper_bound is not None:
if value > upper_bound:
self.output = str(response) + " is too high"
return self.ask_numeric(prompt, lower_bound, upper_bound)
return value
def ask_with_timeout(self, prompt="say BANG", timeout=7):
self.output = prompt
self.waiting_for_user = True
while self.waiting_for_user:
sleep(0.1)
response = self.input.lower().strip()
# TODO measure mic level or type speed
return response, random.randint(1, 7)
def register_default_intents(self):
pass
def handle_yes(self, intent):
return "you sound positive"
def handle_no(self, intent):
return "you sound negative"
def register_core_intents(self):
self.register_keyword_intent("yes", handler=self.handle_yes)
self.register_keyword_intent("no", handler=self.handle_no)
def calc_intent(self, utterance, lang="en-us"):
# check adapt
best_intent = None
utterances = utterance
if isinstance(utterance, str):
utterances = [utterance]
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
ut = self.normalize(utterance, lang)
if not ut:
continue
best_intent = next(
self.adapt.determine_intent(
ut,
100,
include_tags=True,
context_manager=self.context_manager))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
best_intent['normalized_utterance'] = ut
except StopIteration:
# don't show error in log
continue
except Exception as e:
LOG.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
return best_intent
LOG.debug("unknown adapt command: " + str(utterances))
# check padatious
return self.container.calc_intent(utterance)
def register_intent(self, name, samples, handler=None):
self.container.add_intent(name, samples)
self.intents[name] = handler
@staticmethod
def load_resource(name, sep="\n", is_json=False):
path = resolve_resource_file(name)
if path and exists(path):
with open(path, "r") as f:
lines = f.read()
if is_json:
return json.loads(lines)
return lines.split(sep)
return None
def register_keyword_intent(self,
name,
samples=None,
optionals=None,
handler=None,
ignore_default_kw=False):
optionals = optionals or []
samples = samples or [name]
intent_name = self.object_type + ':' + name
intent = IntentBuilder(intent_name)
if samples and isinstance(samples, list):
samples = {samples[0]: samples}
if optionals and isinstance(optionals, list):
optionals = {optionals[0]: optionals}
if not ignore_default_kw:
data = self.load_resource(name)
if data:
# merge file data
for new in data:
if new not in samples[name]:
samples[name].append(new)
for optional in optionals:
data = self.load_resource(optional)
if data:
# merge file data
for new in data:
optionals[optional] = optionals[optional] or []
if new not in optionals[optional]:
optionals[optional].append(new)
for required in samples:
for kw in samples[required]:
self.adapt.register_entity(required, kw)
intent.require(required)
for optional in optionals:
for kw in optionals[optional]:
self.adapt.register_entity(optional, kw)
intent.optionally(optional)
self.adapt.register_intent_parser(intent.build())
self.intents[intent_name] = handler
def parse_command(self, utterance):
# parse intent
intent = self.calc_intent(utterance)
intent_name = intent.get("intent_type", "")
if intent_name in self.intents:
return self.intents[intent_name](intent)
return "?"