def register_alarm_intent(engine: IntentDeterminationEngine):
alarm_keywords = [
"alarm"
]
for ak in alarm_keywords:
engine.register_entity(ak, "AlarmKeyword")
engine.register_regex_entity("(for|at) (?P<Time>.*)")
weekdays = [
"monday",
"tuesday",
"wednesday",
"thursday",
"friday",
"saturday",
"sunday"
]
for w in weekdays:
engine.register_entity(w, "Weekday")
# structure intent
alarm_intent = IntentBuilder("AlarmIntent")\
.require("AlarmKeyword")\
.require("Time")\
.optionally("Weekday")\
.build()
engine.register_intent_parser(alarm_intent)
def testContextAndOneOf(self):
# test to cover https://github.com/MycroftAI/adapt/issues/86
engine = IntentDeterminationEngine()
context_manager = ContextManager()
# define vocabulary
weather_keyword = ["weather"]
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
# structure intent
weather_intent = IntentBuilder("WeatherIntent") \
.require("WeatherKeyword") \
.one_of("Location", "LocationContext").build()
engine.register_intent_parser(weather_intent)
word = 'lizard'
context = 'LocationContext'
entity = {}
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
context_manager.inject_context(entity)
intents = list(
engine.determine_intent('weather',
context_manager=context_manager))
self.assertEqual(1, len(intents), "Incorrect number of intents")
result = intents[0]
self.assertEqual("lizard", result.get("LocationContext"),
"Context not matched")
self.assertEqual(0.75, result.get('confidence'),
"Context confidence not properly applied.")
class JokeIntentParser():
def __init__(self):
self.engine = IntentDeterminationEngine()
joke_verbs = [
"tell",
"crack"
]
for mv in joke_verbs:
self.engine.register_entity(mv, "JokeVerb")
joke_keywords = [
"joke"
]
for mk in joke_keywords:
self.engine.register_entity(mk, "JokeKeyword")
joke_intent = IntentBuilder("JokeIntent")\
.require("JokeVerb")\
.optionally("JokeKeyword")\
.build()
self.engine.register_intent_parser(joke_intent)
def parse(self, sentence):
for intent in self.engine.determine_intent(sentence):
if intent.get('confidence') > 0:
return intent
class MusicPlayerIntentParser():
def __init__(self):
self.engine = IntentDeterminationEngine()
# define music vocabulary
music_verbs = ["listen", "hear", "play", "stop"]
for mv in music_verbs:
self.engine.register_entity(mv, "MusicVerb")
music_keywords = ["songs", "music"]
for mk in music_keywords:
self.engine.register_entity(mk, "MusicKeyword")
self.engine.register_regex_entity(
"(play|hear|listen|listen to)\s*(the)?\s*(song|album)?\s*(?P<Media>.*)$" # NoQA
)
music_intent = IntentBuilder("MusicIntent")\
.require("MusicVerb")\
.optionally("MusicKeyword")\
.optionally("Media")\
.build()
self.engine.register_intent_parser(music_intent)
def parse(self, sentence):
for intent in self.engine.determine_intent(sentence):
if intent.get('confidence') > 0:
return intent
def testEmptyTags(self):
# Validates https://github.com/MycroftAI/adapt/issues/114
engine = IntentDeterminationEngine()
engine.register_entity("Kevin",
"who") # same problem if several entities
builder = IntentBuilder("Buddies")
builder.optionally("who") # same problem if several entity types
engine.register_intent_parser(builder.build())
intents = [i for i in engine.determine_intent("Julien is a friend")]
assert len(intents) == 0
class ContextManagerIntegrationTest(unittest.TestCase):
def setUp(self):
self.context_manager = ContextManager()
self.engine = IntentDeterminationEngine()
def testBasicContextualFollowup(self):
intent1 = IntentBuilder("TimeQueryIntent")\
.require("TimeQuery")\
.require("Location")\
.build()
intent2 = IntentBuilder("WeatherQueryIntent")\
.require("WeatherKeyword")\
.require("Location")\
.build()
self.engine.register_intent_parser(intent1)
self.engine.register_intent_parser(intent2)
self.engine.register_entity("what time is it", "TimeQuery")
self.engine.register_entity("seattle", "Location")
self.engine.register_entity("miami", "Location")
self.engine.register_entity("weather", "WeatherKeyword")
utterance1 = "what time is it in seattle"
intent = next(self.engine.determine_intent(utterance1, include_tags=True, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == 'TimeQueryIntent'
assert '__tags__' in intent
for tag in intent['__tags__']:
context_entity = tag.get('entities')[0]
self.context_manager.inject_context(context_entity)
utterance2 = "what's the weather like?"
intent = next(self.engine.determine_intent(utterance2, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == 'WeatherQueryIntent'
def testContextOnlyUsedOnce(self):
intent_parser = IntentBuilder("DummyIntent")\
.require("Foo")\
.optionally("Foo", "Foo2")\
.build()
context_entity = {'confidence': 1.0, 'data': [('foo', 'Foo')], 'match': 'foo', 'key': 'foo'}
self.context_manager.inject_context(context_entity)
self.engine.register_intent_parser(intent_parser)
self.engine.register_entity("foo", "Foo")
self.engine.register_entity("fop", "Foo")
intent = next(self.engine.determine_intent("foo", include_tags=True, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == "DummyIntent"
assert not (intent.get("Foo") and intent.get("Foo2"))
class ContextManagerIntegrationTest(unittest.TestCase):
def setUp(self):
self.context_manager = ContextManager()
self.engine = IntentDeterminationEngine()
def testBasicContextualFollowup(self):
intent1 = IntentBuilder("TimeQueryIntent")\
.require("TimeQuery")\
.require("Location")\
.build()
intent2 = IntentBuilder("WeatherQueryIntent")\
.require("WeatherKeyword")\
.require("Location")\
.build()
self.engine.register_intent_parser(intent1)
self.engine.register_intent_parser(intent2)
self.engine.register_entity("what time is it", "TimeQuery")
self.engine.register_entity("seattle", "Location")
self.engine.register_entity("miami", "Location")
self.engine.register_entity("weather", "WeatherKeyword")
utterance1 = "what time is it in seattle"
intent = next(self.engine.determine_intent(utterance1, include_tags=True, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == 'TimeQueryIntent'
assert '__tags__' in intent
for tag in intent['__tags__']:
context_entity = tag.get('entities')[0]
self.context_manager.inject_context(context_entity)
utterance2 = "what's the weather like?"
intent = next(self.engine.determine_intent(utterance2, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == 'WeatherQueryIntent'
def testContextOnlyUsedOnce(self):
intent_parser = IntentBuilder("DummyIntent")\
.require("Foo")\
.optionally("Foo", "Foo2")\
.build()
context_entity = {'confidence': 1.0, 'data': [('foo', 'Foo')], 'match': 'foo', 'key': 'foo'}
self.context_manager.inject_context(context_entity)
self.engine.register_intent_parser(intent_parser)
self.engine.register_entity("foo", "Foo")
self.engine.register_entity("fop", "Foo")
intent = next(self.engine.determine_intent("foo", include_tags=True, context_manager=self.context_manager))
assert intent
assert intent['intent_type'] == "DummyIntent"
assert not (intent.get("Foo") and intent.get("Foo2"))
def register_time_intent(engine: IntentDeterminationEngine):
time_keywords = [
"time"
]
for tk in time_keywords:
engine.register_entity(tk, "TimeKeyword")
time_intent = IntentBuilder("TimeIntent")\
.require("TimeKeyword")\
.build()
engine.register_intent_parser(time_intent)
def register_agenda_intent(engine: IntentDeterminationEngine):
# create and register weather vocabulary
agenda_keyword = ["agenda"]
for ak in agenda_keyword:
engine.register_entity(ak, "AgendaKeyword")
# structure intent
agenda_intent = IntentBuilder("AgendaIntent")\
.require("AgendaKeyword")\
.build()
engine.register_intent_parser(agenda_intent)
def register_joke_intent(engine: IntentDeterminationEngine):
# create and register joke vocabulary
joke_keywords = ["joke", "make me laugh"]
for jk in joke_keywords:
engine.register_entity(jk, "JokeKeyword")
# structure intent
joke_intent = IntentBuilder("JokeIntent")\
.require("JokeKeyword")\
.build()
engine.register_intent_parser(joke_intent)
def register_timer_intent(engine: IntentDeterminationEngine):
timer_keywords = ["timer"]
for tk in timer_keywords:
engine.register_entity(tk, "TimerKeyword")
engine.register_regex_entity("for (?P<Time>.*)")
# structure intent
timer_intent = IntentBuilder("TimerIntent")\
.require("TimerKeyword")\
.optionally("Time")\
.build()
engine.register_intent_parser(timer_intent)
def register_todo_intent(engine: IntentDeterminationEngine):
commands = ["add", "remove", "get", "tell me", "clear"]
for c in commands:
engine.register_entity(c, "TodoCommand")
engine.register_regex_entity("(add|remove) (?P<Item>.*) (to|from) my")
engine.register_regex_entity("my (?P<ListType>.*) list")
todo_intent = IntentBuilder("TodoIntent")\
.require("TodoCommand")\
.optionally("Item")\
.require("ListType")\
.build()
engine.register_intent_parser(todo_intent)
def __initialize__():
engine = IntentDeterminationEngine()
launch_keyword = parse_keyword('LaunchKeyword')
for lk in launch_keyword:
engine.register_entity(lk, "LaunchKeyword")
launch_intent = IntentBuilder("LaunchIntent") \
.require("LaunchKeyword") \
.build()
engine.register_intent_parser(launch_intent)
play_keyword = parse_keyword('PlayKeyword')
for pk in play_keyword:
engine.register_entity(pk, "PlayKeyword")
play_intent = IntentBuilder("PlayIntent") \
.require("PlayKeyword") \
.build()
engine.register_intent_parser(play_intent)
def register_weather_intent(engine: IntentDeterminationEngine):
# create and register weather vocabulary
weather_keyword = ["weather"]
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
# create regex to parse out locations
engine.register_regex_entity("in (?P<Location>.*)")
# structure intent
weather_intent = IntentBuilder("WeatherIntent")\
.require("WeatherKeyword")\
.optionally("Location")\
.build()
engine.register_intent_parser(weather_intent)
def build_engine(rdb_conn):
"""Build a recycling intent determination engine."""
engine = IntentDeterminationEngine()
recycle_keywords = ["recycle", "recycled", "recyclable"]
for keyword in recycle_keywords:
engine.register_entity(keyword, "RecycleKeyword")
plastic_keywords = [
item["name"] for item in r.table("items").filter({
"type": "plastic"
}).run(rdb_conn)
]
register_intent("plastic",
engine,
*plastic_keywords,
descriptor=["number"],
numbers=["1", "2", "3", "4", "5", "6", "7"])
glass_keywords = [
item["name"] for item in r.table("items").filter({
"type": "glass"
}).run(rdb_conn)
]
register_intent("glass", engine, *glass_keywords)
paper_keywords = [
item["name"] for item in r.table("items").filter({
"type": "paper"
}).run(rdb_conn)
]
register_intent("paper", engine, *paper_keywords)
other_keywords = [
item["name"] for item in r.table("items").filter(
~r.row.has_fields("type")).run(rdb_conn)
]
register_intent("other", engine, *other_keywords)
return engine
def register_reminder_intent(engine: IntentDeterminationEngine):
reminder_keywords = ["remind"]
for rk in reminder_keywords:
engine.register_entity(rk, "ReminderKeyword")
engine.register_regex_entity("to (?P<Action>.*) in")
engine.register_regex_entity("in (?P<Time>.*)")
# structure intent
reminder_intent = IntentBuilder("ReminderIntent")\
.require("ReminderKeyword")\
.require("Action") \
.require("Time") \
.build()
engine.register_intent_parser(reminder_intent)
def register_what_is_intent(engine: IntentDeterminationEngine):
# create and register what is vocabulary
what_is_keywords = ["what is", "tell me"]
for wk in what_is_keywords:
engine.register_entity(wk, "WhatIsKeyword")
# create regex to parse out subjects
engine.register_regex_entity("is (?P<Subject>.*)")
engine.register_regex_entity("about (?P<Subject>.*)")
# structure intent
what_is_intent = IntentBuilder("WhatIsIntent")\
.require("WhatIsKeyword")\
.require("Subject")\
.build()
engine.register_intent_parser(what_is_intent)
def register_news_intent(engine: IntentDeterminationEngine):
# create and register news vocabulary
news_keyword = ["news"]
for nk in news_keyword:
engine.register_entity(nk, "NewsKeyword")
# create regex to parse out topics
engine.register_regex_entity("about (?P<Topic>.*)")
engine.register_regex_entity("for (?P<Topic>.*)")
# structure intent
news_intent = IntentBuilder("NewsIntent")\
.require("NewsKeyword")\
.optionally("Topic")\
.build()
engine.register_intent_parser(news_intent)
def put_in_engin(text):
engine = IntentDeterminationEngine()
# Register entities on engine
for entity, keywords in entities.items():
for keyword in keywords:
engine.register_entity(keyword, entity)
for entity in multi_regex_entities:
for regex in entity:
engine.register_regex_entity(regex)
# Register intents on engine
for intent in intents:
engine.register_intent_parser(intent)
for intent in engine.determine_intent(text):
# return dict[intenct] # will return function
print(intent)
print('finished')
class input_engine:
"""Manages the intent engine and natural language input parser"""
def __init__(self):
self.engine = IntentDeterminationEngine()
def register_entity(self, keywords, name):
"""Registers an intenty to be found in an input"""
for k in keywords:
self.engine.register_entity(k, name)
def register_intent(self, intent):
"""Registers an intent that can be found in an input"""
self.engine.register_intent_parser(intent)
def get_intent(self, input_string):
"""Returns an intent from an input string if one is found"""
intent = self.engine.determine_intent(input_string)
for intent in self.engine.determine_intent(input_string):
if intent.get("confidence") > 0:
return intent
return None
class SmartHomeIntentParser():
def __init__(self):
self.engine = IntentDeterminationEngine()
# smart home intent vocabalory
switch_tasks_keyword = ["switch", "turn"]
for stk in switch_tasks_keyword:
self.engine.register_entity(stk, "SwitchTasksKeyword")
on_off_keyword = ["on", "off"]
for stk in on_off_keyword:
self.engine.register_entity(stk, "OnOffKeyword")
equipment_keyword = ["lights", "light", "fan"]
for stk in equipment_keyword:
self.engine.register_entity(stk, "EquipmentKeyword")
smart_home_intent = IntentBuilder("SmartHomeIntent")\
.require("SwitchTasksKeyword")\
.require("OnOffKeyword")\
.require("EquipmentKeyword")\
.build()
self.engine.register_intent_parser(smart_home_intent)
def parse(self, sentence):
for intent in self.engine.determine_intent(sentence):
if intent.get('confidence') > 0:
return intent
class IntentEngineTests(unittest.TestCase):
def setUp(self):
self.engine = IntentDeterminationEngine()
def testRegisterIntentParser(self):
assert len(self.engine.intent_parsers) == 0
try:
self.engine.register_intent_parser("NOTAPARSER")
assert "Did not fail to register invalid intent parser" and False
except ValueError as e:
pass
parser = IntentBuilder("Intent").build()
self.engine.register_intent_parser(parser)
assert len(self.engine.intent_parsers) == 1
def testRegisterRegexEntity(self):
assert len(self.engine._regex_strings) == 0
assert len(self.engine.regular_expressions_entities) == 0
self.engine.register_regex_entity(".*")
assert len(self.engine._regex_strings) == 1
assert len(self.engine.regular_expressions_entities) == 1
def testSelectBestIntent(self):
parser1 = IntentBuilder("Parser1").require("Entity1").build()
self.engine.register_intent_parser(parser1)
self.engine.register_entity("tree", "Entity1")
utterance = "go to the tree house"
intent = next(self.engine.determine_intent(utterance))
assert intent
assert intent['intent_type'] == 'Parser1'
parser2 = IntentBuilder("Parser2").require("Entity1").require(
"Entity2").build()
self.engine.register_intent_parser(parser2)
self.engine.register_entity("house", "Entity2")
intent = next(self.engine.determine_intent(utterance))
assert intent
assert intent['intent_type'] == 'Parser2'
class IntentEngineTests(unittest.TestCase):
def setUp(self):
self.engine = IntentDeterminationEngine()
def testRegisterIntentParser(self):
assert len(self.engine.intent_parsers) == 0
try:
self.engine.register_intent_parser("NOTAPARSER")
assert "Did not fail to register invalid intent parser" and False
except ValueError as e:
pass
parser = IntentBuilder("Intent").build()
self.engine.register_intent_parser(parser)
assert len(self.engine.intent_parsers) == 1
def testRegisterRegexEntity(self):
assert len(self.engine._regex_strings) == 0
assert len(self.engine.regular_expressions_entities) == 0
self.engine.register_regex_entity(".*")
assert len(self.engine._regex_strings) == 1
assert len(self.engine.regular_expressions_entities) == 1
def testSelectBestIntent(self):
parser1 = IntentBuilder("Parser1").require("Entity1").build()
self.engine.register_intent_parser(parser1)
self.engine.register_entity("tree", "Entity1")
utterance = "go to the tree house"
intent = next(self.engine.determine_intent(utterance))
assert intent
assert intent['intent_type'] == 'Parser1'
parser2 = IntentBuilder("Parser2").require("Entity1").require("Entity2").build()
self.engine.register_intent_parser(parser2)
self.engine.register_entity("house", "Entity2")
intent = next(self.engine.determine_intent(utterance))
assert intent
assert intent['intent_type'] == 'Parser2'
class WeatherIntentParser():
def __init__(self):
self.engine = IntentDeterminationEngine()
weather_verbs = ["weather", "temperature", "forecast"]
for mv in weather_verbs:
self.engine.register_entity(mv, "WeatherVerb")
self.engine.register_regex_entity(
"in\s*(?P<Location>[A-Z][^\s]*\s*?)+.*$" # NoQA
)
weather_intent = IntentBuilder("WeatherIntent")\
.require("WeatherVerb")\
.require("Location")\
.build()
self.engine.register_intent_parser(weather_intent)
def parse(self, sentence):
for intent in self.engine.determine_intent(sentence):
if intent.get('confidence') > 0:
return intent
def train(keyword, types, locations):
global engine
engine = IntentDeterminationEngine()
for kw in keyword:
engine.register_entity(kw, "Keyword")
for t in types:
engine.register_entity(t, "Type")
for loc in locations:
engine.register_entity(loc, "Location")
intent = IntentBuilder("Intent")\
.require("Keyword")\
.optionally("Type")\
.require("Location")\
.build()
engine.register_intent_parser(intent)
def get_intent(message):
engine = IntentDeterminationEngine()
keywords = [
'service',
'med',
'clinic',
'walk in',
]
for key in keywords:
engine.register_entity(key, "KeyWords")
print(os.getcwd())
with open(os.getcwd() + '/home/addresses.csv', 'rb') as csvfile:
records = csv.reader(csvfile, delimiter=',')
street_number = []
street_name = []
for row in records:
street_number.append(row[8])
street_name.append(row[11])
for key in street_number:
engine.register_entity(key, "StreetNumber")
for key in street_name:
engine.register_entity(key, "StreetName")
address_intent = IntentBuilder("AddressIntent")\
.require("KeyWords")\
.optionally("StreetNumber")\
.optionally("StreetName")\
.build()
engine.register_intent_parser(address_intent)
for intent in engine.determine_intent(''.join(message)):
return intent
def register_add_event_intent(engine: IntentDeterminationEngine):
# <event_name> - done
# <event_location> - done
# <start_date> - done
# <start_time> - done
# <end_date> - done
# <end_time> - done
# called <event_name> from the <start_date> to the <end_date>
# schedule an event called OOP exam from the 21st of May to the 22nd of May
# called <event_name> at <event_location> from the <start_date> to the<end_date>
# schedule an event called OOP exam at FMI 325 from the 21st of May to the 22nd of May
# called <event_name> on the <start_date> at <start_time> until the <end_date> at <end_time>
# schedule an event called OOP exam on the 21st of May at 12:30 p.m. until the 22nd of May at 13:30 p.m.
# called <event_name> at <event_location> on the <start_date> at <start_time> until the <end_date> at <end_time>
# create an event called OOP exam at FMI 325 on the 21st of June at 12:30 p.m. until the 21nd of June at 13:30 p.m.
# other key words:
# from, to, at, on, until
# (from|on)?.+?(?<!until).+?(the (?P<start_date>\d+?)(?=st|nd|td|th))
event_keyword = ["event", "schedule"]
for ek in event_keyword:
engine.register_entity(ek, "AddEventKeyword")
# event name
engine.register_regex_entity(
'called (?P<event_name>.+?)(?=from|to|at|on|until)')
# matches location
engine.register_regex_entity(
"at (?P<location>[a-zA-Z0-9 ]+?)(?=from|to|at|on|until)")
# start_date, when only date is present
engine.register_regex_entity(
'(from the)(?<!to the) ((?P<start_date>\d+?)(?=st|nd|rd|th))')
# start_date, when time is present
engine.register_regex_entity(
'(on the)(?<!until the) ((?P<start_date>\d+?)(?=st|nd|rd|th))')
# start_time
engine.register_regex_entity(
'at (?P<start_time>([0-9]{1,4}:?[0-9]{0,2} ?(p\.m\.|a\.m\.)?)).*')
# end_date, when only date is present
engine.register_regex_entity(
'(to the) ((?P<end_date>\d+?)(?=st|nd|rd|th))')
# end_date, when time is present
engine.register_regex_entity(
'(until the) ((?P<end_date>\d+?)(?=st|nd|rd|th))')
# end_time
engine.register_regex_entity(
'(?<=until the \d{2}(st|nd|rd|th) of) \w*? at (?P<end_time>([0-9]{1,4}:?[0-9]{0,2} ?(p\.m\.|a\.m\.)?))'
)
# structure intent
add_event_intent = IntentBuilder("AddEventIntent") \
.require("AddEventKeyword")\
.optionally("event_name")\
.optionally("location")\
.optionally("start_date")\
.optionally("start_time")\
.optionally("end_date")\
.optionally("end_time")\
.build()
engine.register_intent_parser(add_event_intent)
class IntentService(object):
def __init__(self, emitter):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
# Dictionary for translating a skill id to a name
self.skill_names = {}
# Context related intializations
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.emitter = emitter
self.emitter.on('register_vocab', self.handle_register_vocab)
self.emitter.on('register_intent', self.handle_register_intent)
self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
self.emitter.on('detach_intent', self.handle_detach_intent)
self.emitter.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.emitter.on('add_context', self.handle_add_context)
self.emitter.on('remove_context', self.handle_remove_context)
self.emitter.on('clear_context', self.handle_clear_context)
# Converse method
self.emitter.on('skill.converse.response',
self.handle_converse_response)
self.emitter.on('mycroft.speech.recognition.unknown',
self.reset_converse)
self.emitter.on('mycroft.skills.loaded', self.update_skill_name_dict)
def add_active_skill_handler(message):
self.add_active_skill(message.data['skill_id'])
self.emitter.on('active_skill_request', add_active_skill_handler)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
def update_skill_name_dict(self, message):
"""
Messagebus handler, updates dictionary of if to skill name
conversions.
"""
self.skill_names[message.data['id']] = message.data['name']
def get_skill_name(self, skill_id):
""" Get skill name from skill ID.
Args:
skill_id: a skill id as encoded in Intent handlers.
Returns:
(str) Skill name or the skill id if the skill wasn't found
"""
return self.skill_names.get(skill_id, skill_id)
def reset_converse(self, message):
"""Let skills know there was a problem with speech recognition"""
lang = message.data.get('lang', "en-us")
for skill in self.active_skills:
self.do_converse(None, skill[0], lang)
def do_converse(self, utterances, skill_id, lang):
self.emitter.emit(Message("skill.converse.request", {
"skill_id": skill_id, "utterances": utterances, "lang": lang}))
self.waiting = True
self.result = False
start_time = time.time()
t = 0
while self.waiting and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting = False
return self.result
def handle_converse_response(self, message):
# id = message.data["skill_id"]
# no need to crosscheck id because waiting before new request is made
# no other skill will make this request is safe assumption
result = message.data["result"]
self.result = result
self.waiting = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
""" Updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def send_metrics(self, intent, context, stopwatch):
"""
Send timing metrics to the backend.
NOTE: This only applies to those with Opt In.
"""
LOG.debug('Sending metric if opt_in is enabled')
ident = context['ident'] if context else None
if intent:
# Recreate skill name from skill id
parts = intent.get('intent_type', '').split(':')
intent_type = self.get_skill_name(parts[0])
if len(parts) > 1:
intent_type = ':'.join([intent_type] + parts[1:])
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': intent_type})
else:
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'intent_failure'})
def handle_utterance(self, message):
""" Main entrypoint for handling user utterances with Mycroft skills
Monitor the messagebus for 'recognizer_loop:utterance', typically
generated by a spoken interaction but potentially also from a CLI
or other method of injecting a 'user utterance' into the system.
Utterances then work through this sequence to be handled:
1) Active skills attempt to handle using converse()
2) Adapt intent handlers
3) Padatious intent handlers
4) Other fallbacks
Args:
message (Message): The messagebus data
"""
try:
# Get language of the utterance
lang = message.data.get('lang', "en-us")
utterances = message.data.get('utterances', '')
stopwatch = Stopwatch()
with stopwatch:
# Give active skills an opportunity to handle the utterance
converse = self._converse(utterances, lang)
if not converse:
# No conversation, use intent system to handle utterance
intent = self._adapt_intent_match(utterances, lang)
if converse:
# Report that converse handled the intent and return
ident = message.context['ident'] if message.context else None
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'converse'})
return
elif intent:
# Send the message to the intent handler
reply = message.reply(intent.get('intent_type'), intent)
else:
# Allow fallback system to handle utterance
# NOTE: Padatious intents are handled this way, too
reply = message.reply('intent_failure',
{'utterance': utterances[0],
'lang': lang})
self.emitter.emit(reply)
self.send_metrics(intent, message.context, stopwatch)
except Exception as e:
LOG.exception(e)
def _converse(self, utterances, lang):
""" Give active skills a chance at the utterance
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
bool: True if converse handled it, False if no skill processes it
"""
# check for conversation time-out
self.active_skills = [skill for skill in self.active_skills
if time.time() - skill[
1] <= self.converse_timeout * 60]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return True
return False
def _adapt_intent_match(self, utterances, lang):
""" Run the Adapt engine to search for an matching intent
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
Intent structure, or None if no match was found.
"""
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
best_intent = next(self.engine.determine_intent(
normalize(utterance, lang), 100,
include_tags=True,
context_manager=self.context_manager))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
except StopIteration:
# don't show error in log
continue
except Exception as e:
LOG.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
self.update_context(best_intent)
# update active skills
skill_id = best_intent['intent_type'].split(":")[0]
self.add_active_skill(skill_id)
return best_intent
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers if
not p.name.startswith(skill_id)]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
""" Add context
Args:
message: data contains the 'context' item to add
optionally can include 'word' to be injected as
an alias for the context item.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
# if not a string type try creating a string from it
if not isinstance(word, str):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
""" Remove specific context
Args:
message: data contains the 'context' item to remove
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
""" Clears all keywords from context """
self.context_manager.clear_context()
class AdaptIntentRecognizer(RhasspyActor):
"""Recognize intents with Mycroft Adapt."""
def __init__(self) -> None:
RhasspyActor.__init__(self)
self.engine = None
self.preload = False
def to_started(self, from_state: str) -> None:
"""Transition to started state."""
self.preload = self.config.get("preload", False)
if self.preload:
try:
self.load_engine()
except Exception as e:
self._logger.warning("preload: %s", e)
self.transition("loaded")
def in_loaded(self, message: Any, sender: RhasspyActor) -> None:
"""Handle messages in loaded state."""
if isinstance(message, RecognizeIntent):
try:
self.load_engine()
intent = self.recognize(message.text)
except Exception:
self._logger.exception("in_loaded")
intent = empty_intent()
intent["text"] = message.text
intent["raw_text"] = message.text
intent["speech_confidence"] = message.confidence
self.send(
message.receiver or sender,
IntentRecognized(intent, handle=message.handle),
)
# -------------------------------------------------------------------------
def recognize(self, text: str) -> Dict[str, Any]:
"""Use Adapt engine to recognize intent."""
# Get all intents
assert self.engine is not None, "Adapt engine not loaded"
intents = [
intent for intent in self.engine.determine_intent(text) if intent
]
if len(intents) > 0:
# Return the best intent only
intent = max(intents, key=lambda x: x.get("confidence", 0))
intent_type = intent["intent_type"]
entity_prefix = "{0}.".format(intent_type)
slots = {}
for key, value in intent.items():
if key.startswith(entity_prefix):
key = key[len(entity_prefix):]
slots[key] = value
# Try to match Rasa NLU format for future compatibility
return {
"text":
text,
"intent": {
"name": intent_type,
"confidence": intent.get("confidence", 0),
},
"entities": [{
"entity": name,
"value": value
} for name, value in slots.items()],
}
return empty_intent()
# -------------------------------------------------------------------------
def load_engine(self) -> None:
"""Configure Adapt engine if not already cached."""
if self.engine is None:
from adapt.intent import IntentBuilder
from adapt.engine import IntentDeterminationEngine
config_path = self.profile.read_path("adapt_config.json")
if not os.path.exists(config_path):
return
# Create empty engine
self.engine = IntentDeterminationEngine()
assert self.engine is not None
# { intents: { ... }, entities: [ ... ] }
with open(config_path, "r") as config_file:
config = json.load(config_file)
# Register entities
for entity_name, entity_values in config["entities"].items():
for value in entity_values:
self.engine.register_entity(value, entity_name)
# Register intents
for intent_name, intent_config in config["intents"].items():
intent = IntentBuilder(intent_name)
for required_entity in intent_config["require"]:
intent.require(required_entity)
for optional_entity in intent_config["optionally"]:
intent.optionally(optional_entity)
self.engine.register_intent_parser(intent.build())
self._logger.debug("Loaded engine from config file %s",
config_path)
class VoiceCommands:
ACTIONS = 'Action'
NAME = 'Name'
ACTUATOR_TYPE = 'ActuatorType'
@typechecked()
def __init__(self, job_controll: AsyncActuatorCommands,
logging: RootLogger):
self.__job_controll = job_controll
self.__logging = logging
def configure(self):
self.__engine = IntentDeterminationEngine()
actions = ['on', 'off']
self.__register_entity(actions, self.ACTIONS)
locations = ['living', 'kitchen', 'hollway', 'wemo']
self.__register_entity(locations, self.NAME)
actuator_types = ['light', 'switch', 'courtains', 'door']
self.__register_entity(actuator_types, self.ACTUATOR_TYPE)
actuator_intent = IntentBuilder("ActuatorIntent") \
.require(self.ACTIONS) \
.require(self.ACTUATOR_TYPE) \
.require(self.NAME) \
.build()
self.__engine.register_intent_parser(actuator_intent)
self.__commands_map = [
{
'entities': {
'name': 'living',
'type': 'light'
},
'actuator': 'livingLight'
},
{
'entities': {
'name': 'living',
'type': 'courtains'
},
'actuator': 'livingCourtains'
},
{
'entities': {
'name': 'hollway',
'type': 'light'
},
'actuator': 'holwayLight'
},
{
'entities': {
'name': 'wemo',
'type': 'switch'
},
'actuator': 'wemoSwitch1'
},
]
return self
@typechecked()
def execute(self, command: str) -> None:
command = self.__normalize_command(command)
for intent in self.__engine.determine_intent(command):
if intent and intent.get('confidence') > 0:
self.__logging.info(intent)
command = self.__get_matching_command(intent)
self.__run_command(command, intent)
def __register_entity(self, wordlist, name):
for action in wordlist:
self.__engine.register_entity(action, name)
def __get_matching_command(self, intent):
for command in self.__commands_map:
if command['entities']['name'] == intent[self.NAME] and \
command['entities']['type'] == intent[self.ACTUATOR_TYPE]:
return command
def __run_command(self, command, intent):
if None == command:
return
actuator_state = (False, True)[intent[self.ACTIONS] == 'on']
self.__logging.info('Changin actuator {0} value: {1}'.format(
command['actuator'], actuator_state))
self.__job_controll.change_actuator(command['actuator'],
actuator_state)
def __normalize_command(self, command):
replaces = [('life', 'light'), ('leaving', 'living'),
('hallway', 'hollway'), ('quarters', 'courtains')]
for replace in replaces:
command = re.sub(replace[0], replace[1], command)
return command
event['content']['body']))
for intent in engine.determine_intent(
event['content']['body']):
room.send_text('{} {}'.format(intent.get('intent_type'),
intent.get('confidence')))
if intent.get('intent_type') == 'HideIntent':
if intent.get('confidence') > 0:
#room.send_text(json.dumps(intent, indent=4))
room.send_text('*Apebot hides')
else:
pass
weather_keyword = ['weather']
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
weather_types = ['snow', 'rain', 'wind', 'sleet', 'sun']
for wt in weather_types:
engine.register_entity(wt, "WeatherType")
locations = ['Seattle', 'San Fransisco', 'Tokyo']
for loc in locations:
engine.register_entity(loc, 'Location')
weather_intent = IntentBuilder("WeatherIntent")\
.require("WeatherKeyword")\
.optionally("WeatherType")\
.require('Location')\
.build()
try with the following:
PYTHONPATH=. python examples/single_intent_parser.py "what's the weather like in tokyo"
"""
import json
import sys
from adapt.intent import IntentBuilder
from adapt.engine import IntentDeterminationEngine
engine = IntentDeterminationEngine()
weather_keyword = [
"weather"
]
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
weather_types = [
"snow",
"rain",
"wind",
"sleet",
"sun"
]
for wt in weather_types:
engine.register_entity(wt, "WeatherType")
locations = [
"Seattle",
"San Francisco",
class IntentSkill(BoomerSkill):
def __init__(self):
BoomerSkill.__init__(self, name="IntentSkill")
self.engine = IntentDeterminationEngine()
def initialize(self):
self.emitter.on('register_vocab', self.handle_register_vocab)
self.emitter.on('register_intent', self.handle_register_intent)
self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
self.emitter.on('detach_intent', self.handle_detach_intent)
def handle_utterance(self, message):
timer = Stopwatch()
timer.start()
metrics = MetricsAggregator()
utterances = message.data.get('utterances', '')
best_intent = None
for utterance in utterances:
metrics.increment("utterances.count")
for intent in self.engine.determine_intent(
utterance, num_results=100):
metrics.increment("intents.count")
intent['utterance'] = utterance
best_confidence = best_intent.get('confidence') \
if best_intent else 0.0
cur_confidence = intent.get('confidence', 0.0)
if best_confidence < cur_confidence:
best_intent = intent
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
reply = message.reply(
best_intent.get('intent_type'), data=best_intent)
self.emitter.emit(reply)
elif len(utterances) == 1:
self.emitter.emit(
Message("intent_failure",
data={"utterance": utterances[0]}))
else:
self.emitter.emit(
Message("multi_utterance_intent_failure",
data={"utterances": utterances}))
metrics.timer("parse.time", timer.stop())
metrics.flush()
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def stop(self):
pass
class IntentService(object):
def __init__(self, emitter):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.emitter = emitter
self.emitter.on('register_vocab', self.handle_register_vocab)
self.emitter.on('register_intent', self.handle_register_intent)
self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
self.emitter.on('detach_intent', self.handle_detach_intent)
self.emitter.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.emitter.on('add_context', self.handle_add_context)
self.emitter.on('remove_context', self.handle_remove_context)
self.emitter.on('clear_context', self.handle_clear_context)
# Converse method
self.emitter.on('skill.converse.response',
self.handle_converse_response)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
def do_converse(self, utterances, skill_id, lang):
self.emitter.emit(Message("skill.converse.request", {
"skill_id": skill_id, "utterances": utterances, "lang": lang}))
self.waiting = True
self.result = False
start_time = time.time()
t = 0
while self.waiting and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting = False
return self.result
def handle_converse_response(self, message):
# id = message.data["skill_id"]
# no need to crosscheck id because waiting before new request is made
# no other skill will make this request is safe assumption
result = message.data["result"]
self.result = result
self.waiting = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
"""
updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def handle_utterance(self, message):
# Get language of the utterance
lang = message.data.get('lang', None)
if not lang:
lang = "en-us"
utterances = message.data.get('utterances', '')
# check for conversation time-out
self.active_skills = [skill for skill in self.active_skills
if time.time() - skill[
1] <= self.converse_timeout * 60]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return
# no skill wants to handle utterance
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
best_intent = next(self.engine.determine_intent(
normalize(utterance, lang), 100,
include_tags=True,
context_manager=self.context_manager))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
except StopIteration:
# don't show error in log
continue
except e:
LOG.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
self.update_context(best_intent)
reply = message.reply(
best_intent.get('intent_type'), best_intent)
self.emitter.emit(reply)
# update active skills
skill_id = int(best_intent['intent_type'].split(":")[0])
self.add_active_skill(skill_id)
else:
self.emitter.emit(Message("intent_failure", {
"utterance": utterances[0],
"lang": lang
}))
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
print "Registering: " + str(message.data)
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers if
not p.name.startswith(skill_id)]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
"""
Handles adding context from the message bus.
The data field must contain a context keyword and
may contain a word if a specific word should be injected
as a match for the provided context keyword.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
# if not a string type try creating a string from it
if not isinstance(word, basestring):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
"""
Handles removing context from the message bus. The
data field must contain the 'context' to remove.
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
"""
Clears all keywords from context.
"""
self.context_manager.clear_context()
class IntentService:
def __init__(self, bus):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
# Dictionary for translating a skill id to a name
self.skill_names = {}
# Context related intializations
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.bus = bus
self.bus.on('register_vocab', self.handle_register_vocab)
self.bus.on('register_intent', self.handle_register_intent)
self.bus.on('recognizer_loop:utterance', self.handle_utterance)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.bus.on('add_context', self.handle_add_context)
self.bus.on('remove_context', self.handle_remove_context)
self.bus.on('clear_context', self.handle_clear_context)
# Converse method
self.bus.on('skill.converse.response', self.handle_converse_response)
self.bus.on('skill.converse.error', self.handle_converse_error)
self.bus.on('mycroft.speech.recognition.unknown', self.reset_converse)
self.bus.on('mycroft.skills.loaded', self.update_skill_name_dict)
def add_active_skill_handler(message):
self.add_active_skill(message.data['skill_id'])
self.bus.on('active_skill_request', add_active_skill_handler)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
self.waiting_for_converse = False
self.converse_result = False
self.converse_skill_id = ""
# Intents API
self.registered_intents = []
self.registered_vocab = []
self.bus.on('intent.service.adapt.get', self.handle_get_adapt)
self.bus.on('intent.service.intent.get', self.handle_get_intent)
self.bus.on('intent.service.skills.get', self.handle_get_skills)
self.bus.on('intent.service.active_skills.get',
self.handle_get_active_skills)
self.bus.on('intent.service.adapt.manifest.get', self.handle_manifest)
self.bus.on('intent.service.adapt.vocab.manifest.get',
self.handle_vocab_manifest)
def update_skill_name_dict(self, message):
"""
Messagebus handler, updates dictionary of if to skill name
conversions.
"""
self.skill_names[message.data['id']] = message.data['name']
def get_skill_name(self, skill_id):
""" Get skill name from skill ID.
Args:
skill_id: a skill id as encoded in Intent handlers.
Returns:
(str) Skill name or the skill id if the skill wasn't found
"""
return self.skill_names.get(skill_id, skill_id)
def reset_converse(self, message):
"""Let skills know there was a problem with speech recognition"""
lang = message.data.get('lang', "en-us")
set_active_lang(lang)
for skill in self.active_skills:
self.do_converse(None, skill[0], lang)
def do_converse(self, utterances, skill_id, lang, message):
self.waiting_for_converse = True
self.converse_result = False
self.converse_skill_id = skill_id
self.bus.emit(
message.reply("skill.converse.request", {
"skill_id": skill_id,
"utterances": utterances,
"lang": lang
}))
start_time = time.time()
t = 0
while self.waiting_for_converse and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting_for_converse = False
self.converse_skill_id = ""
return self.converse_result
def handle_converse_error(self, message):
skill_id = message.data["skill_id"]
if message.data["error"] == "skill id does not exist":
self.remove_active_skill(skill_id)
if skill_id == self.converse_skill_id:
self.converse_result = False
self.waiting_for_converse = False
def handle_converse_response(self, message):
skill_id = message.data["skill_id"]
if skill_id == self.converse_skill_id:
self.converse_result = message.data.get("result", False)
self.waiting_for_converse = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
""" Updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def send_metrics(self, intent, context, stopwatch):
"""
Send timing metrics to the backend.
NOTE: This only applies to those with Opt In.
"""
ident = context['ident'] if 'ident' in context else None
if intent:
# Recreate skill name from skill id
parts = intent.get('intent_type', '').split(':')
intent_type = self.get_skill_name(parts[0])
if len(parts) > 1:
intent_type = ':'.join([intent_type] + parts[1:])
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': intent_type})
else:
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'intent_failure'})
def handle_utterance(self, message):
""" Main entrypoint for handling user utterances with Mycroft skills
Monitor the messagebus for 'recognizer_loop:utterance', typically
generated by a spoken interaction but potentially also from a CLI
or other method of injecting a 'user utterance' into the system.
Utterances then work through this sequence to be handled:
1) Active skills attempt to handle using converse()
2) Padatious high match intents (conf > 0.95)
3) Adapt intent handlers
5) Fallbacks:
- Padatious near match intents (conf > 0.8)
- General fallbacks
- Padatious loose match intents (conf > 0.5)
- Unknown intent handler
Args:
message (Message): The messagebus data
"""
try:
# Get language of the utterance
lang = message.data.get('lang', "en-us")
set_active_lang(lang)
utterances = message.data.get('utterances', [])
# normalize() changes "it's a boy" to "it is a boy", etc.
norm_utterances = [
normalize(u.lower(), remove_articles=False) for u in utterances
]
# Build list with raw utterance(s) first, then optionally a
# normalized version following.
combined = utterances + list(
set(norm_utterances) - set(utterances))
LOG.debug("Utterances: {}".format(combined))
stopwatch = Stopwatch()
intent = None
padatious_intent = None
with stopwatch:
# Give active skills an opportunity to handle the utterance
converse = self._converse(combined, lang, message)
if not converse:
# No conversation, use intent system to handle utterance
intent = self._adapt_intent_match(utterances,
norm_utterances, lang)
for utt in combined:
_intent = PadatiousService.instance.calc_intent(utt)
if _intent:
best = padatious_intent.conf if padatious_intent \
else 0.0
if best < _intent.conf:
padatious_intent = _intent
LOG.debug("Padatious intent: {}".format(padatious_intent))
LOG.debug(" Adapt intent: {}".format(intent))
if converse:
# Report that converse handled the intent and return
LOG.debug("Handled in converse()")
ident = None
if message.context and 'ident' in message.context:
ident = message.context['ident']
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'converse'})
return
elif (intent and intent.get('confidence', 0.0) > 0.0 and
not (padatious_intent and padatious_intent.conf >= 0.95)):
# Send the message to the Adapt intent's handler unless
# Padatious is REALLY sure it was directed at it instead.
self.update_context(intent)
# update active skills
skill_id = intent['intent_type'].split(":")[0]
self.add_active_skill(skill_id)
# Adapt doesn't handle context injection for one_of keywords
# correctly. Workaround this issue if possible.
try:
intent = workaround_one_of_context(intent)
except LookupError:
LOG.error('Error during workaround_one_of_context')
reply = message.reply(intent.get('intent_type'), intent)
else:
# Allow fallback system to handle utterance
# NOTE: A matched padatious_intent is handled this way, too
# TODO: Need to redefine intent_failure when STT can return
# multiple hypothesis -- i.e. len(utterances) > 1
reply = message.reply(
'intent_failure', {
'utterance': utterances[0],
'norm_utt': norm_utterances[0],
'lang': lang
})
self.bus.emit(reply)
self.send_metrics(intent, message.context, stopwatch)
except Exception as e:
LOG.exception(e)
def _converse(self, utterances, lang, message):
""" Give active skills a chance at the utterance
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
message (Message): message to use to generate reply
Returns:
bool: True if converse handled it, False if no skill processes it
"""
# check for conversation time-out
self.active_skills = [
skill for skill in self.active_skills
if time.time() - skill[1] <= self.converse_timeout * 60
]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang, message):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return True
return False
def _adapt_intent_match(self, raw_utt, norm_utt, lang):
""" Run the Adapt engine to search for an matching intent
Args:
raw_utt (list): list of utterances
norm_utt (list): same list of utterances, normalized
lang (string): language code, e.g "en-us"
Returns:
Intent structure, or None if no match was found.
"""
best_intent = None
def take_best(intent, utt):
nonlocal best_intent
best = best_intent.get('confidence', 0.0) if best_intent else 0.0
conf = intent.get('confidence', 0.0)
if conf > best:
best_intent = intent
# TODO - Shouldn't Adapt do this?
best_intent['utterance'] = utt
for idx, utt in enumerate(raw_utt):
try:
intents = [
i for i in self.engine.determine_intent(
utt,
100,
include_tags=True,
context_manager=self.context_manager)
]
if intents:
take_best(intents[0], utt)
# Also test the normalized version, but set the utterance to
# the raw version so skill has access to original STT
norm_intents = [
i for i in self.engine.determine_intent(
norm_utt[idx],
100,
include_tags=True,
context_manager=self.context_manager)
]
if norm_intents:
take_best(norm_intents[0], utt)
except Exception as e:
LOG.exception(e)
return best_intent
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(start_concept,
end_concept,
alias_of=alias_of)
self.registered_vocab.append(message.data)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name
]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers
if not p.name.startswith(skill_id)
]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
""" Add context
Args:
message: data contains the 'context' item to add
optionally can include 'word' to be injected as
an alias for the context item.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
origin = message.data.get('origin') or ''
# if not a string type try creating a string from it
if not isinstance(word, str):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
entity['origin'] = origin
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
""" Remove specific context
Args:
message: data contains the 'context' item to remove
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
""" Clears all keywords from context """
self.context_manager.clear_context()
def handle_get_adapt(self, message):
utterance = message.data["utterance"]
lang = message.data.get("lang", "en-us")
norm = normalize(utterance, lang, remove_articles=False)
intent = self._adapt_intent_match([utterance], [norm], lang)
self.bus.emit(
message.reply("intent.service.adapt.reply", {"intent": intent}))
def handle_get_intent(self, message):
utterance = message.data["utterance"]
lang = message.data.get("lang", "en-us")
norm = normalize(utterance, lang, remove_articles=False)
intent = self._adapt_intent_match([utterance], [norm], lang)
# Adapt intent's handler is used unless
# Padatious is REALLY sure it was directed at it instead.
padatious_intent = PadatiousService.instance.calc_intent(utterance)
if not padatious_intent and norm != utterance:
padatious_intent = PadatiousService.instance.calc_intent(norm)
if intent is None or (padatious_intent
and padatious_intent.conf >= 0.95):
intent = padatious_intent.__dict__
self.bus.emit(
message.reply("intent.service.intent.reply", {"intent": intent}))
def handle_get_skills(self, message):
self.bus.emit(
message.reply("intent.service.skills.reply",
{"skills": self.skill_names}))
def handle_get_active_skills(self, message):
self.bus.emit(
message.reply("intent.service.active_skills.reply",
{"skills": [s[0] for s in self.active_skills]}))
def handle_manifest(self, message):
self.bus.emit(
message.reply("intent.service.adapt.manifest",
{"intents": self.registered_intents}))
def handle_vocab_manifest(self, message):
self.bus.emit(
message.reply("intent.service.adapt.vocab.manifest",
{"vocab": self.registered_vocab}))
tokenizer = EnglishTokenizer()
trie = Trie()
tagger = EntityTagger(trie, tokenizer)
parser = Parser(tokenizer, tagger)
engine = IntentDeterminationEngine()
# create and register weather vocabulary
error_keyword = [
"error",
"e",
"E"
]
for er in error_keyword:
engine.register_entity(er, "ErrorKeyword")
error_types = [
"E16",
"E17",
"E19",
"E30"
]
for et in error_types:
engine.register_entity(et, "ErrorType")
# create regex to parse out locations
#engine.register_regex_entity("in (?P<Location>.*)")
tokenizer = EnglishTokenizer()
trie = Trie()
tagger = EntityTagger(trie, tokenizer)
parser = Parser(tokenizer, tagger)
engine = IntentDeterminationEngine()
# create and register weather vocabulary
cartype_keyword = [
"rent",
"transfer"
]
for ck in cartype_keyword:
engine.register_entity(ck, "CarTypeKeyword")
months = [
"January",
"February",
"March",
"April",
"May",
"June",
"July",
"August",
"September",
"October",
"November",
"December"
]
class IntentService(object):
def __init__(self, bus):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
# Dictionary for translating a skill id to a name
self.skill_names = {}
# Context related intializations
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.bus = bus
self.bus.on('register_vocab', self.handle_register_vocab)
self.bus.on('register_intent', self.handle_register_intent)
self.bus.on('recognizer_loop:utterance', self.handle_utterance)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.bus.on('add_context', self.handle_add_context)
self.bus.on('remove_context', self.handle_remove_context)
self.bus.on('clear_context', self.handle_clear_context)
# Converse method
self.bus.on('skill.converse.response', self.handle_converse_response)
self.bus.on('skill.converse.error', self.handle_converse_error)
self.bus.on('mycroft.speech.recognition.unknown', self.reset_converse)
self.bus.on('mycroft.skills.loaded', self.update_skill_name_dict)
def add_active_skill_handler(message):
self.add_active_skill(message.data['skill_id'])
self.bus.on('active_skill_request', add_active_skill_handler)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
self.waiting_for_converse = False
self.converse_result = False
self.converse_skill_id = ""
def update_skill_name_dict(self, message):
"""
Messagebus handler, updates dictionary of if to skill name
conversions.
"""
self.skill_names[message.data['id']] = message.data['name']
def get_skill_name(self, skill_id):
""" Get skill name from skill ID.
Args:
skill_id: a skill id as encoded in Intent handlers.
Returns:
(str) Skill name or the skill id if the skill wasn't found
"""
return self.skill_names.get(skill_id, skill_id)
def reset_converse(self, message):
"""Let skills know there was a problem with speech recognition"""
lang = message.data.get('lang', "en-us")
for skill in self.active_skills:
self.do_converse(None, skill[0], lang)
def do_converse(self, utterances, skill_id, lang):
self.waiting_for_converse = True
self.converse_result = False
self.converse_skill_id = skill_id
self.bus.emit(Message("skill.converse.request", {
"skill_id": skill_id, "utterances": utterances, "lang": lang}))
start_time = time.time()
t = 0
while self.waiting_for_converse and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting_for_converse = False
self.converse_skill_id = ""
return self.converse_result
def handle_converse_error(self, message):
skill_id = message.data["skill_id"]
if message.data["error"] == "skill id does not exist":
self.remove_active_skill(skill_id)
if skill_id == self.converse_skill_id:
self.converse_result = False
self.waiting_for_converse = False
def handle_converse_response(self, message):
skill_id = message.data["skill_id"]
if skill_id == self.converse_skill_id:
self.converse_result = message.data.get("result", False)
self.waiting_for_converse = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
""" Updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def send_metrics(self, intent, context, stopwatch):
"""
Send timing metrics to the backend.
NOTE: This only applies to those with Opt In.
"""
ident = context['ident'] if context else None
if intent:
# Recreate skill name from skill id
parts = intent.get('intent_type', '').split(':')
intent_type = self.get_skill_name(parts[0])
if len(parts) > 1:
intent_type = ':'.join([intent_type] + parts[1:])
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': intent_type})
else:
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'intent_failure'})
def handle_utterance(self, message):
""" Main entrypoint for handling user utterances with Mycroft skills
Monitor the messagebus for 'recognizer_loop:utterance', typically
generated by a spoken interaction but potentially also from a CLI
or other method of injecting a 'user utterance' into the system.
Utterances then work through this sequence to be handled:
1) Active skills attempt to handle using converse()
2) Adapt intent handlers
3) Padatious intent handlers
4) Other fallbacks
Args:
message (Message): The messagebus data
"""
try:
# Get language of the utterance
lang = message.data.get('lang', "en-us")
utterances = message.data.get('utterances', '')
stopwatch = Stopwatch()
with stopwatch:
# Give active skills an opportunity to handle the utterance
converse = self._converse(utterances, lang)
if not converse:
# No conversation, use intent system to handle utterance
intent = self._adapt_intent_match(utterances, lang)
padatious_intent = PadatiousService.instance.calc_intent(
utterances[0])
if converse:
# Report that converse handled the intent and return
ident = message.context['ident'] if message.context else None
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'converse'})
return
elif intent and not (padatious_intent and
padatious_intent.conf >= 0.95):
# Send the message to the Adapt intent's handler unless
# Padatious is REALLY sure it was directed at it instead.
reply = message.reply(intent.get('intent_type'), intent)
else:
# Allow fallback system to handle utterance
# NOTE: Padatious intents are handled this way, too
reply = message.reply('intent_failure',
{'utterance': utterances[0],
'lang': lang})
self.bus.emit(reply)
self.send_metrics(intent, message.context, stopwatch)
except Exception as e:
LOG.exception(e)
def _converse(self, utterances, lang):
""" Give active skills a chance at the utterance
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
bool: True if converse handled it, False if no skill processes it
"""
# check for conversation time-out
self.active_skills = [skill for skill in self.active_skills
if time.time() - skill[
1] <= self.converse_timeout * 60]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return True
return False
def _adapt_intent_match(self, utterances, lang):
""" Run the Adapt engine to search for an matching intent
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
Intent structure, or None if no match was found.
"""
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
best_intent = next(self.engine.determine_intent(
normalize(utterance, lang), 100,
include_tags=True,
context_manager=self.context_manager))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
except StopIteration:
# don't show error in log
continue
except Exception as e:
LOG.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
self.update_context(best_intent)
# update active skills
skill_id = best_intent['intent_type'].split(":")[0]
self.add_active_skill(skill_id)
# adapt doesn't handle context injection for one_of keywords
# correctly. Workaround this issue if possible.
try:
best_intent = workaround_one_of_context(best_intent)
except LookupError:
LOG.error('Error during workaround_one_of_context')
return best_intent
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers if
not p.name.startswith(skill_id)]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
""" Add context
Args:
message: data contains the 'context' item to add
optionally can include 'word' to be injected as
an alias for the context item.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
origin = message.data.get('origin') or ''
# if not a string type try creating a string from it
if not isinstance(word, str):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
entity['origin'] = origin
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
""" Remove specific context
Args:
message: data contains the 'context' item to remove
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
""" Clears all keywords from context """
self.context_manager.clear_context()
def skyAdapt():
engine = IntentDeterminationEngine()
#dota vocabulary
dota_keywords = [ 'dota', 'dotes', 'dote']
for dk in dota_keywords:
engine.register_entity(dk, "DotaKeyword")
happening_keywords = [
'happening',
'anyone up for',
'when is',
'what time',
'tonight',
'this evening?',
'anyone about for',
'around',
'want to',
'fancy some',
'playing some',
'anyone playing'
]
for hk in happening_keywords:
engine.register_entity(hk, "HappeningKeyword")
dota_query_intent = IntentBuilder("DotaIntent")\
.require("DotaKeyword")\
.require("HappeningKeyword")\
.build()
stack_intent_words = [
'stack',
'stacked'
]
for sik in stack_intent_words:
engine.register_entity(hk, "StackKeyword")
stack_optionals = [
'are we',
'do we have a',
'how many',
'who\'s playing'
]
for osk in stack_optionals:
engine.register_entity(hk, "StackOptionalKeyword")
stack_intent = IntentBuilder("StackIntent")\
.require("StackKeyword")\
.optionally("StackOptionalKeyword")\
.build()
engine.register_regex_entity("at (?P<Time>.*)")
new_dota_intent = IntentBuilder("NewDotaIntent")\
.require("DotaKeyword")\
.require("Time")\
.build()
engine.register_intent_parser(dota_query_intent)
engine.register_intent_parser(stack_intent)
engine.register_intent_parser(new_dota_intent)
return engine
# If there's a second argument given, use that to insert an import path
# This enables users to use their own Adapt installation directories.
if len(sys.argv) > 2:
sys.path.insert(0, sys.argv[2])
from adapt.intent import IntentBuilder
from adapt.engine import IntentDeterminationEngine
engine = IntentDeterminationEngine()
schema = json.loads(sys.argv[1])
for entity in schema["entities"]:
if entity["type"] == "string":
for value in entity["values"]:
engine.register_entity(value, entity["name"])
elif entity["type"] == "regex":
engine.register_regex_entity(entity["pattern"])
for intent in schema["intents"]:
ib = IntentBuilder(intent["name"].encode("utf-8"))
for requirement in intent["requirements"]:
ib.require(requirement["entity"], requirement["attribute"])
for optional in intent["optionals"]:
ib.optionally(optional["entity"], optional["attribute"])
engine.register_intent_parser(ib.build())
if __name__ == "__main__":
while True:
line = sys.stdin.readline()
query = json.loads(line)
class IntentService(object):
def __init__(self, emitter):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
# Dictionary for translating a skill id to a name
self.skill_names = {}
# Context related intializations
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.emitter = emitter
self.emitter.on('register_vocab', self.handle_register_vocab)
self.emitter.on('register_intent', self.handle_register_intent)
self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
self.emitter.on('detach_intent', self.handle_detach_intent)
self.emitter.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.emitter.on('add_context', self.handle_add_context)
self.emitter.on('remove_context', self.handle_remove_context)
self.emitter.on('clear_context', self.handle_clear_context)
# Converse method
self.emitter.on('skill.converse.response',
self.handle_converse_response)
self.emitter.on('mycroft.speech.recognition.unknown',
self.reset_converse)
self.emitter.on('mycroft.skills.loaded', self.update_skill_name_dict)
def add_active_skill_handler(message):
self.add_active_skill(message.data['skill_id'])
self.emitter.on('active_skill_request', add_active_skill_handler)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
def update_skill_name_dict(self, message):
"""
Messagebus handler, updates dictionary of if to skill name
conversions.
"""
self.skill_names[message.data['id']] = message.data['name']
def get_skill_name(self, skill_id):
"""
Get skill name from skill ID.
Args
skill_id: a skill id as encoded in Intent handlers.
Returns: (str) Skill name or the skill id if the skill
wasn't found in the dict.
"""
return self.skill_names.get(int(skill_id), skill_id)
def reset_converse(self, message):
"""Let skills know there was a problem with speech recognition"""
lang = message.data.get('lang', "en-us")
for skill in self.active_skills:
self.do_converse(None, skill[0], lang)
def do_converse(self, utterances, skill_id, lang):
self.emitter.emit(Message("skill.converse.request", {
"skill_id": skill_id, "utterances": utterances, "lang": lang}))
self.waiting = True
self.result = False
start_time = time.time()
t = 0
while self.waiting and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting = False
return self.result
def handle_converse_response(self, message):
# id = message.data["skill_id"]
# no need to crosscheck id because waiting before new request is made
# no other skill will make this request is safe assumption
result = message.data["result"]
self.result = result
self.waiting = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
"""
updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def send_metrics(self, intent, context, stopwatch):
"""
Send timing metrics to the backend.
"""
LOG.debug('Sending metric')
ident = context['ident'] if context else None
if intent:
# Recreate skill name from skill id
parts = intent.get('intent_type', '').split(':')
intent_type = self.get_skill_name(parts[0])
if len(parts) > 1:
intent_type = ':'.join([intent_type] + parts[1:])
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': intent_type})
else:
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'intent_failure'})
def handle_utterance(self, message):
"""
Messagebus handler for the recognizer_loop:utterance message
"""
try:
# Get language of the utterance
lang = message.data.get('lang', "en-us")
utterances = message.data.get('utterances', '')
stopwatch = Stopwatch()
with stopwatch:
# Parse the sentence
converse = self.parse_converse(utterances, lang)
if not converse:
# no skill wants to handle utterance
intent = self.parse_utterances(utterances, lang)
if converse:
# Report that converse handled the intent and return
ident = message.context['ident'] if message.context else None
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'converse'})
return
elif intent:
# Send the message on to the intent handler
reply = message.reply(intent.get('intent_type'), intent)
else:
# or if no match send sentence to fallback system
reply = message.reply('intent_failure',
{'utterance': utterances[0],
'lang': lang})
self.emitter.emit(reply)
self.send_metrics(intent, message.context, stopwatch)
except Exception as e:
LOG.exception(e)
def parse_converse(self, utterances, lang):
"""
Converse, check if a recently invoked skill wants to
handle the utterance and override normal adapt handling.
Returns: True if converse handled the utterance, else False.
"""
# check for conversation time-out
self.active_skills = [skill for skill in self.active_skills
if time.time() - skill[
1] <= self.converse_timeout * 60]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return True
return False
def parse_utterances(self, utterances, lang):
"""
Parse the utteracne using adapt to find a matching intent.
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns: Intent structure, or None if no match was found.
"""
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
best_intent = next(self.engine.determine_intent(
normalize(utterance, lang), 100,
include_tags=True,
context_manager=self.context_manager))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
except StopIteration:
# don't show error in log
continue
except Exception as e:
LOG.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
self.update_context(best_intent)
# update active skills
skill_id = int(best_intent['intent_type'].split(":")[0])
self.add_active_skill(skill_id)
return best_intent
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers if
not p.name.startswith(skill_id)]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
"""
Handles adding context from the message bus.
The data field must contain a context keyword and
may contain a word if a specific word should be injected
as a match for the provided context keyword.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
# if not a string type try creating a string from it
if not isinstance(word, basestring):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
"""
Handles removing context from the message bus. The
data field must contain the 'context' to remove.
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
"""
Clears all keywords from context.
"""
self.context_manager.clear_context()
"everything lese is okay right", "anything else"
]
greeting_word = ['hi', 'hello', 'hii', 'Hello', 'Howdy']
greeting_response = [
'hi, how are you', 'hello, any physical illness',
'hii, nice to hear from you', 'Hello', 'Howdy', "Hi", "hello"
]
greeting_word2 = ['thanks', 'thanks you', 'okay', 'nice']
greeting_response2 = [
'You are welcome', 'Happy to help', 'Have a nice day', 'get well soon'
]
for word in greeting_word:
engine.register_entity(word, "greeting")
intent1 = IntentBuilder("greetingIntent")\
.require("greeting")\
.build()
engine.register_intent_parser(intent1)
for word in greeting_word2:
engine.register_entity(word, "greeting2")
intent2 = IntentBuilder("greeting2Intent")\
.require("greeting2")\
.build()
engine.register_intent_parser(intent2)
fever_word = ['temperature', 'fever', 'hot body', 'body hot', 'heat']
for word in fever_word:
engine.register_entity(word, "fever")
intentfeaver = IntentBuilder("feverIntent")\
"""
import json
import sys
from adapt.intent import IntentBuilder
from adapt.engine import IntentDeterminationEngine
engine = IntentDeterminationEngine()
# create and register weather vocabulary
weather_keyword = [
"weather"
]
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
weather_types = [
"snow",
"rain",
"wind",
"sleet",
"sun"
]
for wt in weather_types:
engine.register_entity(wt, "WeatherType")
# create regex to parse out locations
engine.register_regex_entity("in (?P<Location>.*)")
from adapt.engine import IntentDeterminationEngine
tokenizer = EnglishTokenizer()
trie = Trie()
tagger = EntityTagger(trie, tokenizer)
parser = Parser(tokenizer, tagger)
engine = IntentDeterminationEngine()
# define vocabulary
weather_keyword = [
"weather"
]
for wk in weather_keyword:
engine.register_entity(wk, "WeatherKeyword")
weather_types = [
"snow",
"rain",
"wind",
"sleet",
"sun"
]
for wt in weather_types:
engine.register_entity(wt, "WeatherType")
locations = [
"Seattle",
"San Francisco",
class IntentService(object):
def __init__(self, bus):
self.config = Configuration.get().get('context', {})
self.engine = IntentDeterminationEngine()
# Dictionary for translating a skill id to a name
self.skill_names = {}
# Context related intializations
self.context_keywords = self.config.get('keywords', [])
self.context_max_frames = self.config.get('max_frames', 3)
self.context_timeout = self.config.get('timeout', 2)
self.context_greedy = self.config.get('greedy', False)
self.context_manager = ContextManager(self.context_timeout)
self.bus = bus
self.bus.on('register_vocab', self.handle_register_vocab)
self.bus.on('register_intent', self.handle_register_intent)
self.bus.on('recognizer_loop:utterance', self.handle_utterance)
self.bus.on('detach_intent', self.handle_detach_intent)
self.bus.on('detach_skill', self.handle_detach_skill)
# Context related handlers
self.bus.on('add_context', self.handle_add_context)
self.bus.on('remove_context', self.handle_remove_context)
self.bus.on('clear_context', self.handle_clear_context)
# Converse method
self.bus.on('skill.converse.response', self.handle_converse_response)
self.bus.on('skill.converse.error', self.handle_converse_error)
self.bus.on('mycroft.speech.recognition.unknown', self.reset_converse)
self.bus.on('mycroft.skills.loaded', self.update_skill_name_dict)
def add_active_skill_handler(message):
self.add_active_skill(message.data['skill_id'])
self.bus.on('active_skill_request', add_active_skill_handler)
self.active_skills = [] # [skill_id , timestamp]
self.converse_timeout = 5 # minutes to prune active_skills
self.waiting_for_converse = False
self.converse_result = False
self.converse_skill_id = ""
def update_skill_name_dict(self, message):
"""
Messagebus handler, updates dictionary of if to skill name
conversions.
"""
self.skill_names[message.data['id']] = message.data['name']
def get_skill_name(self, skill_id):
""" Get skill name from skill ID.
Args:
skill_id: a skill id as encoded in Intent handlers.
Returns:
(str) Skill name or the skill id if the skill wasn't found
"""
return self.skill_names.get(skill_id, skill_id)
def reset_converse(self, message):
"""Let skills know there was a problem with speech recognition"""
lang = message.data.get('lang', "en-us")
for skill in self.active_skills:
self.do_converse(None, skill[0], lang)
def do_converse(self, utterances, skill_id, lang):
self.waiting_for_converse = True
self.converse_result = False
self.converse_skill_id = skill_id
self.bus.emit(
Message("skill.converse.request", {
"skill_id": skill_id,
"utterances": utterances,
"lang": lang
}))
start_time = time.time()
t = 0
while self.waiting_for_converse and t < 5:
t = time.time() - start_time
time.sleep(0.1)
self.waiting_for_converse = False
self.converse_skill_id = ""
return self.converse_result
def handle_converse_error(self, message):
skill_id = message.data["skill_id"]
if message.data["error"] == "skill id does not exist":
self.remove_active_skill(skill_id)
if skill_id == self.converse_skill_id:
self.converse_result = False
self.waiting_for_converse = False
def handle_converse_response(self, message):
skill_id = message.data["skill_id"]
if skill_id == self.converse_skill_id:
self.converse_result = message.data.get("result", False)
self.waiting_for_converse = False
def remove_active_skill(self, skill_id):
for skill in self.active_skills:
if skill[0] == skill_id:
self.active_skills.remove(skill)
def add_active_skill(self, skill_id):
# search the list for an existing entry that already contains it
# and remove that reference
self.remove_active_skill(skill_id)
# add skill with timestamp to start of skill_list
self.active_skills.insert(0, [skill_id, time.time()])
def update_context(self, intent):
""" Updates context with keyword from the intent.
NOTE: This method currently won't handle one_of intent keywords
since it's not using quite the same format as other intent
keywords. This is under investigation in adapt, PR pending.
Args:
intent: Intent to scan for keywords
"""
for tag in intent['__tags__']:
if 'entities' not in tag:
continue
context_entity = tag['entities'][0]
if self.context_greedy:
self.context_manager.inject_context(context_entity)
elif context_entity['data'][0][1] in self.context_keywords:
self.context_manager.inject_context(context_entity)
def send_metrics(self, intent, context, stopwatch):
"""
Send timing metrics to the backend.
NOTE: This only applies to those with Opt In.
"""
ident = context['ident'] if context else None
if intent:
# Recreate skill name from skill id
parts = intent.get('intent_type', '').split(':')
intent_type = self.get_skill_name(parts[0])
if len(parts) > 1:
intent_type = ':'.join([intent_type] + parts[1:])
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': intent_type})
else:
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'intent_failure'})
def handle_utterance(self, message):
""" Main entrypoint for handling user utterances with Mycroft skills
Monitor the messagebus for 'recognizer_loop:utterance', typically
generated by a spoken interaction but potentially also from a CLI
or other method of injecting a 'user utterance' into the system.
Utterances then work through this sequence to be handled:
1) Active skills attempt to handle using converse()
2) Adapt intent handlers
3) Padatious intent handlers
4) Other fallbacks
Args:
message (Message): The messagebus data
"""
# JN: Code borrowed from get_scheduled_event_status() in core.py
completed_callback = False
completed_status = 'failed' # assume fail
def completion_handler(message): #JN
nonlocal completed_callback
nonlocal completed_status
LOG.debug("Calback called: " + message.serialize())
LOG.debug(' type ' + str(type(message)))
if message.data is not None:
completed_status = message.data['status']
LOG.debug('Completed status is ' + completed_status)
completed_callback = True
def wait_for_reply(): #JN
nonlocal completed_callback
num_tries = 0 # wait upto 30 secs. weather takes e.g. 8 seconds
LOG.debug('Waiting for reply, completed callback is ' +
str(completed_callback))
while completed_callback is False and num_tries < 300:
#LOG.info('Sleepiong')
time.sleep(0.1)
num_tries += 1
LOG.debug('Waited for reply, num_tries is ' + str(num_tries))
LOG.debug(' completed callback is ' + str(completed_callback))
completed_callback = False # for next time
try:
# Get language of the utterance
lang = message.data.get('lang', "en-us")
utterances = message.data.get('utterances', '')
self.bus.on('skill.handler.complete', completion_handler)
#JN: stopwatch doesn't seem to be used, so removed the with stopwatch...
stopwatch = Stopwatch()
#JN: Give active skills an opportunity to handle the utterance
converse = self._converse(utterances, lang)
#JN: code moved to here, finishes the converse stuff
if converse:
# Report that converse handled the intent and return
LOG.debug('Converse handling intent')
ident = message.context['ident'] if message.context else None
report_timing(ident, 'intent_service', stopwatch,
{'intent_type': 'converse'})
return
# if not converse: - redundant
# No conversation, use intent system to handle utterance
for intent in self._adapt_intent_match(utterances,
lang): # JN uses generator
padatious_intent = PadatiousService.instance.calc_intent(
utterances[0])
if intent and not (padatious_intent
and padatious_intent.conf >= 0.95):
# Send the message to the Adapt intent's handler unless
# Padatious is REALLY sure it was directed at it instead.
reply = message.reply(intent.get('intent_type'), intent)
else:
# Allow fallback system to handle utterance
# NOTE: Padatious intents are handled this way, too
LOG.info('Pedatious handing or failure?')
reply = message.reply('intent_failure', {
'utterance': utterances[0],
'lang': lang
})
LOG.debug('Intent bus call msg ' + reply.serialize())
self.bus.emit(reply)
self.send_metrics(intent, message.context, stopwatch)
wait_for_reply()
if completed_status == 'succeeded':
# we are finished now with this utterance
LOG.debug('intent succeeded, utterance handled by ' +
str(intent))
self.bus.remove('skill.handler.complete',
completion_handler)
return
else:
LOG.debug('intent failed, trying next one ' + str(intent))
LOG.info('Intent loop finished')
# we couldn't find a successful handler
# TODO: a handler that says why the semantics of every intent failed
# rather than generic messages
reply = message.reply('intent_failure', {
'utterance': utterances[0],
'lang': lang
})
self.bus.emit(reply)
self.bus.remove('skill.handler.complete', completion_handler)
except Exception as e:
LOG.exception(e)
def _converse(self, utterances, lang):
""" Give active skills a chance at the utterance
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
bool: True if converse handled it, False if no skill processes it
"""
# check for conversation time-out
self.active_skills = [
skill for skill in self.active_skills
if time.time() - skill[1] <= self.converse_timeout * 60
]
# check if any skill wants to handle utterance
for skill in self.active_skills:
if self.do_converse(utterances, skill[0], lang):
# update timestamp, or there will be a timeout where
# intent stops conversing whether its being used or not
self.add_active_skill(skill[0])
return True
return False
def _adapt_intent_match(self, utterances, lang):
""" Run the Adapt engine to search for an matching intent
Args:
utterances (list): list of utterances
lang (string): 4 letter ISO language code
Returns:
Intent structure, or None if no match was found.
"""
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
normal_utterance = normalize(utterance, lang)
#best_intent = next(self.engine.determine_intent(
# normal_utterance, 100,
# include_tags=True,
# context_manager=self.context_manager))
# TODO - Should Adapt handle this?
# JN changed from next(determine_intent), single value only
for best_intent in self.engine.determine_good_intents(
normal_utterance,
100,
include_tags=True,
context_manager=self.context_manager):
best_intent['utterance'] = utterance
best_intent['retry_on_fail'] = True #JN
if best_intent and best_intent.get('confidence',
0.0) > 0.0:
best_intent['utterance'] = utterance
self.update_context(best_intent)
# update active skills
skill_id = best_intent['intent_type'].split(":")[0]
self.add_active_skill(skill_id)
# adapt doesn't handle context injection for one_of keywords
# correctly. Workaround this issue if possible.
try:
best_intent = workaround_one_of_context(
best_intent)
except LookupError:
LOG.error('Error during workaround_one_of_context')
yield best_intent # JN
except StopIteration:
# don't show error in log
continue
except Exception as e:
LOG.exception(e)
continue
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(start_concept,
end_concept,
alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name
]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_id = message.data.get('skill_id')
new_parsers = [
p for p in self.engine.intent_parsers
if not p.name.startswith(skill_id)
]
self.engine.intent_parsers = new_parsers
def handle_add_context(self, message):
""" Add context
Args:
message: data contains the 'context' item to add
optionally can include 'word' to be injected as
an alias for the context item.
"""
entity = {'confidence': 1.0}
context = message.data.get('context')
word = message.data.get('word') or ''
origin = message.data.get('origin') or ''
# if not a string type try creating a string from it
if not isinstance(word, str):
word = str(word)
entity['data'] = [(word, context)]
entity['match'] = word
entity['key'] = word
entity['origin'] = origin
self.context_manager.inject_context(entity)
def handle_remove_context(self, message):
""" Remove specific context
Args:
message: data contains the 'context' item to remove
"""
context = message.data.get('context')
if context:
self.context_manager.remove_context(context)
def handle_clear_context(self, message):
""" Clears all keywords from context """
self.context_manager.clear_context()
class IntentService(object):
def __init__(self, emitter):
self.engine = IntentDeterminationEngine()
self.emitter = emitter
self.emitter.on('register_vocab', self.handle_register_vocab)
self.emitter.on('register_intent', self.handle_register_intent)
self.emitter.on('recognizer_loop:utterance', self.handle_utterance)
self.emitter.on('detach_intent', self.handle_detach_intent)
self.emitter.on('detach_skill', self.handle_detach_skill)
def handle_utterance(self, message):
# Get language of the utterance
lang = message.data.get('lang', None)
if not lang:
lang = "en-us"
utterances = message.data.get('utterances', '')
best_intent = None
for utterance in utterances:
try:
# normalize() changes "it's a boy" to "it is boy", etc.
best_intent = next(self.engine.determine_intent(
normalize(utterance, lang), 100))
# TODO - Should Adapt handle this?
best_intent['utterance'] = utterance
except StopIteration as e:
logger.exception(e)
continue
if best_intent and best_intent.get('confidence', 0.0) > 0.0:
reply = message.reply(
best_intent.get('intent_type'), best_intent)
self.emitter.emit(reply)
elif len(utterances) == 1:
self.emitter.emit(Message("intent_failure", {
"utterance": utterances[0],
"lang": lang
}))
else:
self.emitter.emit(Message("multi_utterance_intent_failure", {
"utterances": utterances,
"lang": lang
}))
def handle_register_vocab(self, message):
start_concept = message.data.get('start')
end_concept = message.data.get('end')
regex_str = message.data.get('regex')
alias_of = message.data.get('alias_of')
if regex_str:
self.engine.register_regex_entity(regex_str)
else:
self.engine.register_entity(
start_concept, end_concept, alias_of=alias_of)
def handle_register_intent(self, message):
intent = open_intent_envelope(message)
self.engine.register_intent_parser(intent)
def handle_detach_intent(self, message):
intent_name = message.data.get('intent_name')
new_parsers = [
p for p in self.engine.intent_parsers if p.name != intent_name]
self.engine.intent_parsers = new_parsers
def handle_detach_skill(self, message):
skill_name = message.data.get('skill_name')
new_parsers = [
p for p in self.engine.intent_parsers if
not p.name.startswith(skill_name)]
self.engine.intent_parsers = new_parsers
