def handle_get_stt(self, message: Message): """ Handles a request for stt. Emits a response to the sender with stt data or error data :param message: Message associated with request """ if message.data.get("audio_data"): wav_file_path = self._write_encoded_file( message.data.pop("audio_data")) else: wav_file_path = message.data.get("audio_file") lang = message.data.get("lang") ident = message.context.get("ident") or "neon.get_stt.response" LOG.info(f"Handling STT request: {ident}") if not wav_file_path: self.bus.emit(message.reply( ident, data={"error": f"audio_file not specified!"})) return if not os.path.isfile(wav_file_path): self.bus.emit(message.reply( ident, data={"error": f"{wav_file_path} Not found!"})) try: _, parser_data, transcriptions = \ self._get_stt_from_file(wav_file_path, lang) self.bus.emit(message.reply(ident, data={"parser_data": parser_data, "transcripts": transcriptions})) except Exception as e: LOG.error(e) self.bus.emit(message.reply(ident, data={"error": repr(e)}))
def walkOnNetwork(selfGraph, startWord, lengthWalk): """ Do a walk on the selfGraph, starting from a concept, during lengthWalk steps, or unless end up at a deadend of the graph """ nStep = 0 deadEnd = False word = startWord while nStep < lengthWalk and not deadEnd: if len( list(selfGraph[word][1].keys()) ) > 0: #reminder: selfGraph[word][1] is a dictionnary of the neighbors. nextWord = random.choice(selfGraph[word][1].keys()) statement = "Path " + str( round(selfGraph[word][1][nextWord], 2)) + " towards " + nextWord + ", " + str( round(selfGraph[nextWord][0], 3)) + ". " print(statement) client.emit(Message('speak', data={'utterance': statement})) #or all answer ? word = nextWord nStep += 1 else: deadEnd = True client.emit( Message('speak', data={'utterance': "It is a dead end of myself." })) #or all answer ? if not deadEnd: client.emit(Message('speak', data={'utterance': "Walk ended here."})) #or all answer ?
def on_no_internet(self, event=None): if connected(): # One last check to see if connection was established return if time.time() - Enclosure._last_internet_notification < 30: # don't bother the user with multiple notifications with 30 secs return Enclosure._last_internet_notification = time.time() # TODO: This should go into EnclosureMark1 subclass of Enclosure. if has_been_paired(): # Handle the translation within that code. self.bus.emit( Message( "speak", { 'utterance': "This device is not connected to the Internet. " "Either plug in a network cable or set up your " "wifi connection." })) else: # enter wifi-setup mode automatically self.bus.emit(Message('system.wifi.setup', {'lang': self.lang}))
def on_manage_skills_pressed(self): """ Adds the checked skills to self.active_skills and the unchecked to self.unactive_skills and activates or deactivates those skills. """ deactivated = [] activated = [] for cb in self.active_skills_checkBoxes: if not cb.isChecked(): self.bus.emit( Message('skillmanager.deactivate', {'skill': cb.text()})) deactivated.append(cb.text()) for cb in self.unactive_skills_checkBoxes: if cb.isChecked(): self.bus.emit( Message('skillmanager.activate', {'skill': cb.text()})) activated.append(cb.text()) self.active_skills = [ skill for skill in self.active_skills if skill not in deactivated ] self.active_skills.extend(activated) self.unactive_skills = [ skill for skill in self.unactive_skills if skill not in activated ] self.unactive_skills.extend(deactivated) self.skills_dialog.hide() self.on_skills_pressed()
def _do_net_check(self): # TODO: This should live in the derived Enclosure, e.g. EnclosureMark1 LOG.info("Checking internet connection") if not connected(): # and self.conn_monitor is None: if has_been_paired(): # TODO: Enclosure/localization self.speak("This unit is not connected to the Internet. " "Either plug in a network cable or setup your " "wifi connection.") else: # Begin the unit startup process, this is the first time it # is being run with factory defaults. # TODO: This logic should be in EnclosureMark1 # TODO: Enclosure/localization # Don't listen to mic during this out-of-box experience self.bus.emit(Message("mycroft.mic.mute")) # Setup handler to unmute mic at the end of on boarding # i.e. after pairing is complete self.bus.once('mycroft.paired', self._handle_pairing_complete) self.speak(mycroft.dialog.get('mycroft.intro')) wait_while_speaking() time.sleep(2) # a pause sounds better than just jumping in # Kick off wifi-setup automatically data = {'allow_timeout': False, 'lang': self.lang} self.bus.emit(Message('system.wifi.setup', data))
def loadSelf(firstTime, audibleSelfQuest, nSearch): """ The VA loads his selfGraph and memory as last saved. Or build it if first time. """ if firstTime: whatVAHeard = "" phrase = "Hatching self in process..." print(phrase) if audibleSelfQuest: client.emit(Message('speak', data={'utterance': phrase})) selfGraph, wordsMemory, description = coreQuest.hatchSelf(nSearch) print(description) if audibleSelfQuest: client.emit(Message('speak', data={'utterance': description})) else: with open('./workshop/data/selfgraph.txt', 'r') as json_file: #selfGraph = json.load(json_file) selfGraph = eval(json_file.read()) # which one RIGHT ? with open('./workshop/data/wordsMemory.txt', "r") as f: wordsMemory = f.readlines() #List of words concepts he looked up with open('./workshop/data/whatVAHeard.txt', "r") as f: #Last historics before Chris learned whatVAHeard = f.read().replace('\n', '') phrase = "I am here. Self Quest can begin." print(phrase) if audibleSelfQuest: client.emit(Message('speak', data={'utterance': phrase})) return selfGraph, wordsMemory, whatVAHeard
def selfMapping(word, selfGraph, wordsMemory, ifMLDrift, lengthML, nSimMax, lengthWalk, walkNetwork, audibleSelfQuest): """ Self Mapping where look if a specific word is related to his selfm and possibly grow his selfGraph. This can be audible, and integrate or not a ML drift and a walk on the network """ answer = "" drift = "" ###(1) Ask Chris about a work on wikipedia if audibleSelfQuest: question = "Tell me about " + word + "." #Dont need wake up word ? phrase = wonder(word) #Phrase to be heard. Can generate Others>>> client.emit(Message('speak', data={'utterance': phrase})) print(phrase) answer = askVA(question) #VA answer and read wikipedia page here ###(2) Possible ML Drift from the last sentence, case where audibleSelf Quest only. if ifMLDrift: drift = oneMLDrift(answer, lengthML) ##(3) Self Awareness Quest: look if this word is related to Self. May have modified the self graph here! print("Looking if {} is related to Self...".format(word)) selfGraph, ifadded, simWord, simScore = coreQuest.isSelf( selfGraph, word, nSimMax) nSelf = len(list(selfGraph.keys())) #(4) State the progress of his selfAwareness quest, aloud if audible Self quest, else only print if ifadded: #Case where the word was found related to self selfAwareness = "Oh, " + word + " is similar to " + simWord + "and hence to me at " + str( round(simScore, 2)) + "Now I know more about myself. " print(selfAwareness) else: #Case where the word was not found related to self selfAwareness = "Whatever, " + word + ", may not be very related to myself. " print(selfAwareness) if audibleSelfQuest: client.emit(Message('speak', data={'utterance': selfAwareness})) #(5) Walk on the network if stated and added the word if walkNetwork and ifadded: walkOnNetwork(selfGraph, word, lengthWalk) #(6) Add word looked for Memory. Just beware if memory full. if len(wordsMemory ) == maxWordsMemory + 1: #Memory Full, erase the older one. if wordsMemory[0] == str(len(wordsMemory) - 1): wordsMemory[0] = str(1) else: wordsMemory[0] = str(int(wordsMemory[0]) + 1) wordsMemory[wordsMemory[0]] = word else: wordsMemory.append( word.lower() ) #Add word to memory when has wiki checked it. Only lowered words here wordsMemory[0] = str(int(wordsMemory[0]) + 1) #in 0 element keep track return selfGraph, wordsMemory, answer, drift, ifadded
def selfQuest(firstTime=False, walkNetwork=False, audibleSelfQuest=True, visualizeGraph=True, ifMLDrift=False, lengthML=100, nSimMax=50, nSearch=100, lengthWalk=10, finetuned_ML_model=False, path_finetuned_ML_model='./workshop/models/gpt-2'): #ifVisualize """ Self Quest with possible ML drits, walks on the network, audible Quest, visualisation of the Graph, etc. The VA is aiming at growing his self Concept, from what he has heard (in the file whatVAHeard). """ # Initialize machine learning tokenizer = GPT2Tokenizer.from_pretrained("gpt2") if finetuned_ML_model: model = GPT2LMHeadModel.from_pretrained(path_finetuned_ML_model) else: model = GPT2LMHeadModel.from_pretrained("gpt2") #(0) Load SELF, memory, and what the VA has heard since last time. ## SelfGraph is a dictionnary, whose keys are concepts, and values are couple (weight, neighbors). selfGraph, wordsMemory, whatVAHeard = loadSelf(firstTime, audibleSelfQuest, nSearch) #(1) Self Quest, with possible walk on the network, possible ML drifts, and parameters, from the text whatVAHear #If not first time (if first time, assume whatVAHear is empty) if not firstTime: selfGraph, wordsMemory, addedWords, blablaQuest = selfMapLoops( whatVAHeard, selfGraph, ifMLDrift, lengthML, nSimMax, wordsMemory, nSearch, walkNetwork, lengthWalk, audibleSelfQuest) #(2) State new Self nN = len(list(selfGraph.keys())) phrase = "Self has now " + str( nN) + " nodes." + "Self is made of:" + ', '.join(list( selfGraph.keys())) print(phrase) if audibleSelfQuest: client.emit(Message('speak', data={'utterance': phrase})) #(3) Update the selfGraph and the memory with open('./workshop/data/selfgraph.txt', 'w') as outfile: json.dump(selfGraph, outfile) with open('./workshop/data/wordsMemory.txt', "w") as f: f.write("\n".join(wordsMemory)) #(4) Visualize if want if visualizeGraph: graph, descriptionSelf = coreQuest.createGraph(selfGraph) coreQuest.drawGraph(graph) if audibleSelfQuest: #Description of himself. Could comment out if too annoying. client.emit(Message('speak', data={'utterance': descriptionSelf}))
def test_serialize_deserialize(self): """Assert that a serized message is recreated when deserialized.""" source = Message('test_type', data={ 'robot': 'marvin', 'android': 'data' }, context={'origin': 'earth'}) msg_string = source.serialize() reassembled = Message.deserialize(msg_string) self.assertEqual(source.msg_type, reassembled.msg_type) self.assertEqual(source.data, reassembled.data) self.assertEqual(source.context, reassembled.context)
def test_response(self): """Assert that the .response is added to the message type for response. """ source = Message('test_type', data={ 'robot': 'marvin', 'android': 'data' }, context={'origin': 'earth'}) response_msg = source.response() self.assertEqual(response_msg.msg_type, "test_type.response") self.assertEqual(response_msg.data, {}) self.assertEqual(response_msg.context, source.context)
def reset(gpio, level, tick): if(pins.read(24) == 1): client.emit(Message('speak', {"utterance": "Shutting Down", "lang": "en-us"})) light.pulse(.2, 100, 0, 10) time.sleep(3) #self.bus.emit(Message('system.shutdown')) #docs say this "force[s] a linux shutdown", but I'll believe it when I see it (it wicked doesn't do that) subprocess.Popen(['sudo','shutdown','-h','now']) else: client.emit(Message('speak', {"utterance": "Restarting", "lang": "en-us"})) light.pulse(.2, 100, 0, 6) time.sleep(3) #self.bus.emit(Message('system.restart')) #same deal here subprocess.Popen(['sudo','shutdown','-r','now'])
def handle_radio_talanthus(self, message): print('Setting up client to connect to a local mycroft instance') client = MessageBusClient() client.run_in_thread() print('Sending speak message...') client.emit(Message('speak', data={'utterance': 'Currently at the top of the bounty board is the calamo five at ten million credits a head'}))
def test_get_stt_no_file(self): context = {"client": "tester", "ident": "123", "user": "******"} stt_resp = self.bus.wait_for_response( Message("neon.get_stt", {}, context), context["ident"]) self.assertEqual(stt_resp.context, context) self.assertIsInstance(stt_resp.data.get("error"), str) self.assertEqual(stt_resp.data["error"], "audio_file not specified!")
def test_get_stt_invalid_file_path(self): context = {"client": "tester", "ident": "1234", "user": "******"} stt_resp = self.bus.wait_for_response( Message("neon.get_stt", {"audio_file": "~/invalid_file.wav"}, context), context["ident"]) self.assertEqual(stt_resp.context, context) self.assertIsInstance(stt_resp.data.get("error"), str)
def play(): print('play') # client.emit(Message('mycroft.audio.service.resume')) client.emit(Message('play:start', {'skill_id': 'mycroft-spotify.forslund', 'phrase': '', 'callback_data': {'type': 'continue'}}))
def mouth_display_png(self, image_absolute_path, invert=False, x=0, y=0, refresh=True): """ Send an image to the enclosure. Args: image_absolute_path (string): The absolute path of the image invert (bool): inverts the image being drawn. x (int): x offset for image y (int): y offset for image refresh (bool): specify whether to clear the faceplate before displaying the new image or not. Useful if you'd like to display muliple images on the faceplate at once. """ self.display_manager.set_active(self.name) self.bus.emit( Message("enclosure.mouth.display_image", { 'img_path': image_absolute_path, 'xOffset': x, 'yOffset': y, 'invert': invert, 'clearPrev': refresh }, context={"destination": ["enclosure"]}))
def test_audio_input_valid(self): handle_utterance = mock.Mock() self.bus.once("recognizer_loop:utterance", handle_utterance) context = { "client": "tester", "ident": "11111", "user": "******", "extra_data": "something" } audio_data = encode_file_to_base64_string( os.path.join(AUDIO_FILE_PATH, "stop.wav")) stt_resp = self.bus.wait_for_response( Message("neon.audio_input", {"audio_data": audio_data}, context), context["ident"], 60.0) self.assertIsInstance(stt_resp, Message) for key in context: self.assertIn(key, stt_resp.context) self.assertEqual(context[key], stt_resp.context[key]) self.assertIsInstance(stt_resp.data.get("skills_recv"), bool, stt_resp.serialize()) handle_utterance.assert_called_once() message = handle_utterance.call_args[0][0] self.assertIsInstance(message, Message) for key in context: self.assertIn(key, message.context) self.assertEqual(context[key], message.context[key]) self.assertIsInstance(message.data["utterances"], list, message.data) self.assertIn("stop", message.data["utterances"], message.data.get("utterances")) self.assertIsInstance(message.context["timing"], dict) self.assertEqual(message.context["destination"], ["skills"])
def on_message(self, message): LOG.info("Received: {}".format(message)) msg = json.loads(message) if (msg.get('type') == "mycroft.events.triggered" and (msg.get('event_name') == 'page_gained_focus' or msg.get('event_name') == 'system.gui.user.interaction')): # System event, a page was changed msg_type = 'gui.page_interaction' msg_data = { 'namespace': msg['namespace'], 'page_number': msg['parameters'].get('number'), 'skill_id': msg['parameters'].get('skillId') } elif msg.get('type') == "mycroft.events.triggered": # A normal event was triggered msg_type = '{}.{}'.format(msg['namespace'], msg['event_name']) msg_data = msg['parameters'] elif msg.get('type') == 'mycroft.session.set': # A value was changed send it back to the skill msg_type = '{}.{}'.format(msg['namespace'], 'set') msg_data = msg['data'] message = Message(msg_type, msg_data) LOG.info('Forwarding to bus...') self.application.enclosure.bus.emit(message) LOG.info('Done!')
def reset(self): """The enclosure should restore itself to a started state. Typically this would be represented by the eyes being 'open' and the mouth reset to its default (smile or blank). """ self.bus.emit( Message("enclosure.reset", context={"destination": ["enclosure"]}))
def simple_cli(lang="en-us"): global bSimple bSimple = True bus.on('speak', handle_speak) try: while True: # Sleep for a while so all the output that results # from the previous command finishes before we print. time.sleep(1.5) print("Input (Ctrl+C to quit):") line = sys.stdin.readline() bus.emit( Message("recognizer_loop:utterance", { 'utterances': [line.strip()], "lang": lang }, { 'client_name': 'mycroft_simple_cli', 'source': 'cli', 'destination': ["skills"] })) except KeyboardInterrupt as e: # User hit Ctrl+C to quit print("") except KeyboardInterrupt as e: LOG.exception(e) event_thread.exit() sys.exit()
def eyes_brightness(self, level=30): """Set the brightness of the eyes in the display. Args: level (int): 1-30, bigger numbers being brighter """ self.bus.emit( Message("enclosure.eyes.level", {'level': level}, context={"destination": ["enclosure"]}))
def send(self, response_type=None, timeout=10): self.response = Message(None, None, None) if response_type is None: response_type = self.query.type + ".reply" self.add_response_type(response_type) self.bus.emit(self.query) self._wait_response(timeout) return self.response
def eyes_blink(self, side): """Make the eyes blink Args: side (str): 'r', 'l', or 'b' for 'right', 'left' or 'both' """ self.bus.emit( Message("enclosure.eyes.blink", {'side': side}, context={"destination": ["enclosure"]}))
def system_blink(self, times): """The 'eyes' should blink the given number of times. Args: times (int): number of times to blink """ self.bus.emit( Message("enclosure.system.blink", {'times': times}, context={"destination": ["enclosure"]}))
def test_get_stt_invalid_file_type(self): context = {"client": "tester", "ident": "123456", "user": "******"} stt_resp = self.bus.wait_for_response( Message("neon.get_stt", {"audio_file": os.path.join(AUDIO_FILE_PATH, "test.txt")}, context), context["ident"]) self.assertEqual(stt_resp.context, context) self.assertIsInstance(stt_resp.data.get("error"), str)
def playback_time(self): """ Request information of current playback time. Returns: seconds (int) if track is playing or None """ response = self.bus.wait_for_response(Message('mycroft.audio.service.playback_time')) data = response.data if response else {"time": 0} return data.get("time") or 0
def available_backends(self): """Return available audio backends. Returns: dict with backend names as keys """ msg = Message('mycroft.audio.service.list_backends') response = self.bus.wait_for_response(msg) return response.data if response else {}
def async_volume_handler(self, vol): LOG.error("ASYNC SET VOL PASSED IN %s" % (vol,)) if vol > 1.0: vol = vol / 10 self.current_volume = vol LOG.error("ASYNC SET VOL TO %s" % (self.current_volume,)) # notify anybody listening on the bus who cares self.bus.emit(Message("hardware.volume", { "volume": self.current_volume}, context={"source": ["enclosure"]}))
def mouth_text(self, text=""): """Display text (scrolling as needed) Args: text (str): text string to display """ self.display_manager.set_active(self.name) self.bus.emit( Message("enclosure.mouth.text", {'text': text}, context={"destination": ["enclosure"]}))
def on_volume_set(self, message): self.current_volume = message.data.get("percent", self.current_volume) LOG.info('Mark2:interface.py set volume to %s' % (self.current_volume,)) self.m2enc.hardware_volume.set_volume(float(self.current_volume)) # notify anybody listening on the bus who cares self.bus.emit(Message("hardware.volume", { "volume": self.current_volume}, context={"source": ["enclosure"]}))