class HandleConnection(ssl_dispatcher): __not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}"} __websearch = {"de-DE": u"Websuche", "en-US": u"Websearch"} def __init__(self, conn): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None self.dbConnection = db.getConnection() self.assistant = None self.sendLock = threading.Lock() self.current_running_plugin = None self.current_location = None self.plugin_lastAceId = None self.logger = logging.getLogger("logger") def handle_ssl_established(self): self.ssled = True def handle_ssl_shutdown(self): self.ssled = False def readable(self): if self.ssled: while self.socket.pending() > 0: self.handle_read_event() return True def handle_read(self): self.data += self.recv(8192) if not self.binary_mode: if "\r\n\r\n" in self.data: endOfHeader = self.data.find("\r\n\r\n")+4 self.header = self.data[:endOfHeader] self.data = self.data[endOfHeader:] self.logger.debug("--------------------------------------Header start------------------------------------") self.logger.debug(self.header) self.logger.debug("---------------------------------------Header end-------------------------------------") self.binary_mode = True self.header_complete = True else: if not self.consumed_ace: self.logger.debug("Received removing ace instruction: {0}".format(repr(self.data[:4]))) self.data = self.data[4:] self.consumed_ace = True self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02" #self.flush_output_buffer() # first process outstanding google answers THIS happens at least on each PING if self.gotGoogleAnswer: self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation) self.lastRequestId = None self.dictation = None self.googleData = None self.gotGoogleAnswer = False self.process_compressed_data() def handle_google_data(self, body, requestId, dictation): self.googleData = json.loads(body) self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def handle_google_failure(self, requestId, dictation): self.googleData = None self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def send_object(self, obj): self.send_plist(obj.to_plist()) def send_plist(self, plist): self.sendLock.acquire() self.logger.debug("Sending:\n{0}".format(pprint.pformat(plist, width=40))) bplist = biplist.writePlistToString(plist); # self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist self.flush_unzipped_output() self.sendLock.release() def send_pong(self, id): self.sendLock.acquire() self.unzipped_output_buffer = struct.pack('>BI', 4, id) self.flush_unzipped_output() self.sendLock.release() def process_recognized_speech(self, googleJson, requestId, dictation): if googleJson == None: # there was a network failure # is this the correct command to send? self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible")) self.send_object(baseObjects.RequestCompleted(requestId)) else: possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match = best_match[0].upper()+best_match[1:] best_match_confidence = possible_matches[0]['confidence'] self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2))) # construct a SpeechRecognized token = speechObjects.Token(best_match, 0, 0, 1000.0, True, True) interpretation = speechObjects.Interpretation([token]) phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = speechObjects.Recognition([phrase]) recognized = speechObjects.SpeechRecognized(requestId, recognition) if not dictation: if self.current_running_plugin == None: (clazz, method) = PluginManager.getPlugin(best_match, self.assistant.language) if clazz != None and method != None: plugin = clazz(method, best_match, self.assistant.language, self.send_object, self.send_plist, self.assistant, self.current_location) plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = uiObjects.AddViews(requestId) errorText = HandleConnection.__not_recognized[self.assistant.language] if self.assistant.language in HandleConnection.__not_recognized else HandleConnection.__not_recognized["en-US"] view.views += [uiObjects.AssistantUtteranceView(errorText.format(best_match), errorText.format(best_match))] websearchText = HandleConnection.__websearch[self.assistant.language] if self.assistant.language in HandleConnection.__websearch else HandleConnection.__websearch["en-US"] button = uiObjects.Button(text=websearchText) cmd = systemObjects.SendCommands() cmd.commands = [systemObjects.StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object(baseObjects.RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(baseObjects.RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(baseObjects.RequestCompleted(requestId)) def process_compressed_data(self): self.unzipped_input += self.decompressor.decompress(self.data) self.data = "" while self.hasNextObj(): reqObject = self.read_next_object_from_unzipped() if reqObject != None: self.logger.debug("Packet with class: {0}".format(reqObject['class'])) self.logger.debug("packet with content:\n{0}".format(pprint.pformat(reqObject, width=40))) # first handle speech stuff if 'refId' in reqObject: # if the following holds, this packet is an answer to a request by a plugin if reqObject['refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.info("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = reqObject self.current_running_plugin.waitForResponse.set() if ObjectIsCommand(reqObject, StartSpeechRequest) or ObjectIsCommand(reqObject, StartSpeechDictation): self.logger.info("New start of speech received") startSpeech = None if ObjectIsCommand(reqObject, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(reqObject) else: dictation = False startSpeech = StartSpeechRequest(reqObject) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(reqObject, SpeechPacket): self.logger.info("Decoding speech packet") speechPacket = SpeechPacket(reqObject) (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) elif reqObject['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': str.lower(reqObject['properties']['utterance'])}]}, reqObject['aceId'], False) elif ObjectIsCommand(reqObject, FinishSpeech): self.logger.info("End of speech received") finishSpeech = FinishSpeech(reqObject) (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] self.logger.info("Sending flac to google for recognition") self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) elif ObjectIsCommand(reqObject, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send cancelRequest = CancelRequest(reqObject) if cancelRequest.refId in self.speech: del self.speech[cancelRequest.refId] self.send_object(CancelSucceeded(cancelRequest.aceId)) elif ObjectIsCommand(reqObject, GetSessionCertificate): getSessionCertificate = GetSessionCertificate(reqObject) response = GetSessionCertificateResponse(getSessionCertificate.aceId) response.caCert = caCert.as_der() response.sessionCert = serverCert.as_der() self.send_object(response) elif ObjectIsCommand(reqObject, CreateSessionInfoRequest): # how does a positive answer look like? createSessionInfoRequest = CreateSessionInfoRequest(reqObject) fail = CommandFailed(createSessionInfoRequest.aceId) fail.reason = "Not authenticated" fail.errorCode = 0 self.send_object(fail) #self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) elif reqObject['class'] == 'CreateAssistant': #create a new assistant helper = Assistant() c = self.dbConnection.cursor() noError = True try: c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error, e: noError = False c.close() if noError: self.assistant = helper self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": reqObject['aceId']}) else: self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"}) elif reqObject['class'] == 'SetAssistantData': # fill assistant if self.assistant != None: c = self.dbConnection.cursor() objProperties = reqObject['properties'] self.assistant.censorSpeech = objProperties['censorSpeech'] self.assistant.timeZoneId = objProperties['timeZoneId'] self.assistant.language = objProperties['language'] self.assistant.region = objProperties['region'] c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() elif reqObject['class'] == 'LoadAssistant': c = self.dbConnection.cursor() c.execute("select assistant from assistants where assistantId = ?", (reqObject['properties']['assistantId'],)) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) else: self.assistant = result[0] self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) c.close() elif reqObject['class'] == 'DestroyAssistant': c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (reqObject['properties']['assistantId'],)) self.dbConnection.commit() c.close() self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": reqObject['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) elif reqObject['class'] == 'StartRequest': #this should also be handeled by special plugins, so lets call the plugin handling stuff self.process_recognized_speech({'hypotheses': [{'utterance': reqObject['properties']['utterance'], 'confidence': 1.0}]}, reqObject['aceId'], False)
class SiriProtocolHandler(Siri): __not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.", "nl-NL": u"Excuses, \"{0}\" versta ik niet."} __websearch = {"de-DE": u"Websuche", "en-US": u"Websearch", "fr-FR": u"Rechercher sur le Web", "nl-NL": u"Zoeken op het web"} __scheduling_interval_timeout__ = 20 __timeout_delay = 10 def __init__(self, server, peer): Siri.__init__(self, server, peer) self.lastPing = 0 self.pong = 0 self.plugin_lastAceId = "" self.current_running_plugin = None self.dbConnection = server.dbConnection self.assistant = None self.speech = dict() self.httpClient = AsyncOpenHttp(self.handle_google_data) self.current_google_request = None self.current_location = None self.lastPingTime = time.time() self.syncAnchors = dict() self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def seconds_since_last_ping(self): return time.time() - self.lastPingTime def connectionLost(self, reason): try: self.timeoutschedule.cancel() except: pass if self.current_google_request != None: self.current_google_request.cancel() #ensure all decoder/encoder attemps are closed for key in self.speech.keys(): (decoder, encoder, _) = self.speech[key] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destroy() del self.speech self.current_running_plugin = None self.dbConnection = None self.httpClient = None Siri.connectionLost(self, reason) def checkTimeout(self): if self.seconds_since_last_ping() > SiriProtocolHandler.__timeout_delay: self.logger.info("Connection timed out") self.transport.loseConnection() else: self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def handle_google_data(self, body, requestId, dictation): self.current_google_request = None if (body != None): googleAnswer = json.loads(body) for i in xrange(0,len(googleAnswer['hypotheses'])-1): utterance = googleAnswer['hypotheses'][i]['utterance'] if len(utterance) == 1: utterance = utterance.upper() else: utterance = utterance[0].upper() + utterance[1:] googleAnswer['hypotheses'][i]['utterance'] = utterance self.process_recognized_speech(googleAnswer, requestId, dictation) else: self.send_object(SpeechFailure(requestId, "No connection to Google possible")) self.send_object(RequestCompleted(requestId)) def received_ping(self, numOfPing): self.pong += 1 self.lastPing = numOfPing self.lastPingTime = time.time() self.send_pong(self.pong) def process_recognized_speech(self, googleJson, requestId, dictation): possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match_confidence = possible_matches[0]['confidence'] self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence) * 100, 2))) # construct a SpeechRecognized token = Token(best_match, 0, 0, 1000.0, True, True) interpretation = Interpretation([token]) phrase = Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = Recognition([phrase]) recognized = SpeechRecognized(requestId, recognition) if not dictation: if self.current_running_plugin == None: plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location)) if plugin != None: plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = UIAddViews(requestId) errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"] errorView = UIAssistantUtteranceView() errorView.text = errorText.format(best_match) errorView.speakableText = errorText.format(best_match) view.views = [errorView] websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"] button = UIButton() button.text = websearchText cmd = SendCommands() cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object(RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) def received_plist(self, plist): self.logger.debug("Got packet with class: {0}".format(plist['class'])) self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40))) # first handle speech stuff if 'refId' in plist: # if the following holds, this packet is an answer to a request by a plugin if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.debug("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance'] self.current_running_plugin.waitForResponse.set() return if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation): self.logger.debug("New start of speech received") startSpeech = None if ObjectIsCommand(plist, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(plist) else: dictation = False startSpeech = StartSpeechRequest(plist) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(plist, SpeechPacket): self.logger.debug("Decoding speech packet") speechPacket = SpeechPacket(plist) if speechPacket.refId in self.speech: (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) else: self.logger.debug("Got a speech packet that did not match any current request") elif plist['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False) elif ObjectIsCommand(plist, FinishSpeech): self.logger.debug("End of speech received") finishSpeech = FinishSpeech(plist) if finishSpeech.refId in self.speech: (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() flacBin = None if encoder: encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] if flacBin != None: self.logger.info("Sending flac to google for recognition") try: self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) except (AttributeError, TypeError): self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.") else: self.logger.info("There was no speech") else: self.logger.debug("Got a finish speech packet that did not match any current request") elif ObjectIsCommand(plist, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send self.logger.debug("Should cancel current request") cancelRequest = CancelRequest(plist) if cancelRequest.refId in self.speech: (decoder, encoder, dictation) = self.speech[cancelRequest.refId] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destroy() del self.speech[cancelRequest.refId] if self.current_google_request != None: self.current_google_request.cancel() # if a google request is running (follow up listening..., plugin might get killed there by user) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: self.current_running_plugin._abortPluginRun() self.current_running_plugin.waitForResponse.set() # if a plugin is running (processing, but not waiting for data from the device we kill it) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse == None: self.current_running_plugin._abortPluginRun() self.send_object(CancelSucceeded(cancelRequest.aceId)) elif ObjectIsCommand(plist, RollbackRequest): pass elif ObjectIsCommand(plist, SyncChunk): chunk = SyncChunk(plist) previous = self.syncAnchors[chunk.key] if chunk.key in self.syncAnchors else None if previous != None: if previous.generation != chunk.preGen: chunkDenied = SyncChunkDenied(chunk.aceId) self.send_object(chunkDenied) return current = SyncAnchor() current.generation = chunk.postGen current.value = chunk.postGen current.validity = chunk.validity current.key = chunk.key self.syncAnchors[current.key] = current chunkAccepted = SyncChunkAccepted(chunk.aceId) chunkAccepted.current = current self.send_object(chunkAccepted) pass elif ObjectIsCommand(plist, GetSessionCertificate): getSessionCertificate = GetSessionCertificate(plist) sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCACert) sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCert) response = GetSessionCertificateResponse(getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER) self.send_object(response) elif ObjectIsCommand(plist, CreateSessionInfoRequest): # how does a positive answer look like? createSessionInfoRequest = CreateSessionInfoRequest(plist) #success = CreateSessionInfoResponse(createSessionInfoRequest.aceId) #success.sessionInfo = biplist.Data("\x01\x02BLABLABLBALBALBALBALBALBALBALBA") #success.validityDuration = 9600 #self.send_object(success) fail = CommandFailed(createSessionInfoRequest.aceId) fail.reason = "Not authenticated" fail.errorCode = 0 self.send_object(fail) ##self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) elif ObjectIsCommand(plist, CreateAssistant): createAssistant = CreateAssistant(plist) #create a new assistant helper = Assistant() helper.assistantId = str.upper(str(uuid.uuid4())) helper.language = createAssistant.language helper.activationToken = createAssistant.activationToken helper.connectionType = createAssistant.connectionType helper.validationData = createAssistant.validationData c = self.dbConnection.cursor() noError = True try: c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error: noError = False c.close() if noError: self.assistant = helper assiCreatedCMD = AssistantCreated(createAssistant.aceId) assiCreatedCMD.assistantId = helper.assistantId assiCreatedCMD.speechId = str(uuid.uuid4()) self.send_object(assiCreatedCMD) else: cmdFailed = CommandFailed(createAssistant.aceId) cmdFailed.reason = "Database Error" cmdFailed.errorCode = 2 self.send_object(cmdFailed) elif ObjectIsCommand(plist, SetAssistantData): setAssistantData = SetAssistantData(plist) # fill assistant if self.assistant != None: try: c = self.dbConnection.cursor() assi_id = self.assistant.assistantId self.assistant.initializeFromPlist(setAssistantData.to_plist()) self.assistant.assistantId = assi_id #Record the user firstName and nickName try: self.assistant.firstName = self.assistant.meCards[0].firstName.encode("utf-8") except: self.assistant.firstName = u'' try: self.assistant.nickName = self.assistant.meCards[0].nickName.encode("utf-8") except: self.assistant.nickName = u'' #Done recording c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() except: cmdFailed = CommandFailed(setAssistantData.aceId) cmdFailed.reason = "Database Error" cmdFailed.errorCode = 2 self.send_object(cmdFailed) self.logger.exception("Database Error on setting assistant data") else: cmdFailed = CommandFailed(setAssistantData.aceId) cmdFailed.reason = "Assistant to set data not found" cmdFailed.errorCode = 2 self.send_object(cmdFailed) self.logger.warning("Trying to set assistant data without having a valid assistant") elif ObjectIsCommand(plist, LoadAssistant): loadAssistant = LoadAssistant(plist) try: c = self.dbConnection.cursor() c.execute("select assistant from assistants where assistantId = ?", (loadAssistant.assistantId,)) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_object(AssistantNotFound(loadAssistant.aceId)) self.logger.warning("Assistant not found in database!!") else: self.assistant = result[0] #update assistant from LoadAssistant self.assistant.language = loadAssistant.language self.assistant.connectionType = loadAssistant.connectionType if self.assistant.language == '' or self.assistant.language == None: self.logger.error ("No language is set for this assistant") c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],)) self.dbConnection.commit() cmdFailed = CommandFailed(loadAssistant.aceId) cmdFailed.reason = "Database error Assistant not found or language settings" cmdFailed.errorCode = 2 self.send_object(cmdFailed) else: loaded = AssistantLoaded(loadAssistant.aceId) loaded.version = "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e" loaded.requestSync = 0 try: loaded.dataAnchor = self.assistant.anchor except: loaded.dataAnchor = "removed" self.send_object(loaded) c.close() except: self.send_object(AssistantNotFound(loadAssistant.aceId)) self.logger.warning("Database error on fetching assistant") elif ObjectIsCommand(plist, DestroyAssistant): destroyAssistant = DestroyAssistant(plist) try: c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],)) self.dbConnection.commit() c.close() destroyed = AssistantDestroyed(destroyAssistant.aceId) destroyed.assistantId = destroyAssistant.assistantId self.send_object(destroyed) except: self.send_object(AssistantNotFound(destroyAssistant.aceId)) self.logger.error("Database error on deleting assistant") elif ObjectIsCommand(plist, StartRequest): startRequest = StartRequest(plist) #this should also be handeled by special plugins, so lets call the plugin handling stuff self.process_recognized_speech({'hypotheses': [{'utterance': startRequest.utterance, 'confidence': 1.0}]}, startRequest.aceId, False) pass
class HandleConnection(ssl_dispatcher): def __init__(self, conn): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None self.dbConnection = db.getConnection() self.assistant = None self.sendLock = threading.Lock() self.current_running_plugin = None self.logger = logging.getLogger("logger") def handle_ssl_established(self): self.ssled = True def handle_ssl_shutdown(self): self.ssled = False def readable(self): if self.ssled: while self.socket.pending() > 0: self.handle_read_event() return True def handle_read(self): self.data += self.recv(8192) if not self.binary_mode: if "\r\n\r\n" in self.data: endOfHeader = self.data.find("\r\n\r\n")+4 self.header = self.data[:endOfHeader] self.data = self.data[endOfHeader:] self.logger.debug("--------------------------------------Header start------------------------------------") self.logger.debug(self.header) self.logger.debug("---------------------------------------Header end-------------------------------------") self.binary_mode = True self.header_complete = True else: if not self.consumed_ace: self.logger.debug("Received removing ace instruction: {0}".format(repr(self.data[:4]))) self.data = self.data[4:] self.consumed_ace = True self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02" #self.flush_output_buffer() # first process outstanding google answers THIS happens at least on each PING if self.gotGoogleAnswer: self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation) self.lastRequestId = None self.dictation = None self.googleData = None self.gotGoogleAnswer = False self.process_compressed_data() def handle_google_data(self, body, requestId, dictation): self.googleData = json.loads(body) self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def handle_google_failure(self, requestId, dictation): self.googleData = None self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def send_object(self, obj): self.send_plist(obj.to_plist()) def send_plist(self, plist): self.sendLock.acquire() self.logger.debug("Sending:\n{0}".format(pprint.pformat(plist, width=40))) bplist = biplist.writePlistToString(plist); # self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist self.flush_unzipped_output() self.sendLock.release() def send_pong(self, id): self.sendLock.acquire() self.unzipped_output_buffer = struct.pack('>BI', 4, id) self.flush_unzipped_output() self.sendLock.release() def process_recognized_speech(self, googleJson, requestId, dictation): if googleJson == None: # there was a network failure self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible")) self.send_object(baseObjects.RequestCompleted(requestID)) else: possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match_confidence = possible_matches[0]['confidence'] self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2))) # construct a SpeechRecognized token = speechObjects.Token(best_match, 0, 0, 1000.0, True, True) interpretation = speechObjects.Interpretation([token]) phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = speechObjects.Recognition([phrase]) recognized = speechObjects.SpeechRecognized(requestId, recognition) if self.current_running_plugin == None: (clazz, method) = PluginManager.getPlugin(best_match, self.assistant.language) if clazz != None and method != None: plugin = clazz(method, best_match, self.assistant.language) plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) self.send_object(baseObjects.RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(baseObjects.RequestCompleted(requestId)) def process_compressed_data(self): self.unzipped_input += self.decompressor.decompress(self.data) self.data = "" while self.hasNextObj(): reqObject = self.read_next_object_from_unzipped() if reqObject != None: self.logger.debug("Packet with class: {0}".format(reqObject['class'])) self.logger.debug("packet with content:\n{0}".format(pprint.pformat(reqObject, width=40))) # first handle speech stuff if reqObject['class'] == 'StartSpeechRequest' or reqObject['class'] == 'StartSpeechDictation': decoder = speex.Decoder() decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder = flac.Encoder() encoder.initialize(16000, 1, 16) #16kHz sample rate, 1 channel, 16 bits per sample dictation=(reqObject['class'] == 'StartSpeechDictation') self.speech[reqObject['aceId']] = (decoder, encoder, dictation) elif reqObject['class'] == 'SpeechPacket': (decoder, encoder, dictation) = self.speech[reqObject['refId']] pcm = decoder.decode(reqObject['properties']['packets']) encoder.encode(pcm) elif reqObject['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': str.lower(reqObject['properties']['utterance'])}]}, reqObject['aceId'], False) elif reqObject['class'] == 'FinishSpeech': (decoder, encoder, dictation) = self.speech[reqObject['refId']] decoder.destroy() encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[reqObject['refId']] self.httpClient.make_google_request(flacBin, reqObject['refId'], dictation, language=self.assistant.language, allowCurses=True) elif reqObject['class'] == 'CancelRequest': # this is probably called when we need to kill a plugin # wait for thread to finish a send if reqObject['refId'] in self.speech: del self.speech[reqObject['refId']] self.send_plist({"class": "CancelSucceeded", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "properties":{"callbacks": []}}) # handle responses to plugin elif self.current_running_plugin != None and reqObject['group'] != "com.apple.ace.system" and "refId" in reqObject: if self.current_running_plugin.waitForResponse != None: # just forward the object to the self.current_running_plugin.response = reqObject self.current_running_plugin.refId = reqObject['refId'] self.current_running_plugin.waitForResponse.set() else: # handle other stuff if reqObject['class'] == 'GetSessionCertificate': caDer = caCert.as_der() serverDer = serverCert.as_der() self.send_plist({"class": "GetSessionCertificateResponse", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "properties":{"certificate": biplist.Data("\x01\x02"+struct.pack(">I", len(caDer))+caDer + struct.pack(">I", len(serverDer))+serverDer)}}) #self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"}) if reqObject['class'] == 'CreateSessionInfoRequest': # how does a positive answer look like? self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"}) #self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) if reqObject['class'] == 'CreateAssistant': #create a new assistant helper = Assistant() c = self.dbConnection.cursor() noError = True try: c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error, e: noError = False c.close() if noError: self.assistant = helper self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": reqObject['aceId']}) else: self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"}) if reqObject['class'] == 'SetAssistantData': # fill assistant if self.assistant != None: c = self.dbConnection.cursor() objProperties = reqObject['properties'] self.assistant.censorSpeech = objProperties['censorSpeech'] self.assistant.timeZoneId = objProperties['timeZoneId'] self.assistant.language = objProperties['language'] self.assistant.region = objProperties['region'] c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() if reqObject['class'] == 'LoadAssistant': c = self.dbConnection.cursor() c.execute("select assistant from assistants where assistantId = ?", (reqObject['properties']['assistantId'],)) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) else: self.assistant = result[0] self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) c.close() if reqObject['class'] == 'DestroyAssistant': c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (reqObject['properties']['assistantId'],)) self.dbConnection.commit() c.close() self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": reqObject['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
class SiriProtocolHandler(Siri): __not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire."} __websearch = {"de-DE": u"Websuche", "en-US": u"Websearch", "fr-FR": u"Rechercher sur le Web"} def __init__(self, server, peer): Siri.__init__(self, server, peer) self.lastPing = 0 self.pong = 0 self.plugin_lastAceId = "" self.current_running_plugin = None self.dbConnection = server.dbConnection self.assistant = None self.speech = dict() self.httpClient = AsyncOpenHttp(self.handle_google_data) self.current_google_request = None self.current_location = None def handle_google_data(self, body, requestId, dictation): self.current_google_request = None if (body != None): googleAnswer = json.loads(body) self.process_recognized_speech(googleAnswer, requestId, dictation) else: self.send_object(SpeechFailure(requestId, "No connection to Google possible")) self.send_object(RequestCompleted(requestId)) def received_ping(self, numOfPing): self.pong += 1 self.lastPing = numOfPing self.send_pong(self.pong) def process_recognized_speech(self, googleJson, requestId, dictation): possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match = best_match[0].upper()+best_match[1:] best_match_confidence = possible_matches[0]['confidence'] self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2))) # construct a SpeechRecognized token = Token(best_match, 0, 0, 1000.0, True, True) interpretation = Interpretation([token]) phrase = Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = Recognition([phrase]) recognized = SpeechRecognized(requestId, recognition) if not dictation: if self.current_running_plugin == None: plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location)) if plugin != None: plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = AddViews(requestId) errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"] view.views += [AssistantUtteranceView(errorText.format(best_match), errorText.format(best_match))] websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"] button = Button(text=websearchText) cmd = SendCommands() cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object(RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) def received_plist(self, plist): self.logger.debug("Packet with class: {0}".format(plist['class'])) self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40))) # first handle speech stuff if 'refId' in plist: # if the following holds, this packet is an answer to a request by a plugin if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.debug("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance'] self.current_running_plugin.waitForResponse.set() return if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation): self.logger.debug("New start of speech received") startSpeech = None if ObjectIsCommand(plist, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(plist) else: dictation = False startSpeech = StartSpeechRequest(plist) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(plist, SpeechPacket): self.logger.debug("Decoding speech packet") speechPacket = SpeechPacket(plist) (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) elif plist['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False) elif ObjectIsCommand(plist, FinishSpeech): self.logger.debug("End of speech received") finishSpeech = FinishSpeech(plist) (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] self.logger.info("Sending flac to google for recognition") try: self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) except AttributeError, TypeError: self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.") elif ObjectIsCommand(plist, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send self.logger.debug("Should cancel current request") cancelRequest = CancelRequest(plist) if cancelRequest.refId in self.speech: del self.speech[cancelRequest.refId] if self.current_google_request != None: self.current_google_request.cancel() self.send_object(CancelSucceeded(cancelRequest.aceId))
class SiriProtocolHandler(Siri): __not_recognized = {"zh-HK": u"对不起,我不知道“{0}”是什么意思。", "zh-TW": u"对不起,我不知道“{0}”是什么意思。", "ja-JP": u"申し訳ありませんが、私は理解していない '{0}", "id-ID": u"Maaf, Saya tidak mengerti apa yang anda maksud dengan '{0}'.", "en-US": u"I don't know what you mean by ‘{0}’.", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.", "zh-CN": u"对不起,我不知道“{0}”是什么意思。"} __websearch = {"zh-HK": u"搜索网页", "zh-TW": u"搜索网页", "ja-JP": u"Webを検索", "id-ID": u"cari di internet", "en-US": u"Search the Web", "fr-FR": u"Rechercher sur le Web", "zh-CN": u"搜索网页"} __scheduling_interval_timeout__ = 20 __timeout_delay = 10 def __init__(self, server, peer): Siri.__init__(self, server, peer) self.lastPing = 0 self.pong = 0 self.plugin_lastAceId = "" self.current_running_plugin = None self.dbConnection = server.dbConnection self.assistant = None self.speech = dict() self.httpClient = AsyncOpenHttp(self.handle_google_data) self.current_google_request = None self.current_location = None self.lastPingTime = time.time() self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def seconds_since_last_ping(self): return time.time() - self.lastPingTime def connectionLost(self, reason): try: self.timeoutschedule.cancel() except: pass if self.current_google_request != None: self.current_google_request.cancel() #ensure all decoder/encoder attemps are closed for key in self.speech.keys(): (decoder, encoder, _) = self.speech[key] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destroy() del self.speech self.current_running_plugin = None self.dbConnection = None self.httpClient = None Siri.connectionLost(self, reason) def checkTimeout(self): if self.seconds_since_last_ping() > SiriProtocolHandler.__timeout_delay: self.logger.info("Connection timed out") self.transport.loseConnection() else: self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def handle_google_data(self, body, requestId, dictation): self.current_google_request = None if (body != None): googleAnswer = json.loads(body) for i in xrange(0,len(googleAnswer['hypotheses'])-1): utterance = googleAnswer['hypotheses'][i]['utterance'] if len(utterance) == 1: utterance = utterance.upper() else: utterance = utterance[0].upper() + utterance[1:] googleAnswer['hypotheses'][i]['utterance'] = utterance self.process_recognized_speech(googleAnswer, requestId, dictation) else: self.send_object(SpeechFailure(requestId, "No connection to Google possible")) self.send_object(RequestCompleted(requestId)) def received_ping(self, numOfPing): self.pong += 1 self.lastPing = numOfPing self.lastPingTime = time.time() self.send_pong(self.pong) def process_recognized_speech(self, googleJson, requestId, dictation): possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match_confidence = possible_matches[0]['confidence'] self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence) * 100, 2))) # construct a SpeechRecognized token = Token(best_match, 0, 0, 1000.0, True, True) interpretation = Interpretation([token]) phrase = Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = Recognition([phrase]) recognized = SpeechRecognized(requestId, recognition) if not dictation: if self.current_running_plugin == None: plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location)) if plugin != None: plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = UIAddViews(requestId) errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"] errorView = UIAssistantUtteranceView() errorView.text = errorText.format(best_match) errorView.speakableText = errorText.format(best_match) view.views = [errorView] websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"] button = UIButton() button.text = websearchText cmd = SendCommands() cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object(RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) def received_plist(self, plist): self.logger.debug("Got packet with class: {0}".format(plist['class'])) self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40))) # first handle speech stuff if 'refId' in plist: # if the following holds, this packet is an answer to a request by a plugin if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.debug("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance'] self.current_running_plugin.waitForResponse.set() return if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation): self.logger.debug("New start of speech received") startSpeech = None if ObjectIsCommand(plist, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(plist) else: dictation = False startSpeech = StartSpeechRequest(plist) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(plist, SpeechPacket): self.logger.debug("Decoding speech packet") speechPacket = SpeechPacket(plist) if speechPacket.refId in self.speech: (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) else: self.logger.debug("Got a speech packet that did not match any current request") elif plist['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False) elif ObjectIsCommand(plist, FinishSpeech): self.logger.debug("End of speech received") finishSpeech = FinishSpeech(plist) if finishSpeech.refId in self.speech: (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() flacBin = None if encoder: encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] if flacBin != None: self.logger.info("Sending flac to google for recognition") try: self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) except (AttributeError, TypeError): self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.") else: self.logger.info("There was no speech") else: self.logger.debug("Got a finish speech packet that did not match any current request") elif ObjectIsCommand(plist, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send self.logger.debug("Should cancel current request") cancelRequest = CancelRequest(plist) if cancelRequest.refId in self.speech: (decoder, encoder, dictation) = self.speech[cancelRequest.refId] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destory() del self.speech[cancelRequest.refId] if self.current_google_request != None: self.current_google_request.cancel() # if a google request is running (follow up listening..., plugin might get killed there by user) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: self.current_running_plugin._abortPluginRun() self.current_running_plugin.waitForResponse.set() # if a plugin is running (processing, but not waiting for data from the device we kill it) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse == None: self.current_running_plugin._abortPluginRun() self.send_object(CancelSucceeded(cancelRequest.aceId)) elif ObjectIsCommand(plist, RollbackRequest): pass elif ObjectIsCommand(plist, GetSessionCertificate): getSessionCertificate = GetSessionCertificate(plist) sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCACert) sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCert) response = GetSessionCertificateResponse(getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER) self.send_object(response) elif ObjectIsCommand(plist, CreateSessionInfoRequest): # how does a positive answer look like? createSessionInfoRequest = CreateSessionInfoRequest(plist) fail = CommandFailed(createSessionInfoRequest.aceId) fail.reason = "Not authenticated" fail.errorCode = 0 self.send_object(fail) #self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) elif plist['class'] == 'CreateAssistant': #create a new assistant helper = Assistant() helper.assistantId = str.upper(str(uuid.uuid4())) c = self.dbConnection.cursor() noError = True try: c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error: noError = False c.close() if noError: self.assistant = helper self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": plist['aceId']}) else: self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"}) elif plist['class'] == 'SetAssistantData': # fill assistant if self.assistant != None: try: c = self.dbConnection.cursor() objProperties = plist['properties'] self.assistant.censorSpeech = objProperties['censorSpeech'] self.assistant.timeZoneId = objProperties['timeZoneId'] self.assistant.language = objProperties['language'] self.assistant.region = objProperties['region'] #Here it is possible to support more languages combined with anyvoice package #We can make assumption that the region will define the language. if config.forcelanguage==True: #Chinese Example if self.assistant.region=='id_ID': self.assistant.language = "id-ID" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='zh_CN': self.assistant.language = "zh-CN" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='zh_TW': self.assistant.language = "zh-TW" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='ko_KR': self.assistant.language = "ko-KR" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='zh_HK': self.assistant.language = "zh-HK" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='ru_RU': self.assistant.language = "ru-RU" self.logger.debug("Forced language to {0}".format(self.assistant.language)) if self.assistant.region=='ja_JP': self.assistant.language = "ja-JP" self.logger.debug("Forced language to {0}".format(self.assistant.language)) #Record the user firstName and nickName try: self.assistant.firstName = objProperties["meCards"][0]["properties"]["firstName"].encode("utf-8") except KeyError: self.assistant.firstName = u'' try: self.assistant.nickName = objProperties["meCards"][0]["properties"]["nickName"].encode("utf-8") except KeyError: self.assistant.nickName = u'' #Done recording c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() except: self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"}) self.logger.error("Database Error on setting assistant data") else: self.send_plist({"class":"CommandFailed", "properties": {"reason":"Assistant to set data not found", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"}) self.logger.warning("Trying to set assistant data without having a valid assistant") elif plist['class'] == 'LoadAssistant': try: c = self.dbConnection.cursor() c.execute("select assistant from assistants where assistantId = ?", (plist['properties']['assistantId'],)) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) self.logger.warning("Assistant not found in database!!") else: self.assistant = result[0] if self.assistant.language == '' or self.assistant.language == None: self.logger.error ("No language is set for this assistant") c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],)) self.dbConnection.commit() self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error Assistant not found or language settings ", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"}) else: self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) c.close() except: self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) self.logger.warning("Database error on fetching assistant") elif plist['class'] == 'DestroyAssistant': try: c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],)) self.dbConnection.commit() c.close() self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": plist['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) except: self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) self.logger.error("Database error on deleting assistant") elif plist['class'] == 'StartRequest': #this should also be handeled by special plugins, so lets call the plugin handling stuff self.process_recognized_speech({'hypotheses': [{'utterance': plist['properties']['utterance'], 'confidence': 1.0}]}, plist['aceId'], False) pass
class HandleConnection(ssl_dispatcher): __not_recognized = { "de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}.\nConfidence: {1}%" } __websearch = {"de-DE": u"Websuche", "en-US": u"Websearch"} def __init__(self, conn, lang): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None self.dbConnection = db.getConnection() self.assistant = None self.sendLock = threading.Lock() self.current_running_plugin = None self.current_location = None self.plugin_lastAceId = None self.logger = logging.getLogger("logger") self.lang = lang def handle_ssl_established(self): self.ssled = True def handle_ssl_shutdown(self): self.ssled = False def readable(self): if self.ssled: while self.socket.pending() > 0: self.handle_read_event() return True def handle_read(self): self.data += self.recv(8192) if not self.binary_mode: if "\r\n\r\n" in self.data: endOfHeader = self.data.find("\r\n\r\n") + 4 self.header = self.data[:endOfHeader] self.data = self.data[endOfHeader:] self.logger.debug( "--------------------------------------Header start------------------------------------" ) self.logger.debug(self.header) self.logger.debug( "---------------------------------------Header end-------------------------------------" ) self.binary_mode = True self.header_complete = True else: if not self.consumed_ace: self.logger.debug( "Received removing ace instruction: {0}".format( repr(self.data[:4]))) self.data = self.data[4:] self.consumed_ace = True self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate( timeval=None, localtime=False, usegmt=True ) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02" #self.flush_output_buffer() # first process outstanding google answers THIS happens at least on each PING if self.gotGoogleAnswer: self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation) self.lastRequestId = None self.dictation = None self.googleData = None self.gotGoogleAnswer = False self.process_compressed_data() def handle_google_data(self, body, requestId, dictation): self.googleData = json.loads(body) self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def handle_google_failure(self, requestId, dictation): self.googleData = None self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def send_object(self, obj): self.send_plist(obj.to_plist()) def send_plist(self, plist): self.sendLock.acquire() self.logger.debug("Sending:\n{0}".format( pprint.pformat(plist, width=40))) bplist = biplist.writePlistToString(plist) # self.unzipped_output_buffer = struct.pack('>BI', 2, len(bplist)) + bplist self.flush_unzipped_output() self.sendLock.release() def send_pong(self, id): self.sendLock.acquire() self.unzipped_output_buffer = struct.pack('>BI', 4, id) self.flush_unzipped_output() self.sendLock.release() def process_recognized_speech(self, googleJson, requestId, dictation): if googleJson == None: # there was a network failure # is this the correct command to send? self.send_object( speechObjects.SpeechFailure( requestId, "No connection to Google possible")) self.send_object(baseObjects.RequestCompleted(requestId)) else: possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match = best_match[0].upper() + best_match[1:] best_match_confidence = possible_matches[0]['confidence'] self.logger.info( u"Best matching result: \"{0}\" with a confidence of {1}%". format(best_match, round(float(best_match_confidence) * 100, 2))) # construct a SpeechRecognized token = speechObjects.Token(best_match, 0, 0, 1000.0, True, True) interpretation = speechObjects.Interpretation([token]) phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = speechObjects.Recognition([phrase]) recognized = speechObjects.SpeechRecognized( requestId, recognition) if not dictation: if self.current_running_plugin == None: (clazz, method) = PluginManager.getPlugin( best_match, self.assistant.language) if clazz != None and method != None: plugin = clazz(method, best_match, self.assistant.language, self.send_object, self.send_plist, self.assistant, self.current_location) plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = uiObjects.AddViews(requestId) errorText = HandleConnection.__not_recognized[ self.assistant. language] if self.assistant.language in HandleConnection.__not_recognized else HandleConnection.__not_recognized[ "en-US"] view.views += [ uiObjects.AssistantUtteranceView( errorText.format( best_match, round( float(best_match_confidence) * 100, 2)), errorText.format( best_match, round( float(best_match_confidence) * 100, 2))) ] websearchText = HandleConnection.__websearch[ self.assistant. language] if self.assistant.language in HandleConnection.__websearch else HandleConnection.__websearch[ "en-US"] button = uiObjects.Button(text=websearchText) cmd = systemObjects.SendCommands() cmd.commands = [ systemObjects.StartRequest( utterance= u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^" .format(best_match)) ] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object( baseObjects.RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object( baseObjects.RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(baseObjects.RequestCompleted(requestId)) def process_compressed_data(self): self.unzipped_input += self.decompressor.decompress(self.data) self.data = "" while self.hasNextObj(): reqObject = self.read_next_object_from_unzipped() if reqObject != None: self.logger.debug("Packet with class: {0}".format( reqObject['class'])) self.logger.debug("packet with content:\n{0}".format( pprint.pformat(reqObject, width=40))) # first handle speech stuff if 'refId' in reqObject: # if the following holds, this packet is an answer to a request by a plugin if reqObject[ 'refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.info("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = reqObject self.current_running_plugin.waitForResponse.set() if ObjectIsCommand(reqObject, StartSpeechRequest) or ObjectIsCommand( reqObject, StartSpeechDictation): self.logger.info("New start of speech received") startSpeech = None if ObjectIsCommand(reqObject, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(reqObject) else: dictation = False startSpeech = StartSpeechRequest(reqObject) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(reqObject, SpeechPacket): self.logger.info("Decoding speech packet") speechPacket = SpeechPacket(reqObject) (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) elif reqObject['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech( { u'hypotheses': [{ 'confidence': 1.0, 'utterance': str.lower(reqObject['properties']['utterance']) }] }, reqObject['aceId'], False) elif ObjectIsCommand(reqObject, FinishSpeech): self.logger.info("End of speech received") finishSpeech = FinishSpeech(reqObject) (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] self.logger.info("Sending flac to google for recognition") self.httpClient.make_google_request( flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) elif ObjectIsCommand(reqObject, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send cancelRequest = CancelRequest(reqObject) if cancelRequest.refId in self.speech: del self.speech[cancelRequest.refId] self.send_object(CancelSucceeded(cancelRequest.aceId)) elif ObjectIsCommand(reqObject, GetSessionCertificate): getSessionCertificate = GetSessionCertificate(reqObject) response = GetSessionCertificateResponse( getSessionCertificate.aceId) response.caCert = caCert.as_der() response.sessionCert = serverCert.as_der() self.send_object(response) elif ObjectIsCommand(reqObject, CreateSessionInfoRequest): # how does a positive answer look like? createSessionInfoRequest = CreateSessionInfoRequest( reqObject) fail = CommandFailed(createSessionInfoRequest.aceId) fail.reason = "Not authenticated" fail.errorCode = 0 self.send_object(fail) #self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"}) elif reqObject['class'] == 'CreateAssistant': #create a new assistant helper = Assistant() c = self.dbConnection.cursor() noError = True try: c.execute( "insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error, e: noError = False c.close() if noError: self.assistant = helper self.send_plist({ "class": "AssistantCreated", "properties": { "speechId": str(uuid.uuid4()), "assistantId": helper.assistantId }, "group": "com.apple.ace.system", "callbacks": [], "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'] }) else: self.send_plist({ "class": "CommandFailed", "properties": { "reason": "Database error", "errorCode": 2, "callbacks": [] }, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group": "com.apple.ace.system" }) elif reqObject['class'] == 'SetAssistantData': # fill assistant if self.assistant != None: c = self.dbConnection.cursor() objProperties = reqObject['properties'] self.assistant.censorSpeech = objProperties[ 'censorSpeech'] self.assistant.timeZoneId = objProperties['timeZoneId'] #self.assistant.language = objProperties['language'] if self.lang != None: self.assistant.language = self.lang else: self.assistant.language = objProperties['language'] self.assistant.region = objProperties['region'] c.execute( "update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() elif reqObject['class'] == 'LoadAssistant': c = self.dbConnection.cursor() c.execute( "select assistant from assistants where assistantId = ?", (reqObject['properties']['assistantId'], )) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_plist({ "class": "AssistantNotFound", "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group": "com.apple.ace.system" }) else: self.assistant = result[0] self.send_plist({ "class": "AssistantLoaded", "properties": { "version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync": False, "dataAnchor": "removed" }, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group": "com.apple.ace.system" }) c.close() elif reqObject['class'] == 'DestroyAssistant': c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (reqObject['properties']['assistantId'], )) self.dbConnection.commit() c.close() self.send_plist({ "class": "AssistantDestroyed", "properties": { "assistantId": reqObject['properties']['assistantId'] }, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group": "com.apple.ace.system" }) elif reqObject['class'] == 'StartRequest': #this should also be handeled by special plugins, so lets call the plugin handling stuff self.process_recognized_speech( { 'hypotheses': [{ 'utterance': reqObject['properties']['utterance'], 'confidence': 1.0 }] }, reqObject['aceId'], False)
class SiriProtocolHandler(Siri): __not_recognized = { "de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.", "nl-NL": u"Excuses, \"{0}\" versta ik niet." } __websearch = { "de-DE": u"Websuche", "en-US": u"Websearch", "fr-FR": u"Rechercher sur le Web", "nl-NL": u"Zoeken op het web" } __scheduling_interval_timeout__ = 20 __timeout_delay = 10 def __init__(self, server, peer): Siri.__init__(self, server, peer) self.lastPing = 0 self.pong = 0 self.plugin_lastAceId = "" self.current_running_plugin = None self.dbConnection = server.dbConnection self.assistant = None self.speech = dict() self.httpClient = AsyncOpenHttp(self.handle_google_data) self.current_google_request = None self.current_location = None self.lastPingTime = time.time() self.syncAnchors = dict() self.timeoutschedule = twisted.internet.reactor.callLater( SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def seconds_since_last_ping(self): return time.time() - self.lastPingTime def connectionLost(self, reason): try: self.timeoutschedule.cancel() except: pass if self.current_google_request != None: self.current_google_request.cancel() #ensure all decoder/encoder attemps are closed for key in self.speech.keys(): (decoder, encoder, _) = self.speech[key] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destroy() del self.speech self.current_running_plugin = None self.dbConnection = None self.httpClient = None Siri.connectionLost(self, reason) def checkTimeout(self): if self.seconds_since_last_ping( ) > SiriProtocolHandler.__timeout_delay: self.logger.info("Connection timed out") self.transport.loseConnection() else: self.timeoutschedule = twisted.internet.reactor.callLater( SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout) def handle_google_data(self, body, requestId, dictation): self.current_google_request = None if (body != None): googleAnswer = json.loads(body) for i in xrange(0, len(googleAnswer['hypotheses']) - 1): utterance = googleAnswer['hypotheses'][i]['utterance'] if len(utterance) == 1: utterance = utterance.upper() else: utterance = utterance[0].upper() + utterance[1:] googleAnswer['hypotheses'][i]['utterance'] = utterance self.process_recognized_speech(googleAnswer, requestId, dictation) else: self.send_object( SpeechFailure(requestId, "No connection to Google possible")) self.send_object(RequestCompleted(requestId)) def received_ping(self, numOfPing): self.pong += 1 self.lastPing = numOfPing self.lastPingTime = time.time() self.send_pong(self.pong) def process_recognized_speech(self, googleJson, requestId, dictation): possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: best_match = possible_matches[0]['utterance'] best_match_confidence = possible_matches[0]['confidence'] self.logger.info( u"Best matching result: \"{0}\" with a confidence of {1}%". format(best_match, round(float(best_match_confidence) * 100, 2))) # construct a SpeechRecognized token = Token(best_match, 0, 0, 1000.0, True, True) interpretation = Interpretation([token]) phrase = Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = Recognition([phrase]) recognized = SpeechRecognized(requestId, recognition) if not dictation: if self.current_running_plugin == None: plugin = PluginManager.getPluginForImmediateExecution( self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location)) if plugin != None: plugin.refId = requestId plugin.connection = self self.current_running_plugin = plugin self.send_object(recognized) self.current_running_plugin.start() else: self.send_object(recognized) view = UIAddViews(requestId) errorText = SiriProtocolHandler.__not_recognized[ self.assistant. language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized[ "en-US"] errorView = UIAssistantUtteranceView() errorView.text = errorText.format(best_match) errorView.speakableText = errorText.format(best_match) view.views = [errorView] websearchText = SiriProtocolHandler.__websearch[ self.assistant. language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch[ "en-US"] button = UIButton() button.text = websearchText cmd = SendCommands() cmd.commands = [ StartRequest( utterance= u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^" .format(best_match)) ] button.commands = [cmd] view.views.append(button) self.send_object(view) self.send_object(RequestCompleted(requestId)) elif self.current_running_plugin.waitForResponse != None: # do we need to send a speech recognized here? i.d.k self.current_running_plugin.response = best_match self.current_running_plugin.refId = requestId self.current_running_plugin.waitForResponse.set() else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) else: self.send_object(recognized) self.send_object(RequestCompleted(requestId)) def received_plist(self, plist): self.logger.debug("Got packet with class: {0}".format(plist['class'])) self.logger.debug("packet with content:\n{0}".format( pprint.pformat(plist, width=40))) # first handle speech stuff if 'refId' in plist: # if the following holds, this packet is an answer to a request by a plugin if plist[ 'refId'] == self.plugin_lastAceId and self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: # just forward the object to the # don't change it's refId, further requests must reference last FinishSpeech self.logger.debug("Forwarding object to plugin") self.plugin_lastAceId = None self.current_running_plugin.response = plist if plist[ 'class'] != "StartRequest" else plist['properties'][ 'utterance'] self.current_running_plugin.waitForResponse.set() return if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand( plist, StartSpeechDictation): self.logger.debug("New start of speech received") startSpeech = None if ObjectIsCommand(plist, StartSpeechDictation): dictation = True startSpeech = StartSpeechDictation(plist) else: dictation = False startSpeech = StartSpeechRequest(plist) decoder = speex.Decoder() encoder = flac.Encoder() speexUsed = False if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value: decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value: decoder.initialize(mode=speex.SPEEX_MODEID_NB) encoder.initialize(16000, 1, 16) speexUsed = True elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue: encoder.initialize(8000, 1, 16) elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue: encoder.initialize(11025, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue: encoder.initialize(16000, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue: encoder.initialize(22050, 1, 16) elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue: encoder.initialize(32000, 1, 16) # we probably need resampling for sample rates other than 16kHz... self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation) elif ObjectIsCommand(plist, SpeechPacket): self.logger.debug("Decoding speech packet") speechPacket = SpeechPacket(plist) if speechPacket.refId in self.speech: (decoder, encoder, dictation) = self.speech[speechPacket.refId] if decoder: pcm = decoder.decode(speechPacket.packets) else: pcm = SpeechPacket.data # <- probably data... if pcm encoder.encode(pcm) else: self.logger.debug( "Got a speech packet that did not match any current request" ) elif plist['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech( { u'hypotheses': [{ 'confidence': 1.0, 'utterance': plist['properties']['utterance'] }] }, plist['aceId'], False) elif ObjectIsCommand(plist, FinishSpeech): self.logger.debug("End of speech received") finishSpeech = FinishSpeech(plist) if finishSpeech.refId in self.speech: (decoder, encoder, dictation) = self.speech[finishSpeech.refId] if decoder: decoder.destroy() flacBin = None if encoder: encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[finishSpeech.refId] if flacBin != None: self.logger.info("Sending flac to google for recognition") try: self.current_google_request = self.httpClient.make_google_request( flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True) except (AttributeError, TypeError): self.logger.warning( "Unable to find language record for this assistant. Try turning Siri off and then back on." ) else: self.logger.info("There was no speech") else: self.logger.debug( "Got a finish speech packet that did not match any current request" ) elif ObjectIsCommand(plist, CancelRequest): # this is probably called when we need to kill a plugin # wait for thread to finish a send self.logger.debug("Should cancel current request") cancelRequest = CancelRequest(plist) if cancelRequest.refId in self.speech: (decoder, encoder, dictation) = self.speech[cancelRequest.refId] if decoder: decoder.destroy() if encoder: encoder.finish() encoder.destroy() del self.speech[cancelRequest.refId] if self.current_google_request != None: self.current_google_request.cancel() # if a google request is running (follow up listening..., plugin might get killed there by user) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse != None: self.current_running_plugin._abortPluginRun() self.current_running_plugin.waitForResponse.set() # if a plugin is running (processing, but not waiting for data from the device we kill it) if self.current_running_plugin != None: if self.current_running_plugin.waitForResponse == None: self.current_running_plugin._abortPluginRun() self.send_object(CancelSucceeded(cancelRequest.aceId)) elif ObjectIsCommand(plist, RollbackRequest): pass elif ObjectIsCommand(plist, SyncChunk): chunk = SyncChunk(plist) previous = self.syncAnchors[ chunk.key] if chunk.key in self.syncAnchors else None if previous != None: if previous.generation != chunk.preGen: chunkDenied = SyncChunkDenied(chunk.aceId) self.send_object(chunkDenied) return current = SyncAnchor() current.generation = chunk.postGen current.value = chunk.postGen current.validity = chunk.validity current.key = chunk.key self.syncAnchors[current.key] = current chunkAccepted = SyncChunkAccepted(chunk.aceId) chunkAccepted.current = current self.send_object(chunkAccepted) pass elif ObjectIsCommand(plist, GetSessionCertificate): getSessionCertificate = GetSessionCertificate(plist) sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCACert) sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCert) response = GetSessionCertificateResponse( getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER) self.send_object(response) elif ObjectIsCommand(plist, CreateSessionInfoRequest): # how does a positive answer look like? createSessionInfoRequest = CreateSessionInfoRequest(plist) #success = CreateSessionInfoResponse(createSessionInfoRequest.aceId) #success.sessionInfo = biplist.Data("\x01\x02BLABLABLBALBALBALBALBALBALBALBA") #success.validityDuration = 9600 #self.send_object(success) fail = CommandFailed(createSessionInfoRequest.aceId) fail.reason = "Not authenticated" fail.errorCode = 0 self.send_object(fail) ##self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"}) elif ObjectIsCommand(plist, CreateAssistant): createAssistant = CreateAssistant(plist) #create a new assistant helper = Assistant() helper.assistantId = str.upper(str(uuid.uuid4())) helper.language = createAssistant.language helper.activationToken = createAssistant.activationToken helper.connectionType = createAssistant.connectionType helper.validationData = createAssistant.validationData c = self.dbConnection.cursor() noError = True try: c.execute( "insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper)) self.dbConnection.commit() except sqlite3.Error: noError = False c.close() if noError: self.assistant = helper assiCreatedCMD = AssistantCreated(createAssistant.aceId) assiCreatedCMD.assistantId = helper.assistantId assiCreatedCMD.speechId = str(uuid.uuid4()) self.send_object(assiCreatedCMD) else: cmdFailed = CommandFailed(createAssistant.aceId) cmdFailed.reason = "Database Error" cmdFailed.errorCode = 2 self.send_object(cmdFailed) elif ObjectIsCommand(plist, SetAssistantData): setAssistantData = SetAssistantData(plist) # fill assistant if self.assistant != None: try: c = self.dbConnection.cursor() assi_id = self.assistant.assistantId self.assistant.initializeFromPlist( setAssistantData.to_plist()) self.assistant.assistantId = assi_id #Record the user firstName and nickName try: self.assistant.firstName = self.assistant.meCards[ 0].firstName.encode("utf-8") except: self.assistant.firstName = u'' try: self.assistant.nickName = self.assistant.meCards[ 0].nickName.encode("utf-8") except: self.assistant.nickName = u'' #Done recording c.execute( "update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId)) self.dbConnection.commit() c.close() except: cmdFailed = CommandFailed(setAssistantData.aceId) cmdFailed.reason = "Database Error" cmdFailed.errorCode = 2 self.send_object(cmdFailed) self.logger.exception( "Database Error on setting assistant data") else: cmdFailed = CommandFailed(setAssistantData.aceId) cmdFailed.reason = "Assistant to set data not found" cmdFailed.errorCode = 2 self.send_object(cmdFailed) self.logger.warning( "Trying to set assistant data without having a valid assistant" ) elif ObjectIsCommand(plist, LoadAssistant): loadAssistant = LoadAssistant(plist) try: c = self.dbConnection.cursor() c.execute( "select assistant from assistants where assistantId = ?", (loadAssistant.assistantId, )) self.dbConnection.commit() result = c.fetchone() if result == None: self.send_object(AssistantNotFound(loadAssistant.aceId)) self.logger.warning("Assistant not found in database!!") else: self.assistant = result[0] #update assistant from LoadAssistant self.assistant.language = loadAssistant.language self.assistant.connectionType = loadAssistant.connectionType if self.assistant.language == '' or self.assistant.language == None: self.logger.error( "No language is set for this assistant") c.execute( "delete from assistants where assistantId = ?", (plist['properties']['assistantId'], )) self.dbConnection.commit() cmdFailed = CommandFailed(loadAssistant.aceId) cmdFailed.reason = "Database error Assistant not found or language settings" cmdFailed.errorCode = 2 self.send_object(cmdFailed) else: loaded = AssistantLoaded(loadAssistant.aceId) loaded.version = "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e" loaded.requestSync = 0 try: loaded.dataAnchor = self.assistant.anchor except: loaded.dataAnchor = "removed" self.send_object(loaded) c.close() except: self.send_object(AssistantNotFound(loadAssistant.aceId)) self.logger.warning("Database error on fetching assistant") elif ObjectIsCommand(plist, DestroyAssistant): destroyAssistant = DestroyAssistant(plist) try: c = self.dbConnection.cursor() c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'], )) self.dbConnection.commit() c.close() destroyed = AssistantDestroyed(destroyAssistant.aceId) destroyed.assistantId = destroyAssistant.assistantId self.send_object(destroyed) except: self.send_object(AssistantNotFound(destroyAssistant.aceId)) self.logger.error("Database error on deleting assistant") elif ObjectIsCommand(plist, StartRequest): startRequest = StartRequest(plist) #this should also be handeled by special plugins, so lets call the plugin handling stuff self.process_recognized_speech( { 'hypotheses': [{ 'utterance': startRequest.utterance, 'confidence': 1.0 }] }, startRequest.aceId, False) pass
class HandleConnection(ssl_dispatcher): def __init__(self, conn): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None def handle_ssl_established(self): self.ssled = True def handle_ssl_shutdown(self): self.ssled = False def readable(self): if self.ssled: while self.socket.pending() > 0: self.handle_read_event() return True def handle_read(self): self.data += self.recv(8192) if not self.binary_mode: if "\r\n\r\n" in self.data: endOfHeader = self.data.find("\r\n\r\n")+4 self.header = self.data[:endOfHeader] self.data = self.data[endOfHeader:] print "Received new header from iDevice" print self.header print "Header end" self.binary_mode = True self.header_complete = True else: if not self.consumed_ace: print "Received removing ace instruction: ", repr(self.data[:4]) self.data = self.data[4:] self.consumed_ace = True self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02" #self.flush_output_buffer() # first process outstanding google answers THIS happens at least on each PING if self.gotGoogleAnswer: self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation) self.lastRequestId = None self.dictation = None self.googleData = None self.gotGoogleAnswer = False self.process_compressed_data() def handle_google_data(self, body, requestId, dictation): self.googleData = json.loads(body) self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def handle_google_failure(self, requestId, dictation): self.googleData = None self.lastRequestId = requestId self.dictation = dictation self.gotGoogleAnswer = True def send_object(self, obj): self.send_plist(obj.to_plist()) def send_plist(self, plist): print "Sending: ", plist bplist = biplist.writePlistToString(plist); self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist self.flush_unzipped_output() def send_pong(self, id): self.unzipped_output_buffer = struct.pack('>BI', 4, id) self.flush_unzipped_output() def process_recognized_speech(self, googleJson, requestId, dictation): if googleJson == None: # there was a network failure self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible")) self.send_object(baseObjects.RequestCompleted(requestID)) else: possible_matches = googleJson['hypotheses'] if len(possible_matches) > 0: answer = possible_matches[0]['utterance'] best_match_confidence = possible_matches[0]['confidence'] print u"Best matching result: \"{0}\" with a confidence of {1}%".format(answer, round(float(best_match_confidence)*100,2)) # construct a SpeechRecognized token = speechObjects.Token(answer, 0, 0, 1000.0, True, True) interpretation = speechObjects.Interpretation([token]) phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation]) recognition = speechObjects.Recognition([phrase]) recognized = speechObjects.SpeechRecognized(requestId, recognition) # Send speechRecognized to iDevice self.send_object(recognized) view = uiObjects.AddViews(requestId) # Example if 'how are you' in answer: view.views += [uiObjects.AssistantUtteranceView(text='Thank you. I\'m fine.')] # elif not dictation: # view.views += [uiObjects.AssistantUtteranceView(text=answer)] else: view.views += [uiObjects.AssistantUtteranceView(text='I don\'t know what that means, but at least it rhymes.')] self.send_object(view) # at the end we need to finish the request self.send_object(baseObjects.RequestCompleted(requestId)) def process_compressed_data(self): self.unzipped_input += self.decompressor.decompress(self.data) self.data = "" while self.hasNextObj(): object = self.read_next_object_from_unzipped() if object != None: print "Packet with class: ", object['class'] print "packet with content: ", object if object['class'] == 'GetSessionCertificate': caDer = caCert.as_der() serverDer = serverCert.as_der() self.send_plist({"class": "GetSessionCertificateResponse", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": object['aceId'], "properties":{"certificate": biplist.Data("\x01\x02"+struct.pack(">I", len(caDer))+caDer + struct.pack(">I", len(serverDer))+serverDer)}}) #self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": object['aceId'], "group":"com.apple.ace.system"}) if object['class'] == 'CreateSessionInfoRequest': # how does a positive answer look like? print "returning response" self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": object['aceId'], "group":"com.apple.ace.system"}) #self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"}) if object['class'] == 'CreateAssistant': self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": str(uuid.uuid4())}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": object['aceId']}) if object['class'] == 'SetAssistantData': # grab assistant data, how is a device identified? pass #probably Create Set and Load assistant work together, first we create one response with success, fill it with set..data and later can load it again using load... however.. what are valid responses to all three requests? if object['class'] == 'LoadAssistant': # reply with a AssistentLoaded # self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"}) self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"}) pass if object['class'] == 'DestroyAssistant': self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": object['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"}) if object['class'] == 'StartSpeechRequest' or object['class'] == 'StartSpeechDictation': decoder = speex.Decoder() decoder.initialize(mode=speex.SPEEX_MODEID_WB) encoder = flac.Encoder() encoder.initialize(16000, 1, 16) #16kHz sample rate, 1 channel, 16 bits per sample dictation=(object['class'] == 'StartSpeechDictation') self.speech[object['aceId']] = (decoder, encoder, dictation) if object['class'] == 'SpeechPacket': (decoder, encoder, dictation) = self.speech[object['refId']] pcm = decoder.decode(object['properties']['packets']) encoder.encode(pcm) if object['class'] == 'CancelRequest': # we should test if this stil exists.. del self.speech[object['refId']] if object['class'] == 'StartCorrectedSpeechRequest': self.process_recognized_speech({u'hypotheses': [{u'confidence': 1.0, u'utterance': str.lower(object['properties']['utterance'])}]}, object['aceId'], False) elif object['class'] == 'FinishSpeech': (decoder, encoder, dictation) = self.speech[object['refId']] decoder.destroy() encoder.finish() flacBin = encoder.getBinary() encoder.destroy() del self.speech[object['refId']] self.httpClient.make_google_request(flacBin, object['refId'], dictation, language='de-DE', allowCurses=True) def hasNextObj(self): if len(self.unzipped_input) == 0: return False cmd, inter1, inter2, data = struct.unpack('>BBBH', self.unzipped_input[:5]) if cmd in (3,4): #ping pong return True if cmd == 2: #print "expect: ", data+5, " received: ", len(self.unzipped_input) return ((data + 5) < len(self.unzipped_input)) def read_next_object_from_unzipped(self): cmd, inter1, inter2, data = struct.unpack('>BBBH', self.unzipped_input[:5]) print cmd, inter1, inter2, data if cmd == 3: #ping self.ping = data self.send_pong(self.pong) self.pong += 1 print "Returning a Pong" self.unzipped_input = self.unzipped_input[5:] return None object_size = data prefix = self.unzipped_input[:5] object_data = self.unzipped_input[5:object_size+5] self.unzipped_input = self.unzipped_input[object_size+5:] return self.parse_object(object_data) def parse_object(self, object_data): #this is a binary plist file plist = biplist.readPlistFromString(object_data) return plist def flush_unzipped_output(self): self.output_buffer += self.compressor.compress(self.unzipped_output_buffer) #make sure everything is compressed self.output_buffer += self.compressor.flush(zlib.Z_SYNC_FLUSH) self.unzipped_output_buffer = "" self.flush_output_buffer() def flush_output_buffer(self): if len(self.output_buffer) > 0: self.send(self.output_buffer) self.output_buffer = ""