def __init__(self, conn, lang):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt",
keyfile="server.passless.key",
server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data,
self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
self.dbConnection = db.getConnection()
self.assistant = None
self.sendLock = threading.Lock()
self.current_running_plugin = None
self.current_location = None
self.plugin_lastAceId = None
self.logger = logging.getLogger("logger")
self.lang = lang
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
self.lastPingTime = time.time()
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def __init__(self, conn):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
self.dbConnection = db.getConnection()
self.assistant = None
self.sendLock = threading.Lock()
self.current_running_plugin = None
self.current_location = None
self.plugin_lastAceId = None
self.logger = logging.getLogger("logger")
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
self.lastPingTime = time.time()
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def __init__(self, conn): asyncore.dispatcher_with_send.__init__(self, conn) self.ssled = False self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True) self.consumed_ace = False self.data = "" self.binary_mode = False self.decompressor = zlib.decompressobj() self.compressor = zlib.compressobj() self.unzipped_input = "" self.unzipped_output_buffer = "" self.output_buffer = "" self.speech = dict() self.pong = 1 self.ping = 0 self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure) self.gotGoogleAnswer = False self.googleData = None self.lastRequestId = None self.dictation = None
class HandleConnection(ssl_dispatcher):
__not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}"}
__websearch = {"de-DE": u"Websuche", "en-US": u"Websearch"}
def __init__(self, conn):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
self.dbConnection = db.getConnection()
self.assistant = None
self.sendLock = threading.Lock()
self.current_running_plugin = None
self.current_location = None
self.plugin_lastAceId = None
self.logger = logging.getLogger("logger")
def handle_ssl_established(self):
self.ssled = True
def handle_ssl_shutdown(self):
self.ssled = False
def readable(self):
if self.ssled:
while self.socket.pending() > 0:
self.handle_read_event()
return True
def handle_read(self):
self.data += self.recv(8192)
if not self.binary_mode:
if "\r\n\r\n" in self.data:
endOfHeader = self.data.find("\r\n\r\n")+4
self.header = self.data[:endOfHeader]
self.data = self.data[endOfHeader:]
self.logger.debug("--------------------------------------Header start------------------------------------")
self.logger.debug(self.header)
self.logger.debug("---------------------------------------Header end-------------------------------------")
self.binary_mode = True
self.header_complete = True
else:
if not self.consumed_ace:
self.logger.debug("Received removing ace instruction: {0}".format(repr(self.data[:4])))
self.data = self.data[4:]
self.consumed_ace = True
self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02"
#self.flush_output_buffer()
# first process outstanding google answers THIS happens at least on each PING
if self.gotGoogleAnswer:
self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation)
self.lastRequestId = None
self.dictation = None
self.googleData = None
self.gotGoogleAnswer = False
self.process_compressed_data()
def handle_google_data(self, body, requestId, dictation):
self.googleData = json.loads(body)
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def handle_google_failure(self, requestId, dictation):
self.googleData = None
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def send_object(self, obj):
self.send_plist(obj.to_plist())
def send_plist(self, plist):
self.sendLock.acquire()
self.logger.debug("Sending:\n{0}".format(pprint.pformat(plist, width=40)))
bplist = biplist.writePlistToString(plist);
#
self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist
self.flush_unzipped_output()
self.sendLock.release()
def send_pong(self, id):
self.sendLock.acquire()
self.unzipped_output_buffer = struct.pack('>BI', 4, id)
self.flush_unzipped_output()
self.sendLock.release()
def process_recognized_speech(self, googleJson, requestId, dictation):
if googleJson == None:
# there was a network failure
# is this the correct command to send?
self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(baseObjects.RequestCompleted(requestId))
else:
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match = best_match[0].upper()+best_match[1:]
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2)))
# construct a SpeechRecognized
token = speechObjects.Token(best_match, 0, 0, 1000.0, True, True)
interpretation = speechObjects.Interpretation([token])
phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = speechObjects.Recognition([phrase])
recognized = speechObjects.SpeechRecognized(requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
(clazz, method) = PluginManager.getPlugin(best_match, self.assistant.language)
if clazz != None and method != None:
plugin = clazz(method, best_match, self.assistant.language, self.send_object, self.send_plist, self.assistant, self.current_location)
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = uiObjects.AddViews(requestId)
errorText = HandleConnection.__not_recognized[self.assistant.language] if self.assistant.language in HandleConnection.__not_recognized else HandleConnection.__not_recognized["en-US"]
view.views += [uiObjects.AssistantUtteranceView(errorText.format(best_match), errorText.format(best_match))]
websearchText = HandleConnection.__websearch[self.assistant.language] if self.assistant.language in HandleConnection.__websearch else HandleConnection.__websearch["en-US"]
button = uiObjects.Button(text=websearchText)
cmd = systemObjects.SendCommands()
cmd.commands = [systemObjects.StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(baseObjects.RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(baseObjects.RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(baseObjects.RequestCompleted(requestId))
def process_compressed_data(self):
self.unzipped_input += self.decompressor.decompress(self.data)
self.data = ""
while self.hasNextObj():
reqObject = self.read_next_object_from_unzipped()
if reqObject != None:
self.logger.debug("Packet with class: {0}".format(reqObject['class']))
self.logger.debug("packet with content:\n{0}".format(pprint.pformat(reqObject, width=40)))
# first handle speech stuff
if 'refId' in reqObject:
# if the following holds, this packet is an answer to a request by a plugin
if reqObject['refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.info("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = reqObject
self.current_running_plugin.waitForResponse.set()
if ObjectIsCommand(reqObject, StartSpeechRequest) or ObjectIsCommand(reqObject, StartSpeechDictation):
self.logger.info("New start of speech received")
startSpeech = None
if ObjectIsCommand(reqObject, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(reqObject)
else:
dictation = False
startSpeech = StartSpeechRequest(reqObject)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation)
elif ObjectIsCommand(reqObject, SpeechPacket):
self.logger.info("Decoding speech packet")
speechPacket = SpeechPacket(reqObject)
(decoder, encoder, dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
elif reqObject['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': str.lower(reqObject['properties']['utterance'])}]}, reqObject['aceId'], False)
elif ObjectIsCommand(reqObject, FinishSpeech):
self.logger.info("End of speech received")
finishSpeech = FinishSpeech(reqObject)
(decoder, encoder, dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
self.logger.info("Sending flac to google for recognition")
self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True)
elif ObjectIsCommand(reqObject, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
cancelRequest = CancelRequest(reqObject)
if cancelRequest.refId in self.speech:
del self.speech[cancelRequest.refId]
self.send_object(CancelSucceeded(cancelRequest.aceId))
elif ObjectIsCommand(reqObject, GetSessionCertificate):
getSessionCertificate = GetSessionCertificate(reqObject)
response = GetSessionCertificateResponse(getSessionCertificate.aceId)
response.caCert = caCert.as_der()
response.sessionCert = serverCert.as_der()
self.send_object(response)
elif ObjectIsCommand(reqObject, CreateSessionInfoRequest):
# how does a positive answer look like?
createSessionInfoRequest = CreateSessionInfoRequest(reqObject)
fail = CommandFailed(createSessionInfoRequest.aceId)
fail.reason = "Not authenticated"
fail.errorCode = 0
self.send_object(fail)
#self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
elif reqObject['class'] == 'CreateAssistant':
#create a new assistant
helper = Assistant()
c = self.dbConnection.cursor()
noError = True
try:
c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error, e:
noError = False
c.close()
if noError:
self.assistant = helper
self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": reqObject['aceId']})
else:
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"})
elif reqObject['class'] == 'SetAssistantData':
# fill assistant
if self.assistant != None:
c = self.dbConnection.cursor()
objProperties = reqObject['properties']
self.assistant.censorSpeech = objProperties['censorSpeech']
self.assistant.timeZoneId = objProperties['timeZoneId']
self.assistant.language = objProperties['language']
self.assistant.region = objProperties['region']
c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
elif reqObject['class'] == 'LoadAssistant':
c = self.dbConnection.cursor()
c.execute("select assistant from assistants where assistantId = ?", (reqObject['properties']['assistantId'],))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
else:
self.assistant = result[0]
self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
c.close()
elif reqObject['class'] == 'DestroyAssistant':
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?", (reqObject['properties']['assistantId'],))
self.dbConnection.commit()
c.close()
self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": reqObject['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
elif reqObject['class'] == 'StartRequest':
#this should also be handeled by special plugins, so lets call the plugin handling stuff
self.process_recognized_speech({'hypotheses': [{'utterance': reqObject['properties']['utterance'], 'confidence': 1.0}]}, reqObject['aceId'], False)
class HandleConnection(ssl_dispatcher):
def __init__(self, conn):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
self.dbConnection = db.getConnection()
self.assistant = None
self.sendLock = threading.Lock()
self.current_running_plugin = None
self.logger = logging.getLogger("logger")
def handle_ssl_established(self):
self.ssled = True
def handle_ssl_shutdown(self):
self.ssled = False
def readable(self):
if self.ssled:
while self.socket.pending() > 0:
self.handle_read_event()
return True
def handle_read(self):
self.data += self.recv(8192)
if not self.binary_mode:
if "\r\n\r\n" in self.data:
endOfHeader = self.data.find("\r\n\r\n")+4
self.header = self.data[:endOfHeader]
self.data = self.data[endOfHeader:]
self.logger.debug("--------------------------------------Header start------------------------------------")
self.logger.debug(self.header)
self.logger.debug("---------------------------------------Header end-------------------------------------")
self.binary_mode = True
self.header_complete = True
else:
if not self.consumed_ace:
self.logger.debug("Received removing ace instruction: {0}".format(repr(self.data[:4])))
self.data = self.data[4:]
self.consumed_ace = True
self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02"
#self.flush_output_buffer()
# first process outstanding google answers THIS happens at least on each PING
if self.gotGoogleAnswer:
self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation)
self.lastRequestId = None
self.dictation = None
self.googleData = None
self.gotGoogleAnswer = False
self.process_compressed_data()
def handle_google_data(self, body, requestId, dictation):
self.googleData = json.loads(body)
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def handle_google_failure(self, requestId, dictation):
self.googleData = None
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def send_object(self, obj):
self.send_plist(obj.to_plist())
def send_plist(self, plist):
self.sendLock.acquire()
self.logger.debug("Sending:\n{0}".format(pprint.pformat(plist, width=40)))
bplist = biplist.writePlistToString(plist);
#
self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist
self.flush_unzipped_output()
self.sendLock.release()
def send_pong(self, id):
self.sendLock.acquire()
self.unzipped_output_buffer = struct.pack('>BI', 4, id)
self.flush_unzipped_output()
self.sendLock.release()
def process_recognized_speech(self, googleJson, requestId, dictation):
if googleJson == None:
# there was a network failure
self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(baseObjects.RequestCompleted(requestID))
else:
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2)))
# construct a SpeechRecognized
token = speechObjects.Token(best_match, 0, 0, 1000.0, True, True)
interpretation = speechObjects.Interpretation([token])
phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = speechObjects.Recognition([phrase])
recognized = speechObjects.SpeechRecognized(requestId, recognition)
if self.current_running_plugin == None:
(clazz, method) = PluginManager.getPlugin(best_match, self.assistant.language)
if clazz != None and method != None:
plugin = clazz(method, best_match, self.assistant.language)
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
self.send_object(baseObjects.RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(baseObjects.RequestCompleted(requestId))
def process_compressed_data(self):
self.unzipped_input += self.decompressor.decompress(self.data)
self.data = ""
while self.hasNextObj():
reqObject = self.read_next_object_from_unzipped()
if reqObject != None:
self.logger.debug("Packet with class: {0}".format(reqObject['class']))
self.logger.debug("packet with content:\n{0}".format(pprint.pformat(reqObject, width=40)))
# first handle speech stuff
if reqObject['class'] == 'StartSpeechRequest' or reqObject['class'] == 'StartSpeechDictation':
decoder = speex.Decoder()
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder = flac.Encoder()
encoder.initialize(16000, 1, 16) #16kHz sample rate, 1 channel, 16 bits per sample
dictation=(reqObject['class'] == 'StartSpeechDictation')
self.speech[reqObject['aceId']] = (decoder, encoder, dictation)
elif reqObject['class'] == 'SpeechPacket':
(decoder, encoder, dictation) = self.speech[reqObject['refId']]
pcm = decoder.decode(reqObject['properties']['packets'])
encoder.encode(pcm)
elif reqObject['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': str.lower(reqObject['properties']['utterance'])}]}, reqObject['aceId'], False)
elif reqObject['class'] == 'FinishSpeech':
(decoder, encoder, dictation) = self.speech[reqObject['refId']]
decoder.destroy()
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[reqObject['refId']]
self.httpClient.make_google_request(flacBin, reqObject['refId'], dictation, language=self.assistant.language, allowCurses=True)
elif reqObject['class'] == 'CancelRequest':
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
if reqObject['refId'] in self.speech:
del self.speech[reqObject['refId']]
self.send_plist({"class": "CancelSucceeded", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "properties":{"callbacks": []}})
# handle responses to plugin
elif self.current_running_plugin != None and reqObject['group'] != "com.apple.ace.system" and "refId" in reqObject:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
self.current_running_plugin.response = reqObject
self.current_running_plugin.refId = reqObject['refId']
self.current_running_plugin.waitForResponse.set()
else:
# handle other stuff
if reqObject['class'] == 'GetSessionCertificate':
caDer = caCert.as_der()
serverDer = serverCert.as_der()
self.send_plist({"class": "GetSessionCertificateResponse", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "properties":{"certificate": biplist.Data("\x01\x02"+struct.pack(">I", len(caDer))+caDer + struct.pack(">I", len(serverDer))+serverDer)}})
#self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"})
if reqObject['class'] == 'CreateSessionInfoRequest':
# how does a positive answer look like?
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"})
#self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
if reqObject['class'] == 'CreateAssistant':
#create a new assistant
helper = Assistant()
c = self.dbConnection.cursor()
noError = True
try:
c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error, e:
noError = False
c.close()
if noError:
self.assistant = helper
self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": reqObject['aceId']})
else:
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": reqObject['aceId'], "group":"com.apple.ace.system"})
if reqObject['class'] == 'SetAssistantData':
# fill assistant
if self.assistant != None:
c = self.dbConnection.cursor()
objProperties = reqObject['properties']
self.assistant.censorSpeech = objProperties['censorSpeech']
self.assistant.timeZoneId = objProperties['timeZoneId']
self.assistant.language = objProperties['language']
self.assistant.region = objProperties['region']
c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
if reqObject['class'] == 'LoadAssistant':
c = self.dbConnection.cursor()
c.execute("select assistant from assistants where assistantId = ?", (reqObject['properties']['assistantId'],))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
else:
self.assistant = result[0]
self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
c.close()
if reqObject['class'] == 'DestroyAssistant':
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?", (reqObject['properties']['assistantId'],))
self.dbConnection.commit()
c.close()
self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": reqObject['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
class SiriProtocolHandler(Siri):
__not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.", "nl-NL": u"Excuses, \"{0}\" versta ik niet."}
__websearch = {"de-DE": u"Websuche", "en-US": u"Websearch", "fr-FR": u"Rechercher sur le Web", "nl-NL": u"Zoeken op het web"}
__scheduling_interval_timeout__ = 20
__timeout_delay = 10
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
self.lastPingTime = time.time()
self.syncAnchors = dict()
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def seconds_since_last_ping(self):
return time.time() - self.lastPingTime
def connectionLost(self, reason):
try:
self.timeoutschedule.cancel()
except:
pass
if self.current_google_request != None:
self.current_google_request.cancel()
#ensure all decoder/encoder attemps are closed
for key in self.speech.keys():
(decoder, encoder, _) = self.speech[key]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destroy()
del self.speech
self.current_running_plugin = None
self.dbConnection = None
self.httpClient = None
Siri.connectionLost(self, reason)
def checkTimeout(self):
if self.seconds_since_last_ping() > SiriProtocolHandler.__timeout_delay:
self.logger.info("Connection timed out")
self.transport.loseConnection()
else:
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def handle_google_data(self, body, requestId, dictation):
self.current_google_request = None
if (body != None):
googleAnswer = json.loads(body)
for i in xrange(0,len(googleAnswer['hypotheses'])-1):
utterance = googleAnswer['hypotheses'][i]['utterance']
if len(utterance) == 1:
utterance = utterance.upper()
else:
utterance = utterance[0].upper() + utterance[1:]
googleAnswer['hypotheses'][i]['utterance'] = utterance
self.process_recognized_speech(googleAnswer, requestId, dictation)
else:
self.send_object(SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(RequestCompleted(requestId))
def received_ping(self, numOfPing):
self.pong += 1
self.lastPing = numOfPing
self.lastPingTime = time.time()
self.send_pong(self.pong)
def process_recognized_speech(self, googleJson, requestId, dictation):
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence) * 100, 2)))
# construct a SpeechRecognized
token = Token(best_match, 0, 0, 1000.0, True, True)
interpretation = Interpretation([token])
phrase = Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = Recognition([phrase])
recognized = SpeechRecognized(requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location))
if plugin != None:
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = UIAddViews(requestId)
errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"]
errorView = UIAssistantUtteranceView()
errorView.text = errorText.format(best_match)
errorView.speakableText = errorText.format(best_match)
view.views = [errorView]
websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"]
button = UIButton()
button.text = websearchText
cmd = SendCommands()
cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
def received_plist(self, plist):
self.logger.debug("Got packet with class: {0}".format(plist['class']))
self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40)))
# first handle speech stuff
if 'refId' in plist:
# if the following holds, this packet is an answer to a request by a plugin
if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.debug("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance']
self.current_running_plugin.waitForResponse.set()
return
if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation):
self.logger.debug("New start of speech received")
startSpeech = None
if ObjectIsCommand(plist, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(plist)
else:
dictation = False
startSpeech = StartSpeechRequest(plist)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation)
elif ObjectIsCommand(plist, SpeechPacket):
self.logger.debug("Decoding speech packet")
speechPacket = SpeechPacket(plist)
if speechPacket.refId in self.speech:
(decoder, encoder, dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
else:
self.logger.debug("Got a speech packet that did not match any current request")
elif plist['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False)
elif ObjectIsCommand(plist, FinishSpeech):
self.logger.debug("End of speech received")
finishSpeech = FinishSpeech(plist)
if finishSpeech.refId in self.speech:
(decoder, encoder, dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
flacBin = None
if encoder:
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
if flacBin != None:
self.logger.info("Sending flac to google for recognition")
try:
self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True)
except (AttributeError, TypeError):
self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.")
else:
self.logger.info("There was no speech")
else:
self.logger.debug("Got a finish speech packet that did not match any current request")
elif ObjectIsCommand(plist, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
self.logger.debug("Should cancel current request")
cancelRequest = CancelRequest(plist)
if cancelRequest.refId in self.speech:
(decoder, encoder, dictation) = self.speech[cancelRequest.refId]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destroy()
del self.speech[cancelRequest.refId]
if self.current_google_request != None:
self.current_google_request.cancel()
# if a google request is running (follow up listening..., plugin might get killed there by user)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
self.current_running_plugin._abortPluginRun()
self.current_running_plugin.waitForResponse.set()
# if a plugin is running (processing, but not waiting for data from the device we kill it)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse == None:
self.current_running_plugin._abortPluginRun()
self.send_object(CancelSucceeded(cancelRequest.aceId))
elif ObjectIsCommand(plist, RollbackRequest):
pass
elif ObjectIsCommand(plist, SyncChunk):
chunk = SyncChunk(plist)
previous = self.syncAnchors[chunk.key] if chunk.key in self.syncAnchors else None
if previous != None:
if previous.generation != chunk.preGen:
chunkDenied = SyncChunkDenied(chunk.aceId)
self.send_object(chunkDenied)
return
current = SyncAnchor()
current.generation = chunk.postGen
current.value = chunk.postGen
current.validity = chunk.validity
current.key = chunk.key
self.syncAnchors[current.key] = current
chunkAccepted = SyncChunkAccepted(chunk.aceId)
chunkAccepted.current = current
self.send_object(chunkAccepted)
pass
elif ObjectIsCommand(plist, GetSessionCertificate):
getSessionCertificate = GetSessionCertificate(plist)
sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCACert)
sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCert)
response = GetSessionCertificateResponse(getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER)
self.send_object(response)
elif ObjectIsCommand(plist, CreateSessionInfoRequest):
# how does a positive answer look like?
createSessionInfoRequest = CreateSessionInfoRequest(plist)
#success = CreateSessionInfoResponse(createSessionInfoRequest.aceId)
#success.sessionInfo = biplist.Data("\x01\x02BLABLABLBALBALBALBALBALBALBALBA")
#success.validityDuration = 9600
#self.send_object(success)
fail = CommandFailed(createSessionInfoRequest.aceId)
fail.reason = "Not authenticated"
fail.errorCode = 0
self.send_object(fail)
##self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
elif ObjectIsCommand(plist, CreateAssistant):
createAssistant = CreateAssistant(plist)
#create a new assistant
helper = Assistant()
helper.assistantId = str.upper(str(uuid.uuid4()))
helper.language = createAssistant.language
helper.activationToken = createAssistant.activationToken
helper.connectionType = createAssistant.connectionType
helper.validationData = createAssistant.validationData
c = self.dbConnection.cursor()
noError = True
try:
c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error:
noError = False
c.close()
if noError:
self.assistant = helper
assiCreatedCMD = AssistantCreated(createAssistant.aceId)
assiCreatedCMD.assistantId = helper.assistantId
assiCreatedCMD.speechId = str(uuid.uuid4())
self.send_object(assiCreatedCMD)
else:
cmdFailed = CommandFailed(createAssistant.aceId)
cmdFailed.reason = "Database Error"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
elif ObjectIsCommand(plist, SetAssistantData):
setAssistantData = SetAssistantData(plist)
# fill assistant
if self.assistant != None:
try:
c = self.dbConnection.cursor()
assi_id = self.assistant.assistantId
self.assistant.initializeFromPlist(setAssistantData.to_plist())
self.assistant.assistantId = assi_id
#Record the user firstName and nickName
try:
self.assistant.firstName = self.assistant.meCards[0].firstName.encode("utf-8")
except:
self.assistant.firstName = u''
try:
self.assistant.nickName = self.assistant.meCards[0].nickName.encode("utf-8")
except:
self.assistant.nickName = u''
#Done recording
c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
except:
cmdFailed = CommandFailed(setAssistantData.aceId)
cmdFailed.reason = "Database Error"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
self.logger.exception("Database Error on setting assistant data")
else:
cmdFailed = CommandFailed(setAssistantData.aceId)
cmdFailed.reason = "Assistant to set data not found"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
self.logger.warning("Trying to set assistant data without having a valid assistant")
elif ObjectIsCommand(plist, LoadAssistant):
loadAssistant = LoadAssistant(plist)
try:
c = self.dbConnection.cursor()
c.execute("select assistant from assistants where assistantId = ?", (loadAssistant.assistantId,))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_object(AssistantNotFound(loadAssistant.aceId))
self.logger.warning("Assistant not found in database!!")
else:
self.assistant = result[0]
#update assistant from LoadAssistant
self.assistant.language = loadAssistant.language
self.assistant.connectionType = loadAssistant.connectionType
if self.assistant.language == '' or self.assistant.language == None:
self.logger.error ("No language is set for this assistant")
c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],))
self.dbConnection.commit()
cmdFailed = CommandFailed(loadAssistant.aceId)
cmdFailed.reason = "Database error Assistant not found or language settings"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
else:
loaded = AssistantLoaded(loadAssistant.aceId)
loaded.version = "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e"
loaded.requestSync = 0
try:
loaded.dataAnchor = self.assistant.anchor
except:
loaded.dataAnchor = "removed"
self.send_object(loaded)
c.close()
except:
self.send_object(AssistantNotFound(loadAssistant.aceId))
self.logger.warning("Database error on fetching assistant")
elif ObjectIsCommand(plist, DestroyAssistant):
destroyAssistant = DestroyAssistant(plist)
try:
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],))
self.dbConnection.commit()
c.close()
destroyed = AssistantDestroyed(destroyAssistant.aceId)
destroyed.assistantId = destroyAssistant.assistantId
self.send_object(destroyed)
except:
self.send_object(AssistantNotFound(destroyAssistant.aceId))
self.logger.error("Database error on deleting assistant")
elif ObjectIsCommand(plist, StartRequest):
startRequest = StartRequest(plist)
#this should also be handeled by special plugins, so lets call the plugin handling stuff
self.process_recognized_speech({'hypotheses': [{'utterance': startRequest.utterance, 'confidence': 1.0}]}, startRequest.aceId, False)
pass
class SiriProtocolHandler(Siri):
__not_recognized = {"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.", "en-US": u"Sorry I don't understand {0}", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire."}
__websearch = {"de-DE": u"Websuche", "en-US": u"Websearch", "fr-FR": u"Rechercher sur le Web"}
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
def handle_google_data(self, body, requestId, dictation):
self.current_google_request = None
if (body != None):
googleAnswer = json.loads(body)
self.process_recognized_speech(googleAnswer, requestId, dictation)
else:
self.send_object(SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(RequestCompleted(requestId))
def received_ping(self, numOfPing):
self.pong += 1
self.lastPing = numOfPing
self.send_pong(self.pong)
def process_recognized_speech(self, googleJson, requestId, dictation):
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match = best_match[0].upper()+best_match[1:]
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence)*100,2)))
# construct a SpeechRecognized
token = Token(best_match, 0, 0, 1000.0, True, True)
interpretation = Interpretation([token])
phrase = Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = Recognition([phrase])
recognized = SpeechRecognized(requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location))
if plugin != None:
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = AddViews(requestId)
errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"]
view.views += [AssistantUtteranceView(errorText.format(best_match), errorText.format(best_match))]
websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"]
button = Button(text=websearchText)
cmd = SendCommands()
cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
def received_plist(self, plist):
self.logger.debug("Packet with class: {0}".format(plist['class']))
self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40)))
# first handle speech stuff
if 'refId' in plist:
# if the following holds, this packet is an answer to a request by a plugin
if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.debug("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance']
self.current_running_plugin.waitForResponse.set()
return
if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation):
self.logger.debug("New start of speech received")
startSpeech = None
if ObjectIsCommand(plist, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(plist)
else:
dictation = False
startSpeech = StartSpeechRequest(plist)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation)
elif ObjectIsCommand(plist, SpeechPacket):
self.logger.debug("Decoding speech packet")
speechPacket = SpeechPacket(plist)
(decoder, encoder, dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
elif plist['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False)
elif ObjectIsCommand(plist, FinishSpeech):
self.logger.debug("End of speech received")
finishSpeech = FinishSpeech(plist)
(decoder, encoder, dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
self.logger.info("Sending flac to google for recognition")
try:
self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True)
except AttributeError, TypeError:
self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.")
elif ObjectIsCommand(plist, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
self.logger.debug("Should cancel current request")
cancelRequest = CancelRequest(plist)
if cancelRequest.refId in self.speech:
del self.speech[cancelRequest.refId]
if self.current_google_request != None:
self.current_google_request.cancel()
self.send_object(CancelSucceeded(cancelRequest.aceId))
class SiriProtocolHandler(Siri):
__not_recognized = {"zh-HK": u"对不起，我不知道“{0}”是什么意思。", "zh-TW": u"对不起，我不知道“{0}”是什么意思。", "ja-JP": u"申し訳ありませんが、私は理解していない '{0}", "id-ID": u"Maaf, Saya tidak mengerti apa yang anda maksud dengan '{0}'.", "en-US": u"I don't know what you mean by ‘{0}’.", "fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.", "zh-CN": u"对不起，我不知道“{0}”是什么意思。"}
__websearch = {"zh-HK": u"搜索网页", "zh-TW": u"搜索网页", "ja-JP": u"Webを検索", "id-ID": u"cari di internet", "en-US": u"Search the Web", "fr-FR": u"Rechercher sur le Web", "zh-CN": u"搜索网页"}
__scheduling_interval_timeout__ = 20
__timeout_delay = 10
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
self.lastPingTime = time.time()
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def seconds_since_last_ping(self):
return time.time() - self.lastPingTime
def connectionLost(self, reason):
try:
self.timeoutschedule.cancel()
except:
pass
if self.current_google_request != None:
self.current_google_request.cancel()
#ensure all decoder/encoder attemps are closed
for key in self.speech.keys():
(decoder, encoder, _) = self.speech[key]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destroy()
del self.speech
self.current_running_plugin = None
self.dbConnection = None
self.httpClient = None
Siri.connectionLost(self, reason)
def checkTimeout(self):
if self.seconds_since_last_ping() > SiriProtocolHandler.__timeout_delay:
self.logger.info("Connection timed out")
self.transport.loseConnection()
else:
self.timeoutschedule = twisted.internet.reactor.callLater(SiriProtocolHandler.__scheduling_interval_timeout__, self.checkTimeout)
def handle_google_data(self, body, requestId, dictation):
self.current_google_request = None
if (body != None):
googleAnswer = json.loads(body)
for i in xrange(0,len(googleAnswer['hypotheses'])-1):
utterance = googleAnswer['hypotheses'][i]['utterance']
if len(utterance) == 1:
utterance = utterance.upper()
else:
utterance = utterance[0].upper() + utterance[1:]
googleAnswer['hypotheses'][i]['utterance'] = utterance
self.process_recognized_speech(googleAnswer, requestId, dictation)
else:
self.send_object(SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(RequestCompleted(requestId))
def received_ping(self, numOfPing):
self.pong += 1
self.lastPing = numOfPing
self.lastPingTime = time.time()
self.send_pong(self.pong)
def process_recognized_speech(self, googleJson, requestId, dictation):
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(u"Best matching result: \"{0}\" with a confidence of {1}%".format(best_match, round(float(best_match_confidence) * 100, 2)))
# construct a SpeechRecognized
token = Token(best_match, 0, 0, 1000.0, True, True)
interpretation = Interpretation([token])
phrase = Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = Recognition([phrase])
recognized = SpeechRecognized(requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
plugin = PluginManager.getPluginForImmediateExecution(self.assistant.assistantId, best_match, self.assistant.language, (self.send_object, self.send_plist, self.assistant, self.current_location))
if plugin != None:
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = UIAddViews(requestId)
errorText = SiriProtocolHandler.__not_recognized[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized["en-US"]
errorView = UIAssistantUtteranceView()
errorView.text = errorText.format(best_match)
errorView.speakableText = errorText.format(best_match)
view.views = [errorView]
websearchText = SiriProtocolHandler.__websearch[self.assistant.language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch["en-US"]
button = UIButton()
button.text = websearchText
cmd = SendCommands()
cmd.commands = [StartRequest(utterance=u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^".format(best_match))]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
def received_plist(self, plist):
self.logger.debug("Got packet with class: {0}".format(plist['class']))
self.logger.debug("packet with content:\n{0}".format(pprint.pformat(plist, width=40)))
# first handle speech stuff
if 'refId' in plist:
# if the following holds, this packet is an answer to a request by a plugin
if plist['refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.debug("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = plist if plist['class'] != "StartRequest" else plist['properties']['utterance']
self.current_running_plugin.waitForResponse.set()
return
if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(plist, StartSpeechDictation):
self.logger.debug("New start of speech received")
startSpeech = None
if ObjectIsCommand(plist, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(plist)
else:
dictation = False
startSpeech = StartSpeechRequest(plist)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else None, encoder, dictation)
elif ObjectIsCommand(plist, SpeechPacket):
self.logger.debug("Decoding speech packet")
speechPacket = SpeechPacket(plist)
if speechPacket.refId in self.speech:
(decoder, encoder, dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
else:
self.logger.debug("Got a speech packet that did not match any current request")
elif plist['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{'confidence': 1.0, 'utterance': plist['properties']['utterance']}]}, plist['aceId'], False)
elif ObjectIsCommand(plist, FinishSpeech):
self.logger.debug("End of speech received")
finishSpeech = FinishSpeech(plist)
if finishSpeech.refId in self.speech:
(decoder, encoder, dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
flacBin = None
if encoder:
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
if flacBin != None:
self.logger.info("Sending flac to google for recognition")
try:
self.current_google_request = self.httpClient.make_google_request(flacBin, finishSpeech.refId, dictation, language=self.assistant.language, allowCurses=True)
except (AttributeError, TypeError):
self.logger.warning("Unable to find language record for this assistant. Try turning Siri off and then back on.")
else:
self.logger.info("There was no speech")
else:
self.logger.debug("Got a finish speech packet that did not match any current request")
elif ObjectIsCommand(plist, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
self.logger.debug("Should cancel current request")
cancelRequest = CancelRequest(plist)
if cancelRequest.refId in self.speech:
(decoder, encoder, dictation) = self.speech[cancelRequest.refId]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destory()
del self.speech[cancelRequest.refId]
if self.current_google_request != None:
self.current_google_request.cancel()
# if a google request is running (follow up listening..., plugin might get killed there by user)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
self.current_running_plugin._abortPluginRun()
self.current_running_plugin.waitForResponse.set()
# if a plugin is running (processing, but not waiting for data from the device we kill it)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse == None:
self.current_running_plugin._abortPluginRun()
self.send_object(CancelSucceeded(cancelRequest.aceId))
elif ObjectIsCommand(plist, RollbackRequest):
pass
elif ObjectIsCommand(plist, GetSessionCertificate):
getSessionCertificate = GetSessionCertificate(plist)
sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCACert)
sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1, self.server.sessionCert)
response = GetSessionCertificateResponse(getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER)
self.send_object(response)
elif ObjectIsCommand(plist, CreateSessionInfoRequest):
# how does a positive answer look like?
createSessionInfoRequest = CreateSessionInfoRequest(plist)
fail = CommandFailed(createSessionInfoRequest.aceId)
fail.reason = "Not authenticated"
fail.errorCode = 0
self.send_object(fail)
#self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
elif plist['class'] == 'CreateAssistant':
#create a new assistant
helper = Assistant()
helper.assistantId = str.upper(str(uuid.uuid4()))
c = self.dbConnection.cursor()
noError = True
try:
c.execute("insert into assistants(assistantId, assistant) values (?,?)", (helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error:
noError = False
c.close()
if noError:
self.assistant = helper
self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": helper.assistantId}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": plist['aceId']})
else:
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"})
elif plist['class'] == 'SetAssistantData':
# fill assistant
if self.assistant != None:
try:
c = self.dbConnection.cursor()
objProperties = plist['properties']
self.assistant.censorSpeech = objProperties['censorSpeech']
self.assistant.timeZoneId = objProperties['timeZoneId']
self.assistant.language = objProperties['language']
self.assistant.region = objProperties['region']
#Here it is possible to support more languages combined with anyvoice package
#We can make assumption that the region will define the language.
if config.forcelanguage==True:
#Chinese Example
if self.assistant.region=='id_ID':
self.assistant.language = "id-ID"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='zh_CN':
self.assistant.language = "zh-CN"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='zh_TW':
self.assistant.language = "zh-TW"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='ko_KR':
self.assistant.language = "ko-KR"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='zh_HK':
self.assistant.language = "zh-HK"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='ru_RU':
self.assistant.language = "ru-RU"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
if self.assistant.region=='ja_JP':
self.assistant.language = "ja-JP"
self.logger.debug("Forced language to {0}".format(self.assistant.language))
#Record the user firstName and nickName
try:
self.assistant.firstName = objProperties["meCards"][0]["properties"]["firstName"].encode("utf-8")
except KeyError:
self.assistant.firstName = u''
try:
self.assistant.nickName = objProperties["meCards"][0]["properties"]["nickName"].encode("utf-8")
except KeyError:
self.assistant.nickName = u''
#Done recording
c.execute("update assistants set assistant = ? where assistantId = ?", (self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
except:
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"})
self.logger.error("Database Error on setting assistant data")
else:
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Assistant to set data not found", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"})
self.logger.warning("Trying to set assistant data without having a valid assistant")
elif plist['class'] == 'LoadAssistant':
try:
c = self.dbConnection.cursor()
c.execute("select assistant from assistants where assistantId = ?", (plist['properties']['assistantId'],))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
self.logger.warning("Assistant not found in database!!")
else:
self.assistant = result[0]
if self.assistant.language == '' or self.assistant.language == None:
self.logger.error ("No language is set for this assistant")
c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],))
self.dbConnection.commit()
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Database error Assistant not found or language settings ", "errorCode":2, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": plist['aceId'], "group":"com.apple.ace.system"})
else:
self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
c.close()
except:
self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
self.logger.warning("Database error on fetching assistant")
elif plist['class'] == 'DestroyAssistant':
try:
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?", (plist['properties']['assistantId'],))
self.dbConnection.commit()
c.close()
self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": plist['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
except:
self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
self.logger.error("Database error on deleting assistant")
elif plist['class'] == 'StartRequest':
#this should also be handeled by special plugins, so lets call the plugin handling stuff
self.process_recognized_speech({'hypotheses': [{'utterance': plist['properties']['utterance'], 'confidence': 1.0}]}, plist['aceId'], False)
pass
class HandleConnection(ssl_dispatcher):
__not_recognized = {
"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.",
"en-US": u"Sorry I don't understand {0}.\nConfidence: {1}%"
}
__websearch = {"de-DE": u"Websuche", "en-US": u"Websearch"}
def __init__(self, conn, lang):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt",
keyfile="server.passless.key",
server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data,
self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
self.dbConnection = db.getConnection()
self.assistant = None
self.sendLock = threading.Lock()
self.current_running_plugin = None
self.current_location = None
self.plugin_lastAceId = None
self.logger = logging.getLogger("logger")
self.lang = lang
def handle_ssl_established(self):
self.ssled = True
def handle_ssl_shutdown(self):
self.ssled = False
def readable(self):
if self.ssled:
while self.socket.pending() > 0:
self.handle_read_event()
return True
def handle_read(self):
self.data += self.recv(8192)
if not self.binary_mode:
if "\r\n\r\n" in self.data:
endOfHeader = self.data.find("\r\n\r\n") + 4
self.header = self.data[:endOfHeader]
self.data = self.data[endOfHeader:]
self.logger.debug(
"--------------------------------------Header start------------------------------------"
)
self.logger.debug(self.header)
self.logger.debug(
"---------------------------------------Header end-------------------------------------"
)
self.binary_mode = True
self.header_complete = True
else:
if not self.consumed_ace:
self.logger.debug(
"Received removing ace instruction: {0}".format(
repr(self.data[:4])))
self.data = self.data[4:]
self.consumed_ace = True
self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(
timeval=None, localtime=False, usegmt=True
) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02"
#self.flush_output_buffer()
# first process outstanding google answers THIS happens at least on each PING
if self.gotGoogleAnswer:
self.process_recognized_speech(self.googleData,
self.lastRequestId,
self.dictation)
self.lastRequestId = None
self.dictation = None
self.googleData = None
self.gotGoogleAnswer = False
self.process_compressed_data()
def handle_google_data(self, body, requestId, dictation):
self.googleData = json.loads(body)
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def handle_google_failure(self, requestId, dictation):
self.googleData = None
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def send_object(self, obj):
self.send_plist(obj.to_plist())
def send_plist(self, plist):
self.sendLock.acquire()
self.logger.debug("Sending:\n{0}".format(
pprint.pformat(plist, width=40)))
bplist = biplist.writePlistToString(plist)
#
self.unzipped_output_buffer = struct.pack('>BI', 2,
len(bplist)) + bplist
self.flush_unzipped_output()
self.sendLock.release()
def send_pong(self, id):
self.sendLock.acquire()
self.unzipped_output_buffer = struct.pack('>BI', 4, id)
self.flush_unzipped_output()
self.sendLock.release()
def process_recognized_speech(self, googleJson, requestId, dictation):
if googleJson == None:
# there was a network failure
# is this the correct command to send?
self.send_object(
speechObjects.SpeechFailure(
requestId, "No connection to Google possible"))
self.send_object(baseObjects.RequestCompleted(requestId))
else:
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match = best_match[0].upper() + best_match[1:]
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(
u"Best matching result: \"{0}\" with a confidence of {1}%".
format(best_match,
round(float(best_match_confidence) * 100, 2)))
# construct a SpeechRecognized
token = speechObjects.Token(best_match, 0, 0, 1000.0, True,
True)
interpretation = speechObjects.Interpretation([token])
phrase = speechObjects.Phrase(lowConfidence=False,
interpretations=[interpretation])
recognition = speechObjects.Recognition([phrase])
recognized = speechObjects.SpeechRecognized(
requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
(clazz, method) = PluginManager.getPlugin(
best_match, self.assistant.language)
if clazz != None and method != None:
plugin = clazz(method, best_match,
self.assistant.language,
self.send_object, self.send_plist,
self.assistant,
self.current_location)
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = uiObjects.AddViews(requestId)
errorText = HandleConnection.__not_recognized[
self.assistant.
language] if self.assistant.language in HandleConnection.__not_recognized else HandleConnection.__not_recognized[
"en-US"]
view.views += [
uiObjects.AssistantUtteranceView(
errorText.format(
best_match,
round(
float(best_match_confidence) * 100,
2)),
errorText.format(
best_match,
round(
float(best_match_confidence) * 100,
2)))
]
websearchText = HandleConnection.__websearch[
self.assistant.
language] if self.assistant.language in HandleConnection.__websearch else HandleConnection.__websearch[
"en-US"]
button = uiObjects.Button(text=websearchText)
cmd = systemObjects.SendCommands()
cmd.commands = [
systemObjects.StartRequest(
utterance=
u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^"
.format(best_match))
]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(
baseObjects.RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(
baseObjects.RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(baseObjects.RequestCompleted(requestId))
def process_compressed_data(self):
self.unzipped_input += self.decompressor.decompress(self.data)
self.data = ""
while self.hasNextObj():
reqObject = self.read_next_object_from_unzipped()
if reqObject != None:
self.logger.debug("Packet with class: {0}".format(
reqObject['class']))
self.logger.debug("packet with content:\n{0}".format(
pprint.pformat(reqObject, width=40)))
# first handle speech stuff
if 'refId' in reqObject:
# if the following holds, this packet is an answer to a request by a plugin
if reqObject[
'refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.info("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = reqObject
self.current_running_plugin.waitForResponse.set()
if ObjectIsCommand(reqObject,
StartSpeechRequest) or ObjectIsCommand(
reqObject, StartSpeechDictation):
self.logger.info("New start of speech received")
startSpeech = None
if ObjectIsCommand(reqObject, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(reqObject)
else:
dictation = False
startSpeech = StartSpeechRequest(reqObject)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else
None, encoder, dictation)
elif ObjectIsCommand(reqObject, SpeechPacket):
self.logger.info("Decoding speech packet")
speechPacket = SpeechPacket(reqObject)
(decoder, encoder,
dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
elif reqObject['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech(
{
u'hypotheses': [{
'confidence':
1.0,
'utterance':
str.lower(reqObject['properties']['utterance'])
}]
}, reqObject['aceId'], False)
elif ObjectIsCommand(reqObject, FinishSpeech):
self.logger.info("End of speech received")
finishSpeech = FinishSpeech(reqObject)
(decoder, encoder,
dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
self.logger.info("Sending flac to google for recognition")
self.httpClient.make_google_request(
flacBin,
finishSpeech.refId,
dictation,
language=self.assistant.language,
allowCurses=True)
elif ObjectIsCommand(reqObject, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
cancelRequest = CancelRequest(reqObject)
if cancelRequest.refId in self.speech:
del self.speech[cancelRequest.refId]
self.send_object(CancelSucceeded(cancelRequest.aceId))
elif ObjectIsCommand(reqObject, GetSessionCertificate):
getSessionCertificate = GetSessionCertificate(reqObject)
response = GetSessionCertificateResponse(
getSessionCertificate.aceId)
response.caCert = caCert.as_der()
response.sessionCert = serverCert.as_der()
self.send_object(response)
elif ObjectIsCommand(reqObject, CreateSessionInfoRequest):
# how does a positive answer look like?
createSessionInfoRequest = CreateSessionInfoRequest(
reqObject)
fail = CommandFailed(createSessionInfoRequest.aceId)
fail.reason = "Not authenticated"
fail.errorCode = 0
self.send_object(fail)
#self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":reqObject['aceId'], "group":"com.apple.ace.system"})
elif reqObject['class'] == 'CreateAssistant':
#create a new assistant
helper = Assistant()
c = self.dbConnection.cursor()
noError = True
try:
c.execute(
"insert into assistants(assistantId, assistant) values (?,?)",
(helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error, e:
noError = False
c.close()
if noError:
self.assistant = helper
self.send_plist({
"class": "AssistantCreated",
"properties": {
"speechId": str(uuid.uuid4()),
"assistantId": helper.assistantId
},
"group": "com.apple.ace.system",
"callbacks": [],
"aceId": str(uuid.uuid4()),
"refId": reqObject['aceId']
})
else:
self.send_plist({
"class": "CommandFailed",
"properties": {
"reason": "Database error",
"errorCode": 2,
"callbacks": []
},
"aceId": str(uuid.uuid4()),
"refId": reqObject['aceId'],
"group": "com.apple.ace.system"
})
elif reqObject['class'] == 'SetAssistantData':
# fill assistant
if self.assistant != None:
c = self.dbConnection.cursor()
objProperties = reqObject['properties']
self.assistant.censorSpeech = objProperties[
'censorSpeech']
self.assistant.timeZoneId = objProperties['timeZoneId']
#self.assistant.language = objProperties['language']
if self.lang != None:
self.assistant.language = self.lang
else:
self.assistant.language = objProperties['language']
self.assistant.region = objProperties['region']
c.execute(
"update assistants set assistant = ? where assistantId = ?",
(self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
elif reqObject['class'] == 'LoadAssistant':
c = self.dbConnection.cursor()
c.execute(
"select assistant from assistants where assistantId = ?",
(reqObject['properties']['assistantId'], ))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_plist({
"class": "AssistantNotFound",
"aceId": str(uuid.uuid4()),
"refId": reqObject['aceId'],
"group": "com.apple.ace.system"
})
else:
self.assistant = result[0]
self.send_plist({
"class": "AssistantLoaded",
"properties": {
"version":
"20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e",
"requestSync": False,
"dataAnchor": "removed"
},
"aceId": str(uuid.uuid4()),
"refId": reqObject['aceId'],
"group": "com.apple.ace.system"
})
c.close()
elif reqObject['class'] == 'DestroyAssistant':
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?",
(reqObject['properties']['assistantId'], ))
self.dbConnection.commit()
c.close()
self.send_plist({
"class": "AssistantDestroyed",
"properties": {
"assistantId":
reqObject['properties']['assistantId']
},
"aceId": str(uuid.uuid4()),
"refId": reqObject['aceId'],
"group": "com.apple.ace.system"
})
elif reqObject['class'] == 'StartRequest':
#this should also be handeled by special plugins, so lets call the plugin handling stuff
self.process_recognized_speech(
{
'hypotheses': [{
'utterance':
reqObject['properties']['utterance'],
'confidence':
1.0
}]
}, reqObject['aceId'], False)
class SiriProtocolHandler(Siri):
__not_recognized = {
"de-DE": u"Entschuldigung, ich verstehe \"{0}\" nicht.",
"en-US": u"Sorry I don't understand {0}",
"fr-FR": u"Désolé je ne comprends pas ce que \"{0}\" veut dire.",
"nl-NL": u"Excuses, \"{0}\" versta ik niet."
}
__websearch = {
"de-DE": u"Websuche",
"en-US": u"Websearch",
"fr-FR": u"Rechercher sur le Web",
"nl-NL": u"Zoeken op het web"
}
__scheduling_interval_timeout__ = 20
__timeout_delay = 10
def __init__(self, server, peer):
Siri.__init__(self, server, peer)
self.lastPing = 0
self.pong = 0
self.plugin_lastAceId = ""
self.current_running_plugin = None
self.dbConnection = server.dbConnection
self.assistant = None
self.speech = dict()
self.httpClient = AsyncOpenHttp(self.handle_google_data)
self.current_google_request = None
self.current_location = None
self.lastPingTime = time.time()
self.syncAnchors = dict()
self.timeoutschedule = twisted.internet.reactor.callLater(
SiriProtocolHandler.__scheduling_interval_timeout__,
self.checkTimeout)
def seconds_since_last_ping(self):
return time.time() - self.lastPingTime
def connectionLost(self, reason):
try:
self.timeoutschedule.cancel()
except:
pass
if self.current_google_request != None:
self.current_google_request.cancel()
#ensure all decoder/encoder attemps are closed
for key in self.speech.keys():
(decoder, encoder, _) = self.speech[key]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destroy()
del self.speech
self.current_running_plugin = None
self.dbConnection = None
self.httpClient = None
Siri.connectionLost(self, reason)
def checkTimeout(self):
if self.seconds_since_last_ping(
) > SiriProtocolHandler.__timeout_delay:
self.logger.info("Connection timed out")
self.transport.loseConnection()
else:
self.timeoutschedule = twisted.internet.reactor.callLater(
SiriProtocolHandler.__scheduling_interval_timeout__,
self.checkTimeout)
def handle_google_data(self, body, requestId, dictation):
self.current_google_request = None
if (body != None):
googleAnswer = json.loads(body)
for i in xrange(0, len(googleAnswer['hypotheses']) - 1):
utterance = googleAnswer['hypotheses'][i]['utterance']
if len(utterance) == 1:
utterance = utterance.upper()
else:
utterance = utterance[0].upper() + utterance[1:]
googleAnswer['hypotheses'][i]['utterance'] = utterance
self.process_recognized_speech(googleAnswer, requestId, dictation)
else:
self.send_object(
SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(RequestCompleted(requestId))
def received_ping(self, numOfPing):
self.pong += 1
self.lastPing = numOfPing
self.lastPingTime = time.time()
self.send_pong(self.pong)
def process_recognized_speech(self, googleJson, requestId, dictation):
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
best_match = possible_matches[0]['utterance']
best_match_confidence = possible_matches[0]['confidence']
self.logger.info(
u"Best matching result: \"{0}\" with a confidence of {1}%".
format(best_match,
round(float(best_match_confidence) * 100, 2)))
# construct a SpeechRecognized
token = Token(best_match, 0, 0, 1000.0, True, True)
interpretation = Interpretation([token])
phrase = Phrase(lowConfidence=False,
interpretations=[interpretation])
recognition = Recognition([phrase])
recognized = SpeechRecognized(requestId, recognition)
if not dictation:
if self.current_running_plugin == None:
plugin = PluginManager.getPluginForImmediateExecution(
self.assistant.assistantId, best_match,
self.assistant.language,
(self.send_object, self.send_plist, self.assistant,
self.current_location))
if plugin != None:
plugin.refId = requestId
plugin.connection = self
self.current_running_plugin = plugin
self.send_object(recognized)
self.current_running_plugin.start()
else:
self.send_object(recognized)
view = UIAddViews(requestId)
errorText = SiriProtocolHandler.__not_recognized[
self.assistant.
language] if self.assistant.language in SiriProtocolHandler.__not_recognized else SiriProtocolHandler.__not_recognized[
"en-US"]
errorView = UIAssistantUtteranceView()
errorView.text = errorText.format(best_match)
errorView.speakableText = errorText.format(best_match)
view.views = [errorView]
websearchText = SiriProtocolHandler.__websearch[
self.assistant.
language] if self.assistant.language in SiriProtocolHandler.__websearch else SiriProtocolHandler.__websearch[
"en-US"]
button = UIButton()
button.text = websearchText
cmd = SendCommands()
cmd.commands = [
StartRequest(
utterance=
u"^webSearchQuery^=^{0}^^webSearchConfirmation^=^Yes^"
.format(best_match))
]
button.commands = [cmd]
view.views.append(button)
self.send_object(view)
self.send_object(RequestCompleted(requestId))
elif self.current_running_plugin.waitForResponse != None:
# do we need to send a speech recognized here? i.d.k
self.current_running_plugin.response = best_match
self.current_running_plugin.refId = requestId
self.current_running_plugin.waitForResponse.set()
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
else:
self.send_object(recognized)
self.send_object(RequestCompleted(requestId))
def received_plist(self, plist):
self.logger.debug("Got packet with class: {0}".format(plist['class']))
self.logger.debug("packet with content:\n{0}".format(
pprint.pformat(plist, width=40)))
# first handle speech stuff
if 'refId' in plist:
# if the following holds, this packet is an answer to a request by a plugin
if plist[
'refId'] == self.plugin_lastAceId and self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
# just forward the object to the
# don't change it's refId, further requests must reference last FinishSpeech
self.logger.debug("Forwarding object to plugin")
self.plugin_lastAceId = None
self.current_running_plugin.response = plist if plist[
'class'] != "StartRequest" else plist['properties'][
'utterance']
self.current_running_plugin.waitForResponse.set()
return
if ObjectIsCommand(plist, StartSpeechRequest) or ObjectIsCommand(
plist, StartSpeechDictation):
self.logger.debug("New start of speech received")
startSpeech = None
if ObjectIsCommand(plist, StartSpeechDictation):
dictation = True
startSpeech = StartSpeechDictation(plist)
else:
dictation = False
startSpeech = StartSpeechRequest(plist)
decoder = speex.Decoder()
encoder = flac.Encoder()
speexUsed = False
if startSpeech.codec == StartSpeech.CodecSpeex_WB_Quality8Value:
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecSpeex_NB_Quality7Value:
decoder.initialize(mode=speex.SPEEX_MODEID_NB)
encoder.initialize(16000, 1, 16)
speexUsed = True
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_8000HzValue:
encoder.initialize(8000, 1, 16)
elif startSpeech.codec == StartSpeech.CodecPCM_Mono_16Bit_11025HzValue:
encoder.initialize(11025, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_16000HzValue:
encoder.initialize(16000, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_22050HzValue:
encoder.initialize(22050, 1, 16)
elif startSpeech.coded == StartSpeech.CodecPCM_Mono_16Bit_32000HzValue:
encoder.initialize(32000, 1, 16)
# we probably need resampling for sample rates other than 16kHz...
self.speech[startSpeech.aceId] = (decoder if speexUsed else None,
encoder, dictation)
elif ObjectIsCommand(plist, SpeechPacket):
self.logger.debug("Decoding speech packet")
speechPacket = SpeechPacket(plist)
if speechPacket.refId in self.speech:
(decoder, encoder, dictation) = self.speech[speechPacket.refId]
if decoder:
pcm = decoder.decode(speechPacket.packets)
else:
pcm = SpeechPacket.data # <- probably data... if pcm
encoder.encode(pcm)
else:
self.logger.debug(
"Got a speech packet that did not match any current request"
)
elif plist['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech(
{
u'hypotheses':
[{
'confidence': 1.0,
'utterance': plist['properties']['utterance']
}]
}, plist['aceId'], False)
elif ObjectIsCommand(plist, FinishSpeech):
self.logger.debug("End of speech received")
finishSpeech = FinishSpeech(plist)
if finishSpeech.refId in self.speech:
(decoder, encoder, dictation) = self.speech[finishSpeech.refId]
if decoder:
decoder.destroy()
flacBin = None
if encoder:
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[finishSpeech.refId]
if flacBin != None:
self.logger.info("Sending flac to google for recognition")
try:
self.current_google_request = self.httpClient.make_google_request(
flacBin,
finishSpeech.refId,
dictation,
language=self.assistant.language,
allowCurses=True)
except (AttributeError, TypeError):
self.logger.warning(
"Unable to find language record for this assistant. Try turning Siri off and then back on."
)
else:
self.logger.info("There was no speech")
else:
self.logger.debug(
"Got a finish speech packet that did not match any current request"
)
elif ObjectIsCommand(plist, CancelRequest):
# this is probably called when we need to kill a plugin
# wait for thread to finish a send
self.logger.debug("Should cancel current request")
cancelRequest = CancelRequest(plist)
if cancelRequest.refId in self.speech:
(decoder, encoder,
dictation) = self.speech[cancelRequest.refId]
if decoder:
decoder.destroy()
if encoder:
encoder.finish()
encoder.destroy()
del self.speech[cancelRequest.refId]
if self.current_google_request != None:
self.current_google_request.cancel()
# if a google request is running (follow up listening..., plugin might get killed there by user)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse != None:
self.current_running_plugin._abortPluginRun()
self.current_running_plugin.waitForResponse.set()
# if a plugin is running (processing, but not waiting for data from the device we kill it)
if self.current_running_plugin != None:
if self.current_running_plugin.waitForResponse == None:
self.current_running_plugin._abortPluginRun()
self.send_object(CancelSucceeded(cancelRequest.aceId))
elif ObjectIsCommand(plist, RollbackRequest):
pass
elif ObjectIsCommand(plist, SyncChunk):
chunk = SyncChunk(plist)
previous = self.syncAnchors[
chunk.key] if chunk.key in self.syncAnchors else None
if previous != None:
if previous.generation != chunk.preGen:
chunkDenied = SyncChunkDenied(chunk.aceId)
self.send_object(chunkDenied)
return
current = SyncAnchor()
current.generation = chunk.postGen
current.value = chunk.postGen
current.validity = chunk.validity
current.key = chunk.key
self.syncAnchors[current.key] = current
chunkAccepted = SyncChunkAccepted(chunk.aceId)
chunkAccepted.current = current
self.send_object(chunkAccepted)
pass
elif ObjectIsCommand(plist, GetSessionCertificate):
getSessionCertificate = GetSessionCertificate(plist)
sessionCA_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1,
self.server.sessionCACert)
sessionCert_DER = crypto.dump_certificate(crypto.FILETYPE_ASN1,
self.server.sessionCert)
response = GetSessionCertificateResponse(
getSessionCertificate.aceId, sessionCA_DER, sessionCert_DER)
self.send_object(response)
elif ObjectIsCommand(plist, CreateSessionInfoRequest):
# how does a positive answer look like?
createSessionInfoRequest = CreateSessionInfoRequest(plist)
#success = CreateSessionInfoResponse(createSessionInfoRequest.aceId)
#success.sessionInfo = biplist.Data("\x01\x02BLABLABLBALBALBALBALBALBALBALBA")
#success.validityDuration = 9600
#self.send_object(success)
fail = CommandFailed(createSessionInfoRequest.aceId)
fail.reason = "Not authenticated"
fail.errorCode = 0
self.send_object(fail)
##self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":plist['aceId'], "group":"com.apple.ace.system"})
elif ObjectIsCommand(plist, CreateAssistant):
createAssistant = CreateAssistant(plist)
#create a new assistant
helper = Assistant()
helper.assistantId = str.upper(str(uuid.uuid4()))
helper.language = createAssistant.language
helper.activationToken = createAssistant.activationToken
helper.connectionType = createAssistant.connectionType
helper.validationData = createAssistant.validationData
c = self.dbConnection.cursor()
noError = True
try:
c.execute(
"insert into assistants(assistantId, assistant) values (?,?)",
(helper.assistantId, helper))
self.dbConnection.commit()
except sqlite3.Error:
noError = False
c.close()
if noError:
self.assistant = helper
assiCreatedCMD = AssistantCreated(createAssistant.aceId)
assiCreatedCMD.assistantId = helper.assistantId
assiCreatedCMD.speechId = str(uuid.uuid4())
self.send_object(assiCreatedCMD)
else:
cmdFailed = CommandFailed(createAssistant.aceId)
cmdFailed.reason = "Database Error"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
elif ObjectIsCommand(plist, SetAssistantData):
setAssistantData = SetAssistantData(plist)
# fill assistant
if self.assistant != None:
try:
c = self.dbConnection.cursor()
assi_id = self.assistant.assistantId
self.assistant.initializeFromPlist(
setAssistantData.to_plist())
self.assistant.assistantId = assi_id
#Record the user firstName and nickName
try:
self.assistant.firstName = self.assistant.meCards[
0].firstName.encode("utf-8")
except:
self.assistant.firstName = u''
try:
self.assistant.nickName = self.assistant.meCards[
0].nickName.encode("utf-8")
except:
self.assistant.nickName = u''
#Done recording
c.execute(
"update assistants set assistant = ? where assistantId = ?",
(self.assistant, self.assistant.assistantId))
self.dbConnection.commit()
c.close()
except:
cmdFailed = CommandFailed(setAssistantData.aceId)
cmdFailed.reason = "Database Error"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
self.logger.exception(
"Database Error on setting assistant data")
else:
cmdFailed = CommandFailed(setAssistantData.aceId)
cmdFailed.reason = "Assistant to set data not found"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
self.logger.warning(
"Trying to set assistant data without having a valid assistant"
)
elif ObjectIsCommand(plist, LoadAssistant):
loadAssistant = LoadAssistant(plist)
try:
c = self.dbConnection.cursor()
c.execute(
"select assistant from assistants where assistantId = ?",
(loadAssistant.assistantId, ))
self.dbConnection.commit()
result = c.fetchone()
if result == None:
self.send_object(AssistantNotFound(loadAssistant.aceId))
self.logger.warning("Assistant not found in database!!")
else:
self.assistant = result[0]
#update assistant from LoadAssistant
self.assistant.language = loadAssistant.language
self.assistant.connectionType = loadAssistant.connectionType
if self.assistant.language == '' or self.assistant.language == None:
self.logger.error(
"No language is set for this assistant")
c.execute(
"delete from assistants where assistantId = ?",
(plist['properties']['assistantId'], ))
self.dbConnection.commit()
cmdFailed = CommandFailed(loadAssistant.aceId)
cmdFailed.reason = "Database error Assistant not found or language settings"
cmdFailed.errorCode = 2
self.send_object(cmdFailed)
else:
loaded = AssistantLoaded(loadAssistant.aceId)
loaded.version = "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e"
loaded.requestSync = 0
try:
loaded.dataAnchor = self.assistant.anchor
except:
loaded.dataAnchor = "removed"
self.send_object(loaded)
c.close()
except:
self.send_object(AssistantNotFound(loadAssistant.aceId))
self.logger.warning("Database error on fetching assistant")
elif ObjectIsCommand(plist, DestroyAssistant):
destroyAssistant = DestroyAssistant(plist)
try:
c = self.dbConnection.cursor()
c.execute("delete from assistants where assistantId = ?",
(plist['properties']['assistantId'], ))
self.dbConnection.commit()
c.close()
destroyed = AssistantDestroyed(destroyAssistant.aceId)
destroyed.assistantId = destroyAssistant.assistantId
self.send_object(destroyed)
except:
self.send_object(AssistantNotFound(destroyAssistant.aceId))
self.logger.error("Database error on deleting assistant")
elif ObjectIsCommand(plist, StartRequest):
startRequest = StartRequest(plist)
#this should also be handeled by special plugins, so lets call the plugin handling stuff
self.process_recognized_speech(
{
'hypotheses': [{
'utterance': startRequest.utterance,
'confidence': 1.0
}]
}, startRequest.aceId, False)
pass
class HandleConnection(ssl_dispatcher):
def __init__(self, conn):
asyncore.dispatcher_with_send.__init__(self, conn)
self.ssled = False
self.secure_connection(certfile="server.passless.crt", keyfile="server.passless.key", server_side=True)
self.consumed_ace = False
self.data = ""
self.binary_mode = False
self.decompressor = zlib.decompressobj()
self.compressor = zlib.compressobj()
self.unzipped_input = ""
self.unzipped_output_buffer = ""
self.output_buffer = ""
self.speech = dict()
self.pong = 1
self.ping = 0
self.httpClient = AsyncOpenHttp(self.handle_google_data, self.handle_google_failure)
self.gotGoogleAnswer = False
self.googleData = None
self.lastRequestId = None
self.dictation = None
def handle_ssl_established(self):
self.ssled = True
def handle_ssl_shutdown(self):
self.ssled = False
def readable(self):
if self.ssled:
while self.socket.pending() > 0:
self.handle_read_event()
return True
def handle_read(self):
self.data += self.recv(8192)
if not self.binary_mode:
if "\r\n\r\n" in self.data:
endOfHeader = self.data.find("\r\n\r\n")+4
self.header = self.data[:endOfHeader]
self.data = self.data[endOfHeader:]
print "Received new header from iDevice"
print self.header
print "Header end"
self.binary_mode = True
self.header_complete = True
else:
if not self.consumed_ace:
print "Received removing ace instruction: ", repr(self.data[:4])
self.data = self.data[4:]
self.consumed_ace = True
self.output_buffer = "HTTP/1.1 200 OK\r\nServer: Apache-Coyote/1.1\r\nDate: " + formatdate(timeval=None, localtime=False, usegmt=True) + "\r\nConnection: close\r\n\r\n\xaa\xcc\xee\x02"
#self.flush_output_buffer()
# first process outstanding google answers THIS happens at least on each PING
if self.gotGoogleAnswer:
self.process_recognized_speech(self.googleData, self.lastRequestId, self.dictation)
self.lastRequestId = None
self.dictation = None
self.googleData = None
self.gotGoogleAnswer = False
self.process_compressed_data()
def handle_google_data(self, body, requestId, dictation):
self.googleData = json.loads(body)
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def handle_google_failure(self, requestId, dictation):
self.googleData = None
self.lastRequestId = requestId
self.dictation = dictation
self.gotGoogleAnswer = True
def send_object(self, obj):
self.send_plist(obj.to_plist())
def send_plist(self, plist):
print "Sending: ", plist
bplist = biplist.writePlistToString(plist);
self.unzipped_output_buffer = struct.pack('>BI', 2,len(bplist)) + bplist
self.flush_unzipped_output()
def send_pong(self, id):
self.unzipped_output_buffer = struct.pack('>BI', 4, id)
self.flush_unzipped_output()
def process_recognized_speech(self, googleJson, requestId, dictation):
if googleJson == None:
# there was a network failure
self.send_object(speechObjects.SpeechFailure(requestId, "No connection to Google possible"))
self.send_object(baseObjects.RequestCompleted(requestID))
else:
possible_matches = googleJson['hypotheses']
if len(possible_matches) > 0:
answer = possible_matches[0]['utterance']
best_match_confidence = possible_matches[0]['confidence']
print u"Best matching result: \"{0}\" with a confidence of {1}%".format(answer, round(float(best_match_confidence)*100,2))
# construct a SpeechRecognized
token = speechObjects.Token(answer, 0, 0, 1000.0, True, True)
interpretation = speechObjects.Interpretation([token])
phrase = speechObjects.Phrase(lowConfidence=False, interpretations=[interpretation])
recognition = speechObjects.Recognition([phrase])
recognized = speechObjects.SpeechRecognized(requestId, recognition)
# Send speechRecognized to iDevice
self.send_object(recognized)
view = uiObjects.AddViews(requestId)
# Example
if 'how are you' in answer:
view.views += [uiObjects.AssistantUtteranceView(text='Thank you. I\'m fine.')]
# elif not dictation:
# view.views += [uiObjects.AssistantUtteranceView(text=answer)]
else:
view.views += [uiObjects.AssistantUtteranceView(text='I don\'t know what that means, but at least it rhymes.')]
self.send_object(view)
# at the end we need to finish the request
self.send_object(baseObjects.RequestCompleted(requestId))
def process_compressed_data(self):
self.unzipped_input += self.decompressor.decompress(self.data)
self.data = ""
while self.hasNextObj():
object = self.read_next_object_from_unzipped()
if object != None:
print "Packet with class: ", object['class']
print "packet with content: ", object
if object['class'] == 'GetSessionCertificate':
caDer = caCert.as_der()
serverDer = serverCert.as_der()
self.send_plist({"class": "GetSessionCertificateResponse", "group": "com.apple.ace.system", "aceId": str(uuid.uuid4()), "refId": object['aceId'], "properties":{"certificate": biplist.Data("\x01\x02"+struct.pack(">I", len(caDer))+caDer + struct.pack(">I", len(serverDer))+serverDer)}})
#self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": object['aceId'], "group":"com.apple.ace.system"})
if object['class'] == 'CreateSessionInfoRequest':
# how does a positive answer look like?
print "returning response"
self.send_plist({"class":"CommandFailed", "properties": {"reason":"Not authenticated", "errorCode":0, "callbacks":[]}, "aceId": str(uuid.uuid4()), "refId": object['aceId'], "group":"com.apple.ace.system"})
#self.send_plist({"class":"SessionValidationFailed", "properties":{"errorCode":"UnsupportedHardwareVersion"}, "aceId": str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"})
if object['class'] == 'CreateAssistant':
self.send_plist({"class": "AssistantCreated", "properties": {"speechId": str(uuid.uuid4()), "assistantId": str(uuid.uuid4())}, "group":"com.apple.ace.system", "callbacks":[], "aceId": str(uuid.uuid4()), "refId": object['aceId']})
if object['class'] == 'SetAssistantData':
# grab assistant data, how is a device identified?
pass
#probably Create Set and Load assistant work together, first we create one response with success, fill it with set..data and later can load it again using load... however.. what are valid responses to all three requests?
if object['class'] == 'LoadAssistant':
# reply with a AssistentLoaded
# self.send_plist({"class": "AssistantNotFound", "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"})
self.send_plist({"class": "AssistantLoaded", "properties": {"version": "20111216-32234-branches/telluride?cnxn=293552c2-8e11-4920-9131-5f5651ce244e", "requestSync":False, "dataAnchor":"removed"}, "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"})
pass
if object['class'] == 'DestroyAssistant':
self.send_plist({"class": "AssistantDestroyed", "properties": {"assistantId": object['properties']['assistantId']}, "aceId":str(uuid.uuid4()), "refId":object['aceId'], "group":"com.apple.ace.system"})
if object['class'] == 'StartSpeechRequest' or object['class'] == 'StartSpeechDictation':
decoder = speex.Decoder()
decoder.initialize(mode=speex.SPEEX_MODEID_WB)
encoder = flac.Encoder()
encoder.initialize(16000, 1, 16) #16kHz sample rate, 1 channel, 16 bits per sample
dictation=(object['class'] == 'StartSpeechDictation')
self.speech[object['aceId']] = (decoder, encoder, dictation)
if object['class'] == 'SpeechPacket':
(decoder, encoder, dictation) = self.speech[object['refId']]
pcm = decoder.decode(object['properties']['packets'])
encoder.encode(pcm)
if object['class'] == 'CancelRequest':
# we should test if this stil exists..
del self.speech[object['refId']]
if object['class'] == 'StartCorrectedSpeechRequest':
self.process_recognized_speech({u'hypotheses': [{u'confidence': 1.0, u'utterance': str.lower(object['properties']['utterance'])}]}, object['aceId'], False)
elif object['class'] == 'FinishSpeech':
(decoder, encoder, dictation) = self.speech[object['refId']]
decoder.destroy()
encoder.finish()
flacBin = encoder.getBinary()
encoder.destroy()
del self.speech[object['refId']]
self.httpClient.make_google_request(flacBin, object['refId'], dictation, language='de-DE', allowCurses=True)
def hasNextObj(self):
if len(self.unzipped_input) == 0:
return False
cmd, inter1, inter2, data = struct.unpack('>BBBH', self.unzipped_input[:5])
if cmd in (3,4): #ping pong
return True
if cmd == 2:
#print "expect: ", data+5, " received: ", len(self.unzipped_input)
return ((data + 5) < len(self.unzipped_input))
def read_next_object_from_unzipped(self):
cmd, inter1, inter2, data = struct.unpack('>BBBH', self.unzipped_input[:5])
print cmd, inter1, inter2, data
if cmd == 3: #ping
self.ping = data
self.send_pong(self.pong)
self.pong += 1
print "Returning a Pong"
self.unzipped_input = self.unzipped_input[5:]
return None
object_size = data
prefix = self.unzipped_input[:5]
object_data = self.unzipped_input[5:object_size+5]
self.unzipped_input = self.unzipped_input[object_size+5:]
return self.parse_object(object_data)
def parse_object(self, object_data):
#this is a binary plist file
plist = biplist.readPlistFromString(object_data)
return plist
def flush_unzipped_output(self):
self.output_buffer += self.compressor.compress(self.unzipped_output_buffer)
#make sure everything is compressed
self.output_buffer += self.compressor.flush(zlib.Z_SYNC_FLUSH)
self.unzipped_output_buffer = ""
self.flush_output_buffer()
def flush_output_buffer(self):
if len(self.output_buffer) > 0:
self.send(self.output_buffer)
self.output_buffer = ""