def __init__(self, keyword_server_url, keyword_extrator):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
class SimluateInput:
def __init__(self):
self.ks = KeywordClient(server_url="http://localhost:5000/")
self.std_spk = "You"
self.last_hyp = ""
self.ks.reset()
def update(self, utterance, delay):
time.sleep(delay)
self.ks.replaceLastUtterance(self.last_hyp,utterance, self.std_spk)
self.last_hyp = utterance
def add_new(self, utterance,delay):
time.sleep(delay)
self.ks.addUtterance(utterance, self.std_spk)
def complete(self, utterance):
self.ks.completeUtterance(utterance, self.std_spk)
def get_delay(self, word):
return len(word) * 0.015
def simulateSentence(self, sentence):
split = sentence.split(" ")
firstword = split[0]
self.add_new(firstword,self.get_delay(firstword))
for x in xrange(2,len(split)+1):
self.update(' '.join(split[:x]),self.get_delay(split[x-1]))
self.complete(sentence)
self.last_hyp = ""
def __init__(self,keyword_server_url, keyword_extrator, lang='en', decay=0.95, max_entries=4, blacklist_file=''):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
#number of times a wikipedia category has been encountered, from all added articles
self.categories = defaultdict(int)
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
self.lang = lang
self.decay = decay
self.max_entries = max_entries
if self.lang == 'en':
self.wiki_category_string = u'Category:'
elif self.lang == 'de':
self.wiki_category_string = u''
else:
print 'WARNING, unknown language', self.lang
self.wiki_category_string = ''
self.blacklist_ids = {}
if blacklist_file != '':
with codecs.open(blacklist_file,'r','utf-8') as infile:
for line in infile:
print 'blacklist wiki id:',line[:-1]
self.blacklist_ids[line[:-1]] = 1
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = '', input_microphone_id=-1):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.paudio = pyaudio.PyAudio()
self.print_devices()
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
self.abort = False
#self.keyword_extractor = extract.TermExtractor()
#self.keyword_extractor.filter = extract.permissiveFilter
if self.send_to_keywordserver:
self.keyword_client.addUtterance('',std_speaker)
self.last_hyp = ''
self.input_microphone_id = input_microphone_id
def __init__(self,keyword_server_url, keyword_extrator):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = ''):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.paudio = pyaudio.PyAudio()
self.print_devices()
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
#self.keyword_extractor = extract.TermExtractor()
#self.keyword_extractor.filter = extract.permissiveFilter
if self.send_to_keywordserver:
self.keyword_client.addUtterance('','You')
self.last_hyp = ''
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url='',
max_sentences=0):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
if self.send_to_keywordserver:
self.keyword_client.addUtterance('', 'You')
self.last_hyp = ''
self.max_sentences = max_sentences
class EventGenerator:
def __init__(self,keyword_server_url, keyword_extrator, lang='en', decay=0.95, max_entries=4, blacklist_file=''):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
#number of times a wikipedia category has been encountered, from all added articles
self.categories = defaultdict(int)
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
self.lang = lang
self.decay = decay
self.max_entries = max_entries
if self.lang == 'en':
self.wiki_category_string = u'Category:'
elif self.lang == 'de':
self.wiki_category_string = u''
else:
print 'WARNING, unknown language', self.lang
self.wiki_category_string = ''
self.blacklist_ids = {}
if blacklist_file != '':
with codecs.open(blacklist_file,'r','utf-8') as infile:
for line in infile:
print 'blacklist wiki id:',line[:-1]
self.blacklist_ids[line[:-1]] = 1
def topCategories(self,maxCategories=6):
topCat = sorted(self.categories.items(), key=lambda x:x[1], reverse=True)
topCat_json = [{'entry_id':idFromTitle(cat[0]),'title':cat[0].replace(u'Kategorie:',u''), 'url': u'https://simple.wikipedia.org/wiki/'
+ self.wiki_category_string + cat[0].replace(' ','_'), 'score': cat[1]} for cat in topCat if cat[1] > 1 and 'Wikipedia:' not in cat[0]]
return topCat_json[:maxCategories]
#Listen loop (redis)
#Todo: for other languages than English, utf8 de and encoding will be needed
def start_listen(self):
pubsub = red.pubsub()
pubsub.subscribe(my_redis_channel)
for message in pubsub.listen():
print 'New message:', message, type(message['data'])
if type(message['data']) == str:
json_message = json.loads(message['data'])
if 'handle' in json_message:
if json_message['handle'] == 'completeUtterance':
print 'handle: completeUtterance'
self.complete_transcript.append(json_message['utterance'])
self.send_relevant_entry_updates(self.max_entries,self.decay)
if json_message['handle'] == 'closed':
print 'handle: closed'
self.delDisplayId(json_message['entry_id'])
print json_message
elif json_message['handle'] == 'reset':
print 'handle: reset all'
self.complete_transcript = []
self.relevant_entries = {}
self.displayed_entries = []
self.categories = defaultdict(int)
self.keyword_client.resetTimer()
elif json_message['handle'] == 'setLanguage':
print 'handle: set language' #todo
# Add a relevant entry to the display, specify how many entries should be allowed maximally
def addDisplayEntry(self, entry_type, entry, max_entries=4):
print 'check to add', entry['title'], entry['score']
#TODO: Refactor the entry_type diretly into entry
if 'type' not in entry:
entry['type'] = entry_type
if 'entry_id' not in entry:
entry['entry_id'] = idFromTitle(entry['title'])
if entry['entry_id'] in self.blacklist_ids:
return False
#Determine position by its score
displayed_entries_get_score = dict_list_index_get_member(self.displayed_entries,'score')
insert_pos = reverse_bisect(displayed_entries_get_score, float(entry['score']))
#Only add entry if we want to insert it into the max_entries best entries
if insert_pos < max_entries:
self.displayed_entries.insert(insert_pos, dict(entry))
len_displayed_entries = len(self.displayed_entries)
#In this case, one of the previous best entries needs to be deleted:
if(len_displayed_entries > max_entries):
#Send delete entry events to entries those score is below the four best showed entries
for display_entry in self.displayed_entries[max_entries:]:
print 'del', display_entry['entry_id'], 'score fell below max_entries'
self.keyword_client.delRelevantEntry(display_entry['type'], display_entry['title'])
self.displayed_entries = self.displayed_entries[:max_entries]
len_displayed_entries = len(self.displayed_entries)
if insert_pos == len_displayed_entries -1:
insert_before = end_marker
print 'Insert',entry['entry_id'],'at the end'
else:
insert_before = self.displayed_entries[insert_pos+1]['entry_id']
print 'Insert',entry['entry_id'],'before',insert_before
print 'add', entry['title'], entry['score']
self.keyword_client.addRelevantEntry('wiki', entry['title'], entry['text'], entry['url'], entry['score'], insert_before)
return True
else:
print "Insert pos is:", insert_pos, "below max_entries for", entry["title"]
return False
# Delete a relevant entry from the display
def delDisplayEntry(self, entry_type, title):
print 'del',title
for i,display_entry in list(enumerate(self.displayed_entries)):
if (display_entry["title"] == title):
print 'del', display_entry["title"]
self.keyword_client.delRelevantEntry(entry_type, title)
del self.displayed_entries[i]
break
# Delete a relevant entry from the model (without sending updates to the display)
def delDisplayId(self, entry_id):
print 'del id',entry_id
for i,display_entry in list(enumerate(self.displayed_entries)):
if (display_entry['entry_id'] == entry_id):
print 'del', entry_id
#self.keyword_client.delRelevantEntry(entry_type, title)
del self.displayed_entries[i]
break
# Send relevant entry updates to the display, given a new full utterance.
# Also specify how many entries we want (max_entries) and how existing keywords should decay their score.
def send_relevant_entry_updates(self,max_entries=4, decay=.9, context_utts=9, extract_top_n_keywords=10, min_found_keywords=3, min_transcript_utts=2):
print 'send_relevant_entry_updates called'
with Timer() as t:
#Do the decay for the displayed entries:
#TODO: handle duplicate keywords and updated scores
for entry in self.displayed_entries:
entry["score"] *= decay
# keywords = self.ke.getKeywordsDruid(self.complete_transcript[-1])
# Take last 10 utterances and combine them
most_recent_transcript = " ".join(self.complete_transcript[-context_utts:])
# Extract top 9 keywords
keywords = self.ke.extract_best_keywords(most_recent_transcript, n_words=extract_top_n_keywords)
print keywords
#abort if we found very little keywords and haven't seen enough utterances
if len(keywords) < min_found_keywords or len(self.complete_transcript) < min_transcript_utts:
return
# Extract top wiki articles
new_relevant_entries = wiki_search_es.extract_best_articles(keywords, n=max_entries)
print "-> Extracted top ", len(new_relevant_entries), " documents", [(entry["title"], entry["score"]) for entry in new_relevant_entries]
new_relevant_entries = dict(zip([entry["title"] for entry in new_relevant_entries],
[entry for entry in new_relevant_entries ] ) )
new_relevant_entries_set = set(new_relevant_entries)
relevant_entries_set = set(self.relevant_entries)
num_added = 0
#generate add relevant entries
for key in new_relevant_entries_set - relevant_entries_set:
entry = new_relevant_entries[key]
if self.addDisplayEntry("wiki", entry):
num_added += 1
for category in entry["categories"]:
self.categories[category] += 1
#now look for changed scores (happens if a keyword got more important and gets mentioned again)
for key in (new_relevant_entries_set & relevant_entries_set):
entry = new_relevant_entries[key]
if entry["score"] > self.relevant_entries[key]["score"]:
print "score change for:",entry["title"], self.relevant_entries[key]["score"], "->", entry["score"]
found_displayed_entry = False
for display_entry in self.displayed_entries:
#already displayed, we could delete and read it, to reflect the new placement
if display_entry["title"] == key:
found_displayed_entry = True
#self.delDisplayEntry("wiki", entry["title"])
#self.addDisplayEntry("wiki", entry)
break
if not found_displayed_entry:
#not displayed, try to see if the higher score gets results in a document that is more important
self.addDisplayEntry("wiki", entry)
for key in new_relevant_entries_set - relevant_entries_set:
self.relevant_entries[key] = new_relevant_entries[key]
topCategories_Event = self.topCategories()
print topCategories_Event
# TODO: only send something if topCategories actually changes
self.keyword_client.sendCategories(topCategories_Event)
print 'send_relevant_entry_updates finished. Time needed:', t.secs, 'seconds.'
print 'Displayed entries should now be:',[entry['title'] for entry in self.displayed_entries]
print 'Added:',num_added
class KaldiClient(WebSocketClient):
def print_devices(self):
info = self.paudio.get_host_api_info_by_index(0)
numdevices = info.get('deviceCount')
#for each audio device, determine if is an input or an output and add it to the appropriate list and dictionary
for i in range (0,numdevices):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
print "Input Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name')
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxOutputChannels')>0:
print "Output Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name')
def getAudioDeviceByString(audioDeviceName):
info = self.paudio.get_host_api_info_by_index(0)
numdevices = info.get('deviceCount')
for i in range (0,numdevices):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
if audioDeviceName in self.paudio.get_device_info_by_host_api_device_index(0,i).get('name'):
return i
print 'No ',audioDeviceName,' microphone found, defaulting to last available input device...'
for i in reversed(range (0,numdevices)):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
return i
print 'No input device found! Please connect a microphone or recording device'
return -1
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = '', input_microphone_id=-1):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.paudio = pyaudio.PyAudio()
self.print_devices()
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
#self.keyword_extractor = extract.TermExtractor()
#self.keyword_extractor.filter = extract.permissiveFilter
if self.send_to_keywordserver:
self.keyword_client.addUtterance('','You')
self.last_hyp = ''
self.input_microphone_id = input_microphone_id
#@rate_limited(4)
def send_data(self, data):
if data is not None:
self.send(data, binary=True)
def opened(self):
#print "Socket opened!"
def send_data_to_ws():
buffer_size = 1024
if self.input_microphone_id == -1:
self.input_microphone_id = self.getAudioDeviceByString("Yamaha")
if self.input_microphone_id == -1:
sys.exit(-1)
else:
print 'Selecting device',self.input_microphone_id,'as input device'
stream = self.paudio.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024, input_device_index = self.input_microphone_id) #buffer
#f = open(self.fn, "rb")
if self.send_adaptation_state_filename is not None:
print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename
try:
adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r"))
self.send(json.dumps(dict(adaptation_state=adaptation_state_props)))
except:
e = sys.exc_info()[0]
print >> sys.stderr, "Failed to send adaptation state: ", e
abort = False
while not abort:
block = stream.read(buffer_size)
self.send_data(block)
print >> sys.stderr, "Audio sent, now sending EOS"
self.send("EOS")
t = threading.Thread(target=send_data_to_ws)
t.start()
# received decoding message from upstream Kaldi server
def received_message(self, m):
try:
response = json.loads(str(m))
#print >> sys.stderr, "RESPONSE:", response
#print >> sys.stderr, "JSON was:", m
if response['status'] == 0:
if 'result' in response:
trans = response['result']['hypotheses'][0]['transcript']
if response['result']['final']:
if trans not in ['a.','I.','i.','the.','but.','one.','it.','she.']:
self.final_hyps.append(trans)
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.keyword_client.completeUtterance(trans, std_speaker)
self.keyword_client.addUtterance('',std_speaker)
self.last_hyp = ''
complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps)
print u'\r\033[K',trans.replace(u'\n', u'\\n')
else:
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.last_hyp = trans
print_trans = trans.replace(u'\n', u'\\n')
print u'\r\033[K',print_trans
if 'adaptation_state' in response:
if self.save_adaptation_state_filename:
print u'Saving adaptation state to %s' % self.save_adaptation_state_filename
with open(self.save_adaptation_state_filename, 'w') as f:
f.write(json.dumps(response['adaptation_state']))
else:
print u'Received error from server (status %d)' % response['status']
if 'message' in response:
print 'Error message:', response['message']
except Exception:
print 'Exception in received_message'
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=10, file=sys.stdout)
def get_full_hyp(self, timeout=60):
return self.final_hyp_queue.get(timeout)
def closed(self, code, reason=None):
#print "Websocket closed() called"
#print >> sys.stderr
self.final_hyp_queue.put(' '.join(self.final_hyps))
class EventGenerator:
def __init__(self,keyword_server_url, keyword_extrator):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
#Listen loop (redis)
#Todo: for other languages than English, utf8 de and encoding will be needed
def start_listen(self):
pubsub = red.pubsub()
pubsub.subscribe(my_redis_channel)
for message in pubsub.listen():
print 'New message:', message, type(message["data"])
if type(message["data"]) == str:
json_message = json.loads(message["data"])
if "handle" in json_message:
print json_message
if json_message["handle"] == "completeUtterance":
self.complete_transcript.append(json_message["utterance"])
self.send_relevant_entry_updates()
elif json_message["handle"] == "reset":
print 'reset all'
self.complete_transcript = []
self.relevant_entries = {}
self.displayed_entries = []
# Add a relevant entry to the display
def addDisplayEntry(entry,max_entries=4):
insert_pos = bisect.bisect(self.displayed_entries, float(entry["score"]))
#Only add entry if we want to insert it into the max_entries best entries
if insert_pos < max_entries:
#Determine position by its score
bisect.insort(self.displayed_entries, float(entry["score"]))
len_displayed_entries = len(displayed_entries)
#In this case, one of the previous best entries needs to be deleted:
if(len_displayed_entries > max_entries):
#Send delete entry events to entries those score is below the four best showed entries
for entry in self.displayed_entries[max_entries:]:
self.delDisplayEntry(entry_type,title)
self.displayed_entries = self.displayed_entries[:max_entries]
len_displayed_entries = len(displayed_entries)
if insert_pos == len_displayed_entries -1:
insert_before = '#end#'
else:
insert_before = self.displayed_entries[insert_before+1]["title"]
self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"], insert_before)
# Delete a relevant entry from the display
def delDisplayEntry(entry_type,title):
self.keyword_client.delRelevantEntry(entry_type, title)
#@profile
def send_relevant_entry_updates(self,max_entries=4):
print 'send_relevant_entry_updates called'
keywords = self.ke.getKeywordsDruid('\n'.join([sentence[:-1] for sentence in self.complete_transcript]))
new_relevant_entries = wiki_search.getSummariesSingleKeyword(keywords,max_entries,lang='en',pics_folder='pics/')
print new_relevant_entries
#generate del relevant entries
for key in set(self.relevant_entries) - set(new_relevant_entries):
entry = self.relevant_entries[key]
self.keyword_client.delRelevantEntry("wiki", entry["title"])
print 'del',key
#generate add relevant entries
for key in set(new_relevant_entries) - set(self.relevant_entries):
entry = new_relevant_entries[key]
self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"])
print 'add',key
#TODO: Update scores of existing entries in self.displayed_entries (?)
self.relevant_entries = new_relevant_entries
class KaldiClient(WebSocketClient):
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url='',
max_sentences=0):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
if self.send_to_keywordserver:
self.keyword_client.addUtterance('', 'You')
self.last_hyp = ''
self.max_sentences = max_sentences
@rate_limited(4)
def send_data(self, data):
self.send(data, binary=True)
def opened(self):
# print "Socket opened!"
def send_data_to_ws():
f = open(self.fn, "rb")
if self.send_adaptation_state_filename is not None:
print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename
try:
adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r"))
self.send(json.dumps(dict(adaptation_state=adaptation_state_props)))
except:
e = sys.exc_info()[0]
print >> sys.stderr, "Failed to send adaptation state: ", e
for block in iter(lambda: f.read(self.byterate / 4), ""):
if self.maximum_sentences_reached():
break
self.send_data(block)
print >> sys.stderr, "Audio sent, now sending EOS"
self.send("EOS")
t = threading.Thread(target=send_data_to_ws)
t.start()
# received decoding message from upstream Kaldi server
def received_message(self, m):
if self.maximum_sentences_reached():
return
try:
response = json.loads(str(m))
# print >> sys.stderr, "RESPONSE:", response
# print >> sys.stderr, "JSON was:", m
if response['status'] == 0:
if 'result' in response:
trans = response['result']['hypotheses'][0]['transcript']
if response['result']['final']:
if trans not in ['a.', 'I.', 'i.', 'the.', 'but.', 'one.', 'it.', 'she.']:
self.final_hyps.append(trans)
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.keyword_client.completeUtterance(trans, std_speaker)
self.keyword_client.addUtterance('', std_speaker)
self.last_hyp = ''
complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps)
print u'\r\033[K', trans.replace(u'\n', u'\\n')
else:
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.last_hyp = trans
print_trans = trans.replace(u'\n', u'\\n')
print u'\r\033[K', print_trans
if 'adaptation_state' in response:
if self.save_adaptation_state_filename:
print u'Saving adaptation state to %s' % self.save_adaptation_state_filename
with open(self.save_adaptation_state_filename, 'w') as f:
f.write(json.dumps(response['adaptation_state']))
else:
print u'Received error from server (status %d)' % response['status']
if 'message' in response:
print 'Error message:', response['message']
except Exception:
print 'Exception in received_message'
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=10, file=sys.stdout)
def get_full_hyp(self, timeout=60):
return self.final_hyp_queue.get(timeout)
# Returns True if the maximum number of sentences defined by the user have been transcribed.
def maximum_sentences_reached(self):
return self.max_sentences != 0 and len(self.final_hyps) >= self.max_sentences
def closed(self, code, reason=None):
# print "Websocket closed() called"
# print >> sys.stderr
self.final_hyp_queue.put(" ".join(self.final_hyps))
class EventGenerator:
def __init__(self, keyword_server_url, keyword_extrator):
self.complete_transcript = []
#dict with all relevant entries
self.relevant_entries = {}
#sorted list of displayed entries
self.displayed_entries = []
self.keyword_client = KeywordClient(keyword_server_url)
self.ke = keyword_extrator
#Listen loop (redis)
#Todo: for other languages than English, utf8 de and encoding will be needed
def start_listen(self):
pubsub = red.pubsub()
pubsub.subscribe(my_redis_channel)
for message in pubsub.listen():
print 'New message:', message, type(message["data"])
if type(message["data"]) == str:
json_message = json.loads(message["data"])
if "handle" in json_message:
print json_message
if json_message["handle"] == "completeUtterance":
self.complete_transcript.append(
json_message["utterance"])
self.send_relevant_entry_updates()
elif json_message["handle"] == "reset":
print 'reset all'
self.complete_transcript = []
self.relevant_entries = {}
self.displayed_entries = []
# Add a relevant entry to the display
def addDisplayEntry(entry, max_entries=4):
insert_pos = bisect.bisect(self.displayed_entries,
float(entry["score"]))
#Only add entry if we want to insert it into the max_entries best entries
if insert_pos < max_entries:
#Determine position by its score
bisect.insort(self.displayed_entries, float(entry["score"]))
len_displayed_entries = len(displayed_entries)
#In this case, one of the previous best entries needs to be deleted:
if (len_displayed_entries > max_entries):
#Send delete entry events to entries those score is below the four best showed entries
for entry in self.displayed_entries[max_entries:]:
self.delDisplayEntry(entry_type, title)
self.displayed_entries = self.displayed_entries[:max_entries]
len_displayed_entries = len(displayed_entries)
if insert_pos == len_displayed_entries - 1:
insert_before = '#end#'
else:
insert_before = self.displayed_entries[insert_before +
1]["title"]
self.keyword_client.addRelevantEntry("wiki", entry["title"],
entry["text"], entry["url"],
entry["score"], insert_before)
# Delete a relevant entry from the display
def delDisplayEntry(entry_type, title):
self.keyword_client.delRelevantEntry(entry_type, title)
#@profile
def send_relevant_entry_updates(self, max_entries=4):
print 'send_relevant_entry_updates called'
keywords = self.ke.getKeywordsDruid('\n'.join(
[sentence[:-1] for sentence in self.complete_transcript]))
new_relevant_entries = wiki_search.getSummariesSingleKeyword(
keywords, max_entries, lang='en', pics_folder='pics/')
print new_relevant_entries
#generate del relevant entries
for key in set(self.relevant_entries) - set(new_relevant_entries):
entry = self.relevant_entries[key]
self.keyword_client.delRelevantEntry("wiki", entry["title"])
print 'del', key
#generate add relevant entries
for key in set(new_relevant_entries) - set(self.relevant_entries):
entry = new_relevant_entries[key]
self.keyword_client.addRelevantEntry("wiki", entry["title"],
entry["text"], entry["url"],
entry["score"])
print 'add', key
#TODO: Update scores of existing entries in self.displayed_entries (?)
self.relevant_entries = new_relevant_entries
class KaldiClient(WebSocketClient):
def print_devices(self):
info = self.paudio.get_host_api_info_by_index(0)
numdevices = info.get('deviceCount')
#for each audio device, determine if is an input or an output and add it to the appropriate list and dictionary
for i in range (0,numdevices):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
print "Input Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name')
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxOutputChannels')>0:
print "Output Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name')
def getYamahaID(self):
info = self.paudio.get_host_api_info_by_index(0)
numdevices = info.get('deviceCount')
for i in range (0,numdevices):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
if 'Yamaha' in self.paudio.get_device_info_by_host_api_device_index(0,i).get('name'):
return i
print 'No yamaha microphone found, defaulting to last available input device...'
for i in reversed(range (0,numdevices)):
if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0:
return i
print 'No input device found! Please connect a microphone or recording device'
return -1
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000,
save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = ''):
super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq)
self.final_hyps = []
self.fn = filename
self.byterate = byterate
self.final_hyp_queue = Queue.Queue()
self.save_adaptation_state_filename = save_adaptation_state_filename
self.send_adaptation_state_filename = send_adaptation_state_filename
self.paudio = pyaudio.PyAudio()
self.print_devices()
self.keyword_client = KeywordClient(keyword_server_url)
self.keyword_client.reset()
self.send_to_keywordserver = not (keyword_server_url == '')
#self.keyword_extractor = extract.TermExtractor()
#self.keyword_extractor.filter = extract.permissiveFilter
if self.send_to_keywordserver:
self.keyword_client.addUtterance('','You')
self.last_hyp = ''
#@rate_limited(4)
def send_data(self, data):
if data is not None:
self.send(data, binary=True)
def opened(self):
#print "Socket opened!"
def send_data_to_ws():
buffer_size = 1024
yamahaID = self.getYamahaID()
if yamahaID == -1:
sys.exit(-1)
else:
print 'Selecting device',yamahaID,'as input device'
stream = self.paudio.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024, input_device_index = yamahaID) #buffer
#f = open(self.fn, "rb")
if self.send_adaptation_state_filename is not None:
print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename
try:
adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r"))
self.send(json.dumps(dict(adaptation_state=adaptation_state_props)))
except:
e = sys.exc_info()[0]
print >> sys.stderr, "Failed to send adaptation state: ", e
abort = False
while not abort:
block = stream.read(buffer_size)
self.send_data(block)
print >> sys.stderr, "Audio sent, now sending EOS"
self.send("EOS")
t = threading.Thread(target=send_data_to_ws)
t.start()
# received decoding message from upstream Kaldi server
def received_message(self, m):
try:
response = json.loads(str(m))
#print >> sys.stderr, "RESPONSE:", response
#print >> sys.stderr, "JSON was:", m
if response['status'] == 0:
if 'result' in response:
trans = response['result']['hypotheses'][0]['transcript']
if response['result']['final']:
if trans not in ['a.','I.','i.','the.','but.','one.','it.','she.']:
self.final_hyps.append(trans)
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.keyword_client.completeUtterance(trans, std_speaker)
self.keyword_client.addUtterance('',std_speaker)
self.last_hyp = ''
complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps)
print u'\r\033[K',trans.replace(u'\n', u'\\n')
else:
if self.send_to_keywordserver:
self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker)
self.last_hyp = trans
print_trans = trans.replace(u'\n', u'\\n')
print u'\r\033[K',print_trans
if 'adaptation_state' in response:
if self.save_adaptation_state_filename:
print u'Saving adaptation state to %s' % self.save_adaptation_state_filename
with open(self.save_adaptation_state_filename, 'w') as f:
f.write(json.dumps(response['adaptation_state']))
else:
print u'Received error from server (status %d)' % response['status']
if 'message' in response:
print 'Error message:', response['message']
except Exception:
print 'Exception in received_message'
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=10, file=sys.stdout)
def get_full_hyp(self, timeout=60):
return self.final_hyp_queue.get(timeout)
def closed(self, code, reason=None):
#print "Websocket closed() called"
#print >> sys.stderr
self.final_hyp_queue.put(' '.join(self.final_hyps))
def __init__(self):
self.ks = KeywordClient(server_url="http://localhost:5000/")
self.std_spk = "You"
self.last_hyp = ""
self.ks.reset()
