def __init__(self, keyword_server_url, keyword_extrator): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator
class SimluateInput: def __init__(self): self.ks = KeywordClient(server_url="http://localhost:5000/") self.std_spk = "You" self.last_hyp = "" self.ks.reset() def update(self, utterance, delay): time.sleep(delay) self.ks.replaceLastUtterance(self.last_hyp,utterance, self.std_spk) self.last_hyp = utterance def add_new(self, utterance,delay): time.sleep(delay) self.ks.addUtterance(utterance, self.std_spk) def complete(self, utterance): self.ks.completeUtterance(utterance, self.std_spk) def get_delay(self, word): return len(word) * 0.015 def simulateSentence(self, sentence): split = sentence.split(" ") firstword = split[0] self.add_new(firstword,self.get_delay(firstword)) for x in xrange(2,len(split)+1): self.update(' '.join(split[:x]),self.get_delay(split[x-1])) self.complete(sentence) self.last_hyp = ""
def __init__(self,keyword_server_url, keyword_extrator, lang='en', decay=0.95, max_entries=4, blacklist_file=''): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] #number of times a wikipedia category has been encountered, from all added articles self.categories = defaultdict(int) self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator self.lang = lang self.decay = decay self.max_entries = max_entries if self.lang == 'en': self.wiki_category_string = u'Category:' elif self.lang == 'de': self.wiki_category_string = u'' else: print 'WARNING, unknown language', self.lang self.wiki_category_string = '' self.blacklist_ids = {} if blacklist_file != '': with codecs.open(blacklist_file,'r','utf-8') as infile: for line in infile: print 'blacklist wiki id:',line[:-1] self.blacklist_ids[line[:-1]] = 1
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = '', input_microphone_id=-1): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.paudio = pyaudio.PyAudio() self.print_devices() self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') self.abort = False #self.keyword_extractor = extract.TermExtractor() #self.keyword_extractor.filter = extract.permissiveFilter if self.send_to_keywordserver: self.keyword_client.addUtterance('',std_speaker) self.last_hyp = '' self.input_microphone_id = input_microphone_id
def __init__(self,keyword_server_url, keyword_extrator): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = ''): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.paudio = pyaudio.PyAudio() self.print_devices() self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') #self.keyword_extractor = extract.TermExtractor() #self.keyword_extractor.filter = extract.permissiveFilter if self.send_to_keywordserver: self.keyword_client.addUtterance('','You') self.last_hyp = ''
def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url='', max_sentences=0): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') if self.send_to_keywordserver: self.keyword_client.addUtterance('', 'You') self.last_hyp = '' self.max_sentences = max_sentences
class EventGenerator: def __init__(self,keyword_server_url, keyword_extrator, lang='en', decay=0.95, max_entries=4, blacklist_file=''): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] #number of times a wikipedia category has been encountered, from all added articles self.categories = defaultdict(int) self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator self.lang = lang self.decay = decay self.max_entries = max_entries if self.lang == 'en': self.wiki_category_string = u'Category:' elif self.lang == 'de': self.wiki_category_string = u'' else: print 'WARNING, unknown language', self.lang self.wiki_category_string = '' self.blacklist_ids = {} if blacklist_file != '': with codecs.open(blacklist_file,'r','utf-8') as infile: for line in infile: print 'blacklist wiki id:',line[:-1] self.blacklist_ids[line[:-1]] = 1 def topCategories(self,maxCategories=6): topCat = sorted(self.categories.items(), key=lambda x:x[1], reverse=True) topCat_json = [{'entry_id':idFromTitle(cat[0]),'title':cat[0].replace(u'Kategorie:',u''), 'url': u'https://simple.wikipedia.org/wiki/' + self.wiki_category_string + cat[0].replace(' ','_'), 'score': cat[1]} for cat in topCat if cat[1] > 1 and 'Wikipedia:' not in cat[0]] return topCat_json[:maxCategories] #Listen loop (redis) #Todo: for other languages than English, utf8 de and encoding will be needed def start_listen(self): pubsub = red.pubsub() pubsub.subscribe(my_redis_channel) for message in pubsub.listen(): print 'New message:', message, type(message['data']) if type(message['data']) == str: json_message = json.loads(message['data']) if 'handle' in json_message: if json_message['handle'] == 'completeUtterance': print 'handle: completeUtterance' self.complete_transcript.append(json_message['utterance']) self.send_relevant_entry_updates(self.max_entries,self.decay) if json_message['handle'] == 'closed': print 'handle: closed' self.delDisplayId(json_message['entry_id']) print json_message elif json_message['handle'] == 'reset': print 'handle: reset all' self.complete_transcript = [] self.relevant_entries = {} self.displayed_entries = [] self.categories = defaultdict(int) self.keyword_client.resetTimer() elif json_message['handle'] == 'setLanguage': print 'handle: set language' #todo # Add a relevant entry to the display, specify how many entries should be allowed maximally def addDisplayEntry(self, entry_type, entry, max_entries=4): print 'check to add', entry['title'], entry['score'] #TODO: Refactor the entry_type diretly into entry if 'type' not in entry: entry['type'] = entry_type if 'entry_id' not in entry: entry['entry_id'] = idFromTitle(entry['title']) if entry['entry_id'] in self.blacklist_ids: return False #Determine position by its score displayed_entries_get_score = dict_list_index_get_member(self.displayed_entries,'score') insert_pos = reverse_bisect(displayed_entries_get_score, float(entry['score'])) #Only add entry if we want to insert it into the max_entries best entries if insert_pos < max_entries: self.displayed_entries.insert(insert_pos, dict(entry)) len_displayed_entries = len(self.displayed_entries) #In this case, one of the previous best entries needs to be deleted: if(len_displayed_entries > max_entries): #Send delete entry events to entries those score is below the four best showed entries for display_entry in self.displayed_entries[max_entries:]: print 'del', display_entry['entry_id'], 'score fell below max_entries' self.keyword_client.delRelevantEntry(display_entry['type'], display_entry['title']) self.displayed_entries = self.displayed_entries[:max_entries] len_displayed_entries = len(self.displayed_entries) if insert_pos == len_displayed_entries -1: insert_before = end_marker print 'Insert',entry['entry_id'],'at the end' else: insert_before = self.displayed_entries[insert_pos+1]['entry_id'] print 'Insert',entry['entry_id'],'before',insert_before print 'add', entry['title'], entry['score'] self.keyword_client.addRelevantEntry('wiki', entry['title'], entry['text'], entry['url'], entry['score'], insert_before) return True else: print "Insert pos is:", insert_pos, "below max_entries for", entry["title"] return False # Delete a relevant entry from the display def delDisplayEntry(self, entry_type, title): print 'del',title for i,display_entry in list(enumerate(self.displayed_entries)): if (display_entry["title"] == title): print 'del', display_entry["title"] self.keyword_client.delRelevantEntry(entry_type, title) del self.displayed_entries[i] break # Delete a relevant entry from the model (without sending updates to the display) def delDisplayId(self, entry_id): print 'del id',entry_id for i,display_entry in list(enumerate(self.displayed_entries)): if (display_entry['entry_id'] == entry_id): print 'del', entry_id #self.keyword_client.delRelevantEntry(entry_type, title) del self.displayed_entries[i] break # Send relevant entry updates to the display, given a new full utterance. # Also specify how many entries we want (max_entries) and how existing keywords should decay their score. def send_relevant_entry_updates(self,max_entries=4, decay=.9, context_utts=9, extract_top_n_keywords=10, min_found_keywords=3, min_transcript_utts=2): print 'send_relevant_entry_updates called' with Timer() as t: #Do the decay for the displayed entries: #TODO: handle duplicate keywords and updated scores for entry in self.displayed_entries: entry["score"] *= decay # keywords = self.ke.getKeywordsDruid(self.complete_transcript[-1]) # Take last 10 utterances and combine them most_recent_transcript = " ".join(self.complete_transcript[-context_utts:]) # Extract top 9 keywords keywords = self.ke.extract_best_keywords(most_recent_transcript, n_words=extract_top_n_keywords) print keywords #abort if we found very little keywords and haven't seen enough utterances if len(keywords) < min_found_keywords or len(self.complete_transcript) < min_transcript_utts: return # Extract top wiki articles new_relevant_entries = wiki_search_es.extract_best_articles(keywords, n=max_entries) print "-> Extracted top ", len(new_relevant_entries), " documents", [(entry["title"], entry["score"]) for entry in new_relevant_entries] new_relevant_entries = dict(zip([entry["title"] for entry in new_relevant_entries], [entry for entry in new_relevant_entries ] ) ) new_relevant_entries_set = set(new_relevant_entries) relevant_entries_set = set(self.relevant_entries) num_added = 0 #generate add relevant entries for key in new_relevant_entries_set - relevant_entries_set: entry = new_relevant_entries[key] if self.addDisplayEntry("wiki", entry): num_added += 1 for category in entry["categories"]: self.categories[category] += 1 #now look for changed scores (happens if a keyword got more important and gets mentioned again) for key in (new_relevant_entries_set & relevant_entries_set): entry = new_relevant_entries[key] if entry["score"] > self.relevant_entries[key]["score"]: print "score change for:",entry["title"], self.relevant_entries[key]["score"], "->", entry["score"] found_displayed_entry = False for display_entry in self.displayed_entries: #already displayed, we could delete and read it, to reflect the new placement if display_entry["title"] == key: found_displayed_entry = True #self.delDisplayEntry("wiki", entry["title"]) #self.addDisplayEntry("wiki", entry) break if not found_displayed_entry: #not displayed, try to see if the higher score gets results in a document that is more important self.addDisplayEntry("wiki", entry) for key in new_relevant_entries_set - relevant_entries_set: self.relevant_entries[key] = new_relevant_entries[key] topCategories_Event = self.topCategories() print topCategories_Event # TODO: only send something if topCategories actually changes self.keyword_client.sendCategories(topCategories_Event) print 'send_relevant_entry_updates finished. Time needed:', t.secs, 'seconds.' print 'Displayed entries should now be:',[entry['title'] for entry in self.displayed_entries] print 'Added:',num_added
class KaldiClient(WebSocketClient): def print_devices(self): info = self.paudio.get_host_api_info_by_index(0) numdevices = info.get('deviceCount') #for each audio device, determine if is an input or an output and add it to the appropriate list and dictionary for i in range (0,numdevices): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: print "Input Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name') if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxOutputChannels')>0: print "Output Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name') def getAudioDeviceByString(audioDeviceName): info = self.paudio.get_host_api_info_by_index(0) numdevices = info.get('deviceCount') for i in range (0,numdevices): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: if audioDeviceName in self.paudio.get_device_info_by_host_api_device_index(0,i).get('name'): return i print 'No ',audioDeviceName,' microphone found, defaulting to last available input device...' for i in reversed(range (0,numdevices)): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: return i print 'No input device found! Please connect a microphone or recording device' return -1 def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = '', input_microphone_id=-1): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.paudio = pyaudio.PyAudio() self.print_devices() self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') #self.keyword_extractor = extract.TermExtractor() #self.keyword_extractor.filter = extract.permissiveFilter if self.send_to_keywordserver: self.keyword_client.addUtterance('','You') self.last_hyp = '' self.input_microphone_id = input_microphone_id #@rate_limited(4) def send_data(self, data): if data is not None: self.send(data, binary=True) def opened(self): #print "Socket opened!" def send_data_to_ws(): buffer_size = 1024 if self.input_microphone_id == -1: self.input_microphone_id = self.getAudioDeviceByString("Yamaha") if self.input_microphone_id == -1: sys.exit(-1) else: print 'Selecting device',self.input_microphone_id,'as input device' stream = self.paudio.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024, input_device_index = self.input_microphone_id) #buffer #f = open(self.fn, "rb") if self.send_adaptation_state_filename is not None: print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename try: adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r")) self.send(json.dumps(dict(adaptation_state=adaptation_state_props))) except: e = sys.exc_info()[0] print >> sys.stderr, "Failed to send adaptation state: ", e abort = False while not abort: block = stream.read(buffer_size) self.send_data(block) print >> sys.stderr, "Audio sent, now sending EOS" self.send("EOS") t = threading.Thread(target=send_data_to_ws) t.start() # received decoding message from upstream Kaldi server def received_message(self, m): try: response = json.loads(str(m)) #print >> sys.stderr, "RESPONSE:", response #print >> sys.stderr, "JSON was:", m if response['status'] == 0: if 'result' in response: trans = response['result']['hypotheses'][0]['transcript'] if response['result']['final']: if trans not in ['a.','I.','i.','the.','but.','one.','it.','she.']: self.final_hyps.append(trans) if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.keyword_client.completeUtterance(trans, std_speaker) self.keyword_client.addUtterance('',std_speaker) self.last_hyp = '' complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps) print u'\r\033[K',trans.replace(u'\n', u'\\n') else: if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.last_hyp = trans print_trans = trans.replace(u'\n', u'\\n') print u'\r\033[K',print_trans if 'adaptation_state' in response: if self.save_adaptation_state_filename: print u'Saving adaptation state to %s' % self.save_adaptation_state_filename with open(self.save_adaptation_state_filename, 'w') as f: f.write(json.dumps(response['adaptation_state'])) else: print u'Received error from server (status %d)' % response['status'] if 'message' in response: print 'Error message:', response['message'] except Exception: print 'Exception in received_message' exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=10, file=sys.stdout) def get_full_hyp(self, timeout=60): return self.final_hyp_queue.get(timeout) def closed(self, code, reason=None): #print "Websocket closed() called" #print >> sys.stderr self.final_hyp_queue.put(' '.join(self.final_hyps))
class EventGenerator: def __init__(self,keyword_server_url, keyword_extrator): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator #Listen loop (redis) #Todo: for other languages than English, utf8 de and encoding will be needed def start_listen(self): pubsub = red.pubsub() pubsub.subscribe(my_redis_channel) for message in pubsub.listen(): print 'New message:', message, type(message["data"]) if type(message["data"]) == str: json_message = json.loads(message["data"]) if "handle" in json_message: print json_message if json_message["handle"] == "completeUtterance": self.complete_transcript.append(json_message["utterance"]) self.send_relevant_entry_updates() elif json_message["handle"] == "reset": print 'reset all' self.complete_transcript = [] self.relevant_entries = {} self.displayed_entries = [] # Add a relevant entry to the display def addDisplayEntry(entry,max_entries=4): insert_pos = bisect.bisect(self.displayed_entries, float(entry["score"])) #Only add entry if we want to insert it into the max_entries best entries if insert_pos < max_entries: #Determine position by its score bisect.insort(self.displayed_entries, float(entry["score"])) len_displayed_entries = len(displayed_entries) #In this case, one of the previous best entries needs to be deleted: if(len_displayed_entries > max_entries): #Send delete entry events to entries those score is below the four best showed entries for entry in self.displayed_entries[max_entries:]: self.delDisplayEntry(entry_type,title) self.displayed_entries = self.displayed_entries[:max_entries] len_displayed_entries = len(displayed_entries) if insert_pos == len_displayed_entries -1: insert_before = '#end#' else: insert_before = self.displayed_entries[insert_before+1]["title"] self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"], insert_before) # Delete a relevant entry from the display def delDisplayEntry(entry_type,title): self.keyword_client.delRelevantEntry(entry_type, title) #@profile def send_relevant_entry_updates(self,max_entries=4): print 'send_relevant_entry_updates called' keywords = self.ke.getKeywordsDruid('\n'.join([sentence[:-1] for sentence in self.complete_transcript])) new_relevant_entries = wiki_search.getSummariesSingleKeyword(keywords,max_entries,lang='en',pics_folder='pics/') print new_relevant_entries #generate del relevant entries for key in set(self.relevant_entries) - set(new_relevant_entries): entry = self.relevant_entries[key] self.keyword_client.delRelevantEntry("wiki", entry["title"]) print 'del',key #generate add relevant entries for key in set(new_relevant_entries) - set(self.relevant_entries): entry = new_relevant_entries[key] self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"]) print 'add',key #TODO: Update scores of existing entries in self.displayed_entries (?) self.relevant_entries = new_relevant_entries
class KaldiClient(WebSocketClient): def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url='', max_sentences=0): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') if self.send_to_keywordserver: self.keyword_client.addUtterance('', 'You') self.last_hyp = '' self.max_sentences = max_sentences @rate_limited(4) def send_data(self, data): self.send(data, binary=True) def opened(self): # print "Socket opened!" def send_data_to_ws(): f = open(self.fn, "rb") if self.send_adaptation_state_filename is not None: print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename try: adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r")) self.send(json.dumps(dict(adaptation_state=adaptation_state_props))) except: e = sys.exc_info()[0] print >> sys.stderr, "Failed to send adaptation state: ", e for block in iter(lambda: f.read(self.byterate / 4), ""): if self.maximum_sentences_reached(): break self.send_data(block) print >> sys.stderr, "Audio sent, now sending EOS" self.send("EOS") t = threading.Thread(target=send_data_to_ws) t.start() # received decoding message from upstream Kaldi server def received_message(self, m): if self.maximum_sentences_reached(): return try: response = json.loads(str(m)) # print >> sys.stderr, "RESPONSE:", response # print >> sys.stderr, "JSON was:", m if response['status'] == 0: if 'result' in response: trans = response['result']['hypotheses'][0]['transcript'] if response['result']['final']: if trans not in ['a.', 'I.', 'i.', 'the.', 'but.', 'one.', 'it.', 'she.']: self.final_hyps.append(trans) if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.keyword_client.completeUtterance(trans, std_speaker) self.keyword_client.addUtterance('', std_speaker) self.last_hyp = '' complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps) print u'\r\033[K', trans.replace(u'\n', u'\\n') else: if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.last_hyp = trans print_trans = trans.replace(u'\n', u'\\n') print u'\r\033[K', print_trans if 'adaptation_state' in response: if self.save_adaptation_state_filename: print u'Saving adaptation state to %s' % self.save_adaptation_state_filename with open(self.save_adaptation_state_filename, 'w') as f: f.write(json.dumps(response['adaptation_state'])) else: print u'Received error from server (status %d)' % response['status'] if 'message' in response: print 'Error message:', response['message'] except Exception: print 'Exception in received_message' exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=10, file=sys.stdout) def get_full_hyp(self, timeout=60): return self.final_hyp_queue.get(timeout) # Returns True if the maximum number of sentences defined by the user have been transcribed. def maximum_sentences_reached(self): return self.max_sentences != 0 and len(self.final_hyps) >= self.max_sentences def closed(self, code, reason=None): # print "Websocket closed() called" # print >> sys.stderr self.final_hyp_queue.put(" ".join(self.final_hyps))
class EventGenerator: def __init__(self, keyword_server_url, keyword_extrator): self.complete_transcript = [] #dict with all relevant entries self.relevant_entries = {} #sorted list of displayed entries self.displayed_entries = [] self.keyword_client = KeywordClient(keyword_server_url) self.ke = keyword_extrator #Listen loop (redis) #Todo: for other languages than English, utf8 de and encoding will be needed def start_listen(self): pubsub = red.pubsub() pubsub.subscribe(my_redis_channel) for message in pubsub.listen(): print 'New message:', message, type(message["data"]) if type(message["data"]) == str: json_message = json.loads(message["data"]) if "handle" in json_message: print json_message if json_message["handle"] == "completeUtterance": self.complete_transcript.append( json_message["utterance"]) self.send_relevant_entry_updates() elif json_message["handle"] == "reset": print 'reset all' self.complete_transcript = [] self.relevant_entries = {} self.displayed_entries = [] # Add a relevant entry to the display def addDisplayEntry(entry, max_entries=4): insert_pos = bisect.bisect(self.displayed_entries, float(entry["score"])) #Only add entry if we want to insert it into the max_entries best entries if insert_pos < max_entries: #Determine position by its score bisect.insort(self.displayed_entries, float(entry["score"])) len_displayed_entries = len(displayed_entries) #In this case, one of the previous best entries needs to be deleted: if (len_displayed_entries > max_entries): #Send delete entry events to entries those score is below the four best showed entries for entry in self.displayed_entries[max_entries:]: self.delDisplayEntry(entry_type, title) self.displayed_entries = self.displayed_entries[:max_entries] len_displayed_entries = len(displayed_entries) if insert_pos == len_displayed_entries - 1: insert_before = '#end#' else: insert_before = self.displayed_entries[insert_before + 1]["title"] self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"], insert_before) # Delete a relevant entry from the display def delDisplayEntry(entry_type, title): self.keyword_client.delRelevantEntry(entry_type, title) #@profile def send_relevant_entry_updates(self, max_entries=4): print 'send_relevant_entry_updates called' keywords = self.ke.getKeywordsDruid('\n'.join( [sentence[:-1] for sentence in self.complete_transcript])) new_relevant_entries = wiki_search.getSummariesSingleKeyword( keywords, max_entries, lang='en', pics_folder='pics/') print new_relevant_entries #generate del relevant entries for key in set(self.relevant_entries) - set(new_relevant_entries): entry = self.relevant_entries[key] self.keyword_client.delRelevantEntry("wiki", entry["title"]) print 'del', key #generate add relevant entries for key in set(new_relevant_entries) - set(self.relevant_entries): entry = new_relevant_entries[key] self.keyword_client.addRelevantEntry("wiki", entry["title"], entry["text"], entry["url"], entry["score"]) print 'add', key #TODO: Update scores of existing entries in self.displayed_entries (?) self.relevant_entries = new_relevant_entries
class KaldiClient(WebSocketClient): def print_devices(self): info = self.paudio.get_host_api_info_by_index(0) numdevices = info.get('deviceCount') #for each audio device, determine if is an input or an output and add it to the appropriate list and dictionary for i in range (0,numdevices): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: print "Input Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name') if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxOutputChannels')>0: print "Output Device id ", i, " - ", self.paudio.get_device_info_by_host_api_device_index(0,i).get('name') def getYamahaID(self): info = self.paudio.get_host_api_info_by_index(0) numdevices = info.get('deviceCount') for i in range (0,numdevices): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: if 'Yamaha' in self.paudio.get_device_info_by_host_api_device_index(0,i).get('name'): return i print 'No yamaha microphone found, defaulting to last available input device...' for i in reversed(range (0,numdevices)): if self.paudio.get_device_info_by_host_api_device_index(0,i).get('maxInputChannels')>0: return i print 'No input device found! Please connect a microphone or recording device' return -1 def __init__(self, filename, url, protocols=None, extensions=None, heartbeat_freq=None, byterate=32000, save_adaptation_state_filename=None, send_adaptation_state_filename=None, keyword_server_url = ''): super(KaldiClient, self).__init__(url, protocols, extensions, heartbeat_freq) self.final_hyps = [] self.fn = filename self.byterate = byterate self.final_hyp_queue = Queue.Queue() self.save_adaptation_state_filename = save_adaptation_state_filename self.send_adaptation_state_filename = send_adaptation_state_filename self.paudio = pyaudio.PyAudio() self.print_devices() self.keyword_client = KeywordClient(keyword_server_url) self.keyword_client.reset() self.send_to_keywordserver = not (keyword_server_url == '') #self.keyword_extractor = extract.TermExtractor() #self.keyword_extractor.filter = extract.permissiveFilter if self.send_to_keywordserver: self.keyword_client.addUtterance('','You') self.last_hyp = '' #@rate_limited(4) def send_data(self, data): if data is not None: self.send(data, binary=True) def opened(self): #print "Socket opened!" def send_data_to_ws(): buffer_size = 1024 yamahaID = self.getYamahaID() if yamahaID == -1: sys.exit(-1) else: print 'Selecting device',yamahaID,'as input device' stream = self.paudio.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024, input_device_index = yamahaID) #buffer #f = open(self.fn, "rb") if self.send_adaptation_state_filename is not None: print >> sys.stderr, "Sending adaptation state from %s" % self.send_adaptation_state_filename try: adaptation_state_props = json.load(open(self.send_adaptation_state_filename, "r")) self.send(json.dumps(dict(adaptation_state=adaptation_state_props))) except: e = sys.exc_info()[0] print >> sys.stderr, "Failed to send adaptation state: ", e abort = False while not abort: block = stream.read(buffer_size) self.send_data(block) print >> sys.stderr, "Audio sent, now sending EOS" self.send("EOS") t = threading.Thread(target=send_data_to_ws) t.start() # received decoding message from upstream Kaldi server def received_message(self, m): try: response = json.loads(str(m)) #print >> sys.stderr, "RESPONSE:", response #print >> sys.stderr, "JSON was:", m if response['status'] == 0: if 'result' in response: trans = response['result']['hypotheses'][0]['transcript'] if response['result']['final']: if trans not in ['a.','I.','i.','the.','but.','one.','it.','she.']: self.final_hyps.append(trans) if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.keyword_client.completeUtterance(trans, std_speaker) self.keyword_client.addUtterance('',std_speaker) self.last_hyp = '' complete_transcript = '\n'.join(sentence[:-1] for sentence in self.final_hyps) print u'\r\033[K',trans.replace(u'\n', u'\\n') else: if self.send_to_keywordserver: self.keyword_client.replaceLastUtterance(self.last_hyp, trans, std_speaker) self.last_hyp = trans print_trans = trans.replace(u'\n', u'\\n') print u'\r\033[K',print_trans if 'adaptation_state' in response: if self.save_adaptation_state_filename: print u'Saving adaptation state to %s' % self.save_adaptation_state_filename with open(self.save_adaptation_state_filename, 'w') as f: f.write(json.dumps(response['adaptation_state'])) else: print u'Received error from server (status %d)' % response['status'] if 'message' in response: print 'Error message:', response['message'] except Exception: print 'Exception in received_message' exc_type, exc_value, exc_traceback = sys.exc_info() traceback.print_exception(exc_type, exc_value, exc_traceback, limit=10, file=sys.stdout) def get_full_hyp(self, timeout=60): return self.final_hyp_queue.get(timeout) def closed(self, code, reason=None): #print "Websocket closed() called" #print >> sys.stderr self.final_hyp_queue.put(' '.join(self.final_hyps))
def __init__(self): self.ks = KeywordClient(server_url="http://localhost:5000/") self.std_spk = "You" self.last_hyp = "" self.ks.reset()