def formulate_response(self,question): grammar = JSGFParser('speech/hark-sphinx/grammar/NielsSebastiaan.gram') language_parsing = nlp() question = language_parsing.remove_name(language_parsing.remove_opts(question)).lower() question = question.replace("?", "").replace(",", "") print question if(grammar.findToken(question) != None): #Question responses = {\ 'what time is it' : "The current time is: " + time.strftime("%H:%M:%S") + ".",\ 'what is the oldest most widely used drug on earth' : 'The oldest, most widely used drug on earth is coffee.',\ 'who are your creators' : 'My creators are Niels and Sebastiaan.'} return responses[question] else: words = question.split(' ') if(grammar.findTokenVar(words[0]) == '<verb>'): #command question = question.replace(words[0] + " to the ", "") if(grammar.findTokenVar(question) == '<location>'): return "I am moving to " + question + "." elif("dining table" in question): #request return "I am approaching the dining table." else: return "I'm sorry, I did not get that. Could you please repeat it?"
def __init__(self, grammarFile): self.grammar = JSGFParser(grammarFile) self.stopWords = self.grammar.findStopwords() #optional words that don't add core meaning to the sentence self.reset()
class Communicator(): def __init__(self, grammarFile): self.grammar = JSGFParser(grammarFile) self.stopWords = self.grammar.findStopwords() #optional words that don't add core meaning to the sentence self.reset() def reset(self): self.status = "idle" def filterStopwords(self, sentence): """ filter the given string by taking out the words in the stopWords list """ for stopWord in self.stopWords: sentence = sentence.replace(" %s " % stopWord, " ") return sentence def resultResponse(self, result): # we already found a string that we consider as understood correctly if result['accepted_string']: responseSentence = "I understood '%s'" % result['accepted_string'] repeat = "no" # we understood something different. The difference is not within the same var elif result['same_var'] is None: responseSentence = "I didn't understand. Please repeat your sentence." repeat = "all" else: var = result['same_var'] print(result['first']['difference_token']) understoodOptions = [result['first']['difference_token']['text'], result['second']['difference_token']['text']] # the two different understood options allOptions = self.grammar.findVariableItems(var, includeVars=False) # all the available options for this var from the grammar if var == "<start>": responseSentence = "Ask: who do you mean? %s " % ' or '.join(understoodOptions) repeat = "part" print ("listen for: %s" % str(allOptions)) elif var == "<object>": responseSentence = "Ask: which object do you mean? %s" % ' or '.join(understoodOptions) repeat = "part" print ("listen for: %s" % str(allOptions)) elif var == "<verb>": responseSentence = "Ask: what do you want me to do? %s" % ' or '.join(understoodOptions) repeat = "part" print ("listen for: %s" % str(allOptions)) elif var == "<location>": responseSentence = "Ask: where do you mean? %s" % ' or '.join(understoodOptions) repeat = "part" print ("listen for: %s" % str(allOptions)) else: responseSentence = "I didn't understand. Please repeat your sentence." repeat = "all" response = { "repeat": repeat, "understoodOptions": understoodOptions, "allOptions": allOptions, "responseSentence": responseSentence } return response def sentenceResponse(self, sentence, filterStopwords=True): if filterStopwords: sentence = self.filterStopwords(sentence) # check for sentences if ("what is your teams name" in sentence): print("My teams name is BORG") return if ("what is your favorite color" in sentence): print("Black as the night") return # check for requests if ("follow me" in sentence): print ("Ok I will follow you") return if ("enter the elevator" in sentence): print("Ok I will go in the elevator") return if ("give me a drink" in sentence): print ("I will give you a drink") return if ("pick up the cup" in sentence): print ("I will pick up the cup") return # specific color cup regex = re.compile(ur'pick up the (.+) cup') matches = re.findall(regex, sentence) if matches: color = matches[0] print("I will pick up the %s cup" % color) return # order actions = self.parseOrderSentence(sentence) if actions: pprint(actions) return def parseOrderSentence(self, sentence): # remove the start of the sentence starts = self.grammar.findVariableItems("<start>", includeVars=False) for start in starts: sentence = sentence.lstrip(start) # split on different locations. these will be the different orders locations = self.grammar.findVariableItems("<location>", includeVars=False) regex = re.compile(" to (%s)" % '|'.join(locations)) matches = re.split(regex, sentence) if not matches: return None # make pairs for each order and location couple order_locations = [] matchCount = len(matches) for idx in range(0, matchCount, 2): if idx + 1 >= matchCount: break order_locations.append((matches[idx].strip().lstrip("and").lstrip(), matches[idx+1].strip())) # build actions from the order location pairs. # each order parts is analyzed (find the verb and different items) # each action is a dict indicating the task verb, items and location verbs = self.grammar.findVariableItems("<verb>", includeVars=False) lastOrderVerb = None actions = [] for order_location in order_locations: orders = order_location[0] location = order_location[1] for verb in verbs: if verb == orders[0:len(verb)]: lastOrderVerb = verb orders = orders[len(verb) + 1:] break action = {} action['verb'] = lastOrderVerb items = orders.split(" and ") action['items'] = items action['location'] = location actions.append(action) # Build a sentence to say for the robot first = True for action in actions: if first: response = "Ok " else: response = "then " response += "I will %s " % action['verb'] firstItem = True for item in action['items']: if firstItem is False: response += "and " response += "the %s " % item firstItem = False response += "to the %s" % location first = False print response return actions
class nlp(): def __init__(self): self.grammar = JSGFParser('speech/hark-sphinx/grammar/NielsSebastiaan.gram') self.names = self.grammar.findVariableItems("<start>", includeVars=False) self.locations = self.grammar.findVariableItems("<location>", includeVars=False) self.verbs = self.grammar.findVariableItems("<verb>", includeVars=False) self.nav = self.grammar.findVariableItems("<nav>", includeVars=False) def remove_name(self,s): for name in self.names: start = "%s " % name if s.startswith(start): s = s.lstrip(start) return s def remove_opts(self, hypstr): return self.grammar.filterOptionals(hypstr) def find_type(self,string): if self.remove_opts(string) in self.locations: return "location" if self.remove_opts(string) in self.verbs: return "verb" return None def split_order(self, s1,s2): splits = [] regex = re.compile(" and ") matches1 = re.split(regex, s1) matches2 = re.split(regex, s2) for x, y in zip(matches1,matches2): splits.append([x,y]) return splits def partial_parse(self,order): parsed_list = [] for s in order: order_verb = None order_location = None order_item = None regex = re.compile("(%s)" % '|'.join(self.locations)) match = re.split(regex, self.remove_name(s)) if len(match)<3: return None order_location = match[1] regex = re.compile("(%s)" % '|'.join(self.verbs)) order_verb_grp = re.search(regex,match[0]) if order_verb_grp != None: order_verb = order_verb_grp.group(0) regex = re.compile("(%s)" % '|'.join(self.items)) order_reg = re.search(regex,match[0]) if order_reg != None: order_item = order_reg.group(0) else: order_item = "it" parsed_list.append({"verb" : order_verb, "location" : order_location}) return parsed_list def needs_question(self,parsed_order): o1 = parsed_order[0] o2 = parsed_order[1] location = None verb = None # we have a perfect match, return to sender if o2 == o1: return {"diff" : False , "question" : "No difference found", "understood_options": None, "action" : o1,'unknown': None} if o1['location'] == o2['location'] : location = o1['location'] if o1['verb'] == o2['verb'] : verb = o1['verb'] # generate questions for the misheard variables TODO not complete # @TODO you should make use of the information here to return a proper action structure # else return {"diff" : True , "question" : "could you repeat this part?", "understood_options": None, "action" : None,'unknown': None} def confirm(self, action): v = action['action']['verb'] l = action['action']['location'] confirmation = "" if v == "approach": confirmation = "I will approach the " + l else: #v in self.nav confirmation = "I will go to the " + l return confirmation def get_actions(self, speech_observation): actions = [] # we first split the order with they keyword "and" orders = self.split_order(self.remove_opts(speech_observation['2best'][0]),self.remove_opts(speech_observation['2best'][1])) for order in orders: parsed_order = self.partial_parse(order) if parsed_order != None: actions.append(self.needs_question(parsed_order)) else: print "could not parse. not in grammar" return actions
def __init__(self): self.grammar = JSGFParser('speech/hark-sphinx/grammar/NielsSebastiaan.gram') self.names = self.grammar.findVariableItems("<start>", includeVars=False) self.locations = self.grammar.findVariableItems("<location>", includeVars=False) self.verbs = self.grammar.findVariableItems("<verb>", includeVars=False) self.nav = self.grammar.findVariableItems("<nav>", includeVars=False)
class DecoderAnalyzer: def __init__(self, grammarFile=None): if grammarFile: self.setGrammar(grammarFile) def setGrammar(self, grammarFile): self.grammar = JSGFParser(grammarFile) self.stopWords = self.grammar.findStopwords( ) #optional words that don't add core meaning to the sentence def getNBestList(self, decoder, n=None): nBestList = {} for oneBest in decoder.nbest(): if not oneBest.hyp() is None: score = oneBest.hyp( ).best_score # log scale. http://stackoverflow.com/questions/20825654/cmusphinx-what-is-the-score-of-a-recognised-hypothesis#comment31476787_20867454 hypstr = oneBest.hyp().hypstr hypstr = self.filterHypstr( hypstr) # remove words that don't add core meaning if not hypstr in nBestList: # nBestList[hypstr] = { # 'count': 0, # 'score': 0, # 'norm_score': 0 # } nBestList[hypstr] = 0 nBestList[hypstr] += 1 # nBestList[hypstr]['count'] += 1 # nBestList[hypstr]['score'] += score decodersBest = self.filterHypstr( decoder.hyp().hypstr) # best hypothesis nBestList[ decodersBest] = 999999 # make sure that the decoders best has the best rank always nBestListOrdered = list( OrderedDict( sorted(nBestList.items(), key=lambda t: t[1], reverse=True))) if n: nBestListOrdered = nBestListOrdered[0:n] # sorted_nBestList = sorted(nBestList.items(), key=operator.itemgetter(1)) # pprint(nBestList[0:2]) # calculate the normalized score for each found sentence # for hypstr in nBestList: # nBestList[hypstr]['norm_score'] = nBestList[hypstr]['score'] / nBestList[hypstr]['count'] # nBestCounter = Counter(nBestList) # order list # take n best return nBestListOrdered def filterHypstr(self, hypstr): """ filter the given string by taking out the words in the stopWords list """ for stopWord in self.stopWords: hypstr = hypstr.replace(" %s " % stopWord, " ") return hypstr def bestResult(self, nBestList): bestResult = None for hypstr in nBestList: if not bestResult: bestResult = hypstr continue if nBestList[hypstr]['norm_score'] > nBestList[bestResult][ 'norm_score']: bestResult = hypstr return (bestResult, nBestList[bestResult]) def difference(self, first, second): diff = difflib.ndiff(first.split(), second.split()) return diff def analyze2Best(self, firstBest, secondBest): # return { # "first": { # "sentence": "alice follow me", # "difference_token": {"text": "alice", "tag": "robot_alice", "var": "<start>"} # }, # "second": { # "sentence": "pseudo follow me", # "difference_token": {"text": "pseudo", "tag": "robot_sudo", "var": "<start>"} # }, # "same_var": "<start>", # "same_tag": None, # "accepted_string": None # } if not secondBest: print( "[DecoderAnalyzer] No second best found, accepting first and only understood sentence" ) return { "first": { "sentence": firstBest, "difference_token": None }, "second": None, "same_var": None, "same_tag": None, "accepted_string": firstBest } diff = self.difference(firstBest, secondBest) first = [] second = [] for a, b in enumerate(diff): if b[0] == '-': first.append(b[2:]) if b[0] == '+': second.append(b[2:]) firstTokenString = ' '.join(first) firstToken = self.grammar.findToken(firstTokenString) secondTokenString = ' '.join(second) secondToken = self.grammar.findToken(secondTokenString) acceptedString = None sameVar = None sameTag = None if not firstToken and not secondToken: # @TODO check if maybe many difference and that's why no token is found... if (not firstTokenString in firstBest) or (not secondTokenString in secondBest): print( "[DecoderAnalyzer] multiple differences between strings at different locations" ) else: print("[DecoderAnalyzer]") print(firstBest) print(secondBest) print( "[DecoderAnalyzer] No significant differences found. Accept first result" ) acceptedString = firstBest elif firstToken and secondToken: print( "[DecoderAnalyzer] Found token difference between the strings") if firstToken["var"] == secondToken["var"]: sameVar = firstToken["var"] print("[DecoderAnalyzer] with the same var") if (firstToken["tag"] and secondToken["tag"] and (firstToken["tag"] == secondToken["tag"])): sameTag = firstToken["tag"] acceptedString = firstBest print("[DecoderAnalyzer] with the same tag") else: print("[DecoderAnalyzer] with different tags") else: print("[DecoderAnalyzer] with different vars") else: differentToken = firstToken if firstToken else secondToken print("[DecoderAnalyzer] Different Token: ") print(differentToken) result = { "first": { "sentence": firstBest, "difference_token": firstToken }, "second": { "sentence": secondBest, "difference_token": secondToken }, "same_var": sameVar, "same_tag": sameTag, "accepted_string": acceptedString } return result def analyze(self, decoder): if decoder.hyp() is None: return None decodersBest = decoder.hyp().hypstr # best hypothesis decodersBest = self.filterHypstr(decodersBest) decoder.get_lattice().write("experiment/lattice_output/" + str(time.time()) + ".lattice") nBestList = self.getNBestList(decoder=decoder, n=2) print("") pprint(nBestList) if len(nBestList) == 1: acceptedString = nBestList[0] else: analyze2BestResult = self.analyze2Best(nBestList[0], nBestList[1]) pprint(analyze2BestResult) acceptedString = analyze2BestResult["accepted_string"] return acceptedString
class Conversator(): grammar = None stopWords = None def __init__(self, grammarFile=None): print("[Conversator] initializing...") self.reset() if grammarFile: self.setGrammar(grammarFile) def setGrammar(self, grammarFile): print("[Conversator] Grammar file set: %s" % grammarFile) self.grammar = JSGFParser(grammarFile) self.stopWords = self.grammar.findStopwords( ) #optional words that don't add core meaning to the sentence def reset(self): self.status = "idle" def filterStopwords(self, sentence): """ filter the given string by taking out the words in the stopWords list """ for stopWord in self.stopWords: sentence = sentence.replace(" %s " % stopWord, " ") return sentence def resultResponse(self, result): # @TODO remove this # return { # "repeat": "part", # "understoodOptions": ['alice', 'pseudo'], # "allOptions": ['alice', 'pseudo', 'rick', 'smith'], # "responseSentence": "Who do you mean? Alice or Pseudo." # } understoodOptions = None allOptions = None # we already found a string that we consider as understood correctly if result['accepted_string']: responseSentence = "I understood '%s'" % result['accepted_string'] repeat = "no" # we understood something different. The difference is not within the same var elif result['same_var'] is None: responseSentence = "I didn't understand. Please repeat your sentence." repeat = "all" else: var = result['same_var'] understoodOptions = [ result['first']['difference_token']['text'], result['second']['difference_token']['text'] ] # the two different understood options allOptions = self.grammar.findVariableItems( var, includeVars=False ) # all the available options for this var from the grammar if var == "<name>": responseSentence = "Who do you mean? %s " % ' or '.join( understoodOptions) repeat = "part" print("[Conversator] listen for: %s" % str(allOptions)) elif var == "<object>": responseSentence = "Which object do you mean? %s" % ' or '.join( understoodOptions) repeat = "part" print("[Conversator] listen for: %s" % str(allOptions)) elif var == "<verb>": responseSentence = "What do you want me to do? %s" % ' or '.join( understoodOptions) repeat = "part" print("[Conversator] listen for: %s" % str(allOptions)) elif var == "<location>": responseSentence = "Where do you mean? %s" % ' or '.join( understoodOptions) repeat = "part" print("[Conversator] listen for: %s" % str(allOptions)) elif var == "<color>": responseSentence = "Which color do you mean? %s" % ' or '.join( understoodOptions) repeat = "part" print("[Conversator] listen for: %s" % str(allOptions)) else: responseSentence = "I didn't understand. Please repeat your sentence." repeat = "all" response = { "repeat": repeat, "understoodOptions": understoodOptions, "allOptions": allOptions, "responseSentence": responseSentence } return response def sentenceResponse(self, sentence, filterStopwords=True): if filterStopwords: sentence = self.filterStopwords(sentence) actions = [] # check for sentences if ("what is your teams name" in sentence): actions.append({"type": "say", "content": "My teams name is BORG"}) return actions elif ("what is your favorite color" in sentence): actions.append({"type": "say", "content": "Black as the night"}) return actions # check for requests elif ("follow me" in sentence): actions.append({"type": "say", "content": "Ok I will follow you"}) return actions elif ("enter the elevator" in sentence): actions.append({ "type": "say", "content": "Ok I will go in the elevator" }) return actions elif ("give me a drink" in sentence): actions.append({ "type": "say", "content": "I will give you a drink" }) return actions elif ("go duck yourself" in sentence): actions.append({ "type": "say", "content": "Just like I did your mom?" }) return actions # specific color cup regex = re.compile(ur'pick up (.+) cup') matches = re.findall(regex, sentence) if matches: color = matches[0] content = "I will pick up the %s cup" % color actions.append({"type": "say", "content": content}) actions.append({ "type": "do", "content": { 'verb': 'pickup', 'items': ["%s cup" % color], 'location': None } }) return actions # order actions = self.parseOrderSentence(sentence) if actions: return actions return [] def parseOrderSentence(self, sentence): # remove the start of the sentence names = self.grammar.findVariableItems("<name>", includeVars=False) for name in names: start = "okay %s" % name if sentence.startswith(start): sentence = sentence.lstrip(start) # split on different locations. these will be the different orders locations = self.grammar.findVariableItems("<location>", includeVars=False) regex = re.compile(" to (%s)" % '|'.join(locations)) matches = re.split(regex, sentence) if not matches: return None # make pairs for each order and location couple order_locations = [] matchCount = len(matches) for idx in range(0, matchCount, 2): if idx + 1 >= matchCount: break order_locations.append( (matches[idx].strip().lstrip("and").lstrip(), matches[idx + 1].strip())) # build actions from the order location pairs. # each order parts is analyzed (find the verb and different items) # each action is a dict indicating the task verb, items and location verbs = self.grammar.findVariableItems("<verb>", includeVars=False) lastOrderVerb = None actions = [] for order_location in order_locations: orders = order_location[0] location = order_location[1] for verb in verbs: if verb == orders[0:len(verb)]: lastOrderVerb = verb orders = orders[len(verb) + 1:] break actionContent = {} actionContent['verb'] = lastOrderVerb items = orders.split(" and ") actionContent['items'] = items actionContent['location'] = location actions.append({"type": "do", "content": actionContent}) # Build a sentence to say for the robot first = True response = "" for action in actions: actionContent = action['content'] if first: response += "Ok " else: response += " then " response += "I will %s " % actionContent['verb'] firstItem = True for item in actionContent['items']: if firstItem is False: response += "and " response += "the %s " % item firstItem = False response += "to the %s" % actionContent['location'] first = False actions.insert(0, { "type": "say", "content": response }) # add as first action return actions def getRepeatAction(self, sentence=None): if not sentence: sentence = "I didn't understand. Please repeat your sentence." return {"type": "say", "content": sentence}
def setGrammar(self, grammarFile): print("[Conversator] Grammar file set: %s" % grammarFile) self.grammar = JSGFParser(grammarFile) self.stopWords = self.grammar.findStopwords( ) #optional words that don't add core meaning to the sentence
class nlp4(): def __init__(self): self.grammar = JSGFParser('speech/hark-sphinx/grammar/NielsSebastiaan.gram') self.names = self.grammar.findVariableItems("<start>", includeVars=False) self.locations = self.grammar.findVariableItems("<location>", includeVars=False) self.verbs = self.grammar.findVariableItems("<verb>", includeVars=False) self.questions = self.grammar.findVariableItems("<sentence>", includeVars=False) def remove_name(self,s): for name in self.names: s = s.lower() start = "%s " % name if s.startswith(start): s = s.lstrip(start) return s def remove_opts(self, hypstr): return self.grammar.filterOptionals(hypstr) def find_type(self,string): if self.remove_opts(string) in self.locations: return "location" if self.remove_opts(string) in self.verbs: return "verb" return None def split_order(self, s1,s2): splits = [] regex = re.compile(" and ") matches1 = re.split(regex, s1) matches2 = re.split(regex, s2) for x, y in zip(matches1,matches2): splits.append([x,y]) return splits def partial_parse(self,order): parsed_list = [] for s in order: order_verb = None order_location = None order_question = None regex = re.compile("(%s)" % '|'.join(self.locations)) match = re.split(regex, self.remove_name(s)) noname = self.remove_name(s) if noname == s: if len(match)<3: #Not a room navigation regex = re.compile("(approach the dining table)") approach_grp = re.search(regex,noname) if approach_grp != None: parsed_list.append({"verb" : "approach", "location" : "dining table", "question" : None}) else: #Question time! if not self.question_parse(parsed_list, noname): return None else: order_location = match[1] regex = re.compile("(%s)" % '|'.join(self.verbs)) order_verb_grp = re.search(regex,match[0]) if order_verb_grp != None: order_verb = order_verb_grp.group(0) print order_location parsed_list.append({"verb" : order_verb, "location" : order_location, "question" : None}) else: #Question time! if not self.question_parse(parsed_list, noname): return None return parsed_list def question_parse(self, parsed_list, question): regex = re.compile("(%s)" % '|'.join(self.questions)) question_grp = re.search(regex,question) if question_grp != None: parsed_list.append({"verb" : None, "location" : None, "question" : question_grp.group(0)}) return True else: return False def needs_question(self,parsed_order): o1 = parsed_order[0] o2 = parsed_order[1] location = None verb = None # we have a perfect match, return to sender if o2 == o1: return {"diff" : False , "question" : "No difference found", "understood_options": None, "action" : o1,'unknown': None} if (not o1['question'] == None and o2['question'] == None) or (not o2['question'] == None and o1['question'] == None): #not sure if question or order return {"diff" : True , "question" : "could you repeat this part?", "understood_options": None, "action" : None,'unknown': None} elif o1['question'] != o2['question'] : return {"diff" : True , "question" : "could you repeat your question?", "understood_options": None, "action" : o1,'unknown': 'question'} elif o1['location'] == o2['location'] : return {"diff" : False , "question" : "Only difference in verb", "understood_options": None, "action" : o1,'unknown': None} else: return {"diff" : True , "question" : "Where did you want me to " + o1['verb'] + "?", "understood_options": None, "action" : o1,'unknown': 'location'} def confirm(self, action): v = action['action']['verb'] l = action['action']['location'] q = action['action']['question'] confirmation = "" if not q == None: responses = {\ 'what time is it' : "The current time is: " + time.strftime("%H:%M:%S") + ".",\ 'what is the oldest most widely used drug on earth' : 'The oldest, most widely used drug on earth is coffee.',\ 'who are your creators' : 'My creators are Niels and Sebastiaan.'} confirmation = responses[q] elif v == "approach": confirmation = "I will approach the " + l else: #v in self.nav confirmation = "I will go to the " + l return confirmation def get_actions(self, speech_observation): actions = [] # we first split the order with they keyword "and" orders = self.split_order(self.remove_opts(speech_observation['2best'][0]),self.remove_opts(speech_observation['2best'][1])) for order in orders: parsed_order = self.partial_parse(order) if parsed_order != None: actions.append(self.needs_question(parsed_order)) else: print "could not parse. not in grammar" return actions