def ActiveListening(this): threshold = None textout.SystemPrint("Started to listen actively") #RECORD A WAV FILE, CUTOFF AT 12s OR FALLS BELOW THRESHOLD #SEND TO WITAI #RECIEVE INPUT #SEND TO PROCESSOR AND RETURN INTENT #FIND ACTION. RATE = 16000 CHUNK = 1024 p = pyaudio.PyAudio() stream = p.open(format=pyaudio.paInt16, channels=1, rate=RATE, input=True, frames_per_buffer=CHUNK) frames = [] lastN = [ 144 * 1.2 for i in range(15) ] #changing array length will determine if average will change faster or not for i in range(0, int(RATE / CHUNK * 5)): #RATE(16000) / CHUNK(1024) * TIME (12s) data = stream.read(CHUNK) frames.append(data) score = getScore(data) lastN.pop(0) lastN.append(score) average = sum(lastN) / float(len(lastN)) print(str(average)) if average < 144 - 20: textout.SystemPrint("Listening stopped, below threshold.") break textout.SystemPrint("Listening Timeout!") stream.stop_stream() stream.close() p.terminate() with tempfile.NamedTemporaryFile(mode='w+b') as f: wav_fp = wave.open(f, 'wb') wav_fp.setnchannels(1) wav_fp.setsampwidth(pyaudio.get_sample_size(pyaudio.paInt16)) wav_fp.setframerate(16000) wav_fp.writeframes(b''.join(frames)) wav_fp.close() f.seek(0) transcriber.TranscribeAudiofile(this, f) textout.SystemPrint("Stopped listening actively")
def __init__(this): textout.SystemPrint("Processor: Starting processor initialization" ) this.data = dataimporter.DataImporter() this.data.PopulateTrainingData() this.outlines = this.data.PopulateOutlinesDict() this.synonyms = this.data.PopulateSynonymsDict() this.entities = this.data.PopulateEntitiesDict() this.LoadAllModules() textout.SystemPrint ("Processor: Processor initialized!\n")
def PopulateTrainingData(this): textout.SystemPrint("DataImporter: Loading Training Data...") textout.SystemPrint("DataImporter: Reading disk training data") if os.path.exists(TRAIN_PATH): with open(TRAIN_PATH, 'r') as fp: this.trainingdata = json.load(fp) textout.SystemPrint("DataImporter: Training data loaded!") else: textout.SystemPrint( "DataImporter: NOTICE! Disk training data is not found. Creating blank training data file." )
def LoadAllModules(this): textout.SystemPrint ("Module Loader: Loading all modules into processor instance...") locations = [MOD_PATH] for finder, name, ispkg in pkgutil.walk_packages(locations): try: loader = finder.find_module(name) mod = loader.load_module(name) except: textout.SystemPrint ("Skipped module '" + name + "' due to an error.") else: textout.SystemPrint ("Found module '" + name + "'") this.modules.append(mod) textout.SystemPrint ("{} modules(s) loaded".format(len(this.modules)))
def UpdateTrainingData(this, phrases, phrasePos, usrinputlength, outlines): #Try to add data into the trainingdata dict, from processor.py when a match is found. #need to convert: #[phrases] >> _outlineKey (string) #[phrasePos] >> [phrasePosDiff] #need to convert [phrases] into one single string _outlineKey = this.CombineStringsList(phrases) phrasePosDiff = [ 0, ] * len(phrasePos) #need to convert [phrasePos] >> [phrasePosDiff] for i in range(1, len( phrasePos)): #index 0 is always value 0, so start loop from 1. phrasePosDiff[i] = phrasePos[i] - phrasePos[i - 1] #Make _data _data = [phrasePosDiff, usrinputlength] #_data is in this format: ([phrasePosDiff], usrinputlength) if _outlineKey in this.trainingdata: if _data not in this.trainingdata[ _outlineKey]: #Add data if OUTLINE exists and _data is not in dict yet textout.SystemPrint( "DataImporter/UpdateTrainingData(): Data does not exist, adding!" ) this.trainingdata[_outlineKey].append(_data) this.WriteTrainingData() this.UpdateOutlinesWithTrainingData(outlines) else: textout.SystemPrint( "DataImporter/UpdateTrainingData(): Data already exists.") else: textout.SystemPrint( "DataImporter/UpdateTrainingData(): Data does not exist, adding!" ) this.trainingdata.setdefault( _outlineKey, []) #If key does not exist, create one and add the data to it. this.trainingdata[_outlineKey].append(_data) this.WriteTrainingData() this.UpdateOutlinesWithTrainingData(outlines)
def PopulateOutlinesDict(this): textout.SystemPrint( "DataImporter: Loading intent outlines from disk...") if os.path.exists(INTENT_PATH): fileDirectories = filter(lambda x: x[-4:] == '.txt', os.listdir(INTENT_PATH)) for filename in fileDirectories: #strip and split ReadTxtFile at the same time. intentTextArray = [ x.strip().lower() for x in this.ReadTxtFile( os.path.join(INTENT_PATH, filename)).split('\n') ] #Shovel each line of intentText into an outline entry for outlineRaw in intentTextArray: phrases = [x.strip() for x in outlineRaw.split(' ')] phraseWeight = [] phrasesCleaned = [ ] #add version without the [] at the back of the phrase #add phraseweight for phrase in phrases: if '[' in phrase and ']' in phrase: #split the string by [ and ], by replacing ] with [, then split all by ['s phraseSplit = phrase.replace(']', '[').split('[') number = phraseSplit[1] phrasesCleaned.append( phraseSplit[0] ) #add version without the [] at the back of the phrase phraseWeight.append(float(number)) else: #No [x] number found, putting default phraseweight as 1 phraseWeight.append(1) phrasesCleaned.append(phrase) this.AddOutline(filename.replace(".txt", ""), phrasesCleaned, phraseWeight) textout.SystemPrint("DataImporter: Intent outlines loaded!") else: textout.SystemWarning( "DataImporter: WARNING! Intent outlines folder is not found. Not loading any outlines!" ) return this.outlines
def PassiveListening(this, callback): def returnfalse(): #hack to remove need for interrupt lmao return False detector = snowboydecoder.HotwordDetector( "/home/pi/Esther/resources/Esther.pmdl", sensitivity=0.5) # main loop, Passive Listening detector.start(detected_callback=callback, interrupt_check=returnfalse, sleep_time=0.03) textout.SystemPrint( "Stopped listening passively, SHOULD NOT PRINT UNLESS PROGRAM STOPPED")
def PopulateEntitiesDict(this): entities = {} textout.SystemPrint("DataImporter: Loading entities from disk...") if os.path.exists(INTENT_PATH): fileDirectories = filter(lambda x: x[-4:] == '.txt', os.listdir(ENTITIES_PATH)) for filename in fileDirectories: #strip and split ReadTxtFile at the same time. entitiesTextArray = [ x.strip().lower() for x in this.ReadTxtFile( os.path.join(ENTITIES_PATH, filename)).split('\n') ] entityRequest = "!" + filename.replace(".txt", "") entities.setdefault(entityRequest, entitiesTextArray) textout.SystemPrint("DataImporter: Entities loaded!") else: textout.SystemWarning( "DataImporter: WARNING! Entities folder is not found. Not loading any entities!" ) return entities
def PopulateSynonymsDict(this): synonyms = {} textout.SystemPrint("DataImporter: Loading synonyms from disk...") if os.path.exists(SYNONYMS_PATH): fileDirectories = filter(lambda x: x[-4:] == '.txt', os.listdir(SYNONYMS_PATH)) for filename in fileDirectories: #strip and split ReadTxtFile at the same time. synonymTextArray = [ x.strip().lower() for x in this.ReadTxtFile( os.path.join(SYNONYMS_PATH, filename)).split('\n') ] #Shovel each line of synonymTextArray into a dict entry for dictEntry in synonymTextArray: phrases = [x.strip() for x in dictEntry.split(':')] synonyms.setdefault(phrases[0], phrases[1]) textout.SystemPrint("DataImporter: Synonyms data loaded") else: textout.SystemWarning( "DataImporter: WARNING! Synonyms folder is not found. Not loading any synonyms!" ) return synonyms
def ProcessInput(this,_usrinput): #Returns a custom item. Refer to "return" at bottom of function usrinput = this.FormatUsrinput(_usrinput) #Create a dict for score sorting #Format: #scores[0] = "phrases together": (scoreValue, [(entityType)], [phrases], [phrasePos], usrinputlength, intentName) scores = {} #Iterate every word in user sentence for i in range(0,len(usrinput)): #if current user word matches keyword if usrinput[i] in this.outlines: #Iterate through all outlines within this starting phrase for nestedOutline in this.outlines.get(usrinput[i]): textout.SystemPrint ("Recognized intent : " + str(nestedOutline)) #create tracker for which position each phrase appears in user's sentence, phasePos phrasePos = [-1,] * len(nestedOutline[1]) this.SetPhrasePos(i, usrinput,nestedOutline, phrasePos) extractedEntities = this.GetExtractedEntities(i, usrinput,nestedOutline, phrasePos) #check if any phrasePos values are -1. Invalidate this entire outline if so. if -1 in phrasePos: textout.SystemPrint ("**Some phrase is not found in this outline: " + str(nestedOutline[1]) + " Skipping this outline**") continue textout.Print ("Phrase's position in user sentence: " + str(phrasePos)) #Calculating scores #Take phraseWeight and apply below formula. #If not the first phrase, do this: phraseBaseScore - ((currindex - previndex - 1) * distMod * phraseWeight) #distMod will range from 0 (no penalty) to 1 (100% score penalty for each length away. 2 words means 200% penalty.) #add all scores together outlineScore = this.CalculateOutlineScore(phrasePos, nestedOutline, len(usrinput)) scores.setdefault(this.CombineStringsList(nestedOutline[1]),(outlineScore,extractedEntities, nestedOutline[1], phrasePos, len(usrinput), nestedOutline[0])) textout.Print ("Final Results: " + str(scores)) #Search for highest score highestKey = "" textout.Print ("---------------Sorting score--------------------") for key, value in scores.items(): if highestKey == "": #First item in loop highestKey = key textout.Print ("First outline set: \"" + highestKey + "\" with values: "+ str(scores[highestKey])) else: if value[0] > 0.3: #Disregard any value below 0.3 score #If number of phrases in value > number of phrases in highest key if len(value[2]) > len(scores[highestKey][2]): highestKey = key #If value of this score is higher than the highest so far elif value[0] > scores[highestKey][0]: highestKey = key if highestKey != "": textout.SystemPrint ("Intent chosen: \"" + highestKey + "\" with values: "+ str(scores[highestKey])) #Update training data for matches. #scores[0] IS TEMPORARY. scores[0] == top probability. #scores[x][2] = phrases | scores[x][3] = phrasePos | scores[x][4] = usrinputlength this.data.UpdateTrainingData(scores[highestKey][2], scores[highestKey][3], scores[highestKey][4], this.outlines) return (scores[highestKey][5],scores[highestKey][1]) #Returns: (intentName, [(entityType)]) return None
def CalculateOutlineScore(this, phrasePos, nestedOutline, usrinputlength): outlineScore = 0 phraseWeight = nestedOutline[3] phrasePosDiffAvgSd = nestedOutline[2] slrAVG = nestedOutline[4] slrSD = nestedOutline[5] phrasePosDiff = [0,] * len(phrasePos) #Calculate [phrasePosDiff] #Convert [phrasePos] to [phrasePosDiff] e.g. [1,3,8] > [0,2,5] for i in range(1,len(phrasePos)): #index 0 is always value 0, so start loop from 1. phrasePosDiff[i] = phrasePos[i] - phrasePos[i-1] #Calculating phraseDistMod for each phrase for i in range(0,len(phraseWeight)): #Check if AVG and SD from training data exists if slrAVG is not -1: #Index 0: Only phraseweight #Index 1 onwards: Phraseweight and phraseDistMod (sd/avg) if i == 0: textout.Print("----------First Phrase------------") textout.Print ("old outline is: " + str(outlineScore)) textout.Print("phraseweight: " + str(phraseWeight[i])) outlineScore = outlineScore + phraseWeight[i] textout.Print ("result outline is: " + str(outlineScore)) textout.Print("----------------------") else: textout.Print("----------Subsequent Phrase------------") textout.Print ("phrasePosDiff is: " + str(phrasePosDiff[i]) ) textout.Print("phrasePosAvg: " + str(phrasePosDiffAvgSd[i][0])) textout.Print("phrasePosSd: " + str(phrasePosDiffAvgSd[i][1])) textout.Print("----------------------") #phraseDistMod = abs(phrasePosDiff[i] - phrasePosDiffAvgSd[i][0]) / phrasePosDiffAvgSd[i][1] #outlineScore = outlineScore + phraseWeight[i] * phraseDistMod #How to use AVG and SD #All values within the range AVG+SD : AVG-SD recieve 0 distance penalty. #All values that fall outside that range will recieve penalty, starting from 0 up to infinity if phrasePosDiffAvgSd[i][0] - phrasePosDiffAvgSd[i][1] <= phrasePosDiff[i] <= phrasePosDiffAvgSd[i][0] + phrasePosDiffAvgSd[i][1]: textout.Print ("No penalty cos phraseposdiff is in range of the SDs") textout.Print ("old outline is: " + str(outlineScore)) textout.Print("phraseweight: " + str(phraseWeight[i])) outlineScore = outlineScore + phraseWeight[i] #No penalty in range textout.Print ("result outline is: " + str(outlineScore)) textout.Print("----------------------") else: #Calculate how far out of the range value is amountOut = 0 if phrasePosDiff[i] > phrasePosDiffAvgSd[i][0]: #If phraseposdiff is at the right side, bigger that AVG + SD amountOut = phrasePosDiff[i] - (phrasePosDiffAvgSd[i][0] + phrasePosDiffAvgSd[i][1]) else: #If phraseposdiff is at the left side, smaller that AVG - SD amountOut = (phrasePosDiffAvgSd[i][0] - phrasePosDiffAvgSd[i][1]) - phrasePosDiff[i] if phrasePosDiffAvgSd[i][1] != 0: #prevent divide by zero error if program saves repeated entries #phraseDistMod = (SD - amountOut) / SD >> IF amountOut is smaller than SD (usually), returns value 1 to 0.00001 #IF (SD - amountOut) is <= 0 (-ve) (exceeds AVG + SD + SD range, very out-there value), oh ho ho big big penalty dude if (phrasePosDiffAvgSd[i][1] - amountOut) > 0: phraseDistMod = (phrasePosDiffAvgSd[i][1] - amountOut) / phrasePosDiffAvgSd[i][1] else: #penalty: put a minimum cap on how low the penalty goes, cos it'll go quite low phraseDistMod = 0.01 else: #If SD is 0, leave phraseDistMod as default 1. phraseDistMod = 1 textout.SystemWarning("Uh-oh, a phrasePosSD has a value of 0. Assigning default value for phraseModDist.") textout.SystemWarning("Please check training data for outline: " + str(nestedOutline[0])) #apply penalty to the phrase then add it to outlinescore textout.Print ("Apply penalty cos phraseposdiff is out of range of the SDs") textout.Print ("old outline is: "+ str(outlineScore)) textout.Print("phraseweight: "+ str(phraseWeight[i])) textout.Print ("phraseDistMod is: "+ str( phraseDistMod)) outlineScore = outlineScore + phraseWeight[i] * phraseDistMod textout.Print ("result outline is: "+ str(outlineScore)) textout.Print("----------------------") else: #No training data exists. Calculate according to distance away. outlineScore = outlineScore + (phraseWeight[i] - (phrasePosDiff[i] * 0.06)) textout.Print ("NO TRAINING DATA. Doing default dist away calculations.") textout.Print ("result outline is: " + str(outlineScore)) #Calculating slrMod for entire outline textout.Print("----------Calculating SLR after phrases------------") textout.Print ("old outline is: " + str(outlineScore)) slr = this.data.GetSumOfList(phrasePosDiff) / usrinputlength textout.Print("slr: " + str(slr)) textout.Print("slrAVG: " + str(slrAVG)) textout.Print("slrSD: " + str(slrSD)) #If slr is out of SD range, apply penalty. If not, ignore and leave alone. if slrAVG - slrSD > slr > slrAVG + slrSD: #Calculate how much out of the range slr is amountOut = 0 if slr > slrAVG: #If slr is at the right side, bigger that AVG + SD amountOut = slr - (slrAVG + slrSD) else: #If slr is at the left side, smaller that AVG - SD amountOut = slr - (slrAVG - slrSD) textout.Print("amountOut: " + str(amountOut)) #slrMod = amountOut / SD * 100 | 10/10 useful comment slrMod = amountOut / slrSD * 100 textout.Print("slrMod: "+ str(slrMod)) #Apply slrMod to entire outline outlineScore = outlineScore * slrMod textout.Print ("result outline is: " + str(outlineScore)) textout.Print("----------------------") else: textout.Print ("No penalty cos slr is in range of the SDs") textout.SystemPrint ("Score for this outline: " + str(outlineScore)) return outlineScore
def WriteTrainingData(this): textout.SystemPrint("DataImporter: Writing training data") with open(TRAIN_PATH, 'w') as fp: json.dump(this.trainingdata, fp, sort_keys=True) textout.SystemPrint("DataImporter: Writing data completed.")