def ActiveListening(this):
threshold = None
textout.SystemPrint("Started to listen actively")
#RECORD A WAV FILE, CUTOFF AT 12s OR FALLS BELOW THRESHOLD
#SEND TO WITAI
#RECIEVE INPUT
#SEND TO PROCESSOR AND RETURN INTENT
#FIND ACTION.
RATE = 16000
CHUNK = 1024
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt16,
channels=1,
rate=RATE,
input=True,
frames_per_buffer=CHUNK)
frames = []
lastN = [
144 * 1.2 for i in range(15)
] #changing array length will determine if average will change faster or not
for i in range(0, int(RATE / CHUNK *
5)): #RATE(16000) / CHUNK(1024) * TIME (12s)
data = stream.read(CHUNK)
frames.append(data)
score = getScore(data)
lastN.pop(0)
lastN.append(score)
average = sum(lastN) / float(len(lastN))
print(str(average))
if average < 144 - 20:
textout.SystemPrint("Listening stopped, below threshold.")
break
textout.SystemPrint("Listening Timeout!")
stream.stop_stream()
stream.close()
p.terminate()
with tempfile.NamedTemporaryFile(mode='w+b') as f:
wav_fp = wave.open(f, 'wb')
wav_fp.setnchannels(1)
wav_fp.setsampwidth(pyaudio.get_sample_size(pyaudio.paInt16))
wav_fp.setframerate(16000)
wav_fp.writeframes(b''.join(frames))
wav_fp.close()
f.seek(0)
transcriber.TranscribeAudiofile(this, f)
textout.SystemPrint("Stopped listening actively")
def __init__(this):
textout.SystemPrint("Processor: Starting processor initialization" )
this.data = dataimporter.DataImporter()
this.data.PopulateTrainingData()
this.outlines = this.data.PopulateOutlinesDict()
this.synonyms = this.data.PopulateSynonymsDict()
this.entities = this.data.PopulateEntitiesDict()
this.LoadAllModules()
textout.SystemPrint ("Processor: Processor initialized!\n")
def PopulateTrainingData(this):
textout.SystemPrint("DataImporter: Loading Training Data...")
textout.SystemPrint("DataImporter: Reading disk training data")
if os.path.exists(TRAIN_PATH):
with open(TRAIN_PATH, 'r') as fp:
this.trainingdata = json.load(fp)
textout.SystemPrint("DataImporter: Training data loaded!")
else:
textout.SystemPrint(
"DataImporter: NOTICE! Disk training data is not found. Creating blank training data file."
)
def LoadAllModules(this):
textout.SystemPrint ("Module Loader: Loading all modules into processor instance...")
locations = [MOD_PATH]
for finder, name, ispkg in pkgutil.walk_packages(locations):
try:
loader = finder.find_module(name)
mod = loader.load_module(name)
except:
textout.SystemPrint ("Skipped module '" + name + "' due to an error.")
else:
textout.SystemPrint ("Found module '" + name + "'")
this.modules.append(mod)
textout.SystemPrint ("{} modules(s) loaded".format(len(this.modules)))
def UpdateTrainingData(this, phrases, phrasePos, usrinputlength, outlines):
#Try to add data into the trainingdata dict, from processor.py when a match is found.
#need to convert:
#[phrases] >> _outlineKey (string)
#[phrasePos] >> [phrasePosDiff]
#need to convert [phrases] into one single string
_outlineKey = this.CombineStringsList(phrases)
phrasePosDiff = [
0,
] * len(phrasePos)
#need to convert [phrasePos] >> [phrasePosDiff]
for i in range(1, len(
phrasePos)): #index 0 is always value 0, so start loop from 1.
phrasePosDiff[i] = phrasePos[i] - phrasePos[i - 1]
#Make _data
_data = [phrasePosDiff, usrinputlength]
#_data is in this format: ([phrasePosDiff], usrinputlength)
if _outlineKey in this.trainingdata:
if _data not in this.trainingdata[
_outlineKey]: #Add data if OUTLINE exists and _data is not in dict yet
textout.SystemPrint(
"DataImporter/UpdateTrainingData(): Data does not exist, adding!"
)
this.trainingdata[_outlineKey].append(_data)
this.WriteTrainingData()
this.UpdateOutlinesWithTrainingData(outlines)
else:
textout.SystemPrint(
"DataImporter/UpdateTrainingData(): Data already exists.")
else:
textout.SystemPrint(
"DataImporter/UpdateTrainingData(): Data does not exist, adding!"
)
this.trainingdata.setdefault(
_outlineKey,
[]) #If key does not exist, create one and add the data to it.
this.trainingdata[_outlineKey].append(_data)
this.WriteTrainingData()
this.UpdateOutlinesWithTrainingData(outlines)
def PopulateOutlinesDict(this):
textout.SystemPrint(
"DataImporter: Loading intent outlines from disk...")
if os.path.exists(INTENT_PATH):
fileDirectories = filter(lambda x: x[-4:] == '.txt',
os.listdir(INTENT_PATH))
for filename in fileDirectories:
#strip and split ReadTxtFile at the same time.
intentTextArray = [
x.strip().lower() for x in this.ReadTxtFile(
os.path.join(INTENT_PATH, filename)).split('\n')
]
#Shovel each line of intentText into an outline entry
for outlineRaw in intentTextArray:
phrases = [x.strip() for x in outlineRaw.split(' ')]
phraseWeight = []
phrasesCleaned = [
] #add version without the [] at the back of the phrase
#add phraseweight
for phrase in phrases:
if '[' in phrase and ']' in phrase:
#split the string by [ and ], by replacing ] with [, then split all by ['s
phraseSplit = phrase.replace(']', '[').split('[')
number = phraseSplit[1]
phrasesCleaned.append(
phraseSplit[0]
) #add version without the [] at the back of the phrase
phraseWeight.append(float(number))
else:
#No [x] number found, putting default phraseweight as 1
phraseWeight.append(1)
phrasesCleaned.append(phrase)
this.AddOutline(filename.replace(".txt", ""),
phrasesCleaned, phraseWeight)
textout.SystemPrint("DataImporter: Intent outlines loaded!")
else:
textout.SystemWarning(
"DataImporter: WARNING! Intent outlines folder is not found. Not loading any outlines!"
)
return this.outlines
def PassiveListening(this, callback):
def returnfalse(): #hack to remove need for interrupt lmao
return False
detector = snowboydecoder.HotwordDetector(
"/home/pi/Esther/resources/Esther.pmdl", sensitivity=0.5)
# main loop, Passive Listening
detector.start(detected_callback=callback,
interrupt_check=returnfalse,
sleep_time=0.03)
textout.SystemPrint(
"Stopped listening passively, SHOULD NOT PRINT UNLESS PROGRAM STOPPED")
def PopulateEntitiesDict(this):
entities = {}
textout.SystemPrint("DataImporter: Loading entities from disk...")
if os.path.exists(INTENT_PATH):
fileDirectories = filter(lambda x: x[-4:] == '.txt',
os.listdir(ENTITIES_PATH))
for filename in fileDirectories:
#strip and split ReadTxtFile at the same time.
entitiesTextArray = [
x.strip().lower() for x in this.ReadTxtFile(
os.path.join(ENTITIES_PATH, filename)).split('\n')
]
entityRequest = "!" + filename.replace(".txt", "")
entities.setdefault(entityRequest, entitiesTextArray)
textout.SystemPrint("DataImporter: Entities loaded!")
else:
textout.SystemWarning(
"DataImporter: WARNING! Entities folder is not found. Not loading any entities!"
)
return entities
def PopulateSynonymsDict(this):
synonyms = {}
textout.SystemPrint("DataImporter: Loading synonyms from disk...")
if os.path.exists(SYNONYMS_PATH):
fileDirectories = filter(lambda x: x[-4:] == '.txt',
os.listdir(SYNONYMS_PATH))
for filename in fileDirectories:
#strip and split ReadTxtFile at the same time.
synonymTextArray = [
x.strip().lower() for x in this.ReadTxtFile(
os.path.join(SYNONYMS_PATH, filename)).split('\n')
]
#Shovel each line of synonymTextArray into a dict entry
for dictEntry in synonymTextArray:
phrases = [x.strip() for x in dictEntry.split(':')]
synonyms.setdefault(phrases[0], phrases[1])
textout.SystemPrint("DataImporter: Synonyms data loaded")
else:
textout.SystemWarning(
"DataImporter: WARNING! Synonyms folder is not found. Not loading any synonyms!"
)
return synonyms
def ProcessInput(this,_usrinput): #Returns a custom item. Refer to "return" at bottom of function
usrinput = this.FormatUsrinput(_usrinput)
#Create a dict for score sorting
#Format:
#scores[0] = "phrases together": (scoreValue, [(entityType)], [phrases], [phrasePos], usrinputlength, intentName)
scores = {}
#Iterate every word in user sentence
for i in range(0,len(usrinput)):
#if current user word matches keyword
if usrinput[i] in this.outlines:
#Iterate through all outlines within this starting phrase
for nestedOutline in this.outlines.get(usrinput[i]):
textout.SystemPrint ("Recognized intent : " + str(nestedOutline))
#create tracker for which position each phrase appears in user's sentence, phasePos
phrasePos = [-1,] * len(nestedOutline[1])
this.SetPhrasePos(i, usrinput,nestedOutline, phrasePos)
extractedEntities = this.GetExtractedEntities(i, usrinput,nestedOutline, phrasePos)
#check if any phrasePos values are -1. Invalidate this entire outline if so.
if -1 in phrasePos:
textout.SystemPrint ("**Some phrase is not found in this outline: " + str(nestedOutline[1]) + " Skipping this outline**")
continue
textout.Print ("Phrase's position in user sentence: " + str(phrasePos))
#Calculating scores
#Take phraseWeight and apply below formula.
#If not the first phrase, do this: phraseBaseScore - ((currindex - previndex - 1) * distMod * phraseWeight)
#distMod will range from 0 (no penalty) to 1 (100% score penalty for each length away. 2 words means 200% penalty.)
#add all scores together
outlineScore = this.CalculateOutlineScore(phrasePos, nestedOutline, len(usrinput))
scores.setdefault(this.CombineStringsList(nestedOutline[1]),(outlineScore,extractedEntities, nestedOutline[1], phrasePos, len(usrinput), nestedOutline[0]))
textout.Print ("Final Results: " + str(scores))
#Search for highest score
highestKey = ""
textout.Print ("---------------Sorting score--------------------")
for key, value in scores.items():
if highestKey == "":
#First item in loop
highestKey = key
textout.Print ("First outline set: \"" + highestKey + "\" with values: "+ str(scores[highestKey]))
else:
if value[0] > 0.3: #Disregard any value below 0.3 score
#If number of phrases in value > number of phrases in highest key
if len(value[2]) > len(scores[highestKey][2]):
highestKey = key
#If value of this score is higher than the highest so far
elif value[0] > scores[highestKey][0]:
highestKey = key
if highestKey != "":
textout.SystemPrint ("Intent chosen: \"" + highestKey + "\" with values: "+ str(scores[highestKey]))
#Update training data for matches.
#scores[0] IS TEMPORARY. scores[0] == top probability.
#scores[x][2] = phrases | scores[x][3] = phrasePos | scores[x][4] = usrinputlength
this.data.UpdateTrainingData(scores[highestKey][2], scores[highestKey][3], scores[highestKey][4], this.outlines)
return (scores[highestKey][5],scores[highestKey][1]) #Returns: (intentName, [(entityType)])
return None
def CalculateOutlineScore(this, phrasePos, nestedOutline, usrinputlength):
outlineScore = 0
phraseWeight = nestedOutline[3]
phrasePosDiffAvgSd = nestedOutline[2]
slrAVG = nestedOutline[4]
slrSD = nestedOutline[5]
phrasePosDiff = [0,] * len(phrasePos)
#Calculate [phrasePosDiff]
#Convert [phrasePos] to [phrasePosDiff] e.g. [1,3,8] > [0,2,5]
for i in range(1,len(phrasePos)): #index 0 is always value 0, so start loop from 1.
phrasePosDiff[i] = phrasePos[i] - phrasePos[i-1]
#Calculating phraseDistMod for each phrase
for i in range(0,len(phraseWeight)):
#Check if AVG and SD from training data exists
if slrAVG is not -1:
#Index 0: Only phraseweight
#Index 1 onwards: Phraseweight and phraseDistMod (sd/avg)
if i == 0:
textout.Print("----------First Phrase------------")
textout.Print ("old outline is: " + str(outlineScore))
textout.Print("phraseweight: " + str(phraseWeight[i]))
outlineScore = outlineScore + phraseWeight[i]
textout.Print ("result outline is: " + str(outlineScore))
textout.Print("----------------------")
else:
textout.Print("----------Subsequent Phrase------------")
textout.Print ("phrasePosDiff is: " + str(phrasePosDiff[i]) )
textout.Print("phrasePosAvg: " + str(phrasePosDiffAvgSd[i][0]))
textout.Print("phrasePosSd: " + str(phrasePosDiffAvgSd[i][1]))
textout.Print("----------------------")
#phraseDistMod = abs(phrasePosDiff[i] - phrasePosDiffAvgSd[i][0]) / phrasePosDiffAvgSd[i][1]
#outlineScore = outlineScore + phraseWeight[i] * phraseDistMod
#How to use AVG and SD
#All values within the range AVG+SD : AVG-SD recieve 0 distance penalty.
#All values that fall outside that range will recieve penalty, starting from 0 up to infinity
if phrasePosDiffAvgSd[i][0] - phrasePosDiffAvgSd[i][1] <= phrasePosDiff[i] <= phrasePosDiffAvgSd[i][0] + phrasePosDiffAvgSd[i][1]:
textout.Print ("No penalty cos phraseposdiff is in range of the SDs")
textout.Print ("old outline is: " + str(outlineScore))
textout.Print("phraseweight: " + str(phraseWeight[i]))
outlineScore = outlineScore + phraseWeight[i] #No penalty in range
textout.Print ("result outline is: " + str(outlineScore))
textout.Print("----------------------")
else:
#Calculate how far out of the range value is
amountOut = 0
if phrasePosDiff[i] > phrasePosDiffAvgSd[i][0]:
#If phraseposdiff is at the right side, bigger that AVG + SD
amountOut = phrasePosDiff[i] - (phrasePosDiffAvgSd[i][0] + phrasePosDiffAvgSd[i][1])
else:
#If phraseposdiff is at the left side, smaller that AVG - SD
amountOut = (phrasePosDiffAvgSd[i][0] - phrasePosDiffAvgSd[i][1]) - phrasePosDiff[i]
if phrasePosDiffAvgSd[i][1] != 0: #prevent divide by zero error if program saves repeated entries
#phraseDistMod = (SD - amountOut) / SD >> IF amountOut is smaller than SD (usually), returns value 1 to 0.00001
#IF (SD - amountOut) is <= 0 (-ve) (exceeds AVG + SD + SD range, very out-there value), oh ho ho big big penalty dude
if (phrasePosDiffAvgSd[i][1] - amountOut) > 0:
phraseDistMod = (phrasePosDiffAvgSd[i][1] - amountOut) / phrasePosDiffAvgSd[i][1]
else:
#penalty: put a minimum cap on how low the penalty goes, cos it'll go quite low
phraseDistMod = 0.01
else:
#If SD is 0, leave phraseDistMod as default 1.
phraseDistMod = 1
textout.SystemWarning("Uh-oh, a phrasePosSD has a value of 0. Assigning default value for phraseModDist.")
textout.SystemWarning("Please check training data for outline: " + str(nestedOutline[0]))
#apply penalty to the phrase then add it to outlinescore
textout.Print ("Apply penalty cos phraseposdiff is out of range of the SDs")
textout.Print ("old outline is: "+ str(outlineScore))
textout.Print("phraseweight: "+ str(phraseWeight[i]))
textout.Print ("phraseDistMod is: "+ str( phraseDistMod))
outlineScore = outlineScore + phraseWeight[i] * phraseDistMod
textout.Print ("result outline is: "+ str(outlineScore))
textout.Print("----------------------")
else:
#No training data exists. Calculate according to distance away.
outlineScore = outlineScore + (phraseWeight[i] - (phrasePosDiff[i] * 0.06))
textout.Print ("NO TRAINING DATA. Doing default dist away calculations.")
textout.Print ("result outline is: " + str(outlineScore))
#Calculating slrMod for entire outline
textout.Print("----------Calculating SLR after phrases------------")
textout.Print ("old outline is: " + str(outlineScore))
slr = this.data.GetSumOfList(phrasePosDiff) / usrinputlength
textout.Print("slr: " + str(slr))
textout.Print("slrAVG: " + str(slrAVG))
textout.Print("slrSD: " + str(slrSD))
#If slr is out of SD range, apply penalty. If not, ignore and leave alone.
if slrAVG - slrSD > slr > slrAVG + slrSD:
#Calculate how much out of the range slr is
amountOut = 0
if slr > slrAVG:
#If slr is at the right side, bigger that AVG + SD
amountOut = slr - (slrAVG + slrSD)
else:
#If slr is at the left side, smaller that AVG - SD
amountOut = slr - (slrAVG - slrSD)
textout.Print("amountOut: " + str(amountOut))
#slrMod = amountOut / SD * 100 | 10/10 useful comment
slrMod = amountOut / slrSD * 100
textout.Print("slrMod: "+ str(slrMod))
#Apply slrMod to entire outline
outlineScore = outlineScore * slrMod
textout.Print ("result outline is: " + str(outlineScore))
textout.Print("----------------------")
else:
textout.Print ("No penalty cos slr is in range of the SDs")
textout.SystemPrint ("Score for this outline: " + str(outlineScore))
return outlineScore
def WriteTrainingData(this):
textout.SystemPrint("DataImporter: Writing training data")
with open(TRAIN_PATH, 'w') as fp:
json.dump(this.trainingdata, fp, sort_keys=True)
textout.SystemPrint("DataImporter: Writing data completed.")