def __init__(self):
###############################################################
#
# Sets up all default requirements
#
# - Helpers: Useful global functions
# - Data: Data functions
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.Data = Data()
def __init__(self, jumpWay):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.jumpwayCl = jumpWay
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def __init__(self, jumpWay):
self.Helpers = Helpers()
self.Logging = Logging()
self.jumpwayCl = jumpWay
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Train/")
self.Logging.logMessage(self.LogFile, "LogFile", "INFO",
"NLU Trainer LogFile Set")
self.Model = Model()
self.Data = Data(self.Logging, self.LogFile)
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.setupData()
self.setupEntities()
def initNLU(self):
###############################################################
#
# Initiates the NLU setting up the data, NLU / entities models
# and required modules such as context and extensions.
#
###############################################################
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
def setup(self):
self.Logging.logMessage(self.LogFile, "NLU", "INFO",
"NLU Classifier Initiating")
self.Data = Data(self.Logging, self.LogFile)
self.Model = Model()
self.Context = Context()
self.user = {}
self.ner = None
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["iMap"][0]
self.restoreEntitiesModel()
self.restoreModel()
self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Ready")
class Trainer():
def __init__(self, jumpWay):
self.Helpers = Helpers()
self.Logging = Logging()
self.jumpwayCl = jumpWay
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Train/")
self.Logging.logMessage(self.LogFile, "LogFile", "INFO",
"NLU Trainer LogFile Set")
self.Model = Model()
self.Data = Data(self.Logging, self.LogFile)
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.setupData()
self.setupEntities()
def setupData(self):
self.trainingData = self.Data.loadTrainingData()
self.Logging.logMessage(self.LogFile, "Trainer", "INFO",
"Loaded NLU Training Data")
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Logging.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Trainer Data Ready")
def setupEntities(self):
if self._confs["ClassifierSettings"]["Entities"] == "Mitie":
self.entityExtractor = Entities()
self.Logging.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entity Extractor Ready")
self.entityExtractor.trainEntities(
self._confs["ClassifierSettings"]["Mitie"]["ModelLocation"],
self.trainingData)
def trainModel(self):
while True:
self.Logging.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
humanStart, trainingStart = self.Helpers.timerStart()
self.Logging.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Model Training At " + humanStart)
self.jumpwayCl.publishToDeviceChannel(
"Training", {
"NeuralNet": "NLU",
"Start": trainingStart,
"End": "In Progress",
"Total": "In Progress",
"Message": "NLU Model Training At " + humanStart
})
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Logging.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
self.jumpwayCl.publishToDeviceChannel(
"Training", {
"NeuralNet":
"NLU",
"Start":
trainingStart,
"End":
trainingEnd,
"Total":
trainingTime,
"Message":
"NLU Model Trained At " + humanEnd + " In " +
str(trainingEnd) + " Seconds"
})
from src.CrossValidation import CrossValidation
from src.evaluation import *
from tools.Data import Data
import sys
import numpy as np
if __name__ == "__main__":
if len(sys.argv) != 5:
sys.stderr('trainfile num_folds learning_rate num_epochs')
sys.exit()
trainfile, num_folds, learning_rate, num_epochs = sys.argv[1:]
num_folds = int(num_folds)
num_epochs = int(num_epochs)
learning_rate = float(learning_rate)
train = Data(trainfile)
train.parse()
cv = CrossValidation(train, num_folds)
cv.stratify()
results = {} # {dataIndex : [fold_of_instance, predicted_class, actual_class, confidence_of_prediction]
for fold_i in range(1, num_folds + 1):
subdataTrain, subdataTest = cv.getSubdata(fold_i)
testDataIndicies = cv.subdataIndicies[fold_i]
perceptron = Perceptron()
perceptron.setWeights([0.1 for i in range(0,len(train.names))])
trainPerceptron(perceptron, subdataTrain, learning_rate, num_epochs)
for i in testDataIndicies:
label = train.npdata[i][-1]
if label == 1:
label = train.variables["class"][-1]
def loadTrain(self, trainFilename):
self.train = Data(trainFilename)
self.train.parse()
class Trainer():
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
def __init__(self, jumpWay):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.jumpwayCl = jumpWay
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
if self._confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self._confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.jumpwayCl.publishToDeviceChannel(
"Training", {
"NeuralNet": "NLU",
"Start": trainingStart,
"End": "In Progress",
"Total": "In Progress",
"Message": "NLU Model Training At " + humanStart
})
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
self.jumpwayCl.publishToDeviceChannel(
"Training", {
"NeuralNet":
"NLU",
"Start":
trainingStart,
"End":
trainingEnd,
"Total":
trainingTime,
"Message":
"NLU Model Trained At " + humanEnd + " In " +
str(trainingEnd) + " Seconds"
})
class BayesNet:
"""
Simple Bayes net model for discrete variables and binary response, supports structure
learning and predicting of Naive Bayes and Tree Augmented Net.
"""
##
# train is a Data object representing a training dataset
# test is a Data object representing a testing dataset
def __init__(self):
self.train = None
self.test = None
self.graph = Graph()
##
# Load the training dataset.
# @param trainFilename A string of filename
def loadTrain(self, trainFilename):
self.train = Data(trainFilename)
self.train.parse()
##
# Load the testing dataset.
# @param testFilename A string of filename
def loadTest(self, testFilename):
self.test = Data(testFilename)
self.test.parse()
##
# Build the graph structure of Naive Bayes
def buildNaiveBayes(self):
for name in self.train.names:
self.graph.addNode(name)
nodes = self.graph.getNodes()
for node in nodes:
if node != 'class':
self.graph.addEdge('class', node.getId())
##
# Build the graph structure of TAN, using Prim's algo.
# Every element in priority queue is (score, to_node, frm_node), where
# score = (-CMI, colIndex(frm), colIndex(to))
# to_node is current node object
# frm_node is the parent node which has edge frm_node -> to_node, with score
def buildTAN(self):
self.buildNaiveBayes() # first make a Naive Bayes structure
pri_q = Q.PriorityQueue()
rootNode = self.graph.getNode(self.train.names[0]) # first attribute
pri_q.put(((0, 0, 0), rootNode, Node(None))) # frm is a dummy node
visited = set() # a set of visited nodes' names
while (not pri_q.empty()) and (len(visited) < self.train.names):
item = pri_q.get() # top element stored in priority queue
score = item[0]
currNode = item[1] # current node object
currName = currNode.getId()
if currName in visited: continue
frmNode = item[2] # frm node object
frmName = frmNode.getId()
self.graph.addEdge(frmName, currName, score)
visited.add(currName)
for attrName in self.train.names:
if attrName not in visited:
CMI = calcCondMI(self.train, currName, attrName, 'class')
indexes = self.train.getColIndex([currName, attrName])
score = (-CMI, indexes[0], indexes[1])
nextNode = self.graph.getNode(attrName)
pri_q.put((score, nextNode, currNode))
##
# Given the values of attributes of a testing instance, predict its class.
# @param instanceVals A list of instance variable values, including the 'class' at the end.
# @return [predicted class value, actual class value, posterior prob of predicted value]
def predictOneInstance(self, instanceVals):
names = self.train.names # including the 'class'
posteriors = []
for y in self.train.variables['class']:
# give the actual posterior porbability in the output
true_y = instanceVals[-1]
Py = calcProb(self.train, ['class'], [y])
Px = calcProb(self.train, names[:-1], instanceVals[:-1])
Px_pa = calcProbsCondParents(self.train, names[:-1],
instanceVals[:-1] + [y], self.graph)
#print Py, Px, Px_pa
pred_p = Py * Px_pa / Px
posteriors.append([y, true_y, pred_p])
# normalize the posteriors
total = sum(x[-1] for x in posteriors)
posteriors = [x[:2] + [round(x[-1] / total, 12)] for x in posteriors]
return sorted(posteriors, key=lambda x: -x[-1])[0]
##
# Predict the class in all instances in testing dataset.
# @return A list of predicted result for instances in testing set.
def predictTestData(self):
results = []
for instance in self.test.data:
results.append(self.predictOneInstance(instance))
return results
## output the result as homework requirement
def printResults(self):
results = self.predictTestData()
# display structures
for attr in self.train.names[:-1]:
node = self.graph.getNode(attr)
parentNodes = node.getParents()
parentIndexes = sorted([self.train.getColIndex([p.getId()])[0] \
for p in parentNodes])
parentNames = [self.train.names[i] for i in parentIndexes]
print attr + ' ' + ' '.join(parentNames) + ' '
print ''
# display predicts
corrects = 0
for r in results:
if r[0] == r[1]:
corrects += 1
print ' '.join([str(x) for x in r])
print ''
# display correct predicts
print str(corrects)
print ''
def loadTest(self, testFilename):
self.test = Data(testFilename)
self.test.parse()
def loadTrain(self, trainFilename):
self.train = Data(trainFilename)
self.train.parse()
class BayesNet:
"""
Simple Bayes net model for discrete variables and binary response, supports structure
learning and predicting of Naive Bayes and Tree Augmented Net.
"""
##
# train is a Data object representing a training dataset
# test is a Data object representing a testing dataset
def __init__(self):
self.train = None
self.test = None
self.graph = Graph()
##
# Load the training dataset.
# @param trainFilename A string of filename
def loadTrain(self, trainFilename):
self.train = Data(trainFilename)
self.train.parse()
##
# Load the testing dataset.
# @param testFilename A string of filename
def loadTest(self, testFilename):
self.test = Data(testFilename)
self.test.parse()
##
# Build the graph structure of Naive Bayes
def buildNaiveBayes(self):
for name in self.train.names:
self.graph.addNode(name)
nodes = self.graph.getNodes()
for node in nodes:
if node != 'class':
self.graph.addEdge('class', node.getId())
##
# Build the graph structure of TAN, using Prim's algo.
# Every element in priority queue is (score, to_node, frm_node), where
# score = (-CMI, colIndex(frm), colIndex(to))
# to_node is current node object
# frm_node is the parent node which has edge frm_node -> to_node, with score
def buildTAN(self):
self.buildNaiveBayes() # first make a Naive Bayes structure
pri_q = Q.PriorityQueue()
rootNode = self.graph.getNode(self.train.names[0]) # first attribute
pri_q.put( ((0,0,0), rootNode, Node(None)) ) # frm is a dummy node
visited = set() # a set of visited nodes' names
while (not pri_q.empty()) and (len(visited) < self.train.names):
item = pri_q.get() # top element stored in priority queue
score = item[0]
currNode = item[1] # current node object
currName = currNode.getId()
if currName in visited: continue
frmNode = item[2] # frm node object
frmName = frmNode.getId()
self.graph.addEdge(frmName, currName, score)
visited.add(currName)
for attrName in self.train.names:
if attrName not in visited:
CMI = calcCondMI(self.train, currName, attrName, 'class')
indexes = self.train.getColIndex([currName, attrName])
score = (-CMI, indexes[0], indexes[1])
nextNode = self.graph.getNode(attrName)
pri_q.put((score, nextNode, currNode))
##
# Given the values of attributes of a testing instance, predict its class.
# @param instanceVals A list of instance variable values, including the 'class' at the end.
# @return [predicted class value, actual class value, posterior prob of predicted value]
def predictOneInstance(self, instanceVals):
names = self.train.names # including the 'class'
posteriors = []
for y in self.train.variables['class']:
# give the actual posterior porbability in the output
true_y = instanceVals[-1]
Py = calcProb(self.train, ['class'], [y])
Px = calcProb(self.train, names[:-1], instanceVals[:-1])
Px_pa = calcProbsCondParents(self.train, names[:-1], instanceVals[:-1] + [y], self.graph)
#print Py, Px, Px_pa
pred_p = Py * Px_pa / Px
posteriors.append([y, true_y, pred_p])
# normalize the posteriors
total = sum(x[-1] for x in posteriors)
posteriors = [x[:2] + [round(x[-1] / total, 12)] for x in posteriors]
return sorted(posteriors, key = lambda x: -x[-1])[0]
##
# Predict the class in all instances in testing dataset.
# @return A list of predicted result for instances in testing set.
def predictTestData(self):
results = []
for instance in self.test.data:
results.append(self.predictOneInstance(instance))
return results
## output the result as homework requirement
def printResults(self):
results = self.predictTestData()
# display structures
for attr in self.train.names[:-1]:
node = self.graph.getNode(attr)
parentNodes = node.getParents()
parentIndexes = sorted([self.train.getColIndex([p.getId()])[0] \
for p in parentNodes])
parentNames = [self.train.names[i] for i in parentIndexes]
print attr + ' ' + ' '.join(parentNames) + ' '
print ''
# display predicts
corrects = 0
for r in results:
if r[0] == r[1]:
corrects += 1
print ' '.join([str(x) for x in r])
print ''
# display correct predicts
print str(corrects)
print ''
def loadTest(self, testFilename):
self.test = Data(testFilename)
self.test.parse()
class NLU():
def __init__(self):
self.Helpers = Helpers()
self.Logging = Logging()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"NLU/")
self.ChatLogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] +
"Chat/")
self.Logging.logMessage(self.LogFile, "NLU", "INFO",
"NLU Classifier LogFile Set")
self.startMQTT()
def commandsCallback(self, topic, payload):
self.Logging.logMessage(
self.LogFile, "iotJumpWay", "INFO",
"Recieved iotJumpWay Command Data : " + str(payload))
commandData = json.loads(payload.decode("utf-8"))
def startMQTT(self):
try:
self.jumpwayClient = jumpWayDevice.DeviceConnection({
"locationID":
self._confs["iotJumpWay"]["Location"],
"zoneID":
self._confs["iotJumpWay"]["Zone"],
"deviceId":
self._confs["iotJumpWay"]["Device"],
"deviceName":
self._confs["iotJumpWay"]["DeviceName"],
"username":
self._confs["iotJumpWay"]["MQTT"]["Username"],
"password":
self._confs["iotJumpWay"]["MQTT"]["Password"]
})
self.jumpwayClient.connectToDevice()
self.jumpwayClient.subscribeToDeviceChannel("Commands")
self.jumpwayClient.deviceCommandsCallback = self.commandsCallback
self.Logging.logMessage(self.LogFile, "iotJumpWay", "INFO",
"iotJumpWay Client Ready")
except Exception as e:
self.Logging.logMessage(self.LogFile, "iotJumpWay", "INFO",
"iotJumpWay Client Initiation Failed")
print(str(e))
sys.exit()
def setup(self):
self.Logging.logMessage(self.LogFile, "NLU", "INFO",
"NLU Classifier Initiating")
self.Data = Data(self.Logging, self.LogFile)
self.Model = Model()
self.Context = Context()
self.user = {}
self.ner = None
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["iMap"][0]
self.restoreEntitiesModel()
self.restoreModel()
self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Ready")
def restoreEntitiesModel(self):
if os.path.exists(self._confs["ClassifierSettings"]["EntitiesDat"]):
self.ner = named_entity_extractor(
self._confs["ClassifierSettings"]["EntitiesDat"])
self.Logging.logMessage(self.LogFile, "NER", "OK",
"Restored NLU NER Model")
def restoreModel(self):
self.tmodel = self.Model.buildDNN(self.x, self.y)
self.Logging.logMessage(self.LogFile, "NLU", "INFO",
"Restored NLU Model")
def setupEntities(self):
if self._confs["ClassifierSettings"]["Entities"] == "Mitie":
self.entityExtractor = Entities()
self.Logging.logMessage(self.LogFile, "NER", "INFO",
"NLU Entity Extractor Initiated")
def initiateSession(self, userID):
self.userID = userID
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
self.Logging.logMessage(self.LogFile, "Session", "INFO",
"NLU Session Ready For User #" + self.userID)
def setThresholds(self, threshold):
self.threshold = float(threshold)
self.entityThrshld = self._confs["ClassifierSettings"]["Mitie"][
"Threshold"]
def predict(self, parsedSentence):
predictions = [[index, confidence] for index, confidence in enumerate(
self.tmodel.predict([
self.Data.makeInferenceBag(parsedSentence, self.trainedWords)
])[0]) if confidence > self.threshold]
predictions.sort(key=lambda x: x[1], reverse=True)
classification = []
for prediction in predictions:
classification.append(
(self.trainedClasses[prediction[0]], prediction[1]))
return classification
def talk(self, sentence, debug=False):
self.Logging.logMessage(self.LogFile, "GeniSys", "STATUS",
"Processing")
parsed, fallback, entityHolder, parsedSentence = self.entityExtractor.parseEntities(
sentence, self.ner, self.trainingData)
classification = self.predict(parsedSentence)
if len(classification) > 0:
clearEntities = False
theIntent = self.trainingData["intents"][self.intentMap[
classification[0][0]]]
if len(entityHolder) and not len(theIntent["entities"]):
clearEntities = True
if (self.Context.checkSessionContext(self.user[self.userID],
theIntent)):
if self.Context.checkClearContext(theIntent, 0):
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and "fallbacks" in theIntent and len(
theIntent["fallbacks"]):
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
action, actionResponses = self.Helpers.setAction(theIntent)
elif "entityType" in theIntent and theIntent[
"entityType"] == "Numbers":
response = random.choice(theIntent["responses"])
action, actionResponses = self.Helpers.setAction(theIntent)
elif not len(entityHolder) and len(theIntent["entities"]):
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
action, actionResponses = self.Helpers.setAction(theIntent)
elif clearEntities:
entityHolder = []
response = random.choice(theIntent["responses"])
action, actionResponses = self.Helpers.setAction(theIntent)
else:
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
action, actionResponses = self.Helpers.setAction(theIntent)
if action != None:
classParts = action.split(".")
classFolder = classParts[0]
className = classParts[1]
module = __import__(classParts[0] + "." + classParts[1],
globals(), locals(), [className])
actionClass = getattr(module, className)()
response = getattr(actionClass, classParts[2])(
random.choice(actionResponses))
return {
"Response":
"OK",
"ResponseData": [{
"Received": sentence,
"Intent": classification[0][0],
"Confidence": str(classification[0][1]),
"Response": response,
"ContextIn": contextIn,
"ContextOut": contextOut,
"Context": contextCurrent,
"Action": action,
"Entities": entityHolder
}]
}
else:
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and fallback in theIntent and len(
theIntent["fallbacks"]):
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
action, actionResponses = None, []
else:
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
action, actionResponses = self.Helpers.setAction(theIntent)
if action != None:
classParts = action.split(".")
classFolder = classParts[0]
className = classParts[1]
module = __import__(classParts[0] + "." + classParts[1],
globals(), locals(), [className])
actionClass = getattr(module, className)()
response = getattr(actionClass, classParts[2])(
random.choice(actionResponses))
else:
response = self.entityExtractor.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
return {
"Response":
"OK",
"ResponseData": [{
"Received": sentence,
"Intent": classification[0][0],
"Confidence": str(classification[0][1]),
"Response": response,
"ContextIn": contextIn,
"ContextOut": contextOut,
"ContextCurrent": contextCurrent,
"Action": action,
"Entities": entityHolder
}]
}
else:
contextCurrent = self.Context.getCurrentContext(
self.user[self.userID])
return {
"Response":
"FAILED",
"ResponseData": [{
"Received":
sentence,
"Intent":
"UNKNOWN",
"Confidence":
"NA",
"Responses": [],
"Response":
random.choice(
self._confs["ClassifierSettings"]["defaultResponses"]),
"ContextIn":
"NA",
"ContextOut":
"NA",
"ContextCurrent":
contextCurrent,
"Action":
"NA",
"Entities":
entityHolder
}]
}
elif sys.argv[1] == 'test':
t = Test()
t.test_scrape_today_buysell()
# elif sys.argv[1] == 'bm':
# b = Benchmark(start_path)
# df, exists = b.get()
# recent_date = list(df.ix[len(df)-1])[0].replace('-', '')
# if exists:
# print('Recent update: ' + recent_date)
# else:
# print('Downloaded data to: ' + recent_date)
#
elif sys.argv[1] == 'data':
d = Data(start_path)
if sys.argv[2] == 'send':
if sys.argv[3] == 'ticker':
d.send_ticker()
elif sys.argv[3] == 'bm':
d.send_bm()
elif sys.argv[3] == 'ohlcv':
d.send_ohlcv()
elif sys.argv[2] == 'update':
if sys.argv[3] == 'ohlcv':
d.update_ohlcv()
elif sys.argv[3] == 'ohlcv_with_date_1':
d.upd_ohlcv_1()
elif sys.argv[3] == 'ohlcv_with_date_2':
d.upd_ohlcv_2()
elif sys.argv[3] == 'ohlcv_with_date_3':
class Model():
def __init__(self):
###############################################################
#
# Sets up all default requirements
#
# - Helpers: Useful global functions
# - Data: Data functions
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.Data = Data()
def createDNNLayers(self, x, y):
###############################################################
#
# Sets up the DNN layers, configuration in required/confs.json
#
###############################################################
net = tflearn.input_data(shape=[None, len(x[0])])
for i in range(self._confs["NLU"]['FcLayers']):
net = tflearn.fully_connected(net, self._confs["NLU"]['FcUnits'])
net = tflearn.fully_connected(net,
len(y[0]),
activation=str(
self._confs["NLU"]['Activation']))
if self._confs["NLU"]['Regression']:
net = tflearn.regression(net)
return net
def trainDNN(self, x, y, words, classes, intentMap):
###############################################################
#
# Trains the DNN, configuration in required/confs.json
#
###############################################################
tf.reset_default_graph()
tmodel = tflearn.DNN(
self.createDNNLayers(x, y),
tensorboard_dir=self._confs["NLU"]['TFLearn']['Logs'],
tensorboard_verbose=self._confs["NLU"]['TFLearn']['LogsLevel'])
tmodel.fit(x,
y,
n_epoch=self._confs["NLU"]['Epochs'],
batch_size=self._confs["NLU"]['BatchSize'],
show_metric=self._confs["NLU"]['ShowMetric'])
self.saveModelData(
self._confs["NLU"]['TFLearn']['Data'], {
'words': words,
'classes': classes,
'x': x,
'y': y,
'intentMap': [intentMap]
}, tmodel)
def saveModelData(self, path, data, tmodel):
###############################################################
#
# Saves the model data for TFLearn and the NLU engine,
# configuration in required/confs.json
#
###############################################################
tmodel.save(self._confs["NLU"]['TFLearn']['Path'])
with open(path, "w") as outfile:
json.dump(data, outfile)
def buildDNN(self, x, y):
###############################################################
#
# Loads the DNN model, configuration in required/confs.json
#
###############################################################
tf.reset_default_graph()
tmodel = tflearn.DNN(self.createDNNLayers(x, y))
tmodel.load(self._confs["NLU"]['TFLearn']['Path'])
return tmodel
def predict(self, tmodel, parsedSentence, trainedWords, trainedClasses):
###############################################################
#
# Makes a prediction against the trained model, checking the
# confidence and then logging the results.
#
###############################################################
predictions = [[index, confidence] for index, confidence in enumerate(
tmodel.predict(
[self.Data.makeBagOfWords(parsedSentence, trainedWords)])[0])]
predictions.sort(key=lambda x: x[1], reverse=True)
classification = []
for prediction in predictions:
classification.append(
(trainedClasses[prediction[0]], prediction[1]))
return classification
class NLU():
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
self.isTraining = False
self.ner = None
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.user = {}
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"NLU/")
self.ChatLogFile = self.Helpers.setLogFile(
self._confs["aiCore"]["Logs"] + "Chat/")
self.jumpWay = JumpWay()
self.jumpWayClient = self.jumpWay.startMQTT()
self.jumpWayClient.subscribeToDeviceChannel(
self._confs["iotJumpWay"]["Channels"]["Commands"])
self.jumpWayClient.deviceCommandsCallback = self.commandsCallback
def initiateSession(self):
###############################################################
#
# Initiates empty guest user session, GeniSys will ask the user
# verify their GeniSys user by speaking or typing if it does
# not know who it is speaking to.
#
###############################################################
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def initNLU(self):
###############################################################
#
# Initiates the NLU setting up the data, NLU / entities models
# and required modules such as context and extensions.
#
###############################################################
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
def commandsCallback(self, topic, payload):
###############################################################
#
# The callback function that is triggerend in the event of a
# command communication from the iotJumpWay.
#
###############################################################
self.Helpers.logMessage(
self.LogFile, "iotJumpWay", "INFO",
"Recieved iotJumpWay Command Data : " + str(payload))
commandData = json.loads(payload.decode("utf-8"))
def restoreData(self):
###############################################################
#
# Sets the local trained data using data retrieved above
#
###############################################################
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def loadEntityController(self):
###############################################################
#
# Initiates the entity extractor class from tools
#
###############################################################
self.entityController = Entities()
def restoreNER(self):
###############################################################
#
# Loads entity controller and restores the NER model
#
###############################################################
self.loadEntityController()
self.ner = self.entityController.restoreNER()
def restoreNLU(self):
###############################################################
#
# Restores the NLU model
#
###############################################################
self.tmodel = self.Model.buildDNN(self.x, self.y)
def setThresholds(self):
###############################################################
#
# Sets the threshold for the NLU engine, this can be changed
# using arguments to commandline programs or paramters for
# API calls.
#
###############################################################
self.threshold = self._confs["NLU"]["Threshold"]
self.entityThrshld = self._confs["NLU"]["Mitie"]["Threshold"]
def communicate(self, sentence):
###############################################################
#
# First checks to ensure that the program is not training,
# then parses any entities that may be in the intent, then
# checks context and extensions before providing a response.
#
###############################################################
if self.isTraining == False:
parsed, fallback, entityHolder, parsedSentence = self.entityController.parseEntities(
sentence, self.ner, self.trainingData)
classification = self.Model.predict(self.tmodel, parsedSentence,
self.trainedWords,
self.trainedClasses)
if len(classification) > 0:
clearEntities = False
theIntent = self.trainingData["intents"][self.intentMap[
classification[0][0]]]
if len(entityHolder) and not len(theIntent["entities"]):
clearEntities = True
if (self.Context.checkSessionContext(self.user[self.userID],
theIntent)):
if self.Context.checkClearContext(theIntent, 0):
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entityHolder = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and fallback in theIntent and len(
theIntent["fallbacks"]):
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses = None, []
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
contextCurrent = self.Context.getCurrentContext(
self.user[self.userID])
return {
"Response":
"FAILED",
"ResponseData": [{
"Received":
sentence,
"Intent":
"UNKNOWN",
"Confidence":
"NA",
"Responses": [],
"Response":
random.choice(self._confs["NLU"]["defaultResponses"]),
"Context": [{
"In": "NA",
"Out": "NA",
"Current": contextCurrent
}],
"Extension":
"NA",
"Entities":
entityHolder
}]
}
else:
return {
"Response":
"FAILED",
"ResponseData": [{
"Status": "Training",
"Message": "NLU Engine is currently training"
}]
}
s.check()
elif sys.argv[2] == 'set':
s.set(sys.argv[3], sys.argv[4])
s.save()
elif sys.argv[1] == 'bm':
b = Benchmark(start_path)
df, exists = b.get()
recent_date = list(df.ix[len(df) - 1])[0].replace('-', '')
if exists:
print('Recent update: ' + recent_date)
else:
print('Downloaded data to: ' + recent_date)
elif sys.argv[1] == 'data':
d = Data(start_path)
if sys.argv[2] == 'send':
if sys.argv[3] == 'ticker':
d.send_ticker()
elif sys.argv[3] == 'bm':
d.send_bm()
elif sys.argv[3] == 'ohlcv':
d.send_ohlcv()
elif sys.argv[2] == 'update':
if sys.argv[3] == 'ohlcv':
d.update_ohlcv()
elif sys.argv[2] == 'clean':
if sys.argv[3] == 'ohlcv':
d.clean_ohlcv()
elif sys.argv[3] == 'bm':
d.clean_bm()
