def __init__(self): ############################################################### # # Sets up all default requirements # # - Helpers: Useful global functions # - Data: Data functions # ############################################################### self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.Data = Data()
def __init__(self, jumpWay): self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] + "Train/") self.jumpwayCl = jumpWay self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.Model = Model() self.Data = Data()
def __init__(self, jumpWay): self.Helpers = Helpers() self.Logging = Logging() self.jumpwayCl = jumpWay self._confs = self.Helpers.loadConfigs() self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] + "Train/") self.Logging.logMessage(self.LogFile, "LogFile", "INFO", "NLU Trainer LogFile Set") self.Model = Model() self.Data = Data(self.Logging, self.LogFile) self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.setupData() self.setupEntities()
def initNLU(self): ############################################################### # # Initiates the NLU setting up the data, NLU / entities models # and required modules such as context and extensions. # ############################################################### self.Data = Data() self.trainingData = self.Data.loadTrainingData() self.trainedData = self.Data.loadTrainedData() self.Model = Model() self.Context = Context() self.Extensions = Extensions() self.restoreData() self.restoreNER() self.restoreNLU() self.initiateSession() self.setThresholds()
def setup(self): self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Classifier Initiating") self.Data = Data(self.Logging, self.LogFile) self.Model = Model() self.Context = Context() self.user = {} self.ner = None self.trainingData = self.Data.loadTrainingData() self.trainedData = self.Data.loadTrainedData() self.trainedWords = self.trainedData["words"] self.trainedClasses = self.trainedData["classes"] self.x = self.trainedData["x"] self.y = self.trainedData["y"] self.intentMap = self.trainedData["iMap"][0] self.restoreEntitiesModel() self.restoreModel() self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Ready")
class Trainer(): def __init__(self, jumpWay): self.Helpers = Helpers() self.Logging = Logging() self.jumpwayCl = jumpWay self._confs = self.Helpers.loadConfigs() self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] + "Train/") self.Logging.logMessage(self.LogFile, "LogFile", "INFO", "NLU Trainer LogFile Set") self.Model = Model() self.Data = Data(self.Logging, self.LogFile) self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.setupData() self.setupEntities() def setupData(self): self.trainingData = self.Data.loadTrainingData() self.Logging.logMessage(self.LogFile, "Trainer", "INFO", "Loaded NLU Training Data") self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData( self.trainingData) self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus, self.words) self.Logging.logMessage(self.LogFile, "TRAIN", "INFO", "NLU Trainer Data Ready") def setupEntities(self): if self._confs["ClassifierSettings"]["Entities"] == "Mitie": self.entityExtractor = Entities() self.Logging.logMessage(self.LogFile, "TRAIN", "OK", "NLU Trainer Entity Extractor Ready") self.entityExtractor.trainEntities( self._confs["ClassifierSettings"]["Mitie"]["ModelLocation"], self.trainingData) def trainModel(self): while True: self.Logging.logMessage(self.LogFile, "TRAIN", "ACTION", "Ready To Begin Training ? (Yes/No)") userInput = input(">") if userInput == 'Yes': break if userInput == 'No': exit() humanStart, trainingStart = self.Helpers.timerStart() self.Logging.logMessage(self.LogFile, "TRAIN", "INFO", "NLU Model Training At " + humanStart) self.jumpwayCl.publishToDeviceChannel( "Training", { "NeuralNet": "NLU", "Start": trainingStart, "End": "In Progress", "Total": "In Progress", "Message": "NLU Model Training At " + humanStart }) self.Model.trainDNN(self.x, self.y, self.words, self.classes, self.intentMap) trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd( trainingStart) self.Logging.logMessage( self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds") self.jumpwayCl.publishToDeviceChannel( "Training", { "NeuralNet": "NLU", "Start": trainingStart, "End": trainingEnd, "Total": trainingTime, "Message": "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds" })
from src.CrossValidation import CrossValidation from src.evaluation import * from tools.Data import Data import sys import numpy as np if __name__ == "__main__": if len(sys.argv) != 5: sys.stderr('trainfile num_folds learning_rate num_epochs') sys.exit() trainfile, num_folds, learning_rate, num_epochs = sys.argv[1:] num_folds = int(num_folds) num_epochs = int(num_epochs) learning_rate = float(learning_rate) train = Data(trainfile) train.parse() cv = CrossValidation(train, num_folds) cv.stratify() results = {} # {dataIndex : [fold_of_instance, predicted_class, actual_class, confidence_of_prediction] for fold_i in range(1, num_folds + 1): subdataTrain, subdataTest = cv.getSubdata(fold_i) testDataIndicies = cv.subdataIndicies[fold_i] perceptron = Perceptron() perceptron.setWeights([0.1 for i in range(0,len(train.names))]) trainPerceptron(perceptron, subdataTrain, learning_rate, num_epochs) for i in testDataIndicies: label = train.npdata[i][-1] if label == 1: label = train.variables["class"][-1]
def loadTrain(self, trainFilename): self.train = Data(trainFilename) self.train.parse()
class Trainer(): ############################################################### # # Sets up all default requirements and placeholders # needed for the NLU engine to run. # # - Helpers: Useful global functions # - JumpWay/jumpWayClient: iotJumpWay class and connection # - Logging: Logging class # ############################################################### def __init__(self, jumpWay): self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] + "Train/") self.jumpwayCl = jumpWay self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.Model = Model() self.Data = Data() def setupData(self): self.trainingData = self.Data.loadTrainingData() self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData( self.trainingData) self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus, self.words) self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO", "NLU Training Data Ready") def setupEntities(self): if self._confs["NLU"]["Entities"] == "Mitie": self.entityController = Entities() self.entityController.trainEntities( self._confs["NLU"]["Mitie"]["ModelLocation"], self.trainingData) self.Helpers.logMessage(self.LogFile, "TRAIN", "OK", "NLU Trainer Entities Ready") def trainModel(self): while True: self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION", "Ready To Begin Training ? (Yes/No)") userInput = input(">") if userInput == 'Yes': break if userInput == 'No': exit() self.setupData() self.setupEntities() humanStart, trainingStart = self.Helpers.timerStart() self.jumpwayCl.publishToDeviceChannel( "Training", { "NeuralNet": "NLU", "Start": trainingStart, "End": "In Progress", "Total": "In Progress", "Message": "NLU Model Training At " + humanStart }) self.Model.trainDNN(self.x, self.y, self.words, self.classes, self.intentMap) trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd( trainingStart) self.Helpers.logMessage( self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds") self.jumpwayCl.publishToDeviceChannel( "Training", { "NeuralNet": "NLU", "Start": trainingStart, "End": trainingEnd, "Total": trainingTime, "Message": "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds" })
class BayesNet: """ Simple Bayes net model for discrete variables and binary response, supports structure learning and predicting of Naive Bayes and Tree Augmented Net. """ ## # train is a Data object representing a training dataset # test is a Data object representing a testing dataset def __init__(self): self.train = None self.test = None self.graph = Graph() ## # Load the training dataset. # @param trainFilename A string of filename def loadTrain(self, trainFilename): self.train = Data(trainFilename) self.train.parse() ## # Load the testing dataset. # @param testFilename A string of filename def loadTest(self, testFilename): self.test = Data(testFilename) self.test.parse() ## # Build the graph structure of Naive Bayes def buildNaiveBayes(self): for name in self.train.names: self.graph.addNode(name) nodes = self.graph.getNodes() for node in nodes: if node != 'class': self.graph.addEdge('class', node.getId()) ## # Build the graph structure of TAN, using Prim's algo. # Every element in priority queue is (score, to_node, frm_node), where # score = (-CMI, colIndex(frm), colIndex(to)) # to_node is current node object # frm_node is the parent node which has edge frm_node -> to_node, with score def buildTAN(self): self.buildNaiveBayes() # first make a Naive Bayes structure pri_q = Q.PriorityQueue() rootNode = self.graph.getNode(self.train.names[0]) # first attribute pri_q.put(((0, 0, 0), rootNode, Node(None))) # frm is a dummy node visited = set() # a set of visited nodes' names while (not pri_q.empty()) and (len(visited) < self.train.names): item = pri_q.get() # top element stored in priority queue score = item[0] currNode = item[1] # current node object currName = currNode.getId() if currName in visited: continue frmNode = item[2] # frm node object frmName = frmNode.getId() self.graph.addEdge(frmName, currName, score) visited.add(currName) for attrName in self.train.names: if attrName not in visited: CMI = calcCondMI(self.train, currName, attrName, 'class') indexes = self.train.getColIndex([currName, attrName]) score = (-CMI, indexes[0], indexes[1]) nextNode = self.graph.getNode(attrName) pri_q.put((score, nextNode, currNode)) ## # Given the values of attributes of a testing instance, predict its class. # @param instanceVals A list of instance variable values, including the 'class' at the end. # @return [predicted class value, actual class value, posterior prob of predicted value] def predictOneInstance(self, instanceVals): names = self.train.names # including the 'class' posteriors = [] for y in self.train.variables['class']: # give the actual posterior porbability in the output true_y = instanceVals[-1] Py = calcProb(self.train, ['class'], [y]) Px = calcProb(self.train, names[:-1], instanceVals[:-1]) Px_pa = calcProbsCondParents(self.train, names[:-1], instanceVals[:-1] + [y], self.graph) #print Py, Px, Px_pa pred_p = Py * Px_pa / Px posteriors.append([y, true_y, pred_p]) # normalize the posteriors total = sum(x[-1] for x in posteriors) posteriors = [x[:2] + [round(x[-1] / total, 12)] for x in posteriors] return sorted(posteriors, key=lambda x: -x[-1])[0] ## # Predict the class in all instances in testing dataset. # @return A list of predicted result for instances in testing set. def predictTestData(self): results = [] for instance in self.test.data: results.append(self.predictOneInstance(instance)) return results ## output the result as homework requirement def printResults(self): results = self.predictTestData() # display structures for attr in self.train.names[:-1]: node = self.graph.getNode(attr) parentNodes = node.getParents() parentIndexes = sorted([self.train.getColIndex([p.getId()])[0] \ for p in parentNodes]) parentNames = [self.train.names[i] for i in parentIndexes] print attr + ' ' + ' '.join(parentNames) + ' ' print '' # display predicts corrects = 0 for r in results: if r[0] == r[1]: corrects += 1 print ' '.join([str(x) for x in r]) print '' # display correct predicts print str(corrects) print ''
def loadTest(self, testFilename): self.test = Data(testFilename) self.test.parse()
class BayesNet: """ Simple Bayes net model for discrete variables and binary response, supports structure learning and predicting of Naive Bayes and Tree Augmented Net. """ ## # train is a Data object representing a training dataset # test is a Data object representing a testing dataset def __init__(self): self.train = None self.test = None self.graph = Graph() ## # Load the training dataset. # @param trainFilename A string of filename def loadTrain(self, trainFilename): self.train = Data(trainFilename) self.train.parse() ## # Load the testing dataset. # @param testFilename A string of filename def loadTest(self, testFilename): self.test = Data(testFilename) self.test.parse() ## # Build the graph structure of Naive Bayes def buildNaiveBayes(self): for name in self.train.names: self.graph.addNode(name) nodes = self.graph.getNodes() for node in nodes: if node != 'class': self.graph.addEdge('class', node.getId()) ## # Build the graph structure of TAN, using Prim's algo. # Every element in priority queue is (score, to_node, frm_node), where # score = (-CMI, colIndex(frm), colIndex(to)) # to_node is current node object # frm_node is the parent node which has edge frm_node -> to_node, with score def buildTAN(self): self.buildNaiveBayes() # first make a Naive Bayes structure pri_q = Q.PriorityQueue() rootNode = self.graph.getNode(self.train.names[0]) # first attribute pri_q.put( ((0,0,0), rootNode, Node(None)) ) # frm is a dummy node visited = set() # a set of visited nodes' names while (not pri_q.empty()) and (len(visited) < self.train.names): item = pri_q.get() # top element stored in priority queue score = item[0] currNode = item[1] # current node object currName = currNode.getId() if currName in visited: continue frmNode = item[2] # frm node object frmName = frmNode.getId() self.graph.addEdge(frmName, currName, score) visited.add(currName) for attrName in self.train.names: if attrName not in visited: CMI = calcCondMI(self.train, currName, attrName, 'class') indexes = self.train.getColIndex([currName, attrName]) score = (-CMI, indexes[0], indexes[1]) nextNode = self.graph.getNode(attrName) pri_q.put((score, nextNode, currNode)) ## # Given the values of attributes of a testing instance, predict its class. # @param instanceVals A list of instance variable values, including the 'class' at the end. # @return [predicted class value, actual class value, posterior prob of predicted value] def predictOneInstance(self, instanceVals): names = self.train.names # including the 'class' posteriors = [] for y in self.train.variables['class']: # give the actual posterior porbability in the output true_y = instanceVals[-1] Py = calcProb(self.train, ['class'], [y]) Px = calcProb(self.train, names[:-1], instanceVals[:-1]) Px_pa = calcProbsCondParents(self.train, names[:-1], instanceVals[:-1] + [y], self.graph) #print Py, Px, Px_pa pred_p = Py * Px_pa / Px posteriors.append([y, true_y, pred_p]) # normalize the posteriors total = sum(x[-1] for x in posteriors) posteriors = [x[:2] + [round(x[-1] / total, 12)] for x in posteriors] return sorted(posteriors, key = lambda x: -x[-1])[0] ## # Predict the class in all instances in testing dataset. # @return A list of predicted result for instances in testing set. def predictTestData(self): results = [] for instance in self.test.data: results.append(self.predictOneInstance(instance)) return results ## output the result as homework requirement def printResults(self): results = self.predictTestData() # display structures for attr in self.train.names[:-1]: node = self.graph.getNode(attr) parentNodes = node.getParents() parentIndexes = sorted([self.train.getColIndex([p.getId()])[0] \ for p in parentNodes]) parentNames = [self.train.names[i] for i in parentIndexes] print attr + ' ' + ' '.join(parentNames) + ' ' print '' # display predicts corrects = 0 for r in results: if r[0] == r[1]: corrects += 1 print ' '.join([str(x) for x in r]) print '' # display correct predicts print str(corrects) print ''
class NLU(): def __init__(self): self.Helpers = Helpers() self.Logging = Logging() self._confs = self.Helpers.loadConfigs() self.LogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] + "NLU/") self.ChatLogFile = self.Logging.setLogFile(self._confs["AI"]["Logs"] + "Chat/") self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Classifier LogFile Set") self.startMQTT() def commandsCallback(self, topic, payload): self.Logging.logMessage( self.LogFile, "iotJumpWay", "INFO", "Recieved iotJumpWay Command Data : " + str(payload)) commandData = json.loads(payload.decode("utf-8")) def startMQTT(self): try: self.jumpwayClient = jumpWayDevice.DeviceConnection({ "locationID": self._confs["iotJumpWay"]["Location"], "zoneID": self._confs["iotJumpWay"]["Zone"], "deviceId": self._confs["iotJumpWay"]["Device"], "deviceName": self._confs["iotJumpWay"]["DeviceName"], "username": self._confs["iotJumpWay"]["MQTT"]["Username"], "password": self._confs["iotJumpWay"]["MQTT"]["Password"] }) self.jumpwayClient.connectToDevice() self.jumpwayClient.subscribeToDeviceChannel("Commands") self.jumpwayClient.deviceCommandsCallback = self.commandsCallback self.Logging.logMessage(self.LogFile, "iotJumpWay", "INFO", "iotJumpWay Client Ready") except Exception as e: self.Logging.logMessage(self.LogFile, "iotJumpWay", "INFO", "iotJumpWay Client Initiation Failed") print(str(e)) sys.exit() def setup(self): self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Classifier Initiating") self.Data = Data(self.Logging, self.LogFile) self.Model = Model() self.Context = Context() self.user = {} self.ner = None self.trainingData = self.Data.loadTrainingData() self.trainedData = self.Data.loadTrainedData() self.trainedWords = self.trainedData["words"] self.trainedClasses = self.trainedData["classes"] self.x = self.trainedData["x"] self.y = self.trainedData["y"] self.intentMap = self.trainedData["iMap"][0] self.restoreEntitiesModel() self.restoreModel() self.Logging.logMessage(self.LogFile, "NLU", "INFO", "NLU Ready") def restoreEntitiesModel(self): if os.path.exists(self._confs["ClassifierSettings"]["EntitiesDat"]): self.ner = named_entity_extractor( self._confs["ClassifierSettings"]["EntitiesDat"]) self.Logging.logMessage(self.LogFile, "NER", "OK", "Restored NLU NER Model") def restoreModel(self): self.tmodel = self.Model.buildDNN(self.x, self.y) self.Logging.logMessage(self.LogFile, "NLU", "INFO", "Restored NLU Model") def setupEntities(self): if self._confs["ClassifierSettings"]["Entities"] == "Mitie": self.entityExtractor = Entities() self.Logging.logMessage(self.LogFile, "NER", "INFO", "NLU Entity Extractor Initiated") def initiateSession(self, userID): self.userID = userID if not self.userID in self.user: self.user[self.userID] = {} self.user[self.userID]["history"] = {} self.Logging.logMessage(self.LogFile, "Session", "INFO", "NLU Session Ready For User #" + self.userID) def setThresholds(self, threshold): self.threshold = float(threshold) self.entityThrshld = self._confs["ClassifierSettings"]["Mitie"][ "Threshold"] def predict(self, parsedSentence): predictions = [[index, confidence] for index, confidence in enumerate( self.tmodel.predict([ self.Data.makeInferenceBag(parsedSentence, self.trainedWords) ])[0]) if confidence > self.threshold] predictions.sort(key=lambda x: x[1], reverse=True) classification = [] for prediction in predictions: classification.append( (self.trainedClasses[prediction[0]], prediction[1])) return classification def talk(self, sentence, debug=False): self.Logging.logMessage(self.LogFile, "GeniSys", "STATUS", "Processing") parsed, fallback, entityHolder, parsedSentence = self.entityExtractor.parseEntities( sentence, self.ner, self.trainingData) classification = self.predict(parsedSentence) if len(classification) > 0: clearEntities = False theIntent = self.trainingData["intents"][self.intentMap[ classification[0][0]]] if len(entityHolder) and not len(theIntent["entities"]): clearEntities = True if (self.Context.checkSessionContext(self.user[self.userID], theIntent)): if self.Context.checkClearContext(theIntent, 0): self.user[self.userID]["context"] = "" contextIn, contextOut, contextCurrent = self.Context.setContexts( theIntent, self.user[self.userID]) if fallback and "fallbacks" in theIntent and len( theIntent["fallbacks"]): response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["fallbacks"]), entityHolder) action, actionResponses = self.Helpers.setAction(theIntent) elif "entityType" in theIntent and theIntent[ "entityType"] == "Numbers": response = random.choice(theIntent["responses"]) action, actionResponses = self.Helpers.setAction(theIntent) elif not len(entityHolder) and len(theIntent["entities"]): response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["fallbacks"]), entityHolder) action, actionResponses = self.Helpers.setAction(theIntent) elif clearEntities: entityHolder = [] response = random.choice(theIntent["responses"]) action, actionResponses = self.Helpers.setAction(theIntent) else: response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) action, actionResponses = self.Helpers.setAction(theIntent) if action != None: classParts = action.split(".") classFolder = classParts[0] className = classParts[1] module = __import__(classParts[0] + "." + classParts[1], globals(), locals(), [className]) actionClass = getattr(module, className)() response = getattr(actionClass, classParts[2])( random.choice(actionResponses)) return { "Response": "OK", "ResponseData": [{ "Received": sentence, "Intent": classification[0][0], "Confidence": str(classification[0][1]), "Response": response, "ContextIn": contextIn, "ContextOut": contextOut, "Context": contextCurrent, "Action": action, "Entities": entityHolder }] } else: self.user[self.userID]["context"] = "" contextIn, contextOut, contextCurrent = self.Context.setContexts( theIntent, self.user[self.userID]) if fallback and fallback in theIntent and len( theIntent["fallbacks"]): response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["fallbacks"]), entityHolder) action, actionResponses = None, [] else: response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) action, actionResponses = self.Helpers.setAction(theIntent) if action != None: classParts = action.split(".") classFolder = classParts[0] className = classParts[1] module = __import__(classParts[0] + "." + classParts[1], globals(), locals(), [className]) actionClass = getattr(module, className)() response = getattr(actionClass, classParts[2])( random.choice(actionResponses)) else: response = self.entityExtractor.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) return { "Response": "OK", "ResponseData": [{ "Received": sentence, "Intent": classification[0][0], "Confidence": str(classification[0][1]), "Response": response, "ContextIn": contextIn, "ContextOut": contextOut, "ContextCurrent": contextCurrent, "Action": action, "Entities": entityHolder }] } else: contextCurrent = self.Context.getCurrentContext( self.user[self.userID]) return { "Response": "FAILED", "ResponseData": [{ "Received": sentence, "Intent": "UNKNOWN", "Confidence": "NA", "Responses": [], "Response": random.choice( self._confs["ClassifierSettings"]["defaultResponses"]), "ContextIn": "NA", "ContextOut": "NA", "ContextCurrent": contextCurrent, "Action": "NA", "Entities": entityHolder }] }
elif sys.argv[1] == 'test': t = Test() t.test_scrape_today_buysell() # elif sys.argv[1] == 'bm': # b = Benchmark(start_path) # df, exists = b.get() # recent_date = list(df.ix[len(df)-1])[0].replace('-', '') # if exists: # print('Recent update: ' + recent_date) # else: # print('Downloaded data to: ' + recent_date) # elif sys.argv[1] == 'data': d = Data(start_path) if sys.argv[2] == 'send': if sys.argv[3] == 'ticker': d.send_ticker() elif sys.argv[3] == 'bm': d.send_bm() elif sys.argv[3] == 'ohlcv': d.send_ohlcv() elif sys.argv[2] == 'update': if sys.argv[3] == 'ohlcv': d.update_ohlcv() elif sys.argv[3] == 'ohlcv_with_date_1': d.upd_ohlcv_1() elif sys.argv[3] == 'ohlcv_with_date_2': d.upd_ohlcv_2() elif sys.argv[3] == 'ohlcv_with_date_3':
class Model(): def __init__(self): ############################################################### # # Sets up all default requirements # # - Helpers: Useful global functions # - Data: Data functions # ############################################################### self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.Data = Data() def createDNNLayers(self, x, y): ############################################################### # # Sets up the DNN layers, configuration in required/confs.json # ############################################################### net = tflearn.input_data(shape=[None, len(x[0])]) for i in range(self._confs["NLU"]['FcLayers']): net = tflearn.fully_connected(net, self._confs["NLU"]['FcUnits']) net = tflearn.fully_connected(net, len(y[0]), activation=str( self._confs["NLU"]['Activation'])) if self._confs["NLU"]['Regression']: net = tflearn.regression(net) return net def trainDNN(self, x, y, words, classes, intentMap): ############################################################### # # Trains the DNN, configuration in required/confs.json # ############################################################### tf.reset_default_graph() tmodel = tflearn.DNN( self.createDNNLayers(x, y), tensorboard_dir=self._confs["NLU"]['TFLearn']['Logs'], tensorboard_verbose=self._confs["NLU"]['TFLearn']['LogsLevel']) tmodel.fit(x, y, n_epoch=self._confs["NLU"]['Epochs'], batch_size=self._confs["NLU"]['BatchSize'], show_metric=self._confs["NLU"]['ShowMetric']) self.saveModelData( self._confs["NLU"]['TFLearn']['Data'], { 'words': words, 'classes': classes, 'x': x, 'y': y, 'intentMap': [intentMap] }, tmodel) def saveModelData(self, path, data, tmodel): ############################################################### # # Saves the model data for TFLearn and the NLU engine, # configuration in required/confs.json # ############################################################### tmodel.save(self._confs["NLU"]['TFLearn']['Path']) with open(path, "w") as outfile: json.dump(data, outfile) def buildDNN(self, x, y): ############################################################### # # Loads the DNN model, configuration in required/confs.json # ############################################################### tf.reset_default_graph() tmodel = tflearn.DNN(self.createDNNLayers(x, y)) tmodel.load(self._confs["NLU"]['TFLearn']['Path']) return tmodel def predict(self, tmodel, parsedSentence, trainedWords, trainedClasses): ############################################################### # # Makes a prediction against the trained model, checking the # confidence and then logging the results. # ############################################################### predictions = [[index, confidence] for index, confidence in enumerate( tmodel.predict( [self.Data.makeBagOfWords(parsedSentence, trainedWords)])[0])] predictions.sort(key=lambda x: x[1], reverse=True) classification = [] for prediction in predictions: classification.append( (trainedClasses[prediction[0]], prediction[1])) return classification
class NLU(): def __init__(self): ############################################################### # # Sets up all default requirements and placeholders # needed for the NLU engine to run. # # - Helpers: Useful global functions # - JumpWay/jumpWayClient: iotJumpWay class and connection # - Logging: Logging class # ############################################################### self.isTraining = False self.ner = None self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.user = {} self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] + "NLU/") self.ChatLogFile = self.Helpers.setLogFile( self._confs["aiCore"]["Logs"] + "Chat/") self.jumpWay = JumpWay() self.jumpWayClient = self.jumpWay.startMQTT() self.jumpWayClient.subscribeToDeviceChannel( self._confs["iotJumpWay"]["Channels"]["Commands"]) self.jumpWayClient.deviceCommandsCallback = self.commandsCallback def initiateSession(self): ############################################################### # # Initiates empty guest user session, GeniSys will ask the user # verify their GeniSys user by speaking or typing if it does # not know who it is speaking to. # ############################################################### self.userID = 0 if not self.userID in self.user: self.user[self.userID] = {} self.user[self.userID]["history"] = {} def initNLU(self): ############################################################### # # Initiates the NLU setting up the data, NLU / entities models # and required modules such as context and extensions. # ############################################################### self.Data = Data() self.trainingData = self.Data.loadTrainingData() self.trainedData = self.Data.loadTrainedData() self.Model = Model() self.Context = Context() self.Extensions = Extensions() self.restoreData() self.restoreNER() self.restoreNLU() self.initiateSession() self.setThresholds() def commandsCallback(self, topic, payload): ############################################################### # # The callback function that is triggerend in the event of a # command communication from the iotJumpWay. # ############################################################### self.Helpers.logMessage( self.LogFile, "iotJumpWay", "INFO", "Recieved iotJumpWay Command Data : " + str(payload)) commandData = json.loads(payload.decode("utf-8")) def restoreData(self): ############################################################### # # Sets the local trained data using data retrieved above # ############################################################### self.trainedWords = self.trainedData["words"] self.trainedClasses = self.trainedData["classes"] self.x = self.trainedData["x"] self.y = self.trainedData["y"] self.intentMap = self.trainedData["intentMap"][0] def loadEntityController(self): ############################################################### # # Initiates the entity extractor class from tools # ############################################################### self.entityController = Entities() def restoreNER(self): ############################################################### # # Loads entity controller and restores the NER model # ############################################################### self.loadEntityController() self.ner = self.entityController.restoreNER() def restoreNLU(self): ############################################################### # # Restores the NLU model # ############################################################### self.tmodel = self.Model.buildDNN(self.x, self.y) def setThresholds(self): ############################################################### # # Sets the threshold for the NLU engine, this can be changed # using arguments to commandline programs or paramters for # API calls. # ############################################################### self.threshold = self._confs["NLU"]["Threshold"] self.entityThrshld = self._confs["NLU"]["Mitie"]["Threshold"] def communicate(self, sentence): ############################################################### # # First checks to ensure that the program is not training, # then parses any entities that may be in the intent, then # checks context and extensions before providing a response. # ############################################################### if self.isTraining == False: parsed, fallback, entityHolder, parsedSentence = self.entityController.parseEntities( sentence, self.ner, self.trainingData) classification = self.Model.predict(self.tmodel, parsedSentence, self.trainedWords, self.trainedClasses) if len(classification) > 0: clearEntities = False theIntent = self.trainingData["intents"][self.intentMap[ classification[0][0]]] if len(entityHolder) and not len(theIntent["entities"]): clearEntities = True if (self.Context.checkSessionContext(self.user[self.userID], theIntent)): if self.Context.checkClearContext(theIntent, 0): self.user[self.userID]["context"] = "" contextIn, contextOut, contextCurrent = self.Context.setContexts( theIntent, self.user[self.userID]) if not len(entityHolder) and len(theIntent["entities"]): response, entities = self.entityController.replaceResponseEntities( random.choice(theIntent["fallbacks"]), entityHolder) extension, extensionResponses, exEntities = self.Extensions.setExtension( theIntent) elif clearEntities: entityHolder = [] response = random.choice(theIntent["responses"]) extension, extensionResponses, exEntities = self.Extensions.setExtension( theIntent) else: response, entities = self.entityController.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) extension, extensionResponses, exEntities = self.Extensions.setExtension( theIntent) if extension != None: classParts = extension.split(".") classFolder = classParts[0] className = classParts[1] theEntities = None if exEntities != False: theEntities = entities module = __import__( classParts[0] + "." + classParts[1], globals(), locals(), [className]) extensionClass = getattr(module, className)() response = getattr(extensionClass, classParts[2])(extensionResponses, theEntities) return { "Response": "OK", "ResponseData": [{ "Received": sentence, "Intent": classification[0][0], "Confidence": str(classification[0][1]), "Response": response, "Context": [{ "In": contextIn, "Out": contextOut, "Current": contextCurrent }], "Extension": extension, "Entities": entityHolder }] } else: self.user[self.userID]["context"] = "" contextIn, contextOut, contextCurrent = self.Context.setContexts( theIntent, self.user[self.userID]) if fallback and fallback in theIntent and len( theIntent["fallbacks"]): response = self.entityController.replaceResponseEntities( random.choice(theIntent["fallbacks"]), entityHolder) extension, extensionResponses = None, [] else: response = self.entityController.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) extension, extensionResponses, exEntities = self.Extensions.setExtension( theIntent) if extension != None: classParts = extension.split(".") classFolder = classParts[0] className = classParts[1] theEntities = None if exEntities != False: theEntities = entities module = __import__( classParts[0] + "." + classParts[1], globals(), locals(), [className]) extensionClass = getattr(module, className)() response = getattr(extensionClass, classParts[2])(extensionResponses, theEntities) else: response = self.entityController.replaceResponseEntities( random.choice(theIntent["responses"]), entityHolder) return { "Response": "OK", "ResponseData": [{ "Received": sentence, "Intent": classification[0][0], "Confidence": str(classification[0][1]), "Response": response, "Context": [{ "In": contextIn, "Out": contextOut, "Current": contextCurrent }], "Extension": extension, "Entities": entityHolder }] } else: contextCurrent = self.Context.getCurrentContext( self.user[self.userID]) return { "Response": "FAILED", "ResponseData": [{ "Received": sentence, "Intent": "UNKNOWN", "Confidence": "NA", "Responses": [], "Response": random.choice(self._confs["NLU"]["defaultResponses"]), "Context": [{ "In": "NA", "Out": "NA", "Current": contextCurrent }], "Extension": "NA", "Entities": entityHolder }] } else: return { "Response": "FAILED", "ResponseData": [{ "Status": "Training", "Message": "NLU Engine is currently training" }] }
s.check() elif sys.argv[2] == 'set': s.set(sys.argv[3], sys.argv[4]) s.save() elif sys.argv[1] == 'bm': b = Benchmark(start_path) df, exists = b.get() recent_date = list(df.ix[len(df) - 1])[0].replace('-', '') if exists: print('Recent update: ' + recent_date) else: print('Downloaded data to: ' + recent_date) elif sys.argv[1] == 'data': d = Data(start_path) if sys.argv[2] == 'send': if sys.argv[3] == 'ticker': d.send_ticker() elif sys.argv[3] == 'bm': d.send_bm() elif sys.argv[3] == 'ohlcv': d.send_ohlcv() elif sys.argv[2] == 'update': if sys.argv[3] == 'ohlcv': d.update_ohlcv() elif sys.argv[2] == 'clean': if sys.argv[3] == 'ohlcv': d.clean_ohlcv() elif sys.argv[3] == 'bm': d.clean_bm()