class Trainer():
""" Trainer Class
Trains the Natural Language Understanding Engine.
"""
def __init__(self):
""" Initializes the class. """
self.Helpers = Helpers("Train")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
self.Helpers.logger.info("Trainer class initialized.")
def setupData(self):
""" Prepares the data. """
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logger.info("NLU Training Data Ready")
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logger.info("NLU Trainer Entities Ready")
def trainModel(self):
""" Trains the model. """
while True:
self.Helpers.logger.info("Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
self.Helpers.logger.info("NLU Model Trained")
def setupEntities(self):
if self._confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self._confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def loadEntityController(self): ############################################################### # # Initiates the entity extractor class from tools # ############################################################### self.entityController = Entities()
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logger.info("NLU Trainer Entities Ready")
class Trainer():
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
def __init__(self):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
if self._confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self._confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
def entities(self):
""" Initializes the entities. """
self.entityController = Entities()
self.ner = self.entityController.restoreNER()
class Engine():
""" Engine Class
Core functions for the NLU Engine.
"""
def __init__(self, isAudio):
""" Initializes the class. """
self.Helpers = Helpers("Engine")
self.ner = None
self.user = {}
#self.bluetoothCon()
self.data()
self.entities()
#self.iotJumpWayCon()
self.model()
self.session()
self.thresholds()
if isAudio:
self.speech()
self.Helpers.logger.info("Engine class initialized.")
def bluetoothCon(self):
""" Initializes the Bluetooth connection. """
self.Bluetooth = BluetoothConnect()
self.Bluetooth.connect()
def data(self):
""" Initializes the data. """
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def doExtension(self, extension, entities, exEntities, extensionResponses):
""" Executes an extension. """
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass, classParts[2])(extensionResponses,
theEntities)
return response
def entities(self):
""" Initializes the entities. """
self.entityController = Entities()
self.ner = self.entityController.restoreNER()
def entitiesCheck(self, entityHolder, theIntent, clearEntities):
""" Checks entities. """
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entities = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
return response, entities, extension, extensionResponses, exEntities
def fallbackCheck(self, fallback, theIntent, entityHolder):
""" Checks if fallback. """
if fallback and fallback in theIntent and len(theIntent["fallbacks"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]), entityHolder)
extension, extensionResponses, exEntities = None, [], None
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]), entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
return response, entities, extension, extensionResponses, exEntities
def model(self):
""" Initializes the model. """
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.tmodel = self.Model.buildDNN(self.x, self.y)
def session(self):
""" Initializes a NLU sesiion.
Initiates empty guest user session, GeniSys will ask the user
verify their GeniSys user by speaking or typing if it does
not know who it is speaking to.
"""
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def respond(self, status, sentence, intent, confidence, response, cIn,
cOut, cCurrent, extension, entities):
""" Forms the response. """
return {
"Response":
status,
"ResponseData": [{
"Received":
sentence,
"Intent":
intent,
"Confidence":
confidence,
"Response":
response,
"Context": [{
"In": cIn,
"Out": cOut,
"Current": cCurrent
}],
"Extension":
extension,
"Entities":
entities
}]
}
def speech(self):
""" Initializes the TTS feature. """
self.TTS = TTS()
def thresholds(self):
""" Sets thresholds
Sets the threshold for the NLU engine, this can be changed
using arguments to commandline programs or paramters for
API calls.
"""
self.threshold = self.Helpers.confs["NLU"]["Threshold"]
self.entityThrshld = self.Helpers.confs["NLU"]["Mitie"]["Threshold"]
class Trainer():
""" ALL Detection System 2019 Chatbot Training Class
Trains the ALL Detection System 2019 Chatbot.
"""
def __init__(self):
""" Initializes the Training class. """
self.Helpers = Helpers()
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
""" Prepares the data. """
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
""" Trains the model. """
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
class Chatbot():
""" ALL Detection System 2019 Chatbot Class
The ALL Detection System 2019 Chatbot.
"""
def __init__(self):
""" Initializes the Chatbot class. """
self.isTraining = False
self.ner = None
self.Helpers = Helpers()
self.user = {}
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "NLU/")
self.ChatLogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Chat/")
def initiateSession(self):
""" Initializes a Chatbot sesiion.
Initiates empty guest user session, GeniSys will ask the user
verify their GeniSys user by speaking or typing if it does
not know who it is speaking to.
"""
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def initNLU(self):
""" Initializes a Chatbot sesiion.
Initiates the NLU setting up the data, NLU / entities models
and required modules such as context and extensions.
"""
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
def commandsCallback(self, topic, payload):
""" iotJumpWay callback function.
The callback function that is triggerend in the event of a
command communication from the iotJumpWay.
"""
self.Helpers.logMessage(
self.LogFile, "iotJumpWay", "INFO",
"Recieved iotJumpWay Command Data : " + str(payload))
commandData = json.loads(payload.decode("utf-8"))
def restoreData(self):
""" Restores the training data.
Sets the local trained data using data retrieved above
"""
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def loadEntityController(self):
""" Initiates the entity extractor class """
self.entityController = Entities()
def restoreNER(self):
""" Loads entity controller and restores the NER model """
self.loadEntityController()
self.ner = self.entityController.restoreNER()
def restoreNLU(self):
""" Restores the NLU model """
self.tmodel = self.Model.buildDNN(self.x, self.y)
def setThresholds(self):
""" Sets thresholds
Sets the threshold for the NLU engine, this can be changed
using arguments to commandline programs or paramters for
API calls.
"""
self.threshold = self.Helpers.confs["NLU"]["Threshold"]
self.entityThrshld = self.Helpers.confs["NLU"]["Mitie"]["Threshold"]
def communicate(self, sentence):
""" Responds to the user
First checks to ensure that the program is not training,
then parses any entities that may be in the intent, then
checks context and extensions before providing a response.
"""
if self.isTraining == False:
parsed, fallback, entityHolder, parsedSentence = self.entityController.parseEntities(
sentence, self.ner, self.trainingData)
classification = self.Model.predict(self.tmodel, parsedSentence,
self.trainedWords,
self.trainedClasses)
if len(classification) > 0:
clearEntities = False
theIntent = self.trainingData["intents"][self.intentMap[
classification[0][0]]]
if len(entityHolder) and not len(theIntent["entities"]):
clearEntities = True
if (self.Context.checkSessionContext(self.user[self.userID],
theIntent)):
if self.Context.checkClearContext(theIntent, 0):
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entityHolder = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and fallback in theIntent and len(
theIntent["fallbacks"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses = None, []
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
if (type(response) == tuple):
response = response[0]
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
contextCurrent = self.Context.getCurrentContext(
self.user[self.userID])
return {
"Response":
"FAILED",
"ResponseData": [{
"Received":
sentence,
"Intent":
"UNKNOWN",
"Confidence":
"NA",
"Responses": [],
"Response":
random.choice(
self.Helpers.confs["NLU"]["defaultResponses"]),
"Context": [{
"In": "NA",
"Out": "NA",
"Current": contextCurrent
}],
"Extension":
"NA",
"Entities":
entityHolder
}]
}
else:
return {
"Response":
"FAILED",
"ResponseData": [{
"Status": "Training",
"Message": "NLU Engine is currently training"
}]
}
def loadEntityController(self):
""" Initiates the entity extractor class """
self.entityController = Entities()