def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
self.isTraining = False
self.ner = None
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.user = {}
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"NLU/")
self.ChatLogFile = self.Helpers.setLogFile(
self._confs["aiCore"]["Logs"] + "Chat/")
self.jumpWay = JumpWay()
self.jumpWayClient = self.jumpWay.startMQTT()
self.jumpWayClient.subscribeToDeviceChannel(
self._confs["iotJumpWay"]["Channels"]["Commands"])
self.jumpWayClient.deviceCommandsCallback = self.commandsCallback
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"/Local")
self.MySql = MySql()
self.OpenCV = OpenCV()
self.OCVframe = None
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.fontColor = (255, 255, 255)
self.fontScale = 1
self.lineType = 1
self.identified = 0
self.Facenet = Facenet()
self.movidius, self.devices, self.device = self.Facenet.CheckDevices()
self.fgraph, self.fgraphfile = self.Facenet.loadGraph(
"Facenet", self.movidius)
self.validDir = self._confs["Classifier"]["NetworkPath"] + self._confs[
"Classifier"]["ValidPath"]
self.testingDir = self._confs["Classifier"][
"NetworkPath"] + self._confs["Classifier"]["TestingPath"]
self.detector = dlib.get_frontal_face_detector()
self.predictor = dlib.shape_predictor(
self._confs["Classifier"]["Dlib"])
self.connectToCamera()
self.tassSocket = None
self.configureSocket()
self.JumpWay = JumpWay()
self.JumpWayCL = self.JumpWay.startMQTT()
self.Helpers.logMessage(self.LogFile, "TASS", "INFO", "TASS Ready")
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.JumpWay = JumpWay()
self.MySql = MySql()
self.MySql.setMysqlCursorRows()
self.Logging = Logging()
self.LogFile = self.Logging.setLogFile(self._confs["aiCore"]["Logs"] +
"Client/")
class Streamer():
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
#
###############################################################
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"/Local")
self.MySql = MySql()
self.OpenCV = OpenCV()
self.OCVframe = None
self.font = cv2.FONT_HERSHEY_SIMPLEX
self.fontColor = (255, 255, 255)
self.fontScale = 1
self.lineType = 1
self.identified = 0
self.Facenet = Facenet()
self.movidius, self.devices, self.device = self.Facenet.CheckDevices()
self.fgraph, self.fgraphfile = self.Facenet.loadGraph(
"Facenet", self.movidius)
self.validDir = self._confs["Classifier"]["NetworkPath"] + self._confs[
"Classifier"]["ValidPath"]
self.testingDir = self._confs["Classifier"][
"NetworkPath"] + self._confs["Classifier"]["TestingPath"]
self.detector = dlib.get_frontal_face_detector()
self.predictor = dlib.shape_predictor(
self._confs["Classifier"]["Dlib"])
self.connectToCamera()
self.tassSocket = None
self.configureSocket()
self.JumpWay = JumpWay()
self.JumpWayCL = self.JumpWay.startMQTT()
self.Helpers.logMessage(self.LogFile, "TASS", "INFO", "TASS Ready")
def connectToCamera(self):
###############################################################
#
# Connects to the Foscam device using the configs in
# required/confs.json
#
###############################################################
self.OCVframe = cv2.VideoCapture(0)
self.Helpers.logMessage(self.LogFile, "TASS", "INFO",
"Connected To Camera")
def configureSocket(self):
###############################################################
#
# Configures the socket we will stream the frames to
#
###############################################################
self.tassSocket = zmq.Context().socket(zmq.PUB)
self.tassSocket.connect("tcp://" + self._confs["Socket"]["host"] +
":" + str(self._confs["Socket"]["port"]))
self.Helpers.logMessage(
self.LogFile, "TASS", "INFO",
"Connected To Socket: tcp://" + self._confs["Socket"]["host"] +
":" + str(self._confs["Socket"]["port"]))
class NLU():
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
self.isTraining = False
self.ner = None
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.user = {}
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"NLU/")
self.ChatLogFile = self.Helpers.setLogFile(
self._confs["aiCore"]["Logs"] + "Chat/")
self.jumpWay = JumpWay()
self.jumpWayClient = self.jumpWay.startMQTT()
self.jumpWayClient.subscribeToDeviceChannel(
self._confs["iotJumpWay"]["Channels"]["Commands"])
self.jumpWayClient.deviceCommandsCallback = self.commandsCallback
def initiateSession(self):
###############################################################
#
# Initiates empty guest user session, GeniSys will ask the user
# verify their GeniSys user by speaking or typing if it does
# not know who it is speaking to.
#
###############################################################
self.userID = 0
if not self.userID in self.user:
self.user[self.userID] = {}
self.user[self.userID]["history"] = {}
def initNLU(self):
###############################################################
#
# Initiates the NLU setting up the data, NLU / entities models
# and required modules such as context and extensions.
#
###############################################################
self.Data = Data()
self.trainingData = self.Data.loadTrainingData()
self.trainedData = self.Data.loadTrainedData()
self.Model = Model()
self.Context = Context()
self.Extensions = Extensions()
self.restoreData()
self.restoreNER()
self.restoreNLU()
self.initiateSession()
self.setThresholds()
def commandsCallback(self, topic, payload):
###############################################################
#
# The callback function that is triggerend in the event of a
# command communication from the iotJumpWay.
#
###############################################################
self.Helpers.logMessage(
self.LogFile, "iotJumpWay", "INFO",
"Recieved iotJumpWay Command Data : " + str(payload))
commandData = json.loads(payload.decode("utf-8"))
def restoreData(self):
###############################################################
#
# Sets the local trained data using data retrieved above
#
###############################################################
self.trainedWords = self.trainedData["words"]
self.trainedClasses = self.trainedData["classes"]
self.x = self.trainedData["x"]
self.y = self.trainedData["y"]
self.intentMap = self.trainedData["intentMap"][0]
def loadEntityController(self):
###############################################################
#
# Initiates the entity extractor class from tools
#
###############################################################
self.entityController = Entities()
def restoreNER(self):
###############################################################
#
# Loads entity controller and restores the NER model
#
###############################################################
self.loadEntityController()
self.ner = self.entityController.restoreNER()
def restoreNLU(self):
###############################################################
#
# Restores the NLU model
#
###############################################################
self.tmodel = self.Model.buildDNN(self.x, self.y)
def setThresholds(self):
###############################################################
#
# Sets the threshold for the NLU engine, this can be changed
# using arguments to commandline programs or paramters for
# API calls.
#
###############################################################
self.threshold = self._confs["NLU"]["Threshold"]
self.entityThrshld = self._confs["NLU"]["Mitie"]["Threshold"]
def communicate(self, sentence):
###############################################################
#
# First checks to ensure that the program is not training,
# then parses any entities that may be in the intent, then
# checks context and extensions before providing a response.
#
###############################################################
if self.isTraining == False:
parsed, fallback, entityHolder, parsedSentence = self.entityController.parseEntities(
sentence, self.ner, self.trainingData)
classification = self.Model.predict(self.tmodel, parsedSentence,
self.trainedWords,
self.trainedClasses)
if len(classification) > 0:
clearEntities = False
theIntent = self.trainingData["intents"][self.intentMap[
classification[0][0]]]
if len(entityHolder) and not len(theIntent["entities"]):
clearEntities = True
if (self.Context.checkSessionContext(self.user[self.userID],
theIntent)):
if self.Context.checkClearContext(theIntent, 0):
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if not len(entityHolder) and len(theIntent["entities"]):
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
elif clearEntities:
entityHolder = []
response = random.choice(theIntent["responses"])
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
else:
response, entities = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
self.user[self.userID]["context"] = ""
contextIn, contextOut, contextCurrent = self.Context.setContexts(
theIntent, self.user[self.userID])
if fallback and fallback in theIntent and len(
theIntent["fallbacks"]):
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["fallbacks"]),
entityHolder)
extension, extensionResponses = None, []
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
extension, extensionResponses, exEntities = self.Extensions.setExtension(
theIntent)
if extension != None:
classParts = extension.split(".")
classFolder = classParts[0]
className = classParts[1]
theEntities = None
if exEntities != False:
theEntities = entities
module = __import__(
classParts[0] + "." + classParts[1], globals(),
locals(), [className])
extensionClass = getattr(module, className)()
response = getattr(extensionClass,
classParts[2])(extensionResponses,
theEntities)
else:
response = self.entityController.replaceResponseEntities(
random.choice(theIntent["responses"]),
entityHolder)
return {
"Response":
"OK",
"ResponseData": [{
"Received":
sentence,
"Intent":
classification[0][0],
"Confidence":
str(classification[0][1]),
"Response":
response,
"Context": [{
"In": contextIn,
"Out": contextOut,
"Current": contextCurrent
}],
"Extension":
extension,
"Entities":
entityHolder
}]
}
else:
contextCurrent = self.Context.getCurrentContext(
self.user[self.userID])
return {
"Response":
"FAILED",
"ResponseData": [{
"Received":
sentence,
"Intent":
"UNKNOWN",
"Confidence":
"NA",
"Responses": [],
"Response":
random.choice(self._confs["NLU"]["defaultResponses"]),
"Context": [{
"In": "NA",
"Out": "NA",
"Current": contextCurrent
}],
"Extension":
"NA",
"Entities":
entityHolder
}]
}
else:
return {
"Response":
"FAILED",
"ResponseData": [{
"Status": "Training",
"Message": "NLU Engine is currently training"
}]
}