class Data():
###############################################################
#
# Core Data class wrapper.
#
###############################################################
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for this class.
#
###############################################################
self.Helpers = Helpers()
self.confs = self.Helpers.loadConfs()
self.logFile = self.Helpers.setLogFile(self.confs["Settings"]["Logs"]["DataLogDir"])
self.DataProcess = DataProcess()
self.labelsToName = {}
self.Helpers.logMessage(self.logFile, "init", "INFO", "Init complete")
def sortData(self):
###############################################################
#
# Sorts the data
#
###############################################################
humanStart, clockStart = self.Helpers.timerStart()
self.Helpers.logMessage(self.logFile, "sortData", "INFO", "Loading & preparing training data")
dataPaths, classes = self.DataProcess.processFilesAndClasses()
classId = [ int(i) for i in classes]
classNamesToIds = dict(zip(classes, classId))
# Divide the training datasets into train and test
numValidation = int(self.confs["Classifier"]["ValidationSize"] * len(dataPaths))
self.Helpers.logMessage(self.logFile, "sortData", "Validation Size", str(numValidation))
self.Helpers.logMessage(self.logFile, "sortData", "Class Size", str(len(classes)))
random.seed(self.confs["Classifier"]["RandomSeed"])
random.shuffle(dataPaths)
trainingFiles = dataPaths[numValidation:]
validationFiles = dataPaths[:numValidation]
# Convert the training and validation sets
self.DataProcess.convertToTFRecord('train', trainingFiles, classNamesToIds)
self.DataProcess.convertToTFRecord('validation', validationFiles, classNamesToIds)
# Write the labels to file
labelsToClassNames = dict(zip(classId, classes))
self.DataProcess.writeLabels(labelsToClassNames)
self.Helpers.logMessage(self.logFile, "sortData", "COMPLETE", "Completed sorting data!")
class Data():
""" Trainer Data Class
Sorts the ALL Detection System 2019 NCS1 Classifier training data.
"""
def __init__(self):
""" Initializes the Data Class """
self.Helpers = Helpers("Data")
self.confs = self.Helpers.confs
self.DataProcess = DataProcess()
self.labelsToName = {}
self.Helpers.logger.info("Data class initialization complete.")
def sortData(self):
""" Sorts the training data """
humanStart, clockStart = self.Helpers.timerStart()
self.Helpers.logger.info("Loading & preparing training data.")
dataPaths, classes = self.DataProcess.processFilesAndClasses()
classId = [int(i) for i in classes]
classNamesToIds = dict(zip(classes, classId))
# Divide the training datasets into train and test
numValidation = int(self.confs["Classifier"]["ValidationSize"] *
len(dataPaths))
self.Helpers.logger.info("Number of classes: " + str(classes))
self.Helpers.logger.info("Validation data size: " + str(numValidation))
random.seed(self.confs["Classifier"]["RandomSeed"])
random.shuffle(dataPaths)
trainingFiles = dataPaths[numValidation:]
validationFiles = dataPaths[:numValidation]
# Convert the training and validation sets
self.DataProcess.convertToTFRecord('train', trainingFiles,
classNamesToIds)
self.DataProcess.convertToTFRecord('validation', validationFiles,
classNamesToIds)
# Write the labels to file
labelsToClassNames = dict(zip(classId, classes))
self.DataProcess.writeLabels(labelsToClassNames)
self.Helpers.logger.info(
"Loading & preparing training data completed.")
def cropTestData(self):
""" Crops the testing data """
self.DataProcess.cropTestDataset()
self.Helpers.logger.info("Testing data resized.")
class Trainer():
###############################################################
#
# Sets up all default requirements and placeholders
# needed for the NLU engine to run.
#
# - Helpers: Useful global functions
# - JumpWay/jumpWayClient: iotJumpWay class and connection
# - Logging: Logging class
#
###############################################################
def __init__(self):
self.Helpers = Helpers()
self._confs = self.Helpers.loadConfigs()
self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] +
"Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
if self._confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self._confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
class Trainer():
""" ALL Detection System 2019 Chatbot Training Class
Trains the ALL Detection System 2019 Chatbot.
"""
def __init__(self):
""" Initializes the Training class. """
self.Helpers = Helpers()
self.LogFile = self.Helpers.setLogFile(
self.Helpers.confs["System"]["Logs"] + "Train/")
self.intentMap = {}
self.words = []
self.classes = []
self.dataCorpus = []
self.Model = Model()
self.Data = Data()
def setupData(self):
""" Prepares the data. """
self.trainingData = self.Data.loadTrainingData()
self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData(
self.trainingData)
self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus,
self.words)
self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO",
"NLU Training Data Ready")
def setupEntities(self):
""" Prepares the entities. """
if self.Helpers.confs["NLU"]["Entities"] == "Mitie":
self.entityController = Entities()
self.entityController.trainEntities(
self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"],
self.trainingData)
self.Helpers.logMessage(self.LogFile, "TRAIN", "OK",
"NLU Trainer Entities Ready")
def trainModel(self):
""" Trains the model. """
while True:
self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION",
"Ready To Begin Training ? (Yes/No)")
userInput = input(">")
if userInput == 'Yes': break
if userInput == 'No': exit()
self.setupData()
self.setupEntities()
humanStart, trainingStart = self.Helpers.timerStart()
self.Model.trainDNN(self.x, self.y, self.words, self.classes,
self.intentMap)
trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd(
trainingStart)
self.Helpers.logMessage(
self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd +
" In " + str(trainingEnd) + " Seconds")
