class Data(): ############################################################### # # Core Data class wrapper. # ############################################################### def __init__(self): ############################################################### # # Sets up all default requirements and placeholders # needed for this class. # ############################################################### self.Helpers = Helpers() self.confs = self.Helpers.loadConfs() self.logFile = self.Helpers.setLogFile(self.confs["Settings"]["Logs"]["DataLogDir"]) self.DataProcess = DataProcess() self.labelsToName = {} self.Helpers.logMessage(self.logFile, "init", "INFO", "Init complete") def sortData(self): ############################################################### # # Sorts the data # ############################################################### humanStart, clockStart = self.Helpers.timerStart() self.Helpers.logMessage(self.logFile, "sortData", "INFO", "Loading & preparing training data") dataPaths, classes = self.DataProcess.processFilesAndClasses() classId = [ int(i) for i in classes] classNamesToIds = dict(zip(classes, classId)) # Divide the training datasets into train and test numValidation = int(self.confs["Classifier"]["ValidationSize"] * len(dataPaths)) self.Helpers.logMessage(self.logFile, "sortData", "Validation Size", str(numValidation)) self.Helpers.logMessage(self.logFile, "sortData", "Class Size", str(len(classes))) random.seed(self.confs["Classifier"]["RandomSeed"]) random.shuffle(dataPaths) trainingFiles = dataPaths[numValidation:] validationFiles = dataPaths[:numValidation] # Convert the training and validation sets self.DataProcess.convertToTFRecord('train', trainingFiles, classNamesToIds) self.DataProcess.convertToTFRecord('validation', validationFiles, classNamesToIds) # Write the labels to file labelsToClassNames = dict(zip(classId, classes)) self.DataProcess.writeLabels(labelsToClassNames) self.Helpers.logMessage(self.logFile, "sortData", "COMPLETE", "Completed sorting data!")
class Data(): """ Trainer Data Class Sorts the ALL Detection System 2019 NCS1 Classifier training data. """ def __init__(self): """ Initializes the Data Class """ self.Helpers = Helpers("Data") self.confs = self.Helpers.confs self.DataProcess = DataProcess() self.labelsToName = {} self.Helpers.logger.info("Data class initialization complete.") def sortData(self): """ Sorts the training data """ humanStart, clockStart = self.Helpers.timerStart() self.Helpers.logger.info("Loading & preparing training data.") dataPaths, classes = self.DataProcess.processFilesAndClasses() classId = [int(i) for i in classes] classNamesToIds = dict(zip(classes, classId)) # Divide the training datasets into train and test numValidation = int(self.confs["Classifier"]["ValidationSize"] * len(dataPaths)) self.Helpers.logger.info("Number of classes: " + str(classes)) self.Helpers.logger.info("Validation data size: " + str(numValidation)) random.seed(self.confs["Classifier"]["RandomSeed"]) random.shuffle(dataPaths) trainingFiles = dataPaths[numValidation:] validationFiles = dataPaths[:numValidation] # Convert the training and validation sets self.DataProcess.convertToTFRecord('train', trainingFiles, classNamesToIds) self.DataProcess.convertToTFRecord('validation', validationFiles, classNamesToIds) # Write the labels to file labelsToClassNames = dict(zip(classId, classes)) self.DataProcess.writeLabels(labelsToClassNames) self.Helpers.logger.info( "Loading & preparing training data completed.") def cropTestData(self): """ Crops the testing data """ self.DataProcess.cropTestDataset() self.Helpers.logger.info("Testing data resized.")
class Trainer(): ############################################################### # # Sets up all default requirements and placeholders # needed for the NLU engine to run. # # - Helpers: Useful global functions # - JumpWay/jumpWayClient: iotJumpWay class and connection # - Logging: Logging class # ############################################################### def __init__(self): self.Helpers = Helpers() self._confs = self.Helpers.loadConfigs() self.LogFile = self.Helpers.setLogFile(self._confs["aiCore"]["Logs"] + "Train/") self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.Model = Model() self.Data = Data() def setupData(self): self.trainingData = self.Data.loadTrainingData() self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData( self.trainingData) self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus, self.words) self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO", "NLU Training Data Ready") def setupEntities(self): if self._confs["NLU"]["Entities"] == "Mitie": self.entityController = Entities() self.entityController.trainEntities( self._confs["NLU"]["Mitie"]["ModelLocation"], self.trainingData) self.Helpers.logMessage(self.LogFile, "TRAIN", "OK", "NLU Trainer Entities Ready") def trainModel(self): while True: self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION", "Ready To Begin Training ? (Yes/No)") userInput = input(">") if userInput == 'Yes': break if userInput == 'No': exit() self.setupData() self.setupEntities() humanStart, trainingStart = self.Helpers.timerStart() self.Model.trainDNN(self.x, self.y, self.words, self.classes, self.intentMap) trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd( trainingStart) self.Helpers.logMessage( self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds")
class Trainer(): """ ALL Detection System 2019 Chatbot Training Class Trains the ALL Detection System 2019 Chatbot. """ def __init__(self): """ Initializes the Training class. """ self.Helpers = Helpers() self.LogFile = self.Helpers.setLogFile( self.Helpers.confs["System"]["Logs"] + "Train/") self.intentMap = {} self.words = [] self.classes = [] self.dataCorpus = [] self.Model = Model() self.Data = Data() def setupData(self): """ Prepares the data. """ self.trainingData = self.Data.loadTrainingData() self.words, self.classes, self.dataCorpus, self.intentMap = self.Data.prepareData( self.trainingData) self.x, self.y = self.Data.finaliseData(self.classes, self.dataCorpus, self.words) self.Helpers.logMessage(self.LogFile, "TRAIN", "INFO", "NLU Training Data Ready") def setupEntities(self): """ Prepares the entities. """ if self.Helpers.confs["NLU"]["Entities"] == "Mitie": self.entityController = Entities() self.entityController.trainEntities( self.Helpers.confs["NLU"]["Mitie"]["ModelLocation"], self.trainingData) self.Helpers.logMessage(self.LogFile, "TRAIN", "OK", "NLU Trainer Entities Ready") def trainModel(self): """ Trains the model. """ while True: self.Helpers.logMessage(self.LogFile, "TRAIN", "ACTION", "Ready To Begin Training ? (Yes/No)") userInput = input(">") if userInput == 'Yes': break if userInput == 'No': exit() self.setupData() self.setupEntities() humanStart, trainingStart = self.Helpers.timerStart() self.Model.trainDNN(self.x, self.y, self.words, self.classes, self.intentMap) trainingEnd, trainingTime, humanEnd = self.Helpers.timerEnd( trainingStart) self.Helpers.logMessage( self.LogFile, "TRAIN", "OK", "NLU Model Trained At " + humanEnd + " In " + str(trainingEnd) + " Seconds")