class Data():
###############################################################
#
# Core Data class wrapper.
#
###############################################################
def __init__(self):
###############################################################
#
# Sets up all default requirements and placeholders
# needed for this class.
#
###############################################################
self.Helpers = Helpers()
self.confs = self.Helpers.loadConfs()
self.logFile = self.Helpers.setLogFile(self.confs["Settings"]["Logs"]["DataLogDir"])
self.DataProcess = DataProcess()
self.labelsToName = {}
self.Helpers.logMessage(self.logFile, "init", "INFO", "Init complete")
def sortData(self):
###############################################################
#
# Sorts the data
#
###############################################################
humanStart, clockStart = self.Helpers.timerStart()
self.Helpers.logMessage(self.logFile, "sortData", "INFO", "Loading & preparing training data")
dataPaths, classes = self.DataProcess.processFilesAndClasses()
classId = [ int(i) for i in classes]
classNamesToIds = dict(zip(classes, classId))
# Divide the training datasets into train and test
numValidation = int(self.confs["Classifier"]["ValidationSize"] * len(dataPaths))
self.Helpers.logMessage(self.logFile, "sortData", "Validation Size", str(numValidation))
self.Helpers.logMessage(self.logFile, "sortData", "Class Size", str(len(classes)))
random.seed(self.confs["Classifier"]["RandomSeed"])
random.shuffle(dataPaths)
trainingFiles = dataPaths[numValidation:]
validationFiles = dataPaths[:numValidation]
# Convert the training and validation sets
self.DataProcess.convertToTFRecord('train', trainingFiles, classNamesToIds)
self.DataProcess.convertToTFRecord('validation', validationFiles, classNamesToIds)
# Write the labels to file
labelsToClassNames = dict(zip(classId, classes))
self.DataProcess.writeLabels(labelsToClassNames)
self.Helpers.logMessage(self.logFile, "sortData", "COMPLETE", "Completed sorting data!")
class Data():
""" Trainer Data Class
Sorts the ALL Detection System 2019 NCS1 Classifier training data.
"""
def __init__(self):
""" Initializes the Data Class """
self.Helpers = Helpers("Data")
self.confs = self.Helpers.confs
self.DataProcess = DataProcess()
self.labelsToName = {}
self.Helpers.logger.info("Data class initialization complete.")
def sortData(self):
""" Sorts the training data """
humanStart, clockStart = self.Helpers.timerStart()
self.Helpers.logger.info("Loading & preparing training data.")
dataPaths, classes = self.DataProcess.processFilesAndClasses()
classId = [int(i) for i in classes]
classNamesToIds = dict(zip(classes, classId))
# Divide the training datasets into train and test
numValidation = int(self.confs["Classifier"]["ValidationSize"] *
len(dataPaths))
self.Helpers.logger.info("Number of classes: " + str(classes))
self.Helpers.logger.info("Validation data size: " + str(numValidation))
random.seed(self.confs["Classifier"]["RandomSeed"])
random.shuffle(dataPaths)
trainingFiles = dataPaths[numValidation:]
validationFiles = dataPaths[:numValidation]
# Convert the training and validation sets
self.DataProcess.convertToTFRecord('train', trainingFiles,
classNamesToIds)
self.DataProcess.convertToTFRecord('validation', validationFiles,
classNamesToIds)
# Write the labels to file
labelsToClassNames = dict(zip(classId, classes))
self.DataProcess.writeLabels(labelsToClassNames)
self.Helpers.logger.info(
"Loading & preparing training data completed.")
def cropTestData(self):
""" Crops the testing data """
self.DataProcess.cropTestDataset()
self.Helpers.logger.info("Testing data resized.")
