Esempio n. 1
0
class Data():
        
    ###############################################################
    #
    # Core Data class wrapper.
    #
    ###############################################################

    def __init__(self):
        
        ###############################################################
        #
        # Sets up all default requirements and placeholders 
        # needed for this class. 
        #
        ###############################################################
        
        self.Helpers = Helpers()
        self.confs = self.Helpers.loadConfs()
        self.logFile = self.Helpers.setLogFile(self.confs["Settings"]["Logs"]["DataLogDir"])
        
        self.DataProcess = DataProcess()
        self.labelsToName = {}
        
        self.Helpers.logMessage(self.logFile, "init", "INFO", "Init complete")

    def sortData(self):
        
        ###############################################################
        #
        # Sorts the data 
        #
        ###############################################################

        humanStart, clockStart = self.Helpers.timerStart()
        self.Helpers.logMessage(self.logFile, "sortData", "INFO", "Loading & preparing training data")
        
        dataPaths, classes = self.DataProcess.processFilesAndClasses()

        classId = [ int(i) for i in classes]
        classNamesToIds = dict(zip(classes, classId))

        # Divide the training datasets into train and test
        numValidation = int(self.confs["Classifier"]["ValidationSize"] * len(dataPaths))
        self.Helpers.logMessage(self.logFile, "sortData", "Validation Size", str(numValidation))
        self.Helpers.logMessage(self.logFile, "sortData", "Class Size", str(len(classes)))
        random.seed(self.confs["Classifier"]["RandomSeed"])
        random.shuffle(dataPaths)
        trainingFiles = dataPaths[numValidation:]
        validationFiles = dataPaths[:numValidation]

        # Convert the training and validation sets
        self.DataProcess.convertToTFRecord('train', trainingFiles, classNamesToIds)
        self.DataProcess.convertToTFRecord('validation', validationFiles, classNamesToIds)

        # Write the labels to file
        labelsToClassNames = dict(zip(classId, classes))
        self.DataProcess.writeLabels(labelsToClassNames)
        self.Helpers.logMessage(self.logFile, "sortData", "COMPLETE", "Completed sorting data!")
Esempio n. 2
0
class Data():
    """ Trainer Data Class

    Sorts the ALL Detection System 2019 NCS1 Classifier training data.
    """
    def __init__(self):
        """ Initializes the Data Class """

        self.Helpers = Helpers("Data")
        self.confs = self.Helpers.confs

        self.DataProcess = DataProcess()
        self.labelsToName = {}

        self.Helpers.logger.info("Data class initialization complete.")

    def sortData(self):
        """ Sorts the training data """

        humanStart, clockStart = self.Helpers.timerStart()

        self.Helpers.logger.info("Loading & preparing training data.")

        dataPaths, classes = self.DataProcess.processFilesAndClasses()

        classId = [int(i) for i in classes]
        classNamesToIds = dict(zip(classes, classId))

        # Divide the training datasets into train and test
        numValidation = int(self.confs["Classifier"]["ValidationSize"] *
                            len(dataPaths))
        self.Helpers.logger.info("Number of classes: " + str(classes))
        self.Helpers.logger.info("Validation data size: " + str(numValidation))
        random.seed(self.confs["Classifier"]["RandomSeed"])
        random.shuffle(dataPaths)
        trainingFiles = dataPaths[numValidation:]
        validationFiles = dataPaths[:numValidation]

        # Convert the training and validation sets
        self.DataProcess.convertToTFRecord('train', trainingFiles,
                                           classNamesToIds)
        self.DataProcess.convertToTFRecord('validation', validationFiles,
                                           classNamesToIds)

        # Write the labels to file
        labelsToClassNames = dict(zip(classId, classes))
        self.DataProcess.writeLabels(labelsToClassNames)

        self.Helpers.logger.info(
            "Loading & preparing training data completed.")

    def cropTestData(self):
        """ Crops the testing data """

        self.DataProcess.cropTestDataset()

        self.Helpers.logger.info("Testing data resized.")