def testClassifyHTMAsExpected(self): """ Tests ClassificationModelHTM. Training on the first five samples of the dataset, and testing on the rest, the model's classifications should match those in the expected classes data file. """ runner = HTMRunner(dataPath=os.path.join(DATA_DIR, "responses_network.csv"), resultsDir="", experimentName="fingerprints_test", load=False, modelName="ClassificationModelHTM", modelModuleName="fluent.models.classify_htm", numClasses=3, plots=0, orderedSplit=True, trainSize=[5], verbosity=0, generateData=False, votingMethod="last", classificationFile=os.path.join( DATA_DIR, "responses_classifications.json"), classifierType="KNN") self.runExperiment(runner) expectedClasses, resultClasses = self.getExpectedClassifications( runner, os.path.join(DATA_DIR, "responses_expected_classes_htm.csv")) [ self.assertEqual( sorted(e), sorted(r), "HTM model predicted classes other than what we expect.") for e, r in zip(expectedClasses, resultClasses) ]
def testClassifyHTMAsExpectedWithKNN(self): """ Tests ClassificationModelHTM. Training on the first five samples of the dataset, and testing on the rest, the model's classifications should match those in the expected classes data file. """ modelName = "HTMNetwork" runner = HTMRunner( dataPath=os.path.join(DATA_DIR, "responses_network.csv"), networkConfigPath=os.path.join(DATA_DIR, "network_config_sp_tm_knn.json"), resultsDir="", experimentName="htm_test", loadPath=None, modelName=modelName, numClasses=3, plots=0, orderedSplit=True, trainSizes=[5], verbosity=0, generateData=False, votingMethod="last", classificationFile=os.path.join(DATA_DIR, "responses_classifications.json")) runner.initModel(0) runner.runExperiment() expectedClasses, resultClasses = self.getExpectedClassifications( runner, os.path.join(DATA_DIR, "responses_expected_classes_htm.csv")) [ self.assertEqual( sorted(e), sorted(r), "HTM model predicted classes other than what we expect.") for e, r in zip(expectedClasses, resultClasses) ]
def run(args): start = time.time() if (not isinstance(args.kFolds, int)) or (args.kFolds < 1): raise ValueError("Invalid value for number of cross-validation folds.") root = os.path.dirname(os.path.realpath(__file__)) resultsDir = os.path.join(root, args.resultsDir) if args.modelName == "HTMNetwork": runner = HTMRunner(dataPath=args.dataPath, networkConfigPath=args.networkConfigPath, resultsDir=resultsDir, experimentName=args.experimentName, loadPath=args.loadPath, modelName=args.modelName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSizes=[], verbosity=args.verbosity, generateData=args.generateData, votingMethod=args.votingMethod, classificationFile=args.classificationFile, classifierType=args.classifierType) else: runner = Runner(dataPath=args.dataPath, resultsDir=resultsDir, experimentName=args.experimentName, loadPath=args.loadPath, modelName=args.modelName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSizes=[], verbosity=args.verbosity) # HTM network data isn't ready yet to initialize the model runner.initModel(args.modelName) print "Reading in data and preprocessing." dataTime = time.time() runner.setupData(args.textPreprocess) # TODO: move kfolds splitting to Runner random = False if args.orderedSplit else True runner.partitions = KFolds(args.kFolds).split( range(len(runner.samples)), randomize=random) runner.trainSizes = [len(x[0]) for x in runner.partitions] print ("Data setup complete; elapsed time is {0:.2f} seconds.\nNow encoding " "the data".format(time.time() - dataTime)) encodeTime = time.time() runner.encodeSamples() print ("Encoding complete; elapsed time is {0:.2f} seconds.\nNow running the " "experiment.".format(time.time() - encodeTime)) runner.runExperiment() print "Experiment complete in {0:.2f} seconds.".format(time.time() - start) resultCalcs = runner.calculateResults() _ = runner.evaluateCumulativeResults(resultCalcs) print "Saving..." runner.saveModel() if args.validation: print "Validating experiment against expected classifications..." print runner.validateExperiment(args.validation)
def run(args): start = time.time() root = os.path.dirname(os.path.realpath(__file__)) resultsDir = os.path.join(root, args.resultsDir) if os.path.isdir(args.dataPath): runner = MultiRunner(dataPath=args.dataPath, resultsDir=resultsDir, experimentName=args.experimentName, load=args.load, modelName=args.modelName, modelModuleName=args.modelModuleName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSize=args.trainSize, verbosity=args.verbosity, test=args.test) elif args.modelName == "ClassificationModelHTM": runner = HTMRunner(dataPath=args.dataPath, resultsDir=resultsDir, experimentName=args.experimentName, load=args.load, modelName=args.modelName, modelModuleName=args.modelModuleName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSize=args.trainSize, verbosity=args.verbosity, generateData=args.generateData, votingMethod=args.votingMethod, classificationFile=args.classificationFile, classifierType=args.classifierType) else: runner = Runner(dataPath=args.dataPath, resultsDir=resultsDir, experimentName=args.experimentName, load=args.load, modelName=args.modelName, modelModuleName=args.modelModuleName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSize=args.trainSize, verbosity=args.verbosity) if args.modelName != "ClassificationModelHTM": # The data isn't ready yet to initialize an htm model runner.initModel() print "Reading in data and preprocessing." dataTime = time.time() runner.setupData(args.textPreprocess) print( "Data setup complete; elapsed time is {0:.2f} seconds.\nNow encoding " "the data".format(time.time() - dataTime)) encodeTime = time.time() runner.encodeSamples() print( "Encoding complete; elapsed time is {0:.2f} seconds.\nNow running the " "experiment.".format(time.time() - encodeTime)) runner.runExperiment() runner.writeOutClassifications() runner.calculateResults() print "Saving..." runner.save() print "Experiment complete in {0:.2f} seconds.".format(time.time() - start) if args.validation: print "Validating experiment against expected classifications..." print runner.validateExperiment(args.validation)
def run(args): start = time.time() if (not isinstance(args.kFolds, int)) or (args.kFolds < 1): raise ValueError("Invalid value for number of cross-validation folds.") root = os.path.dirname(os.path.realpath(__file__)) resultsDir = os.path.join(root, args.resultsDir) if args.modelName == "HTMNetwork": runner = HTMRunner(dataPath=args.dataPath, networkConfigPath=args.networkConfigPath, resultsDir=resultsDir, experimentName=args.experimentName, loadPath=args.loadPath, modelName=args.modelName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSizes=[], verbosity=args.verbosity, generateData=args.generateData, votingMethod=args.votingMethod, classificationFile=args.classificationFile, classifierType=args.classifierType) else: runner = Runner(dataPath=args.dataPath, resultsDir=resultsDir, experimentName=args.experimentName, loadPath=args.loadPath, modelName=args.modelName, numClasses=args.numClasses, plots=args.plots, orderedSplit=args.orderedSplit, trainSizes=[], verbosity=args.verbosity) # HTM network data isn't ready yet to initialize the model runner.initModel(args.modelName) print "Reading in data and preprocessing." dataTime = time.time() runner.setupData(args.textPreprocess) # TODO: move kfolds splitting to Runner random = False if args.orderedSplit else True runner.partitions = KFolds(args.kFolds).split(range(len(runner.samples)), randomize=random) runner.trainSizes = [len(x[0]) for x in runner.partitions] print( "Data setup complete; elapsed time is {0:.2f} seconds.\nNow encoding " "the data".format(time.time() - dataTime)) encodeTime = time.time() runner.encodeSamples() print( "Encoding complete; elapsed time is {0:.2f} seconds.\nNow running the " "experiment.".format(time.time() - encodeTime)) runner.runExperiment() print "Experiment complete in {0:.2f} seconds.".format(time.time() - start) resultCalcs = runner.calculateResults() _ = runner.evaluateCumulativeResults(resultCalcs) print "Saving..." runner.saveModel() if args.validation: print "Validating experiment against expected classifications..." print runner.validateExperiment(args.validation)