def createRunner(resultsDir, profileName, resultsName=None): """Create a nab.runner.Runner object for testing.""" root = os.path.dirname(os.path.realpath(__file__)) labelPath = os.path.abspath( os.path.join(root, "..", "test_labels/labels.json")) testRunner = Runner(dataDir=None, resultsDir=resultsDir, labelPath=labelPath, profilesPath=None, thresholdPath=None) testRunner.profiles = { profileName: { "CostMatrix": { "tpWeight": 1.0 } } } if resultsName is not None: resultsFile = resultsName+'_'+profileName+'_scores.csv' resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile) testRunner.resultsFiles = [resultsFilePath] return testRunner
def main(args): root = os.path.dirname(os.path.realpath(__file__)) numCPUs = int(args.numCPUs) if args.numCPUs is not None else None dataDir = os.path.join(root, args.dataDir) labelFile = os.path.join(root, args.labelFile) resultsDir = os.path.join(root, args.resultsDir) profilesFile = os.path.join(root, args.profilesFile) thresholdsFile = os.path.join(root, args.thresholdsFile) runner = Runner(dataDir=dataDir, labelPath=labelFile, resultsDir=resultsDir, profilesPath=profilesFile, thresholdPath=thresholdsFile, numCPUs=numCPUs) runner.initialize() if args.detect: detectorConstructors = getDetectorClassConstructors(args.detectors) runner.detect(detectorConstructors) if args.optimize: runner.optimize(args.detectors) if args.score: with open(args.thresholdsFile) as thresholdConfigFile: detectorThresholds = json.load(thresholdConfigFile) runner.score(args.detectors, detectorThresholds)
def createRunner(resultsDir, profileName, resultsName=None): """Create a nab.runner.Runner object for testing.""" testRunner = Runner(dataDir=None, resultsDir=resultsDir, labelPath=None, profilesPath=None, thresholdPath=None) testRunner.profiles = json.loads('{"' + profileName + '": {}}') if resultsName is not None: resultsFile = resultsName + "_" + profileName + "_scores.csv" resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile) testRunner.resultsFiles = [resultsFilePath] return testRunner
def createRunner(resultsDir, profileName, resultsName=None): """Create a nab.runner.Runner object for testing.""" testRunner = Runner(dataDir=None, resultsDir=resultsDir, labelPath=None, profilesPath=None, thresholdPath=None) testRunner.profiles = json.loads('{"' + profileName + '": {}}') if resultsName is not None: resultsFile = resultsName + '_' + profileName + '_scores.csv' resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile) testRunner.resultsFiles = [resultsFilePath] return testRunner
def testBaselineScoreLoading(self): """Tests that we fail apropriately if baseline scores are absent.""" testRunner = Runner(dataDir=None, resultsDir="", labelPath=None, profilesPath=None, thresholdPath=None) # Should fail due to resultsDir/baseline not being a directory. with self.assertRaises(IOError): testRunner.normalize() tmpResultsDir = self._createTemporaryResultsDir() os.makedirs(os.path.join(tmpResultsDir, "baseline")) testRunner2 = createRunner(tmpResultsDir, "standard") # Should fail due to resultsDir/baseline being empty. with self.assertRaises(IOError): testRunner2.normalize()
def createRunner(resultsDir, profileName, resultsName=None): """Create a nab.runner.Runner object for testing.""" root = os.path.dirname(os.path.realpath(__file__)) labelPath = os.path.abspath( os.path.join(root, "..", "test_labels/labels.json")) testRunner = Runner(dataDir=None, resultsDir=resultsDir, labelPath=labelPath, profilesPath=None, thresholdPath=None) testRunner.profiles = {profileName: {"CostMatrix": {"tpWeight": 1.0}}} if resultsName is not None: resultsFile = resultsName + '_' + profileName + '_scores.csv' resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile) testRunner.resultsFiles = [resultsFilePath] return testRunner
def testNullScoreLoading(self): """Tests that we fail apropriately if null detector scores are absent.""" testRunner = Runner(dataDir=None, resultsDir='', labelPath=None, profilesPath=None, thresholdPath=None) # Should fail due to resultsDir/null not being a directory. with self.assertRaises(IOError): testRunner.normalize() tmpResultsDir = self._createTemporaryResultsDir() os.makedirs(os.path.join(tmpResultsDir,'null')) testRunner2 = createRunner(tmpResultsDir, 'standard') # Should fail due to resultsDir/null being empty. with self.assertRaises(IOError): testRunner2.normalize()
def testBaselineScoreLoading(self): """Tests that we fail apropriately if baseline scores are absent.""" testRunner = Runner(dataDir=None, resultsDir='', labelPath=None, profilesPath=None, thresholdPath=None) # Should fail due to resultsDir/baseline not being a directory. with self.assertRaises(IOError): testRunner.normalize() tmpResultsDir = self._createTemporaryResultsDir() os.makedirs(os.path.join(tmpResultsDir, 'baseline')) testRunner2 = createRunner(tmpResultsDir, 'standard') # Should fail due to resultsDir/baseline being empty. with self.assertRaises(IOError): testRunner2.normalize()
def main(args): filepath = os.path.realpath(__file__) # Find the main NAB folder # Assuming `filepath` is ~ <...>/NAB/nab/detectors/htmjava/run.py root = get_nth_parent_dir(3, filepath) numCPUs = int(args.numCPUs) if args.numCPUs is not None else None dataDir = os.path.join(root, args.dataDir) windowsFile = os.path.join(root, args.windowsFile) resultsDir = os.path.join(root, args.resultsDir) profilesFile = os.path.join(root, args.profilesFile) thresholdsFile = os.path.join(root, args.thresholdsFile) runner = Runner(dataDir=dataDir, labelPath=windowsFile, resultsDir=resultsDir, profilesPath=profilesFile, thresholdPath=thresholdsFile, numCPUs=numCPUs) runner.initialize() runner.detect({'htmjava': HtmjavaDetector})
def main(args): filepath = os.path.realpath(__file__) # Find the main NAB folder # Assuming `filepath` is ~ <...>/NAB/nab/detectors/numenta/run.py root = get_nth_parent_dir(3, filepath) numCPUs = int(args.numCPUs) if args.numCPUs is not None else None dataDir = os.path.join(root, args.dataDir) windowsFile = os.path.join(root, args.windowsFile) resultsDir = os.path.join(root, args.resultsDir) profilesFile = os.path.join(root, args.profilesFile) runner = Runner(dataDir=dataDir, labelPath=windowsFile, resultsDir=resultsDir, profilesPath=profilesFile, numCPUs=numCPUs) runner.initialize() detectorConstructors = getDetectorClassConstructors(args.detectors) runner.detect(detectorConstructors)
def main(args): if not args.detect and not args.score and not args.optimize: args.detect = True args.optimize = True args.score = True detectors = args.detectors numCPUs = int(args.numCPUs) if args.numCPUs is not None else None probationaryPercent = float(args.probationaryPercent) dataDir = os.path.join(root, args.dataDir) labelDir = os.path.join(root, args.labelDir) resultsDir = os.path.join(root, args.resultsDir) profilesPath = os.path.join(root, args.profilesPath) thresholdPath = os.path.join(root, args.thresholdPath) runner = Runner(dataDir=dataDir, labelDir=labelDir, resultsDir=resultsDir, profilesPath=profilesPath, thresholdPath=thresholdPath, probationaryPercent=probationaryPercent, numCPUs=numCPUs) runner.initialize() if args.detect: detectorConstructors = getDetectorClassConstructors(args.detectors) runner.detect(detectorConstructors) if args.optimize: runner.optimize(args.detectors) if args.score: with open(args.thresholdPath) as thresholdConfigFile: detectorThresholds = yaml.load(thresholdConfigFile) runner.score(args.detectors, detectorThresholds)
def main(args): root = os.path.dirname(os.path.realpath(__file__)) numCPUs = int(args.numCPUs) if args.numCPUs is not None else None dataDir = os.path.join(root, args.dataDir) windowsFile = os.path.join(root, args.windowsFile) resultsDir = os.path.join(root, args.resultsDir) profilesFile = os.path.join(root, args.profilesFile) thresholdsFile = os.path.join(root, args.thresholdsFile) runner = Runner(dataDir=dataDir, labelPath=windowsFile, resultsDir=resultsDir, profilesPath=profilesFile, thresholdPath=thresholdsFile, numCPUs=numCPUs) runner.initialize() if args.detect: detectorConstructors = getDetectorClassConstructors(args.detectors) runner.detect(detectorConstructors) if args.optimize: runner.optimize(args.detectors) if args.score: with open(args.thresholdsFile) as thresholdConfigFile: detectorThresholds = json.load(thresholdConfigFile) runner.score(args.detectors, detectorThresholds) if args.normalize: try: runner.normalize() except AttributeError("Error: you must run the scoring step with the " "normalization step."): return
def main(args): root = os.path.dirname(os.path.realpath(__file__)) numCPUs = int(args.numCPUs) if args.numCPUs is not None else None dataDir = os.path.join(root, args.dataDir) windowsFile = os.path.join(root, args.windowsFile) resultsDir = os.path.join(root, args.resultsDir) profilesFile = os.path.join(root, args.profilesFile) thresholdsFile = os.path.join(root, args.thresholdsFile) runner = Runner(dataDir=dataDir, labelPath=windowsFile, resultsDir=resultsDir, profilesPath=profilesFile, thresholdPath=thresholdsFile, numCPUs=numCPUs) runner.initialize() if args.detect: detectorConstructors = getDetectorClassConstructors(args.detectors) print("Starting Evaluation!!\n") start_time = datetime.datetime.now() runner.detect(detectorConstructors) delta = datetime.datetime.now() - start_time print("Total time for classification in milliseconds " + str(int(delta.total_seconds() * 1000)) + '\n') #print("Total time for classification: " + str(delta) + '\n') if args.optimize: runner.optimize(args.detectors) if args.score: with open(args.thresholdsFile) as thresholdConfigFile: detectorThresholds = json.load(thresholdConfigFile) runner.score(args.detectors, detectorThresholds) if args.normalize: try: runner.normalize() except AttributeError("Error: you must run the scoring step with the " "normalization step."): return