def createRunner(resultsDir, profileName, resultsName=None):
"""Create a nab.runner.Runner object for testing."""
root = os.path.dirname(os.path.realpath(__file__))
labelPath = os.path.abspath(
os.path.join(root, "..", "test_labels/labels.json"))
testRunner = Runner(dataDir=None,
resultsDir=resultsDir,
labelPath=labelPath,
profilesPath=None,
thresholdPath=None)
testRunner.profiles = {
profileName: {
"CostMatrix": {
"tpWeight": 1.0
}
}
}
if resultsName is not None:
resultsFile = resultsName+'_'+profileName+'_scores.csv'
resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile)
testRunner.resultsFiles = [resultsFilePath]
return testRunner
def main(args):
root = os.path.dirname(os.path.realpath(__file__))
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
labelFile = os.path.join(root, args.labelFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
thresholdsFile = os.path.join(root, args.thresholdsFile)
runner = Runner(dataDir=dataDir,
labelPath=labelFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
thresholdPath=thresholdsFile,
numCPUs=numCPUs)
runner.initialize()
if args.detect:
detectorConstructors = getDetectorClassConstructors(args.detectors)
runner.detect(detectorConstructors)
if args.optimize:
runner.optimize(args.detectors)
if args.score:
with open(args.thresholdsFile) as thresholdConfigFile:
detectorThresholds = json.load(thresholdConfigFile)
runner.score(args.detectors, detectorThresholds)
def createRunner(resultsDir, profileName, resultsName=None):
"""Create a nab.runner.Runner object for testing."""
testRunner = Runner(dataDir=None, resultsDir=resultsDir, labelPath=None, profilesPath=None, thresholdPath=None)
testRunner.profiles = json.loads('{"' + profileName + '": {}}')
if resultsName is not None:
resultsFile = resultsName + "_" + profileName + "_scores.csv"
resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile)
testRunner.resultsFiles = [resultsFilePath]
return testRunner
def createRunner(resultsDir, profileName, resultsName=None):
"""Create a nab.runner.Runner object for testing."""
testRunner = Runner(dataDir=None,
resultsDir=resultsDir,
labelPath=None,
profilesPath=None,
thresholdPath=None)
testRunner.profiles = json.loads('{"' + profileName + '": {}}')
if resultsName is not None:
resultsFile = resultsName + '_' + profileName + '_scores.csv'
resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile)
testRunner.resultsFiles = [resultsFilePath]
return testRunner
def testBaselineScoreLoading(self):
"""Tests that we fail apropriately if baseline scores are absent."""
testRunner = Runner(dataDir=None, resultsDir="", labelPath=None, profilesPath=None, thresholdPath=None)
# Should fail due to resultsDir/baseline not being a directory.
with self.assertRaises(IOError):
testRunner.normalize()
tmpResultsDir = self._createTemporaryResultsDir()
os.makedirs(os.path.join(tmpResultsDir, "baseline"))
testRunner2 = createRunner(tmpResultsDir, "standard")
# Should fail due to resultsDir/baseline being empty.
with self.assertRaises(IOError):
testRunner2.normalize()
def createRunner(resultsDir, profileName, resultsName=None):
"""Create a nab.runner.Runner object for testing."""
root = os.path.dirname(os.path.realpath(__file__))
labelPath = os.path.abspath(
os.path.join(root, "..", "test_labels/labels.json"))
testRunner = Runner(dataDir=None,
resultsDir=resultsDir,
labelPath=labelPath,
profilesPath=None,
thresholdPath=None)
testRunner.profiles = {profileName: {"CostMatrix": {"tpWeight": 1.0}}}
if resultsName is not None:
resultsFile = resultsName + '_' + profileName + '_scores.csv'
resultsFilePath = os.path.join(resultsDir, resultsName, resultsFile)
testRunner.resultsFiles = [resultsFilePath]
return testRunner
def testNullScoreLoading(self):
"""Tests that we fail apropriately if null detector scores are absent."""
testRunner = Runner(dataDir=None,
resultsDir='',
labelPath=None,
profilesPath=None,
thresholdPath=None)
# Should fail due to resultsDir/null not being a directory.
with self.assertRaises(IOError):
testRunner.normalize()
tmpResultsDir = self._createTemporaryResultsDir()
os.makedirs(os.path.join(tmpResultsDir,'null'))
testRunner2 = createRunner(tmpResultsDir, 'standard')
# Should fail due to resultsDir/null being empty.
with self.assertRaises(IOError):
testRunner2.normalize()
def testBaselineScoreLoading(self):
"""Tests that we fail apropriately if baseline scores are absent."""
testRunner = Runner(dataDir=None,
resultsDir='',
labelPath=None,
profilesPath=None,
thresholdPath=None)
# Should fail due to resultsDir/baseline not being a directory.
with self.assertRaises(IOError):
testRunner.normalize()
tmpResultsDir = self._createTemporaryResultsDir()
os.makedirs(os.path.join(tmpResultsDir, 'baseline'))
testRunner2 = createRunner(tmpResultsDir, 'standard')
# Should fail due to resultsDir/baseline being empty.
with self.assertRaises(IOError):
testRunner2.normalize()
def main(args):
filepath = os.path.realpath(__file__)
# Find the main NAB folder
# Assuming `filepath` is ~ <...>/NAB/nab/detectors/htmjava/run.py
root = get_nth_parent_dir(3, filepath)
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
windowsFile = os.path.join(root, args.windowsFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
thresholdsFile = os.path.join(root, args.thresholdsFile)
runner = Runner(dataDir=dataDir,
labelPath=windowsFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
thresholdPath=thresholdsFile,
numCPUs=numCPUs)
runner.initialize()
runner.detect({'htmjava': HtmjavaDetector})
def main(args):
filepath = os.path.realpath(__file__)
# Find the main NAB folder
# Assuming `filepath` is ~ <...>/NAB/nab/detectors/numenta/run.py
root = get_nth_parent_dir(3, filepath)
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
windowsFile = os.path.join(root, args.windowsFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
runner = Runner(dataDir=dataDir,
labelPath=windowsFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
numCPUs=numCPUs)
runner.initialize()
detectorConstructors = getDetectorClassConstructors(args.detectors)
runner.detect(detectorConstructors)
def main(args):
if not args.detect and not args.score and not args.optimize:
args.detect = True
args.optimize = True
args.score = True
detectors = args.detectors
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
probationaryPercent = float(args.probationaryPercent)
dataDir = os.path.join(root, args.dataDir)
labelDir = os.path.join(root, args.labelDir)
resultsDir = os.path.join(root, args.resultsDir)
profilesPath = os.path.join(root, args.profilesPath)
thresholdPath = os.path.join(root, args.thresholdPath)
runner = Runner(dataDir=dataDir,
labelDir=labelDir,
resultsDir=resultsDir,
profilesPath=profilesPath,
thresholdPath=thresholdPath,
probationaryPercent=probationaryPercent,
numCPUs=numCPUs)
runner.initialize()
if args.detect:
detectorConstructors = getDetectorClassConstructors(args.detectors)
runner.detect(detectorConstructors)
if args.optimize:
runner.optimize(args.detectors)
if args.score:
with open(args.thresholdPath) as thresholdConfigFile:
detectorThresholds = yaml.load(thresholdConfigFile)
runner.score(args.detectors, detectorThresholds)
def main(args):
root = os.path.dirname(os.path.realpath(__file__))
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
windowsFile = os.path.join(root, args.windowsFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
thresholdsFile = os.path.join(root, args.thresholdsFile)
runner = Runner(dataDir=dataDir,
labelPath=windowsFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
thresholdPath=thresholdsFile,
numCPUs=numCPUs)
runner.initialize()
if args.detect:
detectorConstructors = getDetectorClassConstructors(args.detectors)
runner.detect(detectorConstructors)
if args.optimize:
runner.optimize(args.detectors)
if args.score:
with open(args.thresholdsFile) as thresholdConfigFile:
detectorThresholds = json.load(thresholdConfigFile)
runner.score(args.detectors, detectorThresholds)
if args.normalize:
try:
runner.normalize()
except AttributeError("Error: you must run the scoring step with the "
"normalization step."):
return
def main(args):
root = os.path.dirname(os.path.realpath(__file__))
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
windowsFile = os.path.join(root, args.windowsFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
thresholdsFile = os.path.join(root, args.thresholdsFile)
runner = Runner(dataDir=dataDir,
labelPath=windowsFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
thresholdPath=thresholdsFile,
numCPUs=numCPUs)
runner.initialize()
if args.detect:
detectorConstructors = getDetectorClassConstructors(args.detectors)
print("Starting Evaluation!!\n")
start_time = datetime.datetime.now()
runner.detect(detectorConstructors)
delta = datetime.datetime.now() - start_time
print("Total time for classification in milliseconds " +
str(int(delta.total_seconds() * 1000)) + '\n')
#print("Total time for classification: " + str(delta) + '\n')
if args.optimize:
runner.optimize(args.detectors)
if args.score:
with open(args.thresholdsFile) as thresholdConfigFile:
detectorThresholds = json.load(thresholdConfigFile)
runner.score(args.detectors, detectorThresholds)
if args.normalize:
try:
runner.normalize()
except AttributeError("Error: you must run the scoring step with the "
"normalization step."):
return
def main(args):
root = os.path.dirname(os.path.realpath(__file__))
numCPUs = int(args.numCPUs) if args.numCPUs is not None else None
dataDir = os.path.join(root, args.dataDir)
windowsFile = os.path.join(root, args.windowsFile)
resultsDir = os.path.join(root, args.resultsDir)
profilesFile = os.path.join(root, args.profilesFile)
thresholdsFile = os.path.join(root, args.thresholdsFile)
runner = Runner(dataDir=dataDir,
labelPath=windowsFile,
resultsDir=resultsDir,
profilesPath=profilesFile,
thresholdPath=thresholdsFile,
numCPUs=numCPUs)
runner.initialize()
if args.detect:
detectorConstructors = getDetectorClassConstructors(args.detectors)
runner.detect(detectorConstructors)
if args.optimize:
runner.optimize(args.detectors)
if args.score:
with open(args.thresholdsFile) as thresholdConfigFile:
detectorThresholds = json.load(thresholdConfigFile)
runner.score(args.detectors, detectorThresholds)
if args.normalize:
try:
runner.normalize()
except AttributeError("Error: you must run the scoring step with the "
"normalization step."):
return
