def main(): """Run according to options in sys.argv""" # Init the NuPic logging configuration from the nupic-logging.conf configuration # file. This is found either in the NTA_CONF_DIR directory (if defined) or # in the 'conf' subdirectory of the NuPic install location. nupic.support.initLogging(verbose=True) # Initialize PRNGs initExperimentPrng() # Run it! runExperiment(sys.argv[1:])
def main(): """Run according to options in sys.argv""" nupic.support.initLogging(verbose=True) # Initialize pseudo-random number generators (PRNGs) # # This will fix the seed that is used by numpy when generating 'random' # numbers. This allows for repeatability across experiments. initExperimentPrng() # Run it! runExperiment(sys.argv[1:])
def runReducedExperiment(path, reduced=True): """ Run the experiment in the <path> with a reduced iteration count """ initExperimentPrng() # Load experiment if reduced: args = [path, '--testMode'] else: args = [path] runExperiment(args)
def main(): """Run according to options in sys.argv""" # Init the NuPic logging configuration from the nupic-logging.conf configuration # file. This is found either in the NTA_CONF_DIR directory (if defined) or # in the 'conf' subdirectory of the NuPic install location. nupic.support.initLogging(verbose=True) # Initialize pseudo-random number generators (PRNGs) # # This will fix the seed that is used by numpy when generating 'random' # numbers. This allows for repeatability across experiments. initExperimentPrng() # Run it! runExperiment(sys.argv[1:])
def main(): """Run according to options in sys.argv and diff classifiers.""" initLogging(verbose=True) # Initialize PRNGs initExperimentPrng() # Mock out the creation of the CLAClassifier. @staticmethod def _mockCreate(*args, **kwargs): kwargs.pop('implementation', None) return CLAClassifierDiff(*args, **kwargs) CLAClassifierFactory.create = _mockCreate # Run it! runExperiment(sys.argv[1:])
def main(): """Run according to options in sys.argv and diff classifiers.""" initLogging(verbose=True) # Initialize PRNGs initExperimentPrng() # Mock out the creation of the SDRClassifier. @staticmethod def _mockCreate(*args, **kwargs): kwargs.pop('implementation', None) return SDRClassifierDiff(*args, **kwargs) SDRClassifierFactory.create = _mockCreate # Run it! runExperiment(sys.argv[1:])
def testHotgymRegression(self): experimentDir = os.path.join( os.path.dirname(__file__).partition( os.path.normpath("tests/integration/nupic/opf"))[0], "examples", "opf", "experiments", "multistep", "hotgym") resultsDir = os.path.join(experimentDir, "inference") savedModelsDir = os.path.join(experimentDir, "savedmodels") try: _model = experiment_runner.runExperiment([experimentDir]) resultsPath = os.path.join( resultsDir, "DefaultTask.TemporalMultiStep.predictionLog.csv") with open(resultsPath) as f: reader = csv.reader(f) headers = reader.next() self.assertEqual(headers[14], "multiStepBestPredictions:multiStep:errorMetric='aae':" "steps=1:window=1000:field=consumption") lastRow = collections.deque(reader, 1)[0] # Changes that affect prediction results will cause this test to fail. If # the change is understood and reviewers agree that there has not been a # regression then this value can be updated to reflect the new result. self.assertAlmostEqual(float(lastRow[14]), 5.89191585339) finally: shutil.rmtree(resultsDir, ignore_errors=True) shutil.rmtree(savedModelsDir, ignore_errors=True)
def testHotgymRegression(self): experimentDir = os.path.join( os.path.dirname(__file__).partition( os.path.normpath("tests/integration/nupic/opf"))[0], "examples", "opf", "experiments", "multistep", "hotgym") resultsDir = os.path.join(experimentDir, "inference") savedModelsDir = os.path.join(experimentDir, "savedmodels") try: _model = experiment_runner.runExperiment([experimentDir]) resultsPath = os.path.join( resultsDir, "DefaultTask.TemporalMultiStep.predictionLog.csv") with open(resultsPath) as f: reader = csv.reader(f) headers = reader.next() self.assertEqual( headers[14], "multiStepBestPredictions:multiStep:errorMetric='aae':" "steps=1:window=1000:field=consumption") lastRow = collections.deque(reader, 1)[0] # Changes that affect prediction results will cause this test to fail. If # the change is understood and reviewers agree that there has not been a # regression then this value can be updated to reflect the new result. self.assertAlmostEqual(float(lastRow[14]), 5.84456654247) finally: shutil.rmtree(resultsDir, ignore_errors=True) shutil.rmtree(savedModelsDir, ignore_errors=True)
def main(): """Run according to options in sys.argv and diff classifiers.""" # Init the NuPic logging configuration from the nupic-logging.conf # configuration file. This is found either in the NTA_CONF_DIR directory # (if defined) or in the 'conf' subdirectory of the NuPic install location. initLogging(verbose=True) # Initialize PRNGs initExperimentPrng() # Mock out the creation of the CLAClassifier. @staticmethod def _mockCreate(*args, **kwargs): kwargs.pop('implementation', None) return CLAClassifierDiff(*args, **kwargs) CLAClassifierFactory.create = _mockCreate # Run it! runExperiment(sys.argv[1:])
def _testBackwardsCompatibility(experiment, checkpointName): """ Test that we can load in a checkpoint saved by an earlier version of the OPF. Parameters: ----------------------------------------------------------------------- experiment: Directory of the experiment. checkpointName: which checkpoint to verify """ # Get the experiment directories expDir = os.path.join(_EXPERIMENT_BASE, experiment) # Copy the pertinent checkpoint if os.path.exists(os.path.join(expDir, 'savedmodels')): shutil.rmtree(os.path.join(expDir, 'savedmodels')) shutil.copytree(src=os.path.join(expDir, checkpointName), dst=os.path.join(expDir, 'savedmodels')) # Run it from the checkpoint _aPlusBExp = runExperiment(args=[expDir, '--load=DefaultTask', '--noCheckpoint'])
def _testBackwardsCompatibility(experiment, checkpointName): """ Test that we can load in a checkpoint saved by an earlier version of the OPF. Parameters: ----------------------------------------------------------------------- experiment: Directory of the experiment. checkpointName: which checkpoint to verify """ # Get the experiment directories expDir = os.path.join(_EXPERIMENT_BASE, experiment) # Copy the pertinent checkpoint if os.path.exists(os.path.join(expDir, 'savedmodels')): shutil.rmtree(os.path.join(expDir, 'savedmodels')) shutil.copytree(src=os.path.join(expDir, checkpointName), dst=os.path.join(expDir, 'savedmodels')) # Run it from the checkpoint _aPlusBExp = runExperiment( args=[expDir, '--load=DefaultTask', '--noCheckpoint'])
def _testSamePredictions(self, experiment, predSteps, checkpointAt, predictionsFilename, additionalFields=None, newSerialization=False): """ Test that we get the same predictions out from the following two scenarios: a_plus_b: Run the network for 'a' iterations followed by 'b' iterations a, followed by b: Run the network for 'a' iterations, save it, load it back in, then run for 'b' iterations. Parameters: ----------------------------------------------------------------------- experiment: base directory of the experiment. This directory should contain the following: base.py a_plus_b/description.py a/description.py b/description.py The sub-directory description files should import the base.py and only change the first and last record used from the data file. predSteps: Number of steps ahead predictions are for checkpointAt: Number of iterations that 'a' runs for. IMPORTANT: This must match the number of records that a/description.py runs for - it is NOT dynamically stuffed into the a/description.py. predictionsFilename: The name of the predictions file that the OPF generates for this experiment (for example 'DefaulTask.NontemporalMultiStep.predictionLog.csv') newSerialization: Whether to use new capnproto serialization. """ # Get the 3 sub-experiment directories aPlusBExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "a_plus_b") aExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "a") bExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "b") # Run a+b args = self._createExperimentArgs(aPlusBExpDir, newSerialization=newSerialization) _aPlusBExp = runExperiment(args) # Run a, the copy the saved checkpoint into the b directory args = self._createExperimentArgs(aExpDir, newSerialization=newSerialization) _aExp = runExperiment(args) if os.path.exists(os.path.join(bExpDir, 'savedmodels')): shutil.rmtree(os.path.join(bExpDir, 'savedmodels')) shutil.copytree(src=os.path.join(aExpDir, 'savedmodels'), dst=os.path.join(bExpDir, 'savedmodels')) args = self._createExperimentArgs(bExpDir, newSerialization=newSerialization, additionalArgs=['--load=DefaultTask']) _bExp = runExperiment(args) # Now, compare the predictions at the end of a+b to those in b. aPlusBPred = FileRecordStream(os.path.join(aPlusBExpDir, 'inference', predictionsFilename)) bPred = FileRecordStream(os.path.join(bExpDir, 'inference', predictionsFilename)) colNames = [x[0] for x in aPlusBPred.getFields()] actValueColIdx = colNames.index('multiStepPredictions.actual') predValueColIdx = colNames.index('multiStepPredictions.%d' % (predSteps)) # Skip past the 'a' records in aPlusB for i in range(checkpointAt): aPlusBPred.next() # Now, read through the records that don't have predictions yet for i in range(predSteps): aPlusBPred.next() bPred.next() # Now, compare predictions in the two files rowIdx = checkpointAt + predSteps + 4 - 1 epsilon = 0.0001 while True: rowIdx += 1 try: rowAPB = aPlusBPred.next() rowB = bPred.next() # Compare actuals self.assertEqual(rowAPB[actValueColIdx], rowB[actValueColIdx], "Mismatch in actual values: row %d of a+b has %s and row %d of " "b has %s" % (rowIdx, rowAPB[actValueColIdx], rowIdx-checkpointAt, rowB[actValueColIdx])) # Compare predictions, within nearest epsilon predAPB = eval(rowAPB[predValueColIdx]) predB = eval(rowB[predValueColIdx]) # Sort with highest probabilities first predAPB = [(a, b) for b, a in predAPB.items()] predB = [(a, b) for b, a in predB.items()] predAPB.sort(reverse=True) predB.sort(reverse=True) if additionalFields is not None: for additionalField in additionalFields: fieldIdx = colNames.index(additionalField) self.assertEqual(rowAPB[fieldIdx], rowB[fieldIdx], "Mismatch in field \'%s\' values: row %d of a+b has value: (%s)\n" " and row %d of b has value: %s" % \ (additionalField, rowIdx, rowAPB[fieldIdx], rowIdx-checkpointAt, rowB[fieldIdx])) self.assertEqual(len(predAPB), len(predB), "Mismatch in predicted values: row %d of a+b has %d predictions: " "\n (%s) and row %d of b has %d predictions:\n (%s)" % \ (rowIdx, len(predAPB), predAPB, rowIdx-checkpointAt, len(predB), predB)) for i in range(len(predAPB)): (aProb, aValue) = predAPB[i] (bProb, bValue) = predB[i] self.assertLess(abs(aValue-bValue), epsilon, "Mismatch in predicted values: row %d of a+b predicts value %s " "and row %d of b predicts %s" % (rowIdx, aValue, rowIdx-checkpointAt, bValue)) self.assertLess(abs(aProb-bProb), epsilon, "Mismatch in probabilities: row %d of a+b predicts %s with " "probability %s and row %d of b predicts %s with probability %s" \ % (rowIdx, aValue, aProb, rowIdx-checkpointAt, bValue, bProb)) except StopIteration: break # clean up model checkpoint directories shutil.rmtree(getCheckpointParentDir(aExpDir)) shutil.rmtree(getCheckpointParentDir(bExpDir)) shutil.rmtree(getCheckpointParentDir(aPlusBExpDir)) print "Predictions match!"
def _testSamePredictions(self, experiment, predSteps, checkpointAt, predictionsFilename, additionalFields=None): """ Test that we get the same predictions out from the following two scenarios: a_plus_b: Run the network for 'a' iterations followed by 'b' iterations a, followed by b: Run the network for 'a' iterations, save it, load it back in, then run for 'b' iterations. Parameters: ----------------------------------------------------------------------- experiment: base directory of the experiment. This directory should contain the following: base.py a_plus_b/description.py a/description.py b/description.py The sub-directory description files should import the base.py and only change the first and last record used from the data file. predSteps: Number of steps ahead predictions are for checkpointAt: Number of iterations that 'a' runs for. IMPORTANT: This must match the number of records that a/description.py runs for - it is NOT dynamically stuffed into the a/description.py. predictionsFilename: The name of the predictions file that the OPF generates for this experiment (for example 'DefaulTask.NontemporalMultiStep.predictionLog.csv') """ # Get the 3 sub-experiment directories aPlusBExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "a_plus_b") aExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "a") bExpDir = os.path.join(_EXPERIMENT_BASE, experiment, "b") # Run a+b _aPlusBExp = runExperiment(args=[aPlusBExpDir]) # Run a, the copy the saved checkpoint into the b directory _aExp = runExperiment(args=[aExpDir]) if os.path.exists(os.path.join(bExpDir, 'savedmodels')): shutil.rmtree(os.path.join(bExpDir, 'savedmodels')) shutil.copytree(src=os.path.join(aExpDir, 'savedmodels'), dst=os.path.join(bExpDir, 'savedmodels')) _bExp = runExperiment(args=[bExpDir, '--load=DefaultTask']) # Now, compare the predictions at the end of a+b to those in b. aPlusBPred = FileRecordStream( os.path.join(aPlusBExpDir, 'inference', predictionsFilename)) bPred = FileRecordStream( os.path.join(bExpDir, 'inference', predictionsFilename)) colNames = [x[0] for x in aPlusBPred.getFields()] actValueColIdx = colNames.index('multiStepPredictions.actual') predValueColIdx = colNames.index('multiStepPredictions.%d' % (predSteps)) # Skip past the 'a' records in aPlusB for i in range(checkpointAt): aPlusBPred.next() # Now, read through the records that don't have predictions yet for i in range(predSteps): aPlusBPred.next() bPred.next() # Now, compare predictions in the two files rowIdx = checkpointAt + predSteps + 4 - 1 epsilon = 0.0001 while True: rowIdx += 1 try: rowAPB = aPlusBPred.next() rowB = bPred.next() # Compare actuals self.assertEqual( rowAPB[actValueColIdx], rowB[actValueColIdx], "Mismatch in actual values: row %d of a+b has %s and row %d of " "b has %s" % (rowIdx, rowAPB[actValueColIdx], rowIdx - checkpointAt, rowB[actValueColIdx])) # Compare predictions, within nearest epsilon predAPB = eval(rowAPB[predValueColIdx]) predB = eval(rowB[predValueColIdx]) # Sort with highest probabilities first predAPB = [(a, b) for b, a in predAPB.items()] predB = [(a, b) for b, a in predB.items()] predAPB.sort(reverse=True) predB.sort(reverse=True) if additionalFields is not None: for additionalField in additionalFields: fieldIdx = colNames.index(additionalField) self.assertEqual(rowAPB[fieldIdx], rowB[fieldIdx], "Mismatch in field \'%s\' values: row %d of a+b has value: (%s)\n" " and row %d of b has value: %s" % \ (additionalField, rowIdx, rowAPB[fieldIdx], rowIdx-checkpointAt, rowB[fieldIdx])) self.assertEqual(len(predAPB), len(predB), "Mismatch in predicted values: row %d of a+b has %d predictions: " "\n (%s) and row %d of b has %d predictions:\n (%s)" % \ (rowIdx, len(predAPB), predAPB, rowIdx-checkpointAt, len(predB), predB)) for i in range(len(predAPB)): (aProb, aValue) = predAPB[i] (bProb, bValue) = predB[i] self.assertLess( abs(aValue - bValue), epsilon, "Mismatch in predicted values: row %d of a+b predicts value %s " "and row %d of b predicts %s" % (rowIdx, aValue, rowIdx - checkpointAt, bValue)) self.assertLess(abs(aProb-bProb), epsilon, "Mismatch in probabilities: row %d of a+b predicts %s with " "probability %s and row %d of b predicts %s with probability %s" \ % (rowIdx, aValue, aProb, rowIdx-checkpointAt, bValue, bProb)) except StopIteration: break # clean up model checkpoint directories shutil.rmtree(getCheckpointParentDir(aExpDir)) shutil.rmtree(getCheckpointParentDir(bExpDir)) shutil.rmtree(getCheckpointParentDir(aPlusBExpDir)) print "Predictions match!"