Пример #1
0
    def newPosition(self, globalBestPosition, rng):
        """See comments in base class."""
        # First, update the velocity. The new velocity is given as:
        # v = (inertia * v)  + (cogRate * r1 * (localBest-pos))
        #                    + (socRate * r2 * (globalBest-pos))
        #
        # where r1 and r2 are random numbers between 0 and 1.0
        lb = float(Configuration.get("nupic.hypersearch.randomLowerBound"))
        ub = float(Configuration.get("nupic.hypersearch.randomUpperBound"))

        self._velocity = (self._velocity * self._inertia +
                          rng.uniform(lb, ub) * self._cogRate *
                          (self._bestPosition - self.getPosition()))
        if globalBestPosition is not None:
            self._velocity += rng.uniform(lb, ub) * self._socRate * (
                globalBestPosition - self.getPosition())

        # update position based on velocity
        self._position += self._velocity

        # Clip it
        self._position = max(self.min, self._position)
        self._position = min(self.max, self._position)

        # Return it
        return self.getPosition()
Пример #2
0
def dbValidator():
    """
    Let the user know what NuPIC config file is being used
    and whether or not they have mysql set up correctly for
    swarming.
  """
    fileused = getFileUsed()

    # Get the values we need from NuPIC's configuration
    host = Configuration.get("nupic.cluster.database.host")
    port = int(Configuration.get("nupic.cluster.database.port"))
    user = Configuration.get("nupic.cluster.database.user")
    passwd = Configuration.get("nupic.cluster.database.passwd")

    print("This script will validate that your MySQL is setup correctly for ")
    print("NuPIC. MySQL is required for NuPIC swarming. The settings are")
    print("defined in a configuration file found in  ")
    print("$NUPIC/src/nupic/support/nupic-default.xml Out of the box those ")
    print("settings contain MySQL's default access credentials.")
    print()
    print("The nupic-default.xml can be duplicated to define user specific ")
    print("changes calling the copied file ")
    print("$NUPIC/src/nupic/support/nupic-site.xml Refer to the ")
    print("nupic-default.xml for additional instructions.")
    print()
    print("Defaults: localhost, 3306, root, no password")
    print()
    print("Retrieved the following NuPIC configuration using: ", fileused)
    print("    host   :    ", host)
    print("    port   :    ", port)
    print("    user   :    "******"    passwd :    ", "*" * len(passwd))

    testDbConnection(host, port, user, passwd)
    print("Connection successful!!")
Пример #3
0
  def newPosition(self, globalBestPosition, rng):
    """See comments in base class."""
    # First, update the velocity. The new velocity is given as:
    # v = (inertia * v)  + (cogRate * r1 * (localBest-pos))
    #                    + (socRate * r2 * (globalBest-pos))
    #
    # where r1 and r2 are random numbers between 0 and 1.0
    lb=float(Configuration.get("nupic.hypersearch.randomLowerBound"))
    ub=float(Configuration.get("nupic.hypersearch.randomUpperBound"))

    self._velocity = (self._velocity * self._inertia + rng.uniform(lb, ub) *
                      self._cogRate * (self._bestPosition - self.getPosition()))
    if globalBestPosition is not None:
      self._velocity += rng.uniform(lb, ub) * self._socRate * (
          globalBestPosition - self.getPosition())

    # update position based on velocity
    self._position += self._velocity

    # Clip it
    self._position = max(self.min, self._position)
    self._position = min(self.max, self._position)

    # Return it
    return self.getPosition()
Пример #4
0
    def __init__(self, clamodel, anomalyParams={}):
        if anomalyParams is None:
            anomalyParams = {}
        self.clamodel = clamodel

        self._version = CLAModelClassifierHelper.__VERSION__

        self._classificationMaxDist = 0.1

        if 'autoDetectWaitRecords' not in anomalyParams or \
            anomalyParams['autoDetectWaitRecords'] is None:
            self._autoDetectWaitRecords = int(
                Configuration.get('nupic.model.temporalAnomaly.wait_records'))
        else:
            self._autoDetectWaitRecords = anomalyParams[
                'autoDetectWaitRecords']

        if 'autoDetectThreshold' not in anomalyParams or \
            anomalyParams['autoDetectThreshold'] is None:
            self._autoDetectThreshold = float(
                Configuration.get(
                    'nupic.model.temporalAnomaly.auto_detect_threshold'))
        else:
            self._autoDetectThreshold = anomalyParams['autoDetectThreshold']

        if 'anomalyCacheRecords' not in anomalyParams or \
            anomalyParams['anomalyCacheRecords'] is None:
            self._history_length = int(
                Configuration.get('nupic.model.temporalAnomaly.window_length'))
        else:
            self._history_length = anomalyParams['anomalyCacheRecords']

        if 'anomalyVectorType' not in anomalyParams or \
            anomalyParams['anomalyVectorType'] is None:
            self._vectorType = str(
                Configuration.get(
                    'nupic.model.temporalAnomaly.anomaly_vector'))
        else:
            self._vectorType = anomalyParams['anomalyVectorType']

        self._activeColumnCount = \
          self.clamodel._getSPRegion().getSelf().getParameter('numActivePerInhArea')

        # Storage for last run
        self._anomalyVectorLength = None
        self._classificationVector = numpy.array([])
        self._prevPredictedColumns = numpy.array([])
        self._prevTPCells = numpy.array([])

        # Array of CLAClassificationRecord's used to recompute and get history
        self.saved_states = []
        self.saved_categories = []
Пример #5
0
def _getCommonSteadyDBArgsDict():
    """ Returns a dictionary of arguments for DBUtils.SteadyDB.SteadyDBConnection
  constructor.
  """

    return dict(creator=pymysql,
                host=Configuration.get('nupic.cluster.database.host'),
                port=int(Configuration.get('nupic.cluster.database.port')),
                user=Configuration.get('nupic.cluster.database.user'),
                passwd=Configuration.get('nupic.cluster.database.passwd'),
                charset='utf8',
                use_unicode=True,
                setsession=['SET AUTOCOMMIT = 1'])
Пример #6
0
  def __init__(self, clamodel, anomalyParams=None):
    if anomalyParams is None:
      anomalyParams = {}
    if anomalyParams is None:
      anomalyParams = {}
    self.clamodel = clamodel

    self._version = CLAModelClassifierHelper.__VERSION__

    self._classificationMaxDist = 0.1

    if 'autoDetectWaitRecords' not in anomalyParams or \
        anomalyParams['autoDetectWaitRecords'] is None:
      self._autoDetectWaitRecords = int(Configuration.get(
        'nupic.model.temporalAnomaly.wait_records'))
    else:
      self._autoDetectWaitRecords = anomalyParams['autoDetectWaitRecords']

    if 'autoDetectThreshold' not in anomalyParams or \
        anomalyParams['autoDetectThreshold'] is None:
      self._autoDetectThreshold = float(Configuration.get(
        'nupic.model.temporalAnomaly.auto_detect_threshold'))
    else:
      self._autoDetectThreshold = anomalyParams['autoDetectThreshold']

    if 'anomalyCacheRecords' not in anomalyParams or \
        anomalyParams['anomalyCacheRecords'] is None:
      self._history_length = int(Configuration.get(
        'nupic.model.temporalAnomaly.window_length'))
    else:
      self._history_length = anomalyParams['anomalyCacheRecords']

    if 'anomalyVectorType' not in anomalyParams or \
        anomalyParams['anomalyVectorType'] is None:
      self._vectorType = str(Configuration.get(
        'nupic.model.temporalAnomaly.anomaly_vector'))
    else:
      self._vectorType = anomalyParams['anomalyVectorType']

    self._activeColumnCount = \
      self.clamodel._getSPRegion().getSelf().getParameter('numActiveColumnsPerInhArea')

    # Storage for last run
    self._anomalyVectorLength = None
    self._classificationVector = numpy.array([])
    self._prevPredictedColumns = numpy.array([])
    self._prevTPCells = numpy.array([])

    # Array of CLAClassificationRecord's used to recompute and get history
    self.saved_states = []
    self.saved_categories = []
Пример #7
0
def _getCommonSteadyDBArgsDict():
  """ Returns a dictionary of arguments for DBUtils.SteadyDB.SteadyDBConnection
  constructor.
  """

  return dict(
      creator = pymysql,
      host = Configuration.get('nupic.cluster.database.host'),
      port = int(Configuration.get('nupic.cluster.database.port')),
      user = Configuration.get('nupic.cluster.database.user'),
      passwd = Configuration.get('nupic.cluster.database.passwd'),
      charset = 'utf8',
      use_unicode = True,
      setsession = ['SET AUTOCOMMIT = 1'])
Пример #8
0
  def __setstate__(self, state):
    version = 1
    if "_version" in state:
      version = state["_version"]

    # Migrate from version 1 to version 2
    if version == 1:
      self._vectorType = str(Configuration.get(
        'nupic.model.temporalAnomaly.anomaly_vector'))
      self._autoDetectWaitRecords = state['_classificationDelay']
    elif version == 2:
      self._autoDetectWaitRecords = state['_classificationDelay']
    elif version == 3:
      pass
    else:
      raise Exception("Error while deserializing %s: Invalid version %s"
                      %(self.__class__, version))

    if '_autoDetectThreshold' not in state:
      self._autoDetectThreshold = 1.1

    for attr, value in state.iteritems():
      setattr(self, attr, value)

    self._version = CLAModelClassifierHelper.__VERSION__
Пример #9
0
    def __init__(self, steps="1", alpha=0.001, verbosity=0, implementation=None, maxCategoryCount=None):

        # Set default implementation
        if implementation is None:
            implementation = Configuration.get("nupic.opf.claClassifier.implementation")

        # Convert the steps designation to a list
        self.classifierImp = implementation
        self.steps = steps
        self.stepsList = eval("[%s]" % (steps))
        self.alpha = alpha
        self.verbosity = verbosity

        # Initialize internal structures
        self._claClassifier = CLAClassifierFactory.create(
            steps=self.stepsList, alpha=self.alpha, verbosity=self.verbosity, implementation=implementation
        )
        self.learningMode = True
        self.inferenceMode = False
        self.maxCategoryCount = maxCategoryCount
        self.recordNum = 0
        self._initEphemerals()

        # Flag to know if the compute() function is ever called. This is to
        # prevent backward compatibilities issues with the customCompute() method
        # being called at the same time as the compute() method. Only compute()
        # should be called via network.run(). This flag will be removed once we
        # get to cleaning up the clamodel.py file.
        self._computeFlag = False
Пример #10
0
    def __setstate__(self, state):
        version = 1
        if "_version" in state:
            version = state["_version"]

        # Migrate from version 1 to version 2
        if version == 1:
            self._vectorType = str(
                Configuration.get(
                    'nupic.model.temporalAnomaly.anomaly_vector'))
            self._autoDetectWaitRecords = state['_classificationDelay']
        elif version == 2:
            self._autoDetectWaitRecords = state['_classificationDelay']
        elif version == 3:
            pass
        else:
            raise Exception(
                "Error while deserializing %s: Invalid version %s" %
                (self.__class__, version))

        if '_autoDetectThreshold' not in state:
            self._autoDetectThreshold = 1.1

        for attr, value in state.iteritems():
            setattr(self, attr, value)

        self._version = CLAModelClassifierHelper.__VERSION__
Пример #11
0
    def __init__(self, steps="1", alpha=0.001, clVerbosity=0, implementation=None, maxCategoryCount=None):

        # Set default implementation
        if implementation is None:
            implementation = Configuration.get("nupic.opf.sdrClassifier.implementation")

        self.implementation = implementation
        # Convert the steps designation to a list
        self.steps = steps
        self.stepsList = [int(i) for i in steps.split(",")]
        self.alpha = alpha
        self.verbosity = clVerbosity

        # Initialize internal structures
        self._sdrClassifier = None
        self.learningMode = True
        self.inferenceMode = False
        self.maxCategoryCount = maxCategoryCount
        self.recordNum = 0

        # Flag to know if the compute() function is ever called. This is to
        # prevent backward compatibilities issues with the customCompute() method
        # being called at the same time as the compute() method. Only compute()
        # should be called via network.run(). This flag will be removed once we
        # get to cleaning up the clamodel.py file.
        self._computeFlag = False
Пример #12
0
    def __init__(self,
                 steps='1',
                 alpha=0.001,
                 verbosity=0,
                 implementation=None,
                 maxCategoryCount=None):

        # Set default implementation
        if implementation is None:
            implementation = Configuration.get(
                'nupic.opf.sdrClassifier.implementation')

        self.implementation = implementation
        # Convert the steps designation to a list
        self.steps = steps
        self.stepsList = [int(i) for i in steps.split(",")]
        self.alpha = alpha
        self.verbosity = verbosity

        # Initialize internal structures
        self._sdrClassifier = None
        self.learningMode = True
        self.inferenceMode = False
        self.maxCategoryCount = maxCategoryCount
        self.recordNum = 0

        # Flag to know if the compute() function is ever called. This is to
        # prevent backward compatibilities issues with the customCompute() method
        # being called at the same time as the compute() method. Only compute()
        # should be called via network.run(). This flag will be removed once we
        # get to cleaning up the clamodel.py file.
        self._computeFlag = False
Пример #13
0
    def __createModelCheckpoint(self):
        """ Create a checkpoint from the current model, and store it in a dir named
    after checkpoint GUID, and finally store the GUID in the Models DB """

        if self._model is None or self._modelCheckpointGUID is None:
            return

        # Create an output store, if one doesn't exist already
        if self._predictionLogger is None:
            self._createPredictionLogger()

        predictions = io.StringIO()
        self._predictionLogger.checkpoint(
            checkpointSink=predictions,
            maxRows=int(
                Configuration.get('nupic.model.checkpoint.maxPredictionRows')))

        self._model.save(
            os.path.join(self._experimentDir, str(self._modelCheckpointGUID)))
        self._jobsDAO.modelSetFields(
            modelID, {'modelCheckpointId': str(self._modelCheckpointGUID)},
            ignoreUnchanged=True)

        self._logger.info(
            "Checkpointed Hypersearch Model: modelID: %r, "
            "checkpointID: %r", self._modelID, checkpointID)
        return
Пример #14
0
def dbValidator():
  """
    Let the user know what NuPIC config file is being used
    and whether or not they have mysql set up correctly for
    swarming.
  """
  fileused = getFileUsed()

  # Get the values we need from NuPIC's configuration
  host = Configuration.get("nupic.cluster.database.host")
  port = int(Configuration.get("nupic.cluster.database.port"))
  user = Configuration.get("nupic.cluster.database.user")
  
  #BUG was here
  #passwd =len(Configuration.get("nupic.cluster.database.passwd")) * '*'
  passwd =Configuration.get("nupic.cluster.database.passwd")


  print "This script will validate that your MySQL is setup correctly for "
  print "NuPIC. MySQL is required for NuPIC swarming. The settings are"
  print "defined in a configuration file found in  "
  print "$NUPIC/src/nupic/support/nupic-default.xml Out of the box those "
  print "settings contain MySQL's default access credentials."
  print
  print "The nupic-default.xml can be duplicated to define user specific "
  print "changes calling the copied file "
  print "$NUPIC/src/nupic/support/nupic-site.xml Refer to the "
  print "nupic-default.xml for additional instructions."
  print
  print "Defaults: localhost, 3306, root, no password"
  print
  print "Retrieved the following NuPIC configuration using: ", fileused
  print "    host   :    ", host
  print "    port   :    ", port
  print "    user   :    "******"    passwd :    ", len(passwd)* '*'
  
  if testDbConnection(host, port, user, passwd):
    print "Connection successful!!"
  else:
    print ("Couldn't connect to the database or you don't have the "
           "permissions required to create databases and tables. "
           "Please ensure you have MySQL\n installed, running, "
           "accessible using the NuPIC configuration settings, "
           "and the user specified has permission to create both "
           "databases and tables.")
Пример #15
0
    def __init__(self,
                 min,
                 max,
                 stepSize=None,
                 inertia=None,
                 cogRate=None,
                 socRate=None):
        """Construct a variable that permutes over floating point values using
    the Particle Swarm Optimization (PSO) algorithm. See descriptions of
    PSO (i.e. http://en.wikipedia.org/wiki/Particle_swarm_optimization)
    for references to the inertia, cogRate, and socRate parameters.

    Parameters:
    -----------------------------------------------------------------------
    min:          min allowed value of position
    max:          max allowed value of position
    stepSize:     if not None, the position must be at min + N * stepSize,
                    where N is an integer
    inertia:      The inertia for the particle.
    cogRate:      This parameter controls how much the particle is affected
                    by its distance from it's local best position
    socRate:      This parameter controls how much the particle is affected
                    by its distance from the global best position

    """
        super(PermuteFloat, self).__init__()
        self.min = min
        self.max = max
        self.stepSize = stepSize

        # The particle's initial position and velocity.
        self._position = (self.max + self.min) / 2.0
        self._velocity = (self.max - self.min) / 5.0

        # The inertia, cognitive, and social components of the particle
        self._inertia = (float(Configuration.get("nupic.hypersearch.inertia"))
                         if inertia is None else inertia)
        self._cogRate = (float(Configuration.get("nupic.hypersearch.cogRate"))
                         if cogRate is None else cogRate)
        self._socRate = (float(Configuration.get("nupic.hypersearch.socRate"))
                         if socRate is None else socRate)

        # The particle's local best position and the best global position.
        self._bestPosition = self.getPosition()
        self._bestResult = None
Пример #16
0
def dbValidator():
  """
    Let the user know what NuPIC config file is being used
    and whether or not they have mysql set up correctly for
    swarming.
  """
  fileused = getFileUsed()

  # Get the values we need from NuPIC's configuration
  host = Configuration.get("nupic.cluster.database.host")
  port = int(Configuration.get("nupic.cluster.database.port"))
  user = Configuration.get("nupic.cluster.database.user")
  passwd = "*" * len(Configuration.get("nupic.cluster.database.passwd"))


  print "This script will validate that your MySQL is setup correctly for "
  print "NuPIC. MySQL is required for NuPIC swarming. The settings are"
  print "defined in a configuration file found in  "
  print "$NUPIC/src/nupic/support/nupic-default.xml Out of the box those "
  print "settings contain MySQL's default access credentials."
  print
  print "The nupic-default.xml can be duplicated to define user specific "
  print "changes calling the copied file "
  print "$NUPIC/src/nupic/support/nupic-site.xml Refer to the "
  print "nupic-default.xml for additional instructions."
  print
  print "Defaults: localhost, 3306, root, no password"
  print
  print "Retrieved the following NuPIC configuration using: ", fileused
  print "    host   :    ", host
  print "    port   :    ", port
  print "    user   :    "******"    passwd :    ", passwd


  if testDbConnection(host, port, user, passwd):
    print "Connection successful!!"
  else:
    print ("Couldn't connect to the database or you don't have the "
           "permissions required to create databases and tables. "
           "Please ensure you have MySQL\n installed, running, "
           "accessible using the NuPIC configuration settings, "
           "and the user specified has permission to create both "
           "databases and tables.")
 def create(*args, **kwargs):
     impl = kwargs.pop("implementation", None)
     if impl is None:
         impl = Configuration.get("nupic.opf.claClassifier.implementation")
     if impl == "py":
         return SequenceClassifier(*args, **kwargs)
     elif impl == "cpp":
         raise ValueError("cpp version not yet implemented")
     else:
         raise ValueError("Invalid classifier implementation (%r). Value must be " '"py" or "cpp".' % impl)
Пример #18
0
 def create(*args, **kwargs):
   impl = kwargs.pop('implementation', None)
   if impl is None:
     impl = Configuration.get('nupic.opf.claClassifier.implementation')
   if impl == 'py':
     return SequenceClassifier(*args, **kwargs)
   elif impl == 'cpp':
     raise ValueError('cpp version not yet implemented')
   else:
     raise ValueError('Invalid classifier implementation (%r). Value must be '
                      '"py" or "cpp".' % impl)
Пример #19
0
def dask_main(argv):
  logging.setLoggerClass(ExtendedLogger)
  buildID = Configuration.get('nupic.software.buildNumber', 'N/A')
  logPrefix = '<BUILDID=%s, WORKER=HS, WRKID=N/A, JOBID=N/A> ' % buildID
  ExtendedLogger.setLogPrefix(logPrefix)
  try:
    main(argv)
  except:
    logging.exception("HypersearchWorker is exiting with unhandled exception; "
                      "argv=%r", sys.argv)
    raise
 def create(*args, **kwargs):
   impl = kwargs.pop('implementation', None)
   if impl is None:
     impl = Configuration.get('nupic.opf.claClassifier.implementation')
   if impl == 'py':
     return SequenceClassifier(*args, **kwargs)
   elif impl == 'cpp':
     raise ValueError('cpp version not yet implemented')
   else:
     raise ValueError('Invalid classifier implementation (%r). Value must be '
                      '"py" or "cpp".' % impl)
Пример #21
0
def _getCommonSteadyDBArgsDict():
  """ Returns a dictionary of arguments for DBUtils.SteadyDB.SteadyDBConnection
  constructor.
  """

  from nupic.database.client_jobs_dao import ClientJobsDAO
  dbname = ClientJobsDAO._getDBName()
  print(dbname)
  host = Configuration.get('nupic.cluster.database.host')
  port = int(Configuration.get('nupic.cluster.database.port'))
  user = Configuration.get('nupic.cluster.database.user')
  password = Configuration.get('nupic.cluster.database.passwd')

  os.environ['PGPASSWORD'] = password
  command = "SELECT 1 FROM pg_database WHERE datname= '%s'" % (dbname)
  args = ['psql','-At', '-U', user,  '-d', 'postgres', '-h', host, '-p', str(port), '-c', command]
  proc = Popen(args, stdout=PIPE, stderr=PIPE)
  out, err = proc.communicate()
  if out == b'1\n':
      print("database exists")
  else:
      print("create database")
      command = 'CREATE DATABASE %s' % (dbname)
      call(['psql','-U', user,  '-d', 'postgres', '-h', host, '-p', str(port), '-c', command])

  return dict(
      creator = psycopg2,
      ping = 7,
      maxusage = 1,
      dbname = dbname,
      host = host,
      port = port,
      user = user,
      password = password,
      connect_timeout = 0,
      client_encoding = 'utf-8',
      keepalives_idle = 2,
      keepalives_interval = 2
      #setsession = ['SET AUTOCOMMIT TO ON']
      #use_unicode = True,
      )
Пример #22
0
  def __init__(self, min, max, stepSize=None, inertia=None, cogRate=None,
               socRate=None):
    """Construct a variable that permutes over floating point values using
    the Particle Swarm Optimization (PSO) algorithm. See descriptions of
    PSO (i.e. http://en.wikipedia.org/wiki/Particle_swarm_optimization)
    for references to the inertia, cogRate, and socRate parameters.

    Parameters:
    -----------------------------------------------------------------------
    min:          min allowed value of position
    max:          max allowed value of position
    stepSize:     if not None, the position must be at min + N * stepSize,
                    where N is an integer
    inertia:      The inertia for the particle.
    cogRate:      This parameter controls how much the particle is affected
                    by its distance from it's local best position
    socRate:      This parameter controls how much the particle is affected
                    by its distance from the global best position

    """
    super(PermuteFloat, self).__init__()
    self.min = min
    self.max = max
    self.stepSize = stepSize

    # The particle's initial position and velocity.
    self._position = (self.max + self.min) / 2.0
    self._velocity = (self.max - self.min) / 5.0


    # The inertia, cognitive, and social components of the particle
    self._inertia = (float(Configuration.get("nupic.hypersearch.inertia"))
                     if inertia is None else inertia)
    self._cogRate = (float(Configuration.get("nupic.hypersearch.cogRate"))
                     if cogRate is None else cogRate)
    self._socRate = (float(Configuration.get("nupic.hypersearch.socRate"))
                     if socRate is None else socRate)

    # The particle's local best position and the best global position.
    self._bestPosition = self.getPosition()
    self._bestResult = None
Пример #23
0
 def create(*args, **kwargs):
   impl = kwargs.pop('implementation', None)
   if impl is None:
     impl = Configuration.get('nupic.opf.claClassifier.implementation')
   if impl == 'py':
     return CLAClassifier(*args, **kwargs)
   elif impl == 'cpp':
     return FastCLAClassifier(*args, **kwargs)
   elif impl == 'diff':
     return CLAClassifierDiff(*args, **kwargs)
   else:
     raise ValueError('Invalid classifier implementation (%r). Value must be '
                      '"py" or "cpp".' % impl)
Пример #24
0
def createAndStartSwarm(client,
                        clientInfo="",
                        clientKey="",
                        params="",
                        minimumWorkers=None,
                        maximumWorkers=None,
                        alreadyRunning=False):
    """Create and start a swarm job.

  Args:
    client - A string identifying the calling client. There is a small limit
        for the length of the value. See ClientJobsDAO.CLIENT_MAX_LEN.
    clientInfo - JSON encoded dict of client specific information.
    clientKey - Foreign key. Limited in length, see ClientJobsDAO._initTables.
    params - JSON encoded dict of the parameters for the job. This can be
        fetched out of the database by the worker processes based on the jobID.
    minimumWorkers - The minimum workers to allocate to the swarm. Set to None
        to use the default.
    maximumWorkers - The maximum workers to allocate to the swarm. Set to None
        to use the swarm default. Set to 0 to use the maximum scheduler value.
    alreadyRunning - Insert a job record for an already running process. Used
        for testing.
  """
    if minimumWorkers is None:
        minimumWorkers = Configuration.getInt(
            "nupic.hypersearch.minWorkersPerSwarm")
    if maximumWorkers is None:
        maximumWorkers = Configuration.getInt(
            "nupic.hypersearch.maxWorkersPerSwarm")

    return ClientJobsDAO.get().jobInsert(client=client,
                                         cmdLine="$HYPERSEARCH",
                                         clientInfo=clientInfo,
                                         clientKey=clientKey,
                                         alreadyRunning=alreadyRunning,
                                         params=params,
                                         minimumWorkers=minimumWorkers,
                                         maximumWorkers=maximumWorkers,
                                         jobType=ClientJobsDAO.JOB_TYPE_HS)
Пример #25
0
def getFileUsed():
  """
    Determine which NuPIC configuration file is being used and returns the
    name of the configuration file it is using. Either DEFAULT_CONFIG or
    USER_CONFIG.
  """

  # output will be {} if the file passed into Configuration._readConfigFile
  # can not be found in the standard paths returned by
  # Configuration._getConfigPaths.
  output = Configuration._readConfigFile(USER_CONFIG) #pylint: disable=protected-access
  if output != {}:
    return USER_CONFIG
  return DEFAULT_CONFIG
Пример #26
0
def getFileUsed():
  """
    Determine which NuPIC configuration file is being used and returns the
    name of the configuration file it is using. Either DEFAULT_CONFIG or
    USER_CONFIG.
  """

  # output will be {} if the file passed into Configuration._readConfigFile
  # can not be found in the standard paths returned by
  # Configuration._getConfigPaths.
  output = Configuration._readConfigFile(USER_CONFIG) #pylint: disable=protected-access
  if output != {}:
    return USER_CONFIG
  return DEFAULT_CONFIG
Пример #27
0
def createAndStartSwarm(client, clientInfo="", clientKey="", params="",
                        minimumWorkers=None, maximumWorkers=None,
                        alreadyRunning=False):
  """Create and start a swarm job.

  Args:
    client - A string identifying the calling client. There is a small limit
        for the length of the value. See ClientJobsDAO.CLIENT_MAX_LEN.
    clientInfo - JSON encoded dict of client specific information.
    clientKey - Foreign key. Limited in length, see ClientJobsDAO._initTables.
    params - JSON encoded dict of the parameters for the job. This can be
        fetched out of the database by the worker processes based on the jobID.
    minimumWorkers - The minimum workers to allocate to the swarm. Set to None
        to use the default.
    maximumWorkers - The maximum workers to allocate to the swarm. Set to None
        to use the swarm default. Set to 0 to use the maximum scheduler value.
    alreadyRunning - Insert a job record for an already running process. Used
        for testing.
  """
  if minimumWorkers is None:
    minimumWorkers = Configuration.getInt(
        "nupic.hypersearch.minWorkersPerSwarm")
  if maximumWorkers is None:
    maximumWorkers = Configuration.getInt(
        "nupic.hypersearch.maxWorkersPerSwarm")

  return ClientJobsDAO.get().jobInsert(
      client=client,
      cmdLine="$HYPERSEARCH",
      clientInfo=clientInfo,
      clientKey=clientKey,
      alreadyRunning=alreadyRunning,
      params=params,
      minimumWorkers=minimumWorkers,
      maximumWorkers=maximumWorkers,
      jobType=ClientJobsDAO.JOB_TYPE_HS)
Пример #28
0
    def create(*args, **kwargs):
        """
    Create a SDR classifier factory.
    The implementation of the SDR Classifier can be specified with
    the "implementation" keyword argument.

    The SDRClassifierFactory uses the implementation as specified in
     src/nupic/support/nupic-default.xml
    """
        impl = kwargs.pop("implementation", None)
        if impl is None:
            impl = Configuration.get("nupic.opf.sdrClassifier.implementation")
        if impl == "py":
            return SDRClassifier(*args, **kwargs)
        else:
            raise ValueError("Invalid classifier implementation (%r). Value must be " '"py".' % impl)
Пример #29
0
    def create(*args, **kwargs):
        """
    Create a SDR classifier factory.
    The implementation of the SDR Classifier can be specified with
    the "implementation" keyword argument.

    The SDRClassifierFactory uses the implementation as specified in
     src/nupic/support/nupic-default.xml
    """
        impl = kwargs.pop('implementation', None)
        if impl is None:
            impl = Configuration.get('nupic.opf.sdrClassifier.implementation')
        if impl == 'py':
            return SDRClassifier(*args, **kwargs)
        else:
            raise ValueError(
                'Invalid classifier implementation (%r). Value must be '
                '"py".' % impl)
Пример #30
0
  def create(*args, **kwargs):
    """
    Create a SDR classifier factory.
    The implementation of the SDR Classifier can be specified with
    the "implementation" keyword argument.

    The SDRClassifierFactory uses the implementation as specified in
     `Default NuPIC Configuration <default-config.html>`_.
    """
    impl = kwargs.pop('implementation', None)
    if impl is None:
      impl = Configuration.get('nupic.opf.sdrClassifier.implementation')
    if impl == 'py':
      return SDRClassifier(*args, **kwargs)
    elif impl == 'cpp':
      return FastSDRClassifier(*args, **kwargs)
    else:
      raise ValueError('Invalid classifier implementation (%r). Value must be '
                       '"py" or "cpp".' % impl)
Пример #31
0
    def create(*args, **kwargs):
        """
    Create a SDR classifier factory.
    The implementation of the SDR Classifier can be specified with
    the "implementation" keyword argument.

    The SDRClassifierFactory uses the implementation as specified in
     `Default NuPIC Configuration <default-config.html>`_.
    """
        impl = kwargs.pop('implementation', None)
        if impl is None:
            impl = Configuration.get('nupic.opf.sdrClassifier.implementation')
        if impl == 'py':
            return SDRClassifier(*args, **kwargs)
        elif impl == 'cpp':
            return FastSDRClassifier(*args, **kwargs)
        else:
            raise ValueError(
                'Invalid classifier implementation (%r). Value must be '
                '"py" or "cpp".' % impl)
Пример #32
0
  def __init__(self,
               steps='1',
               alpha=0.001,
               clVerbosity=0,
               implementation=None,
               maxCategoryCount=None
               ):

    # Set default implementation
    if implementation is None:
      implementation = Configuration.get('nupic.opf.claClassifier.implementation')

    # Convert the steps designation to a list
    self.classifierImp = implementation
    self.steps = steps
    self.stepsList = eval("[%s]" % (steps))
    self.alpha = alpha
    self.verbosity = clVerbosity

    # Initialize internal structures
    self._claClassifier = CLAClassifierFactory.create(
      steps=self.stepsList,
      alpha=self.alpha,
      verbosity=self.verbosity,
      implementation=implementation,
    )
    self.learningMode = True
    self.inferenceMode = False
    self.maxCategoryCount = maxCategoryCount
    self.recordNum = 0
    self._initEphemerals()

    # Flag to know if the compute() function is ever called. This is to 
    # prevent backward compatibilities issues with the customCompute() method
    # being called at the same time as the compute() method. Only compute() 
    # should be called via network.run(). This flag will be removed once we 
    # get to cleaning up the clamodel.py file.
    self._computeFlag = False
Пример #33
0
    def run(self):
        """ Run this worker.

    Parameters:
    ----------------------------------------------------------------------
    retval:     jobID of the job we ran. This is used by unit test code
                  when calling this working using the --params command
                  line option (which tells this worker to insert the job
                  itself).
    """
        # Easier access to options
        options = self._options

        # ---------------------------------------------------------------------
        # Connect to the jobs database
        self.logger.info("Connecting to the jobs database")
        cjDAO = ClientJobsDAO.get()

        # Get our worker ID
        self._workerID = cjDAO.getConnectionID()

        if options.clearModels:
            cjDAO.modelsClearAll()

        # -------------------------------------------------------------------------
        # if params were specified on the command line, insert a new job using
        #  them.
        if options.params is not None:
            options.jobID = cjDAO.jobInsert(
                client="hwTest",
                cmdLine="echo 'test mode'",
                params=options.params,
                alreadyRunning=True,
                minimumWorkers=1,
                maximumWorkers=1,
                jobType=cjDAO.JOB_TYPE_HS,
            )
        if options.workerID is not None:
            wID = options.workerID
        else:
            wID = self._workerID

        buildID = Configuration.get("nupic.software.buildNumber", "N/A")
        logPrefix = "<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> " % (buildID, wID, options.jobID)
        ExtendedLogger.setLogPrefix(logPrefix)

        # ---------------------------------------------------------------------
        # Get the search parameters
        # If asked to reset the job status, do that now
        if options.resetJobStatus:
            cjDAO.jobSetFields(
                options.jobID,
                fields={
                    "workerCompletionReason": ClientJobsDAO.CMPL_REASON_SUCCESS,
                    "cancel": False,
                    #'engWorkerState': None
                },
                useConnectionID=False,
                ignoreUnchanged=True,
            )
        jobInfo = cjDAO.jobInfo(options.jobID)
        self.logger.info("Job info retrieved: %s" % (str(clippedObj(jobInfo))))

        # ---------------------------------------------------------------------
        # Instantiate the Hypersearch object, which will handle the logic of
        #  which models to create when we need more to evaluate.
        jobParams = json.loads(jobInfo.params)

        # Validate job params
        jsonSchemaPath = os.path.join(os.path.dirname(__file__), "jsonschema", "jobParamsSchema.json")
        validate(jobParams, schemaPath=jsonSchemaPath)

        hsVersion = jobParams.get("hsVersion", None)
        if hsVersion == "v2":
            self._hs = HypersearchV2(
                searchParams=jobParams,
                workerID=self._workerID,
                cjDAO=cjDAO,
                jobID=options.jobID,
                logLevel=options.logLevel,
            )
        else:
            raise RuntimeError("Invalid Hypersearch implementation (%s) specified" % (hsVersion))

        # =====================================================================
        # The main loop.
        try:
            exit = False
            numModelsTotal = 0
            print >>sys.stderr, "reporter:status:Evaluating first model..."
            while not exit:

                # ------------------------------------------------------------------
                # Choose a model to evaluate
                batchSize = 10  # How many to try at a time.
                modelIDToRun = None
                while modelIDToRun is None:

                    if options.modelID is None:
                        # -----------------------------------------------------------------
                        # Get the latest results on all running models and send them to
                        #  the Hypersearch implementation
                        # This calls cjDAO.modelsGetUpdateCounters(), compares the
                        # updateCounters with what we have cached, fetches the results for the
                        # changed and new models, and sends those to the Hypersearch
                        # implementation's self._hs.recordModelProgress() method.
                        self._processUpdatedModels(cjDAO)

                        # --------------------------------------------------------------------
                        # Create a new batch of models
                        (exit, newModels) = self._hs.createModels(numModels=batchSize)
                        if exit:
                            break

                        # No more models left to create, just loop. The _hs is waiting for
                        #   all remaining running models to complete, and may pick up on an
                        #  orphan if it detects one.
                        if len(newModels) == 0:
                            continue

                        # Try and insert one that we will run
                        for (modelParams, modelParamsHash, particleHash) in newModels:
                            jsonModelParams = json.dumps(modelParams)
                            (modelID, ours) = cjDAO.modelInsertAndStart(
                                options.jobID, jsonModelParams, modelParamsHash, particleHash
                            )

                            # Some other worker is already running it, tell the Hypersearch object
                            #  so that it doesn't try and insert it again
                            if not ours:
                                mParamsAndHash = cjDAO.modelsGetParams([modelID])[0]
                                mResult = cjDAO.modelsGetResultAndStatus([modelID])[0]
                                results = mResult.results
                                if results is not None:
                                    results = json.loads(results)

                                modelParams = json.loads(mParamsAndHash.params)
                                particleHash = cjDAO.modelsGetFields(modelID, ["engParticleHash"])[0]
                                particleInst = "%s.%s" % (
                                    modelParams["particleState"]["id"],
                                    modelParams["particleState"]["genIdx"],
                                )
                                self.logger.info(
                                    "Adding model %d to our internal DB "
                                    "because modelInsertAndStart() failed to insert it: "
                                    "paramsHash=%s, particleHash=%s, particleId='%s'",
                                    modelID,
                                    mParamsAndHash.engParamsHash.encode("hex"),
                                    particleHash.encode("hex"),
                                    particleInst,
                                )
                                self._hs.recordModelProgress(
                                    modelID=modelID,
                                    modelParams=modelParams,
                                    modelParamsHash=mParamsAndHash.engParamsHash,
                                    results=results,
                                    completed=(mResult.status == cjDAO.STATUS_COMPLETED),
                                    completionReason=mResult.completionReason,
                                    matured=mResult.engMatured,
                                    numRecords=mResult.numRecords,
                                )
                            else:
                                modelIDToRun = modelID
                                break

                    else:
                        # A specific modelID was passed on the command line
                        modelIDToRun = int(options.modelID)
                        mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun])[0]
                        modelParams = json.loads(mParamsAndHash.params)
                        modelParamsHash = mParamsAndHash.engParamsHash

                        # Make us the worker
                        cjDAO.modelSetFields(modelIDToRun, dict(engWorkerConnId=self._workerID))
                        if False:
                            # Change the hash and params of the old entry so that we can
                            #  create a new model with the same params
                            for attempt in range(1000):
                                paramsHash = hashlib.md5("OrphanParams.%d.%d" % (modelIDToRun, attempt)).digest()
                                particleHash = hashlib.md5("OrphanParticle.%d.%d" % (modelIDToRun, attempt)).digest()
                                try:
                                    cjDAO.modelSetFields(
                                        modelIDToRun, dict(engParamsHash=paramsHash, engParticleHash=particleHash)
                                    )
                                    success = True
                                except:
                                    success = False
                                if success:
                                    break
                            if not success:
                                raise RuntimeError(
                                    "Unexpected failure to change paramsHash and " "particleHash of orphaned model"
                                )

                            (modelIDToRun, ours) = cjDAO.modelInsertAndStart(
                                options.jobID, mParamsAndHash.params, modelParamsHash
                            )

                        # ^^^ end while modelIDToRun ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

                # ---------------------------------------------------------------
                # We have a model, evaluate it now
                # All done?
                if exit:
                    break

                # Run the model now
                self.logger.info(
                    "RUNNING MODEL GID=%d, paramsHash=%s, params=%s",
                    modelIDToRun,
                    modelParamsHash.encode("hex"),
                    modelParams,
                )

                # ---------------------------------------------------------------------
                # Construct model checkpoint GUID for this model:
                # jobParams['persistentJobGUID'] contains the client's (e.g., API Server)
                # persistent, globally-unique model identifier, which is what we need;
                persistentJobGUID = jobParams["persistentJobGUID"]
                assert persistentJobGUID, "persistentJobGUID: %r" % (persistentJobGUID,)

                modelCheckpointGUID = jobInfo.client + "_" + persistentJobGUID + ("_" + str(modelIDToRun))

                self._hs.runModel(
                    modelID=modelIDToRun,
                    jobID=options.jobID,
                    modelParams=modelParams,
                    modelParamsHash=modelParamsHash,
                    jobsDAO=cjDAO,
                    modelCheckpointGUID=modelCheckpointGUID,
                )

                # TODO: don't increment for orphaned models
                numModelsTotal += 1

                self.logger.info("COMPLETED MODEL GID=%d; EVALUATED %d MODELs", modelIDToRun, numModelsTotal)
                print >>sys.stderr, "reporter:status:Evaluated %d models..." % (numModelsTotal)
                print >>sys.stderr, "reporter:counter:HypersearchWorker,numModels,1"

                if options.modelID is not None:
                    exit = True
                # ^^^ end while not exit

        finally:
            # Provide Hypersearch instance an opportunity to clean up temporary files
            self._hs.close()

        self.logger.info("FINISHED. Evaluated %d models." % (numModelsTotal))
        print >>sys.stderr, "reporter:status:Finished, evaluated %d models" % (numModelsTotal)
        return options.jobID
Пример #34
0
        jobID = None
        completionReason = ClientJobsDAO.CMPL_REASON_SUCCESS
        completionMsg = "Success"

        try:
            jobID = hst.run()
        except Exception, e:
            jobID = hst._options.jobID
            completionReason = ClientJobsDAO.CMPL_REASON_ERROR
            completionMsg = "ERROR: %s" % (e,)
            raise
        finally:
            if jobID is not None:
                cjDAO = ClientJobsDAO.get()
                cjDAO.jobSetCompleted(jobID=jobID, completionReason=completionReason, completionMsg=completionMsg)

    return jobID


if __name__ == "__main__":
    logging.setLoggerClass(ExtendedLogger)
    buildID = Configuration.get("nupic.software.buildNumber", "N/A")
    logPrefix = "<BUILDID=%s, WORKER=HS, WRKID=N/A, JOBID=N/A> " % buildID
    ExtendedLogger.setLogPrefix(logPrefix)

    try:
        main(sys.argv)
    except:
        logging.exception("HypersearchWorker is exiting with unhandled exception; " "argv=%r", sys.argv)
        raise
Пример #35
0
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import pymysql
from nupic.support.configuration import Configuration

# Get the values we need from NuPIC's configuration
host = Configuration.get('nupic.cluster.database.host')
port = int(Configuration.get('nupic.cluster.database.port'))
user = Configuration.get('nupic.cluster.database.user')
passwd = Configuration.get('nupic.cluster.database.passwd')

print 
print "This script will validate that your MySQL is setup correctly for NuPIC."
print "MySQL is required for NuPIC swarming. The settings are defined in "
print "NuPIC's configuration files in $NUPIC/conf/default/nupic-default.xml "
print "Out of the box those settings contain MySQL's default access "
print "credentials."
print
print "Retrieved the following settings from NuPIC configuration:"
print "    host   :    ", host
print "    port   :    ", port
print "    user   :    ", user
  def run(self):
    """ Run this worker.

    Parameters:
    ----------------------------------------------------------------------
    retval:     jobID of the job we ran. This is used by unit test code
                  when calling this working using the --params command
                  line option (which tells this worker to insert the job
                  itself).
    """
    # Easier access to options
    options = self._options

    # ---------------------------------------------------------------------
    # Connect to the jobs database
    self.logger.info("Connecting to the jobs database")
    cjDAO = ClientJobsDAO.get()

    # Get our worker ID
    self._workerID = cjDAO.getConnectionID()

    if options.clearModels:
      cjDAO.modelsClearAll()

    # -------------------------------------------------------------------------
    # if params were specified on the command line, insert a new job using
    #  them.
    if options.params is not None:
      options.jobID = cjDAO.jobInsert(client='hwTest', cmdLine="echo 'test mode'",
                  params=options.params, alreadyRunning=True,
                  minimumWorkers=1, maximumWorkers=1,
                  jobType = cjDAO.JOB_TYPE_HS)
    if options.workerID is not None:
      wID = options.workerID
    else:
      wID = self._workerID
    
    buildID = Configuration.get('nupic.software.buildNumber', 'N/A')
    logPrefix = '<BUILDID=%s, WORKER=HW, WRKID=%s, JOBID=%s> ' % \
                (buildID, wID, options.jobID)
    ExtendedLogger.setLogPrefix(logPrefix)

    # ---------------------------------------------------------------------
    # Get the search parameters
    # If asked to reset the job status, do that now
    if options.resetJobStatus:
      cjDAO.jobSetFields(options.jobID,
           fields={'workerCompletionReason': ClientJobsDAO.CMPL_REASON_SUCCESS,
                   'cancel': False,
                   #'engWorkerState': None
                   },
           useConnectionID=False,
           ignoreUnchanged=True)
    jobInfo = cjDAO.jobInfo(options.jobID)
    self.logger.info("Job info retrieved: %s" % (str(clippedObj(jobInfo))))


    # ---------------------------------------------------------------------
    # Instantiate the Hypersearch object, which will handle the logic of
    #  which models to create when we need more to evaluate.
    jobParams = json.loads(jobInfo.params)

    # Validate job params
    jsonSchemaPath = os.path.join(os.path.dirname(__file__),
                                  "jsonschema",
                                  "jobParamsSchema.json")
    validate(jobParams, schemaPath=jsonSchemaPath)


    hsVersion = jobParams.get('hsVersion', None)
    if hsVersion == 'v2':
      self._hs = HypersearchV2(searchParams=jobParams, workerID=self._workerID,
              cjDAO=cjDAO, jobID=options.jobID, logLevel=options.logLevel)
    else:
      raise RuntimeError("Invalid Hypersearch implementation (%s) specified" \
                          % (hsVersion))


    # =====================================================================
    # The main loop.
    try:
      exit = False
      numModelsTotal = 0
      print >>sys.stderr, "reporter:status:Evaluating first model..."
      while not exit:

        # ------------------------------------------------------------------
        # Choose a model to evaluate
        batchSize = 10              # How many to try at a time.
        modelIDToRun = None
        while modelIDToRun is None:

          if options.modelID is None:
            # -----------------------------------------------------------------
            # Get the latest results on all running models and send them to
            #  the Hypersearch implementation
            # This calls cjDAO.modelsGetUpdateCounters(), compares the
            # updateCounters with what we have cached, fetches the results for the
            # changed and new models, and sends those to the Hypersearch
            # implementation's self._hs.recordModelProgress() method.
            self._processUpdatedModels(cjDAO)
  
            # --------------------------------------------------------------------
            # Create a new batch of models
            (exit, newModels) = self._hs.createModels(numModels = batchSize)
            if exit:
              break

            # No more models left to create, just loop. The _hs is waiting for
            #   all remaining running models to complete, and may pick up on an
            #  orphan if it detects one.
            if len(newModels) == 0:
              continue
  
            # Try and insert one that we will run
            for (modelParams, modelParamsHash, particleHash) in newModels:
              jsonModelParams = json.dumps(modelParams)
              (modelID, ours) = cjDAO.modelInsertAndStart(options.jobID,
                                  jsonModelParams, modelParamsHash, particleHash)
  
              # Some other worker is already running it, tell the Hypersearch object
              #  so that it doesn't try and insert it again
              if not ours:
                mParamsAndHash = cjDAO.modelsGetParams([modelID])[0]
                mResult = cjDAO.modelsGetResultAndStatus([modelID])[0]
                results = mResult.results
                if results is not None:
                  results = json.loads(results)
  
                modelParams = json.loads(mParamsAndHash.params)
                particleHash = cjDAO.modelsGetFields(modelID, 
                                  ['engParticleHash'])[0]
                particleInst = "%s.%s" % (
                          modelParams['particleState']['id'],
                          modelParams['particleState']['genIdx'])
                self.logger.info("Adding model %d to our internal DB " \
                      "because modelInsertAndStart() failed to insert it: " \
                      "paramsHash=%s, particleHash=%s, particleId='%s'", modelID, 
                      mParamsAndHash.engParamsHash.encode('hex'),
                      particleHash.encode('hex'), particleInst)
                self._hs.recordModelProgress(modelID = modelID,
                      modelParams = modelParams,
                      modelParamsHash = mParamsAndHash.engParamsHash,
                      results = results,
                      completed = (mResult.status == cjDAO.STATUS_COMPLETED),
                      completionReason = mResult.completionReason,
                      matured = mResult.engMatured,
                      numRecords = mResult.numRecords)
              else:
                modelIDToRun = modelID
                break
  
          else:
            # A specific modelID was passed on the command line
            modelIDToRun = int(options.modelID)
            mParamsAndHash = cjDAO.modelsGetParams([modelIDToRun])[0]
            modelParams = json.loads(mParamsAndHash.params)
            modelParamsHash = mParamsAndHash.engParamsHash
            
            # Make us the worker
            cjDAO.modelSetFields(modelIDToRun,
                                     dict(engWorkerConnId=self._workerID))
            if False:
              # Change the hash and params of the old entry so that we can
              #  create a new model with the same params
              for attempt in range(1000):
                paramsHash = hashlib.md5("OrphanParams.%d.%d" % (modelIDToRun,
                                                                 attempt)).digest()
                particleHash = hashlib.md5("OrphanParticle.%d.%d" % (modelIDToRun,
                                                                  attempt)).digest()
                try:
                  cjDAO.modelSetFields(modelIDToRun,
                                           dict(engParamsHash=paramsHash,
                                                engParticleHash=particleHash))
                  success = True
                except:
                  success = False
                if success:
                  break
              if not success:
                raise RuntimeError("Unexpected failure to change paramsHash and "
                                   "particleHash of orphaned model")
              
              (modelIDToRun, ours) = cjDAO.modelInsertAndStart(options.jobID,
                                  mParamsAndHash.params, modelParamsHash)

            
            
            # ^^^ end while modelIDToRun ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

        # ---------------------------------------------------------------
        # We have a model, evaluate it now
        # All done?
        if exit:
          break

        # Run the model now
        self.logger.info("RUNNING MODEL GID=%d, paramsHash=%s, params=%s",
              modelIDToRun, modelParamsHash.encode('hex'), modelParams)

        # ---------------------------------------------------------------------
        # Construct model checkpoint GUID for this model:
        # jobParams['persistentJobGUID'] contains the client's (e.g., API Server)
        # persistent, globally-unique model identifier, which is what we need;
        persistentJobGUID = jobParams['persistentJobGUID']
        assert persistentJobGUID, "persistentJobGUID: %r" % (persistentJobGUID,)

        modelCheckpointGUID = jobInfo.client + "_" + persistentJobGUID + (
          '_' + str(modelIDToRun))


        self._hs.runModel(modelID=modelIDToRun, jobID = options.jobID,
                          modelParams=modelParams, modelParamsHash=modelParamsHash,
                          jobsDAO=cjDAO, modelCheckpointGUID=modelCheckpointGUID)

        # TODO: don't increment for orphaned models
        numModelsTotal += 1

        self.logger.info("COMPLETED MODEL GID=%d; EVALUATED %d MODELs",
          modelIDToRun, numModelsTotal)
        print >>sys.stderr, "reporter:status:Evaluated %d models..." % \
                                    (numModelsTotal)
        print >>sys.stderr, "reporter:counter:HypersearchWorker,numModels,1"

        if options.modelID is not None:
          exit = True
        # ^^^ end while not exit

    finally:
      # Provide Hypersearch instance an opportunity to clean up temporary files
      self._hs.close()

    self.logger.info("FINISHED. Evaluated %d models." % (numModelsTotal))
    print >>sys.stderr, "reporter:status:Finished, evaluated %d models" % (numModelsTotal)
    return options.jobID
    try:
      jobID = hst.run()
    except Exception, e:
      jobID = hst._options.jobID
      completionReason = ClientJobsDAO.CMPL_REASON_ERROR
      completionMsg = "ERROR: %s" % (e,)
      raise
    finally:
      if jobID is not None:
        cjDAO = ClientJobsDAO.get()
        cjDAO.jobSetCompleted(jobID=jobID,
                              completionReason=completionReason,
                              completionMsg=completionMsg)

  return jobID



if __name__ == "__main__":
  logging.setLoggerClass(ExtendedLogger)
  buildID = Configuration.get('nupic.software.buildNumber', 'N/A')
  logPrefix = '<BUILDID=%s, WORKER=HS, WRKID=N/A, JOBID=N/A> ' % buildID
  ExtendedLogger.setLogPrefix(logPrefix)
  
  try:
    main(sys.argv)
  except:
    logging.exception("HypersearchWorker is exiting with unhandled exception; "
                      "argv=%r", sys.argv)
    raise
Пример #38
0
    def __init__(self,
                 modelID,
                 jobID,
                 predictedField,
                 experimentDir,
                 reportKeyPatterns,
                 optimizeKeyPattern,
                 jobsDAO,
                 modelCheckpointGUID,
                 logLevel=None,
                 predictionCacheMaxRecords=None):
        """
    Parameters:
    -------------------------------------------------------------------------
    modelID:            ID for this model in the models table

    jobID:              ID for this hypersearch job in the jobs table
    predictedField:     Name of the input field for which this model is being
                        optimized
    experimentDir:      Directory path containing the experiment's
                        description.py script
    reportKeyPatterns:  list of items from the results dict to include in
                        the report. These can be regular expressions.
    optimizeKeyPattern: Which report item, if any, we will be optimizing for.
                        This can also be a regular expression, but is an error
                        if it matches more than one key from the experiment's
                        results.
    jobsDAO:            Jobs data access object - the interface to the
                        jobs database which has the model's table.
    modelCheckpointGUID:
                        A persistent, globally-unique identifier for
                        constructing the model checkpoint key. If None, then
                        don't bother creating a model checkpoint.
    logLevel:           override logging level to this value, if not None
    predictionCacheMaxRecords:
                        Maximum number of records for the prediction output cache.
                        Pass None for default value.
    """

        # -----------------------------------------------------------------------
        # Initialize class constants
        # -----------------------------------------------------------------------
        self._MIN_RECORDS_TO_BE_BEST = int(
            Configuration.get('nupic.hypersearch.bestModelMinRecords'))
        self._MATURITY_MAX_CHANGE = float(
            Configuration.get('nupic.hypersearch.maturityPctChange'))
        self._MATURITY_NUM_POINTS = int(
            Configuration.get('nupic.hypersearch.maturityNumPoints'))

        # -----------------------------------------------------------------------
        # Initialize instance variables
        # -----------------------------------------------------------------------
        self._modelID = modelID
        self._jobID = jobID
        self._predictedField = predictedField
        self._experimentDir = experimentDir
        self._reportKeyPatterns = reportKeyPatterns
        self._optimizeKeyPattern = optimizeKeyPattern
        self._jobsDAO = jobsDAO
        self._modelCheckpointGUID = modelCheckpointGUID
        self._predictionCacheMaxRecords = predictionCacheMaxRecords

        self._isMaturityEnabled = bool(
            int(Configuration.get('nupic.hypersearch.enableModelMaturity')))

        self._logger = logging.getLogger(".".join([
            'com.numenta', self.__class__.__module__, self.__class__.__name__
        ]))

        self._optimizedMetricLabel = None
        self._reportMetricLabels = []

        # Our default completion reason
        self._cmpReason = ClientJobsDAO.CMPL_REASON_EOF

        if logLevel is not None:
            self._logger.setLevel(logLevel)

        # The manager object to compute the metrics for this model
        self.__metricMgr = None

        # Will be set to a new instance of OPFTaskDriver by __runTask()
        #self.__taskDriver = None

        # Current task control parameters. Will be set by __runTask()
        self.__task = None

        # Will be set to a new instance of PeriodicActivityManager by __runTask()
        self._periodic = None

        # Will be set to streamDef string by _runTask()
        self._streamDef = None

        # Will be set to new OpfExperiment instance by run()
        self._model = None

        # Will be set to new InputSource by __runTask()
        self._inputSource = None

        # 0-based index of the record being processed;
        # Initialized and updated by __runTask()
        self._currentRecordIndex = None

        # Interface to write predictions to a persistent storage
        self._predictionLogger = None

        # In-memory cache for predictions. Predictions are written here for speed
        # when they don't need to be written to a persistent store
        self.__predictionCache = deque()

        # Flag to see if this is the best model in the job (as determined by the
        # model chooser logic). This is essentially a cache of the value in the
        # ClientJobsDB
        self._isBestModel = False

        # Flag to see if there is a best model (not necessarily this one)
        # stored in the DB
        self._isBestModelStored = False

        # -----------------------------------------------------------------------
        # Flags for model cancelation/checkpointing
        # -----------------------------------------------------------------------

        # Flag to see if the job that this model is part of
        self._isCanceled = False

        # Flag to see if model was killed, either by the model terminator or by the
        # hypsersearch implementation (ex. the a swarm is killed/matured)
        self._isKilled = False

        # Flag to see if the model is matured. In most cases, this means that we
        # should stop running the model. The only execption is if this model is the
        # best model for the job, in which case it should continue running.
        self._isMature = False

        # Event to see if interrupt signal has been sent
        self._isInterrupted = threading.Event()

        # -----------------------------------------------------------------------
        # Facilities for measuring model maturity
        # -----------------------------------------------------------------------
        # List of tuples, (iteration, metric), used to see if the model has 'matured'
        self._metricRegression = regression.AveragePctChange(
            windowSize=self._MATURITY_NUM_POINTS)

        self.__loggedMetricPatterns = []
Пример #39
0
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
# See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see http://www.gnu.org/licenses.
#
# http://numenta.org/licenses/
# ----------------------------------------------------------------------
import pymysql
from nupic.support.configuration import Configuration

# Get the values we need from NuPIC's configuration
host = Configuration.get('nupic.cluster.database.host')
port = int(Configuration.get('nupic.cluster.database.port'))
user = Configuration.get('nupic.cluster.database.user')
passwd = Configuration.get('nupic.cluster.database.passwd')

print
print "This script will validate that your MySQL is setup correctly for NuPIC."
print "MySQL is required for NuPIC swarming. The settings are defined in "
print "NuPIC's configuration files in $NUPIC/nupic/support/nupic-default.xml "
print "Out of the box those settings contain MySQL's default access "
print "credentials."
print
print "Retrieved the following settings from NuPIC configuration:"
print "    host   :    ", host
print "    port   :    ", port
print "    user   :    ", user