Пример #1
0
def reinitLoggingDir():
  """ (Re-)Initialize the loging directory for the calling application that
  uses initLogging() for logging configuration
  
  NOTE: It's typially unnecessary to call this function directly since
   initLogging takes care of it for you. This function is exposed primarily for
   the benefit of nupic-services.py to allow it to restore its logging directory
   after the hard-reset operation.
  """
  if gLoggingInitialized:
    makeDirectoryFromAbsolutePath(os.path.dirname(_genLoggingFilePath()))
Пример #2
0
    def __call__(self, experiment):
        import pickle

        tfdr = experiment.network.regions['level1'].getSelf()._tfdr

        initArgsDict = tfdr._initArgsDict

        # Write it out to a file as json
        absFilePath = os.path.abspath(self.__filePath)

        absDir = os.path.dirname(absFilePath)
        makeDirectoryFromAbsolutePath(absDir)

        with open(absFilePath, 'wb') as pickleFile:
            pickle.dump(initArgsDict, pickleFile)

        return
Пример #3
0
  def __call__(self, experiment):
    import pickle

    tfdr = experiment.network.regions['level1'].getSelf()._tfdr

    initArgsDict = tfdr._initArgsDict


    # Write it out to a file as json
    absFilePath = os.path.abspath(self.__filePath)

    absDir = os.path.dirname(absFilePath)
    makeDirectoryFromAbsolutePath(absDir)

    with open(absFilePath, 'wb') as pickleFile:
      pickle.dump(initArgsDict, pickleFile)

    return
Пример #4
0
    def __call__(self, claModel):

        import pickle

        # Get the TP args dictionary
        assert isinstance(claModel, CLAModel)

        tpRegion = claModel._getTPRegion().getSelf()

        tfdr = tpRegion._tfdr

        initArgsDict = tfdr._initArgsDict

        # Write it out to a file as json
        absFilePath = os.path.abspath(self.__filePath)

        absDir = os.path.dirname(absFilePath)
        makeDirectoryFromAbsolutePath(absDir)

        with open(absFilePath, "wb") as pickleFile:
            pickle.dump(initArgsDict, pickleFile)

        return
Пример #5
0
    def __call__(self, claModel):

        import pickle

        # Get the TP args dictionary
        assert isinstance(claModel, CLAModel)

        tpRegion = claModel._getTPRegion().getSelf()

        tfdr = tpRegion._tfdr

        initArgsDict = tfdr._initArgsDict

        # Write it out to a file as json
        absFilePath = os.path.abspath(self.__filePath)

        absDir = os.path.dirname(absFilePath)
        makeDirectoryFromAbsolutePath(absDir)

        with open(absFilePath, 'wb') as pickleFile:
            pickle.dump(initArgsDict, pickleFile)

        return
    def __call__(self, htmPredictionModel):

        import pickle

        # Get the SP args dictionary
        assert isinstance(htmPredictionModel, HTMPredictionModel)

        spRegion = htmPredictionModel._getSPRegion().getSelf()

        sfdr = spRegion._sfdr

        initArgsDict = sfdr._initArgsDict

        # Write it out to a file as json
        absFilePath = os.path.abspath(self.__filePath)

        absDir = os.path.dirname(absFilePath)
        makeDirectoryFromAbsolutePath(absDir)

        with open(absFilePath, 'wb') as pickleFile:
            pickle.dump(initArgsDict, pickleFile)

        return
Пример #7
0
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
  """
  Initilize NuPic logging by reading in from the logging configuration file. The
  logging configuration file is named 'nupic-logging.conf' and is expected to be
  in the format defined by the python logging module.

  If the environment variable 'NTA_CONF_DIR' is defined, then the logging
  configuration file is expected to be in the NTA_CONF_DIR directory. If
  NTA_CONF_DIR is not defined, then it is found in the 'conf/default'
  subdirectory of the NuPic installation directory (typically
  ~/nta/current/conf/default)

  The logging configuration file can use the environment variable 'NTA_LOG_DIR'
  to set the locations of log files. If this variable is not defined already in
  the environment, this method will set it to the 'logs' subdirectory of the
  NuPic install directory (typically ~/nta/eng/logs) before loading in the
  configuration file.
  
  console:    Defines console output for the default "root" logging
              configuration; this may be one of 'stdout', 'stderr', or None;
              Use None to suppress console logging output
  consoleLevel:
              Logging-level filter string for console output corresponding to
              logging levels in the logging module; may be one of:
              'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
              E.g.,  a value of'WARNING' suppresses DEBUG and INFO level output
              to console, but allows WARNING, ERROR, and CRITICAL 
  """

  # NOTE: If you call this twice from the same process there seems to be a
  # bug - logged messages don't show up for loggers that you do another
  # logging.getLogger() on.
  global gLoggingInitialized
  if gLoggingInitialized:
    if verbose:
      print >> sys.stderr, "Logging already initialized, doing nothing."
    return
  
  consoleStreamMappings = {
    'stdout'  : 'stdoutConsoleHandler',
    'stderr'  : 'stderrConsoleHandler',
  }
  
  consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',
                      'FATAL']
  
  assert console is None or console in consoleStreamMappings.keys(), (
    'Unexpected console arg value: %r') % (console,)
  
  assert consoleLevel in consoleLogLevels, (
    'Unexpected consoleLevel arg value: %r') % (consoleLevel)

  # -----------------------------------------------------------------------
  # Setup logging. Look for the nupic-logging.conf file, first in the
  #   NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
  #   module
  # TODO: move into nupic.support
  configFilename = 'nupic-logging.conf'
  try:
    configFilePath = Configuration.findConfigFile(configFilename)
  except:
    configFilePath = None


  # If NTA_LOG_DIR is not defined, set it now. This is used by the logging
  #   config file to set the path for the log files
  if 'NTA_LOG_DIR' not in os.environ:
    os.environ['NTA_LOG_DIR'] = os.path.join(nupic.rootDir, 'logs')
  if not os.path.exists(os.environ['NTA_LOG_DIR']):
    makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR']))

  # Load in the logging configuration file
  if configFilePath is None:
    print >> sys.stderr, (
      "WARNING: Could not find the logging configuration file " \
      "(filename: '%s', expected to be in search path: %s). Logging is " \
      " disabled.") % (configFilename, Configuration.getConfigPaths())
  else:
    if verbose:
      print >> sys.stderr, (
        "Using logging configuration file: %s") % (configFilePath)
      
    # This dict will hold our replacement strings for logging configuration
    replacements = dict()
    
    def makeKey(name):
      """ Makes replacement key """
      return "$$%s$$" % (name)
    
    platform = sys.platform.lower()
    if platform.startswith('java'):
      # Jython
      import java.lang
      platform = java.lang.System.getProperty("os.name").lower()
      if platform.startswith('mac os x'):
        platform = 'darwin'

    if platform.startswith('darwin'):
      replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
    elif platform.startswith('linux'):
      replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
    else:
      raise RuntimeError("This platform is neither darwin nor linux: %s" % (
        sys.platform,))
    
    if False: #os.path.isdir('/var/log/numenta/nupic'):
      # NOTE: Not using syslogHandler for now because it either truncates or
      #  drops messages over ~1,400 bytes (depending on platform)
      #  Nupic logs go to syslog. Also, SysLogHandler raises an exception
      #  on jython (at least on 2.5.2): "AttributeError: 'module' object has no
      #  attribute 'AF_UNIX'" (jython is used by a sub-moduleof
      #  ClientJobManager)
      replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'syslogHandler'
    else:
      # Nupic logs go to file
      replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'
    
      # Set up log file path for the default file handler
      logFilePath = _genLoggingFilePath()
      makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
      replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)
    
    # Set up root logger
    replacements[makeKey('ROOT_LOGGER_HANDLERS')] = (
      replacements[makeKey('PERSISTENT_LOG_HANDLER')])
    if console is not None:
      replacements[makeKey('ROOT_LOGGER_HANDLERS')] += (
        ',' + consoleStreamMappings[console])
    
    # Set up log level for console handlers
    replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel
    
    customConfig = StringIO()
    with open(configFilePath) as src:
      for lineNum, line in enumerate(src):
        if "$$" in line:
          for (key, value) in replacements.items():
            line = line.replace(key, value)
            
        # If there is still a replacement string in the line, we're missing it
        #  from our replacements dict
        if "$$" in line and "$$<key>$$" not in line:
          raise RuntimeError(("The text %r, found at line #%d of file %r, "
                              "contains a string not found in our replacement "
                              "dict.") % (line, lineNum, configFilePath))
        
        customConfig.write(line)
    
    customConfig.seek(0)
    if version.StrictVersion(python_version()) >= version.StrictVersion('2.6'):
      # NOTE: the disable_existing_loggers arg is new as of python 2.6, so it's
      #  not supported on our jython interperter, which was v2.5.x as of this
      #  writing
      logging.config.fileConfig(customConfig, disable_existing_loggers=False)
    else:
      logging.config.fileConfig(customConfig)

  gLoggingInitialized = True
        # Add unmatched remaining properties to custom config store
        for propertyName, value in copyOfProperties.iteritems():
            newProp = ElementTree.Element('property')
            nameTag = ElementTree.Element('name')
            nameTag.text = propertyName
            newProp.append(nameTag)

            valueTag = ElementTree.Element('value')
            valueTag.text = str(value)
            newProp.append(valueTag)

            elements.append(newProp)

        try:
            makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
            with open(configFilePath, 'w') as fp:
                fp.write(ElementTree.tostring(elements))
        except Exception, e:
            _getLogger().exception(
                "Error while saving custom configuration "
                "properties %s in %s.", properties, configFilePath)
            raise

    @classmethod
    def _setPath(cls):
        """ Sets the path of the custom configuration file
    """
        cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
                                 cls.customFileName)
Пример #9
0
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
  """
  Initilize NuPic logging by reading in from the logging configuration file. The
  logging configuration file is named 'nupic-logging.conf' and is expected to be
  in the format defined by the python logging module.

  If the environment variable 'NTA_CONF_PATH' is defined, then the logging
  configuration file is expected to be in the NTA_CONF_PATH directory. If
  NTA_CONF_PATH is not defined, then it is found in the 'conf/default'
  subdirectory of the NuPic installation directory (typically
  ~/nupic/current/conf/default)

  The logging configuration file can use the environment variable 'NTA_LOG_DIR'
  to set the locations of log files. If this variable is not defined already in
  the environment, this method will set it to the 'logs' subdirectory of the
  NuPic install directory (typically ~/nupic/eng/logs) before loading in the
  configuration file.
  
  console:    Defines console output for the default "root" logging
              configuration; this may be one of 'stdout', 'stderr', or None;
              Use None to suppress console logging output
  consoleLevel:
              Logging-level filter string for console output corresponding to
              logging levels in the logging module; may be one of:
              'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
              E.g.,  a value of'WARNING' suppresses DEBUG and INFO level output
              to console, but allows WARNING, ERROR, and CRITICAL 
  """

  # NOTE: If you call this twice from the same process there seems to be a
  # bug - logged messages don't show up for loggers that you do another
  # logging.getLogger() on.
  global gLoggingInitialized
  if gLoggingInitialized:
    if verbose:
      print >> sys.stderr, "Logging already initialized, doing nothing."
    return
  
  consoleStreamMappings = {
    'stdout'  : 'stdoutConsoleHandler',
    'stderr'  : 'stderrConsoleHandler',
  }
  
  consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',
                      'FATAL']
  
  assert console is None or console in consoleStreamMappings.keys(), (
    'Unexpected console arg value: %r') % (console,)
  
  assert consoleLevel in consoleLogLevels, (
    'Unexpected consoleLevel arg value: %r') % (consoleLevel)

  # -----------------------------------------------------------------------
  # Setup logging. Look for the nupic-logging.conf file, first in the
  #   NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
  #   module
  configFilename = 'nupic-logging.conf'

  configFilePath = resource_filename("nupic.support", configFilename);

  # If NTA_LOG_DIR is not defined, set it now. This is used by the logging
  #   config file to set the path for the log files
  if 'NTA_LOG_DIR' not in os.environ:
    os.environ['NTA_LOG_DIR'] = os.path.join(os.environ['NUPIC'], 'logs')
  if not os.path.exists(os.environ['NTA_LOG_DIR']):
    makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR']))

  # Load in the logging configuration file
  if verbose:
    print >> sys.stderr, (
      "Using logging configuration file: %s") % (configFilePath)

  # This dict will hold our replacement strings for logging configuration
  replacements = dict()

  def makeKey(name):
    """ Makes replacement key """
    return "$$%s$$" % (name)

  platform = sys.platform.lower()
  if platform.startswith('java'):
    # Jython
    import java.lang
    platform = java.lang.System.getProperty("os.name").lower()
    if platform.startswith('mac os x'):
      platform = 'darwin'

  if platform.startswith('darwin'):
    replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
  elif platform.startswith('linux'):
    replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
  else:
    raise RuntimeError("This platform is neither darwin nor linux: %s" % (
      sys.platform,))

  # Nupic logs go to file
  replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'

  # Set up log file path for the default file handler
  logFilePath = _genLoggingFilePath()
  makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
  replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)

  # Set up root logger
  replacements[makeKey('ROOT_LOGGER_HANDLERS')] = (
    replacements[makeKey('PERSISTENT_LOG_HANDLER')])
  if console is not None:
    replacements[makeKey('ROOT_LOGGER_HANDLERS')] += (
      ',' + consoleStreamMappings[console])

  # Set up log level for console handlers
  replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel

  customConfig = StringIO()

  # Using pkg_resources to get the logging file, which should be packaged and
  # associated with this source file name.
  loggingFileContents = resource_string(__name__, configFilename)

  for lineNum, line in enumerate(loggingFileContents.splitlines()):
    if "$$" in line:
      for (key, value) in replacements.items():
        line = line.replace(key, value)

    # If there is still a replacement string in the line, we're missing it
    #  from our replacements dict
    if "$$" in line and "$$<key>$$" not in line:
      raise RuntimeError(("The text %r, found at line #%d of file %r, "
                          "contains a string not found in our replacement "
                          "dict.") % (line, lineNum, configFilePath))

    customConfig.write("%s\n" % line)
    
  customConfig.seek(0)
  if python_version()[:3] >= '2.6':
    logging.config.fileConfig(customConfig, disable_existing_loggers=False)
  else:
    logging.config.fileConfig(customConfig)

  gLoggingInitialized = True
Пример #10
0
def initLogging(verbose=False, console="stdout", consoleLevel="DEBUG"):
    """
  Initilize NuPic logging by reading in from the logging configuration file. The
  logging configuration file is named 'nupic-logging.conf' and is expected to be
  in the format defined by the python logging module.

  If the environment variable 'NTA_CONF_PATH' is defined, then the logging
  configuration file is expected to be in the NTA_CONF_PATH directory. If
  NTA_CONF_PATH is not defined, then it is found in the 'conf/default'
  subdirectory of the NuPic installation directory (typically
  ~/nupic/current/conf/default)

  The logging configuration file can use the environment variable 'NTA_LOG_DIR'
  to set the locations of log files. If this variable is not defined, logging to
  files will be disabled.
  
  console:    Defines console output for the default "root" logging
              configuration; this may be one of 'stdout', 'stderr', or None;
              Use None to suppress console logging output
  consoleLevel:
              Logging-level filter string for console output corresponding to
              logging levels in the logging module; may be one of:
              'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
              E.g.,  a value of'WARNING' suppresses DEBUG and INFO level output
              to console, but allows WARNING, ERROR, and CRITICAL 
  """

    # NOTE: If you call this twice from the same process there seems to be a
    # bug - logged messages don't show up for loggers that you do another
    # logging.getLogger() on.
    global gLoggingInitialized
    if gLoggingInitialized:
        if verbose:
            print >>sys.stderr, "Logging already initialized, doing nothing."
        return

    consoleStreamMappings = {"stdout": "stdoutConsoleHandler", "stderr": "stderrConsoleHandler"}

    consoleLogLevels = ["DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"]

    assert console is None or console in consoleStreamMappings.keys(), ("Unexpected console arg value: %r") % (console,)

    assert consoleLevel in consoleLogLevels, ("Unexpected consoleLevel arg value: %r") % (consoleLevel)

    # -----------------------------------------------------------------------
    # Setup logging. Look for the nupic-logging.conf file, first in the
    #   NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
    #   module
    configFilename = "nupic-logging.conf"
    configFilePath = resource_filename("nupic.support", configFilename)

    configLogDir = os.environ.get("NTA_LOG_DIR", None)

    # Load in the logging configuration file
    if verbose:
        print >>sys.stderr, ("Using logging configuration file: %s") % (configFilePath)

    # This dict will hold our replacement strings for logging configuration
    replacements = dict()

    def makeKey(name):
        """ Makes replacement key """
        return "$$%s$$" % (name)

    platform = sys.platform.lower()
    if platform.startswith("java"):
        # Jython
        import java.lang

        platform = java.lang.System.getProperty("os.name").lower()
        if platform.startswith("mac os x"):
            platform = "darwin"

    if platform.startswith("darwin"):
        replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/var/run/syslog"'
    elif platform.startswith("linux"):
        replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/dev/log"'
    elif platform.startswith("win"):
        replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"log"'
    else:
        raise RuntimeError("This platform is neither darwin, win32, nor linux: %s" % (sys.platform,))

    # Nupic logs go to file
    replacements[makeKey("PERSISTENT_LOG_HANDLER")] = "fileHandler"
    if platform.startswith("win"):
        replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"NUL"'
    else:
        replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"/dev/null"'

    # Set up log file path for the default file handler and configure handlers
    handlers = list()

    if configLogDir is not None:
        logFilePath = _genLoggingFilePath()
        makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
        replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = repr(logFilePath)

        handlers.append(replacements[makeKey("PERSISTENT_LOG_HANDLER")])

    if console is not None:
        handlers.append(consoleStreamMappings[console])

    replacements[makeKey("ROOT_LOGGER_HANDLERS")] = ", ".join(handlers)

    # Set up log level for console handlers
    replacements[makeKey("CONSOLE_LOG_LEVEL")] = consoleLevel

    customConfig = StringIO()

    # Using pkg_resources to get the logging file, which should be packaged and
    # associated with this source file name.
    loggingFileContents = resource_string(__name__, configFilename)

    for lineNum, line in enumerate(loggingFileContents.splitlines()):
        if "$$" in line:
            for (key, value) in replacements.items():
                line = line.replace(key, value)

        # If there is still a replacement string in the line, we're missing it
        #  from our replacements dict
        if "$$" in line and "$$<key>$$" not in line:
            raise RuntimeError(
                (
                    "The text %r, found at line #%d of file %r, "
                    "contains a string not found in our replacement "
                    "dict."
                )
                % (line, lineNum, configFilePath)
            )

        customConfig.write("%s\n" % line)

    customConfig.seek(0)
    if python_version()[:3] >= "2.6":
        logging.config.fileConfig(customConfig, disable_existing_loggers=False)
    else:
        logging.config.fileConfig(customConfig)

    gLoggingInitialized = True
Пример #11
0
    # Add unmatched remaining properties to custom config store
    for propertyName, value in copyOfProperties.iteritems():
      newProp = ElementTree.Element('property')
      nameTag = ElementTree.Element('name')
      nameTag.text = propertyName
      newProp.append(nameTag)

      valueTag = ElementTree.Element('value')
      valueTag.text = str(value)
      newProp.append(valueTag)

      elements.append(newProp)
    
    try:
      makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
      with open(configFilePath,'w') as fp:
        fp.write(ElementTree.tostring(elements))
    except Exception, e:
      _getLogger().exception("Error while saving custom configuration "
        "properties %s in %s.", properties, configFilePath)
      raise


  @classmethod
  def _setPath(cls):
    """ Sets the path of the custom configuration file
    """
    cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
                             cls.customFileName)