def reinitLoggingDir(): """ (Re-)Initialize the loging directory for the calling application that uses initLogging() for logging configuration NOTE: It's typially unnecessary to call this function directly since initLogging takes care of it for you. This function is exposed primarily for the benefit of nupic-services.py to allow it to restore its logging directory after the hard-reset operation. """ if gLoggingInitialized: makeDirectoryFromAbsolutePath(os.path.dirname(_genLoggingFilePath()))
def __call__(self, experiment): import pickle tfdr = experiment.network.regions['level1'].getSelf()._tfdr initArgsDict = tfdr._initArgsDict # Write it out to a file as json absFilePath = os.path.abspath(self.__filePath) absDir = os.path.dirname(absFilePath) makeDirectoryFromAbsolutePath(absDir) with open(absFilePath, 'wb') as pickleFile: pickle.dump(initArgsDict, pickleFile) return
def __call__(self, claModel): import pickle # Get the TP args dictionary assert isinstance(claModel, CLAModel) tpRegion = claModel._getTPRegion().getSelf() tfdr = tpRegion._tfdr initArgsDict = tfdr._initArgsDict # Write it out to a file as json absFilePath = os.path.abspath(self.__filePath) absDir = os.path.dirname(absFilePath) makeDirectoryFromAbsolutePath(absDir) with open(absFilePath, "wb") as pickleFile: pickle.dump(initArgsDict, pickleFile) return
def __call__(self, claModel): import pickle # Get the TP args dictionary assert isinstance(claModel, CLAModel) tpRegion = claModel._getTPRegion().getSelf() tfdr = tpRegion._tfdr initArgsDict = tfdr._initArgsDict # Write it out to a file as json absFilePath = os.path.abspath(self.__filePath) absDir = os.path.dirname(absFilePath) makeDirectoryFromAbsolutePath(absDir) with open(absFilePath, 'wb') as pickleFile: pickle.dump(initArgsDict, pickleFile) return
def __call__(self, htmPredictionModel): import pickle # Get the SP args dictionary assert isinstance(htmPredictionModel, HTMPredictionModel) spRegion = htmPredictionModel._getSPRegion().getSelf() sfdr = spRegion._sfdr initArgsDict = sfdr._initArgsDict # Write it out to a file as json absFilePath = os.path.abspath(self.__filePath) absDir = os.path.dirname(absFilePath) makeDirectoryFromAbsolutePath(absDir) with open(absFilePath, 'wb') as pickleFile: pickle.dump(initArgsDict, pickleFile) return
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'): """ Initilize NuPic logging by reading in from the logging configuration file. The logging configuration file is named 'nupic-logging.conf' and is expected to be in the format defined by the python logging module. If the environment variable 'NTA_CONF_DIR' is defined, then the logging configuration file is expected to be in the NTA_CONF_DIR directory. If NTA_CONF_DIR is not defined, then it is found in the 'conf/default' subdirectory of the NuPic installation directory (typically ~/nta/current/conf/default) The logging configuration file can use the environment variable 'NTA_LOG_DIR' to set the locations of log files. If this variable is not defined already in the environment, this method will set it to the 'logs' subdirectory of the NuPic install directory (typically ~/nta/eng/logs) before loading in the configuration file. console: Defines console output for the default "root" logging configuration; this may be one of 'stdout', 'stderr', or None; Use None to suppress console logging output consoleLevel: Logging-level filter string for console output corresponding to logging levels in the logging module; may be one of: 'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'. E.g., a value of'WARNING' suppresses DEBUG and INFO level output to console, but allows WARNING, ERROR, and CRITICAL """ # NOTE: If you call this twice from the same process there seems to be a # bug - logged messages don't show up for loggers that you do another # logging.getLogger() on. global gLoggingInitialized if gLoggingInitialized: if verbose: print >> sys.stderr, "Logging already initialized, doing nothing." return consoleStreamMappings = { 'stdout' : 'stdoutConsoleHandler', 'stderr' : 'stderrConsoleHandler', } consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL', 'FATAL'] assert console is None or console in consoleStreamMappings.keys(), ( 'Unexpected console arg value: %r') % (console,) assert consoleLevel in consoleLogLevels, ( 'Unexpected consoleLevel arg value: %r') % (consoleLevel) # ----------------------------------------------------------------------- # Setup logging. Look for the nupic-logging.conf file, first in the # NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic # module # TODO: move into nupic.support configFilename = 'nupic-logging.conf' try: configFilePath = Configuration.findConfigFile(configFilename) except: configFilePath = None # If NTA_LOG_DIR is not defined, set it now. This is used by the logging # config file to set the path for the log files if 'NTA_LOG_DIR' not in os.environ: os.environ['NTA_LOG_DIR'] = os.path.join(nupic.rootDir, 'logs') if not os.path.exists(os.environ['NTA_LOG_DIR']): makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR'])) # Load in the logging configuration file if configFilePath is None: print >> sys.stderr, ( "WARNING: Could not find the logging configuration file " \ "(filename: '%s', expected to be in search path: %s). Logging is " \ " disabled.") % (configFilename, Configuration.getConfigPaths()) else: if verbose: print >> sys.stderr, ( "Using logging configuration file: %s") % (configFilePath) # This dict will hold our replacement strings for logging configuration replacements = dict() def makeKey(name): """ Makes replacement key """ return "$$%s$$" % (name) platform = sys.platform.lower() if platform.startswith('java'): # Jython import java.lang platform = java.lang.System.getProperty("os.name").lower() if platform.startswith('mac os x'): platform = 'darwin' if platform.startswith('darwin'): replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"' elif platform.startswith('linux'): replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"' else: raise RuntimeError("This platform is neither darwin nor linux: %s" % ( sys.platform,)) if False: #os.path.isdir('/var/log/numenta/nupic'): # NOTE: Not using syslogHandler for now because it either truncates or # drops messages over ~1,400 bytes (depending on platform) # Nupic logs go to syslog. Also, SysLogHandler raises an exception # on jython (at least on 2.5.2): "AttributeError: 'module' object has no # attribute 'AF_UNIX'" (jython is used by a sub-moduleof # ClientJobManager) replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'syslogHandler' else: # Nupic logs go to file replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler' # Set up log file path for the default file handler logFilePath = _genLoggingFilePath() makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath)) replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath) # Set up root logger replacements[makeKey('ROOT_LOGGER_HANDLERS')] = ( replacements[makeKey('PERSISTENT_LOG_HANDLER')]) if console is not None: replacements[makeKey('ROOT_LOGGER_HANDLERS')] += ( ',' + consoleStreamMappings[console]) # Set up log level for console handlers replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel customConfig = StringIO() with open(configFilePath) as src: for lineNum, line in enumerate(src): if "$$" in line: for (key, value) in replacements.items(): line = line.replace(key, value) # If there is still a replacement string in the line, we're missing it # from our replacements dict if "$$" in line and "$$<key>$$" not in line: raise RuntimeError(("The text %r, found at line #%d of file %r, " "contains a string not found in our replacement " "dict.") % (line, lineNum, configFilePath)) customConfig.write(line) customConfig.seek(0) if version.StrictVersion(python_version()) >= version.StrictVersion('2.6'): # NOTE: the disable_existing_loggers arg is new as of python 2.6, so it's # not supported on our jython interperter, which was v2.5.x as of this # writing logging.config.fileConfig(customConfig, disable_existing_loggers=False) else: logging.config.fileConfig(customConfig) gLoggingInitialized = True
# Add unmatched remaining properties to custom config store for propertyName, value in copyOfProperties.iteritems(): newProp = ElementTree.Element('property') nameTag = ElementTree.Element('name') nameTag.text = propertyName newProp.append(nameTag) valueTag = ElementTree.Element('value') valueTag.text = str(value) newProp.append(valueTag) elements.append(newProp) try: makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath)) with open(configFilePath, 'w') as fp: fp.write(ElementTree.tostring(elements)) except Exception, e: _getLogger().exception( "Error while saving custom configuration " "properties %s in %s.", properties, configFilePath) raise @classmethod def _setPath(cls): """ Sets the path of the custom configuration file """ cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'], cls.customFileName)
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'): """ Initilize NuPic logging by reading in from the logging configuration file. The logging configuration file is named 'nupic-logging.conf' and is expected to be in the format defined by the python logging module. If the environment variable 'NTA_CONF_PATH' is defined, then the logging configuration file is expected to be in the NTA_CONF_PATH directory. If NTA_CONF_PATH is not defined, then it is found in the 'conf/default' subdirectory of the NuPic installation directory (typically ~/nupic/current/conf/default) The logging configuration file can use the environment variable 'NTA_LOG_DIR' to set the locations of log files. If this variable is not defined already in the environment, this method will set it to the 'logs' subdirectory of the NuPic install directory (typically ~/nupic/eng/logs) before loading in the configuration file. console: Defines console output for the default "root" logging configuration; this may be one of 'stdout', 'stderr', or None; Use None to suppress console logging output consoleLevel: Logging-level filter string for console output corresponding to logging levels in the logging module; may be one of: 'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'. E.g., a value of'WARNING' suppresses DEBUG and INFO level output to console, but allows WARNING, ERROR, and CRITICAL """ # NOTE: If you call this twice from the same process there seems to be a # bug - logged messages don't show up for loggers that you do another # logging.getLogger() on. global gLoggingInitialized if gLoggingInitialized: if verbose: print >> sys.stderr, "Logging already initialized, doing nothing." return consoleStreamMappings = { 'stdout' : 'stdoutConsoleHandler', 'stderr' : 'stderrConsoleHandler', } consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL', 'FATAL'] assert console is None or console in consoleStreamMappings.keys(), ( 'Unexpected console arg value: %r') % (console,) assert consoleLevel in consoleLogLevels, ( 'Unexpected consoleLevel arg value: %r') % (consoleLevel) # ----------------------------------------------------------------------- # Setup logging. Look for the nupic-logging.conf file, first in the # NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic # module configFilename = 'nupic-logging.conf' configFilePath = resource_filename("nupic.support", configFilename); # If NTA_LOG_DIR is not defined, set it now. This is used by the logging # config file to set the path for the log files if 'NTA_LOG_DIR' not in os.environ: os.environ['NTA_LOG_DIR'] = os.path.join(os.environ['NUPIC'], 'logs') if not os.path.exists(os.environ['NTA_LOG_DIR']): makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR'])) # Load in the logging configuration file if verbose: print >> sys.stderr, ( "Using logging configuration file: %s") % (configFilePath) # This dict will hold our replacement strings for logging configuration replacements = dict() def makeKey(name): """ Makes replacement key """ return "$$%s$$" % (name) platform = sys.platform.lower() if platform.startswith('java'): # Jython import java.lang platform = java.lang.System.getProperty("os.name").lower() if platform.startswith('mac os x'): platform = 'darwin' if platform.startswith('darwin'): replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"' elif platform.startswith('linux'): replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"' else: raise RuntimeError("This platform is neither darwin nor linux: %s" % ( sys.platform,)) # Nupic logs go to file replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler' # Set up log file path for the default file handler logFilePath = _genLoggingFilePath() makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath)) replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath) # Set up root logger replacements[makeKey('ROOT_LOGGER_HANDLERS')] = ( replacements[makeKey('PERSISTENT_LOG_HANDLER')]) if console is not None: replacements[makeKey('ROOT_LOGGER_HANDLERS')] += ( ',' + consoleStreamMappings[console]) # Set up log level for console handlers replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel customConfig = StringIO() # Using pkg_resources to get the logging file, which should be packaged and # associated with this source file name. loggingFileContents = resource_string(__name__, configFilename) for lineNum, line in enumerate(loggingFileContents.splitlines()): if "$$" in line: for (key, value) in replacements.items(): line = line.replace(key, value) # If there is still a replacement string in the line, we're missing it # from our replacements dict if "$$" in line and "$$<key>$$" not in line: raise RuntimeError(("The text %r, found at line #%d of file %r, " "contains a string not found in our replacement " "dict.") % (line, lineNum, configFilePath)) customConfig.write("%s\n" % line) customConfig.seek(0) if python_version()[:3] >= '2.6': logging.config.fileConfig(customConfig, disable_existing_loggers=False) else: logging.config.fileConfig(customConfig) gLoggingInitialized = True
def initLogging(verbose=False, console="stdout", consoleLevel="DEBUG"): """ Initilize NuPic logging by reading in from the logging configuration file. The logging configuration file is named 'nupic-logging.conf' and is expected to be in the format defined by the python logging module. If the environment variable 'NTA_CONF_PATH' is defined, then the logging configuration file is expected to be in the NTA_CONF_PATH directory. If NTA_CONF_PATH is not defined, then it is found in the 'conf/default' subdirectory of the NuPic installation directory (typically ~/nupic/current/conf/default) The logging configuration file can use the environment variable 'NTA_LOG_DIR' to set the locations of log files. If this variable is not defined, logging to files will be disabled. console: Defines console output for the default "root" logging configuration; this may be one of 'stdout', 'stderr', or None; Use None to suppress console logging output consoleLevel: Logging-level filter string for console output corresponding to logging levels in the logging module; may be one of: 'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'. E.g., a value of'WARNING' suppresses DEBUG and INFO level output to console, but allows WARNING, ERROR, and CRITICAL """ # NOTE: If you call this twice from the same process there seems to be a # bug - logged messages don't show up for loggers that you do another # logging.getLogger() on. global gLoggingInitialized if gLoggingInitialized: if verbose: print >>sys.stderr, "Logging already initialized, doing nothing." return consoleStreamMappings = {"stdout": "stdoutConsoleHandler", "stderr": "stderrConsoleHandler"} consoleLogLevels = ["DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"] assert console is None or console in consoleStreamMappings.keys(), ("Unexpected console arg value: %r") % (console,) assert consoleLevel in consoleLogLevels, ("Unexpected consoleLevel arg value: %r") % (consoleLevel) # ----------------------------------------------------------------------- # Setup logging. Look for the nupic-logging.conf file, first in the # NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic # module configFilename = "nupic-logging.conf" configFilePath = resource_filename("nupic.support", configFilename) configLogDir = os.environ.get("NTA_LOG_DIR", None) # Load in the logging configuration file if verbose: print >>sys.stderr, ("Using logging configuration file: %s") % (configFilePath) # This dict will hold our replacement strings for logging configuration replacements = dict() def makeKey(name): """ Makes replacement key """ return "$$%s$$" % (name) platform = sys.platform.lower() if platform.startswith("java"): # Jython import java.lang platform = java.lang.System.getProperty("os.name").lower() if platform.startswith("mac os x"): platform = "darwin" if platform.startswith("darwin"): replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/var/run/syslog"' elif platform.startswith("linux"): replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/dev/log"' elif platform.startswith("win"): replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"log"' else: raise RuntimeError("This platform is neither darwin, win32, nor linux: %s" % (sys.platform,)) # Nupic logs go to file replacements[makeKey("PERSISTENT_LOG_HANDLER")] = "fileHandler" if platform.startswith("win"): replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"NUL"' else: replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"/dev/null"' # Set up log file path for the default file handler and configure handlers handlers = list() if configLogDir is not None: logFilePath = _genLoggingFilePath() makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath)) replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = repr(logFilePath) handlers.append(replacements[makeKey("PERSISTENT_LOG_HANDLER")]) if console is not None: handlers.append(consoleStreamMappings[console]) replacements[makeKey("ROOT_LOGGER_HANDLERS")] = ", ".join(handlers) # Set up log level for console handlers replacements[makeKey("CONSOLE_LOG_LEVEL")] = consoleLevel customConfig = StringIO() # Using pkg_resources to get the logging file, which should be packaged and # associated with this source file name. loggingFileContents = resource_string(__name__, configFilename) for lineNum, line in enumerate(loggingFileContents.splitlines()): if "$$" in line: for (key, value) in replacements.items(): line = line.replace(key, value) # If there is still a replacement string in the line, we're missing it # from our replacements dict if "$$" in line and "$$<key>$$" not in line: raise RuntimeError( ( "The text %r, found at line #%d of file %r, " "contains a string not found in our replacement " "dict." ) % (line, lineNum, configFilePath) ) customConfig.write("%s\n" % line) customConfig.seek(0) if python_version()[:3] >= "2.6": logging.config.fileConfig(customConfig, disable_existing_loggers=False) else: logging.config.fileConfig(customConfig) gLoggingInitialized = True
# Add unmatched remaining properties to custom config store for propertyName, value in copyOfProperties.iteritems(): newProp = ElementTree.Element('property') nameTag = ElementTree.Element('name') nameTag.text = propertyName newProp.append(nameTag) valueTag = ElementTree.Element('value') valueTag.text = str(value) newProp.append(valueTag) elements.append(newProp) try: makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath)) with open(configFilePath,'w') as fp: fp.write(ElementTree.tostring(elements)) except Exception, e: _getLogger().exception("Error while saving custom configuration " "properties %s in %s.", properties, configFilePath) raise @classmethod def _setPath(cls): """ Sets the path of the custom configuration file """ cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'], cls.customFileName)