def reinitLoggingDir():
""" (Re-)Initialize the loging directory for the calling application that
uses initLogging() for logging configuration
NOTE: It's typially unnecessary to call this function directly since
initLogging takes care of it for you. This function is exposed primarily for
the benefit of nupic-services.py to allow it to restore its logging directory
after the hard-reset operation.
"""
if gLoggingInitialized:
makeDirectoryFromAbsolutePath(os.path.dirname(_genLoggingFilePath()))
def __call__(self, experiment):
import pickle
tfdr = experiment.network.regions['level1'].getSelf()._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def __call__(self, experiment):
import pickle
tfdr = experiment.network.regions['level1'].getSelf()._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def __call__(self, claModel):
import pickle
# Get the TP args dictionary
assert isinstance(claModel, CLAModel)
tpRegion = claModel._getTPRegion().getSelf()
tfdr = tpRegion._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, "wb") as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def __call__(self, claModel):
import pickle
# Get the TP args dictionary
assert isinstance(claModel, CLAModel)
tpRegion = claModel._getTPRegion().getSelf()
tfdr = tpRegion._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def __call__(self, htmPredictionModel):
import pickle
# Get the SP args dictionary
assert isinstance(htmPredictionModel, HTMPredictionModel)
spRegion = htmPredictionModel._getSPRegion().getSelf()
sfdr = spRegion._sfdr
initArgsDict = sfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
"""
Initilize NuPic logging by reading in from the logging configuration file. The
logging configuration file is named 'nupic-logging.conf' and is expected to be
in the format defined by the python logging module.
If the environment variable 'NTA_CONF_DIR' is defined, then the logging
configuration file is expected to be in the NTA_CONF_DIR directory. If
NTA_CONF_DIR is not defined, then it is found in the 'conf/default'
subdirectory of the NuPic installation directory (typically
~/nta/current/conf/default)
The logging configuration file can use the environment variable 'NTA_LOG_DIR'
to set the locations of log files. If this variable is not defined already in
the environment, this method will set it to the 'logs' subdirectory of the
NuPic install directory (typically ~/nta/eng/logs) before loading in the
configuration file.
console: Defines console output for the default "root" logging
configuration; this may be one of 'stdout', 'stderr', or None;
Use None to suppress console logging output
consoleLevel:
Logging-level filter string for console output corresponding to
logging levels in the logging module; may be one of:
'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
E.g., a value of'WARNING' suppresses DEBUG and INFO level output
to console, but allows WARNING, ERROR, and CRITICAL
"""
# NOTE: If you call this twice from the same process there seems to be a
# bug - logged messages don't show up for loggers that you do another
# logging.getLogger() on.
global gLoggingInitialized
if gLoggingInitialized:
if verbose:
print >> sys.stderr, "Logging already initialized, doing nothing."
return
consoleStreamMappings = {
'stdout' : 'stdoutConsoleHandler',
'stderr' : 'stderrConsoleHandler',
}
consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',
'FATAL']
assert console is None or console in consoleStreamMappings.keys(), (
'Unexpected console arg value: %r') % (console,)
assert consoleLevel in consoleLogLevels, (
'Unexpected consoleLevel arg value: %r') % (consoleLevel)
# -----------------------------------------------------------------------
# Setup logging. Look for the nupic-logging.conf file, first in the
# NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
# module
# TODO: move into nupic.support
configFilename = 'nupic-logging.conf'
try:
configFilePath = Configuration.findConfigFile(configFilename)
except:
configFilePath = None
# If NTA_LOG_DIR is not defined, set it now. This is used by the logging
# config file to set the path for the log files
if 'NTA_LOG_DIR' not in os.environ:
os.environ['NTA_LOG_DIR'] = os.path.join(nupic.rootDir, 'logs')
if not os.path.exists(os.environ['NTA_LOG_DIR']):
makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR']))
# Load in the logging configuration file
if configFilePath is None:
print >> sys.stderr, (
"WARNING: Could not find the logging configuration file " \
"(filename: '%s', expected to be in search path: %s). Logging is " \
" disabled.") % (configFilename, Configuration.getConfigPaths())
else:
if verbose:
print >> sys.stderr, (
"Using logging configuration file: %s") % (configFilePath)
# This dict will hold our replacement strings for logging configuration
replacements = dict()
def makeKey(name):
""" Makes replacement key """
return "$$%s$$" % (name)
platform = sys.platform.lower()
if platform.startswith('java'):
# Jython
import java.lang
platform = java.lang.System.getProperty("os.name").lower()
if platform.startswith('mac os x'):
platform = 'darwin'
if platform.startswith('darwin'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
elif platform.startswith('linux'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
else:
raise RuntimeError("This platform is neither darwin nor linux: %s" % (
sys.platform,))
if False: #os.path.isdir('/var/log/numenta/nupic'):
# NOTE: Not using syslogHandler for now because it either truncates or
# drops messages over ~1,400 bytes (depending on platform)
# Nupic logs go to syslog. Also, SysLogHandler raises an exception
# on jython (at least on 2.5.2): "AttributeError: 'module' object has no
# attribute 'AF_UNIX'" (jython is used by a sub-moduleof
# ClientJobManager)
replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'syslogHandler'
else:
# Nupic logs go to file
replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'
# Set up log file path for the default file handler
logFilePath = _genLoggingFilePath()
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)
# Set up root logger
replacements[makeKey('ROOT_LOGGER_HANDLERS')] = (
replacements[makeKey('PERSISTENT_LOG_HANDLER')])
if console is not None:
replacements[makeKey('ROOT_LOGGER_HANDLERS')] += (
',' + consoleStreamMappings[console])
# Set up log level for console handlers
replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel
customConfig = StringIO()
with open(configFilePath) as src:
for lineNum, line in enumerate(src):
if "$$" in line:
for (key, value) in replacements.items():
line = line.replace(key, value)
# If there is still a replacement string in the line, we're missing it
# from our replacements dict
if "$$" in line and "$$<key>$$" not in line:
raise RuntimeError(("The text %r, found at line #%d of file %r, "
"contains a string not found in our replacement "
"dict.") % (line, lineNum, configFilePath))
customConfig.write(line)
customConfig.seek(0)
if version.StrictVersion(python_version()) >= version.StrictVersion('2.6'):
# NOTE: the disable_existing_loggers arg is new as of python 2.6, so it's
# not supported on our jython interperter, which was v2.5.x as of this
# writing
logging.config.fileConfig(customConfig, disable_existing_loggers=False)
else:
logging.config.fileConfig(customConfig)
gLoggingInitialized = True
# Add unmatched remaining properties to custom config store
for propertyName, value in copyOfProperties.iteritems():
newProp = ElementTree.Element('property')
nameTag = ElementTree.Element('name')
nameTag.text = propertyName
newProp.append(nameTag)
valueTag = ElementTree.Element('value')
valueTag.text = str(value)
newProp.append(valueTag)
elements.append(newProp)
try:
makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
with open(configFilePath, 'w') as fp:
fp.write(ElementTree.tostring(elements))
except Exception, e:
_getLogger().exception(
"Error while saving custom configuration "
"properties %s in %s.", properties, configFilePath)
raise
@classmethod
def _setPath(cls):
""" Sets the path of the custom configuration file
"""
cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
cls.customFileName)
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
"""
Initilize NuPic logging by reading in from the logging configuration file. The
logging configuration file is named 'nupic-logging.conf' and is expected to be
in the format defined by the python logging module.
If the environment variable 'NTA_CONF_PATH' is defined, then the logging
configuration file is expected to be in the NTA_CONF_PATH directory. If
NTA_CONF_PATH is not defined, then it is found in the 'conf/default'
subdirectory of the NuPic installation directory (typically
~/nupic/current/conf/default)
The logging configuration file can use the environment variable 'NTA_LOG_DIR'
to set the locations of log files. If this variable is not defined already in
the environment, this method will set it to the 'logs' subdirectory of the
NuPic install directory (typically ~/nupic/eng/logs) before loading in the
configuration file.
console: Defines console output for the default "root" logging
configuration; this may be one of 'stdout', 'stderr', or None;
Use None to suppress console logging output
consoleLevel:
Logging-level filter string for console output corresponding to
logging levels in the logging module; may be one of:
'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
E.g., a value of'WARNING' suppresses DEBUG and INFO level output
to console, but allows WARNING, ERROR, and CRITICAL
"""
# NOTE: If you call this twice from the same process there seems to be a
# bug - logged messages don't show up for loggers that you do another
# logging.getLogger() on.
global gLoggingInitialized
if gLoggingInitialized:
if verbose:
print >> sys.stderr, "Logging already initialized, doing nothing."
return
consoleStreamMappings = {
'stdout' : 'stdoutConsoleHandler',
'stderr' : 'stderrConsoleHandler',
}
consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',
'FATAL']
assert console is None or console in consoleStreamMappings.keys(), (
'Unexpected console arg value: %r') % (console,)
assert consoleLevel in consoleLogLevels, (
'Unexpected consoleLevel arg value: %r') % (consoleLevel)
# -----------------------------------------------------------------------
# Setup logging. Look for the nupic-logging.conf file, first in the
# NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
# module
configFilename = 'nupic-logging.conf'
configFilePath = resource_filename("nupic.support", configFilename);
# If NTA_LOG_DIR is not defined, set it now. This is used by the logging
# config file to set the path for the log files
if 'NTA_LOG_DIR' not in os.environ:
os.environ['NTA_LOG_DIR'] = os.path.join(os.environ['NUPIC'], 'logs')
if not os.path.exists(os.environ['NTA_LOG_DIR']):
makeDirectoryFromAbsolutePath(os.path.abspath(os.environ['NTA_LOG_DIR']))
# Load in the logging configuration file
if verbose:
print >> sys.stderr, (
"Using logging configuration file: %s") % (configFilePath)
# This dict will hold our replacement strings for logging configuration
replacements = dict()
def makeKey(name):
""" Makes replacement key """
return "$$%s$$" % (name)
platform = sys.platform.lower()
if platform.startswith('java'):
# Jython
import java.lang
platform = java.lang.System.getProperty("os.name").lower()
if platform.startswith('mac os x'):
platform = 'darwin'
if platform.startswith('darwin'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
elif platform.startswith('linux'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
else:
raise RuntimeError("This platform is neither darwin nor linux: %s" % (
sys.platform,))
# Nupic logs go to file
replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'
# Set up log file path for the default file handler
logFilePath = _genLoggingFilePath()
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)
# Set up root logger
replacements[makeKey('ROOT_LOGGER_HANDLERS')] = (
replacements[makeKey('PERSISTENT_LOG_HANDLER')])
if console is not None:
replacements[makeKey('ROOT_LOGGER_HANDLERS')] += (
',' + consoleStreamMappings[console])
# Set up log level for console handlers
replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel
customConfig = StringIO()
# Using pkg_resources to get the logging file, which should be packaged and
# associated with this source file name.
loggingFileContents = resource_string(__name__, configFilename)
for lineNum, line in enumerate(loggingFileContents.splitlines()):
if "$$" in line:
for (key, value) in replacements.items():
line = line.replace(key, value)
# If there is still a replacement string in the line, we're missing it
# from our replacements dict
if "$$" in line and "$$<key>$$" not in line:
raise RuntimeError(("The text %r, found at line #%d of file %r, "
"contains a string not found in our replacement "
"dict.") % (line, lineNum, configFilePath))
customConfig.write("%s\n" % line)
customConfig.seek(0)
if python_version()[:3] >= '2.6':
logging.config.fileConfig(customConfig, disable_existing_loggers=False)
else:
logging.config.fileConfig(customConfig)
gLoggingInitialized = True
def initLogging(verbose=False, console="stdout", consoleLevel="DEBUG"):
"""
Initilize NuPic logging by reading in from the logging configuration file. The
logging configuration file is named 'nupic-logging.conf' and is expected to be
in the format defined by the python logging module.
If the environment variable 'NTA_CONF_PATH' is defined, then the logging
configuration file is expected to be in the NTA_CONF_PATH directory. If
NTA_CONF_PATH is not defined, then it is found in the 'conf/default'
subdirectory of the NuPic installation directory (typically
~/nupic/current/conf/default)
The logging configuration file can use the environment variable 'NTA_LOG_DIR'
to set the locations of log files. If this variable is not defined, logging to
files will be disabled.
console: Defines console output for the default "root" logging
configuration; this may be one of 'stdout', 'stderr', or None;
Use None to suppress console logging output
consoleLevel:
Logging-level filter string for console output corresponding to
logging levels in the logging module; may be one of:
'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
E.g., a value of'WARNING' suppresses DEBUG and INFO level output
to console, but allows WARNING, ERROR, and CRITICAL
"""
# NOTE: If you call this twice from the same process there seems to be a
# bug - logged messages don't show up for loggers that you do another
# logging.getLogger() on.
global gLoggingInitialized
if gLoggingInitialized:
if verbose:
print >>sys.stderr, "Logging already initialized, doing nothing."
return
consoleStreamMappings = {"stdout": "stdoutConsoleHandler", "stderr": "stderrConsoleHandler"}
consoleLogLevels = ["DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"]
assert console is None or console in consoleStreamMappings.keys(), ("Unexpected console arg value: %r") % (console,)
assert consoleLevel in consoleLogLevels, ("Unexpected consoleLevel arg value: %r") % (consoleLevel)
# -----------------------------------------------------------------------
# Setup logging. Look for the nupic-logging.conf file, first in the
# NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
# module
configFilename = "nupic-logging.conf"
configFilePath = resource_filename("nupic.support", configFilename)
configLogDir = os.environ.get("NTA_LOG_DIR", None)
# Load in the logging configuration file
if verbose:
print >>sys.stderr, ("Using logging configuration file: %s") % (configFilePath)
# This dict will hold our replacement strings for logging configuration
replacements = dict()
def makeKey(name):
""" Makes replacement key """
return "$$%s$$" % (name)
platform = sys.platform.lower()
if platform.startswith("java"):
# Jython
import java.lang
platform = java.lang.System.getProperty("os.name").lower()
if platform.startswith("mac os x"):
platform = "darwin"
if platform.startswith("darwin"):
replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/var/run/syslog"'
elif platform.startswith("linux"):
replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"/dev/log"'
elif platform.startswith("win"):
replacements[makeKey("SYSLOG_HANDLER_ADDRESS")] = '"log"'
else:
raise RuntimeError("This platform is neither darwin, win32, nor linux: %s" % (sys.platform,))
# Nupic logs go to file
replacements[makeKey("PERSISTENT_LOG_HANDLER")] = "fileHandler"
if platform.startswith("win"):
replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"NUL"'
else:
replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = '"/dev/null"'
# Set up log file path for the default file handler and configure handlers
handlers = list()
if configLogDir is not None:
logFilePath = _genLoggingFilePath()
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
replacements[makeKey("FILE_HANDLER_LOG_FILENAME")] = repr(logFilePath)
handlers.append(replacements[makeKey("PERSISTENT_LOG_HANDLER")])
if console is not None:
handlers.append(consoleStreamMappings[console])
replacements[makeKey("ROOT_LOGGER_HANDLERS")] = ", ".join(handlers)
# Set up log level for console handlers
replacements[makeKey("CONSOLE_LOG_LEVEL")] = consoleLevel
customConfig = StringIO()
# Using pkg_resources to get the logging file, which should be packaged and
# associated with this source file name.
loggingFileContents = resource_string(__name__, configFilename)
for lineNum, line in enumerate(loggingFileContents.splitlines()):
if "$$" in line:
for (key, value) in replacements.items():
line = line.replace(key, value)
# If there is still a replacement string in the line, we're missing it
# from our replacements dict
if "$$" in line and "$$<key>$$" not in line:
raise RuntimeError(
(
"The text %r, found at line #%d of file %r, "
"contains a string not found in our replacement "
"dict."
)
% (line, lineNum, configFilePath)
)
customConfig.write("%s\n" % line)
customConfig.seek(0)
if python_version()[:3] >= "2.6":
logging.config.fileConfig(customConfig, disable_existing_loggers=False)
else:
logging.config.fileConfig(customConfig)
gLoggingInitialized = True
# Add unmatched remaining properties to custom config store
for propertyName, value in copyOfProperties.iteritems():
newProp = ElementTree.Element('property')
nameTag = ElementTree.Element('name')
nameTag.text = propertyName
newProp.append(nameTag)
valueTag = ElementTree.Element('value')
valueTag.text = str(value)
newProp.append(valueTag)
elements.append(newProp)
try:
makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
with open(configFilePath,'w') as fp:
fp.write(ElementTree.tostring(elements))
except Exception, e:
_getLogger().exception("Error while saving custom configuration "
"properties %s in %s.", properties, configFilePath)
raise
@classmethod
def _setPath(cls):
""" Sets the path of the custom configuration file
"""
cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
cls.customFileName)