def __call__(self, htmPredictionModel):
import pickle
# Get the TM args dictionary
assert isinstance(htmPredictionModel, HTMPredictionModel)
tpRegion = htmPredictionModel._getTPRegion().getSelf()
tfdr = tpRegion._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
def __call__(self, htmPredictionModel):
import pickle
# Get the TM args dictionary
assert isinstance(htmPredictionModel, HTMPredictionModel)
tpRegion = htmPredictionModel._getTPRegion().getSelf()
tfdr = tpRegion._tfdr
initArgsDict = tfdr._initArgsDict
# Write it out to a file as json
absFilePath = os.path.abspath(self.__filePath)
absDir = os.path.dirname(absFilePath)
makeDirectoryFromAbsolutePath(absDir)
with open(absFilePath, 'wb') as pickleFile:
pickle.dump(initArgsDict, pickleFile)
return
# Add unmatched remaining properties to custom config store
for propertyName, value in copyOfProperties.iteritems():
newProp = ElementTree.Element('property')
nameTag = ElementTree.Element('name')
nameTag.text = propertyName
newProp.append(nameTag)
valueTag = ElementTree.Element('value')
valueTag.text = str(value)
newProp.append(valueTag)
elements.append(newProp)
try:
makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
with open(configFilePath,'w') as fp:
fp.write(ElementTree.tostring(elements))
except Exception, e:
_getLogger().exception("Error while saving custom configuration "
"properties %s in %s.", properties, configFilePath)
raise
@classmethod
def _setPath(cls):
""" Sets the path of the custom configuration file
"""
cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
cls.customFileName)
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
"""
Initilize NuPic logging by reading in from the logging configuration file. The
logging configuration file is named ``nupic-logging.conf`` and is expected to
be in the format defined by the python logging module.
If the environment variable ``NTA_CONF_PATH`` is defined, then the logging
configuration file is expected to be in the ``NTA_CONF_PATH`` directory. If
``NTA_CONF_PATH`` is not defined, then it is found in the 'conf/default'
subdirectory of the NuPic installation directory (typically
~/nupic/current/conf/default)
The logging configuration file can use the environment variable
``NTA_LOG_DIR`` to set the locations of log files. If this variable is not
defined, logging to files will be disabled.
:param console: Defines console output for the default "root" logging
configuration; this may be one of 'stdout', 'stderr', or None;
Use None to suppress console logging output
:param consoleLevel:
Logging-level filter string for console output corresponding to
logging levels in the logging module; may be one of:
'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
E.g., a value of'WARNING' suppresses DEBUG and INFO level output
to console, but allows WARNING, ERROR, and CRITICAL
"""
# NOTE: If you call this twice from the same process there seems to be a
# bug - logged messages don't show up for loggers that you do another
# logging.getLogger() on.
global gLoggingInitialized
if gLoggingInitialized:
if verbose:
print("Logging already initialized, doing nothing.",
file=sys.stderr)
return
consoleStreamMappings = {
'stdout': 'stdoutConsoleHandler',
'stderr': 'stderrConsoleHandler',
}
consoleLogLevels = [
'DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL', 'FATAL'
]
assert console is None or console in list(consoleStreamMappings.keys()), (
'Unexpected console arg value: %r') % (console, )
assert consoleLevel in consoleLogLevels, (
'Unexpected consoleLevel arg value: %r') % (consoleLevel)
# -----------------------------------------------------------------------
# Setup logging. Look for the nupic-logging.conf file, first in the
# NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
# module
configFilename = 'nupic-logging.conf'
configFilePath = resource_filename("nupic.support", configFilename)
configLogDir = os.environ.get('NTA_LOG_DIR', None)
# Load in the logging configuration file
if verbose:
print(("Using logging configuration file: %s") % (configFilePath),
file=sys.stderr)
# This dict will hold our replacement strings for logging configuration
replacements = dict()
def makeKey(name):
""" Makes replacement key """
return "$$%s$$" % (name)
platform = sys.platform.lower()
if platform.startswith('java'):
# Jython
import java.lang
platform = java.lang.System.getProperty("os.name").lower()
if platform.startswith('mac os x'):
platform = 'darwin'
if platform.startswith('darwin'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
elif platform.startswith('linux'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
elif platform.startswith('win'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"log"'
else:
raise RuntimeError(
"This platform is neither darwin, win32, nor linux: %s" %
(sys.platform, ))
# Nupic logs go to file
replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'
if platform.startswith('win'):
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '"NUL"'
else:
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '"/dev/null"'
# Set up log file path for the default file handler and configure handlers
handlers = list()
if configLogDir is not None:
logFilePath = _genLoggingFilePath()
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)
handlers.append(replacements[makeKey('PERSISTENT_LOG_HANDLER')])
if console is not None:
handlers.append(consoleStreamMappings[console])
replacements[makeKey('ROOT_LOGGER_HANDLERS')] = ", ".join(handlers)
# Set up log level for console handlers
replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel
customConfig = StringIO()
# Using pkg_resources to get the logging file, which should be packaged and
# associated with this source file name.
loggingFileContents = resource_string(__name__, configFilename)
for lineNum, line in enumerate(loggingFileContents.splitlines()):
if "$$" in line:
for (key, value) in list(replacements.items()):
line = line.replace(key, value)
# If there is still a replacement string in the line, we're missing it
# from our replacements dict
if "$$" in line and "$$<key>$$" not in line:
raise RuntimeError(
("The text %r, found at line #%d of file %r, "
"contains a string not found in our replacement "
"dict.") % (line, lineNum, configFilePath))
customConfig.write("%s\n" % line)
customConfig.seek(0)
if python_version()[:3] >= '2.6':
logging.config.fileConfig(customConfig, disable_existing_loggers=False)
else:
logging.config.fileConfig(customConfig)
gLoggingInitialized = True
def edit(cls, properties):
""" Edits the XML configuration file with the parameters specified by
properties
Parameters:
----------------------------------------------------------------
properties: dict of settings to be applied to the custom configuration store
(key is property name, value is value)
"""
copyOfProperties = copy(properties)
configFilePath = cls.getPath()
try:
with open(configFilePath, 'r') as fp:
contents = fp.read()
except IOError as e:
if e.errno != errno.ENOENT:
_getLogger().exception(
"Error %s reading custom configuration store "
"from %s, while editing properties %s.", e.errno,
configFilePath, properties)
raise
contents = '<configuration/>'
try:
elements = ElementTree.XML(contents)
ElementTree.tostring(elements)
except Exception as e:
# Raising error as RuntimeError with custom message since ElementTree
# exceptions aren't clear.
msg = "File contents of custom configuration is corrupt. File " \
"location: %s; Contents: '%s'. Original Error (%s): %s." % \
(configFilePath, contents, type(e), e)
_getLogger().exception(msg)
raise RuntimeError(msg).with_traceback(sys.exc_info()[2])
if elements.tag != 'configuration':
e = "Expected top-level element to be 'configuration' but got '%s'" % \
(elements.tag)
_getLogger().error(e)
raise RuntimeError(e)
# Apply new properties to matching settings in the custom config store;
# pop matching properties from our copy of the properties dict
for propertyItem in elements.findall('./property'):
propInfo = dict((attr.tag, attr.text) for attr in propertyItem)
name = propInfo['name']
if name in copyOfProperties:
foundValues = propertyItem.findall('./value')
if len(foundValues) > 0:
foundValues[0].text = str(copyOfProperties.pop(name))
if not copyOfProperties:
break
else:
e = "Property %s missing value tag." % (name, )
_getLogger().error(e)
raise RuntimeError(e)
# Add unmatched remaining properties to custom config store
for propertyName, value in copyOfProperties.items():
newProp = ElementTree.Element('property')
nameTag = ElementTree.Element('name')
nameTag.text = propertyName
newProp.append(nameTag)
valueTag = ElementTree.Element('value')
valueTag.text = str(value)
newProp.append(valueTag)
elements.append(newProp)
try:
makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
with open(configFilePath, 'w') as fp:
fp.write(ElementTree.tostring(elements))
except Exception as e:
_getLogger().exception(
"Error while saving custom configuration "
"properties %s in %s.", properties, configFilePath)
raise
# Add unmatched remaining properties to custom config store
for propertyName, value in copyOfProperties.iteritems():
newProp = ElementTree.Element('property')
nameTag = ElementTree.Element('name')
nameTag.text = propertyName
newProp.append(nameTag)
valueTag = ElementTree.Element('value')
valueTag.text = str(value)
newProp.append(valueTag)
elements.append(newProp)
try:
makeDirectoryFromAbsolutePath(os.path.dirname(configFilePath))
with open(configFilePath, 'w') as fp:
fp.write(ElementTree.tostring(elements))
except Exception, e:
_getLogger().exception(
"Error while saving custom configuration "
"properties %s in %s.", properties, configFilePath)
raise
@classmethod
def _setPath(cls):
""" Sets the path of the custom configuration file
"""
cls._path = os.path.join(os.environ['NTA_DYNAMIC_CONF_DIR'],
cls.customFileName)
def initLogging(verbose=False, console='stdout', consoleLevel='DEBUG'):
"""
Initilize NuPic logging by reading in from the logging configuration file. The
logging configuration file is named ``nupic-logging.conf`` and is expected to
be in the format defined by the python logging module.
If the environment variable ``NTA_CONF_PATH`` is defined, then the logging
configuration file is expected to be in the ``NTA_CONF_PATH`` directory. If
``NTA_CONF_PATH`` is not defined, then it is found in the 'conf/default'
subdirectory of the NuPic installation directory (typically
~/nupic/current/conf/default)
The logging configuration file can use the environment variable
``NTA_LOG_DIR`` to set the locations of log files. If this variable is not
defined, logging to files will be disabled.
:param console: Defines console output for the default "root" logging
configuration; this may be one of 'stdout', 'stderr', or None;
Use None to suppress console logging output
:param consoleLevel:
Logging-level filter string for console output corresponding to
logging levels in the logging module; may be one of:
'DEBUG', 'INFO', 'WARNING', 'ERROR', or 'CRITICAL'.
E.g., a value of'WARNING' suppresses DEBUG and INFO level output
to console, but allows WARNING, ERROR, and CRITICAL
"""
# NOTE: If you call this twice from the same process there seems to be a
# bug - logged messages don't show up for loggers that you do another
# logging.getLogger() on.
global gLoggingInitialized
if gLoggingInitialized:
if verbose:
print >> sys.stderr, "Logging already initialized, doing nothing."
return
consoleStreamMappings = {
'stdout' : 'stdoutConsoleHandler',
'stderr' : 'stderrConsoleHandler',
}
consoleLogLevels = ['DEBUG', 'INFO', 'WARNING', 'WARN', 'ERROR', 'CRITICAL',
'FATAL']
assert console is None or console in consoleStreamMappings.keys(), (
'Unexpected console arg value: %r') % (console,)
assert consoleLevel in consoleLogLevels, (
'Unexpected consoleLevel arg value: %r') % (consoleLevel)
# -----------------------------------------------------------------------
# Setup logging. Look for the nupic-logging.conf file, first in the
# NTA_CONFIG_DIR path (if defined), then in a subdirectory of the nupic
# module
configFilename = 'nupic-logging.conf'
configFilePath = resource_filename("nupic.support", configFilename)
configLogDir = os.environ.get('NTA_LOG_DIR', None)
# Load in the logging configuration file
if verbose:
print >> sys.stderr, (
"Using logging configuration file: %s") % (configFilePath)
# This dict will hold our replacement strings for logging configuration
replacements = dict()
def makeKey(name):
""" Makes replacement key """
return "$$%s$$" % (name)
platform = sys.platform.lower()
if platform.startswith('java'):
# Jython
import java.lang
platform = java.lang.System.getProperty("os.name").lower()
if platform.startswith('mac os x'):
platform = 'darwin'
if platform.startswith('darwin'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/var/run/syslog"'
elif platform.startswith('linux'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"/dev/log"'
elif platform.startswith('win'):
replacements[makeKey('SYSLOG_HANDLER_ADDRESS')] = '"log"'
else:
raise RuntimeError("This platform is neither darwin, win32, nor linux: %s" % (
sys.platform,))
# Nupic logs go to file
replacements[makeKey('PERSISTENT_LOG_HANDLER')] = 'fileHandler'
if platform.startswith('win'):
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '"NUL"'
else:
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = '"/dev/null"'
# Set up log file path for the default file handler and configure handlers
handlers = list()
if configLogDir is not None:
logFilePath = _genLoggingFilePath()
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
replacements[makeKey('FILE_HANDLER_LOG_FILENAME')] = repr(logFilePath)
handlers.append(replacements[makeKey('PERSISTENT_LOG_HANDLER')])
if console is not None:
handlers.append(consoleStreamMappings[console])
replacements[makeKey('ROOT_LOGGER_HANDLERS')] = ", ".join(handlers)
# Set up log level for console handlers
replacements[makeKey('CONSOLE_LOG_LEVEL')] = consoleLevel
customConfig = StringIO()
# Using pkg_resources to get the logging file, which should be packaged and
# associated with this source file name.
loggingFileContents = resource_string(__name__, configFilename)
for lineNum, line in enumerate(loggingFileContents.splitlines()):
if "$$" in line:
for (key, value) in replacements.items():
line = line.replace(key, value)
# If there is still a replacement string in the line, we're missing it
# from our replacements dict
if "$$" in line and "$$<key>$$" not in line:
raise RuntimeError(("The text %r, found at line #%d of file %r, "
"contains a string not found in our replacement "
"dict.") % (line, lineNum, configFilePath))
customConfig.write("%s\n" % line)
customConfig.seek(0)
if python_version()[:3] >= '2.6':
logging.config.fileConfig(customConfig, disable_existing_loggers=False)
else:
logging.config.fileConfig(customConfig)
gLoggingInitialized = True