def save(self):
""" Save the current override-only configuration to the override config
file
:raises TypeError: if mode != MODE_OVERRIDE_ONLY
"""
if self._mode != self.MODE_OVERRIDE_ONLY:
raise TypeError(
"Config=%s was not loaded in MODE_OVERRIDE_ONLY; mode=%s" %
(self._configName, self._mode))
configOverrideDir = self._getConfigOverrideDir()
makeDirectoryFromAbsolutePath(configOverrideDir)
overrideConfigPath = os.path.join(configOverrideDir, self._configName)
# NOTE: we open with os.O_RDWR | os.O_CREAT so that we can acquire the
# file lock before altering contents of the file
with os.fdopen(
os.open(overrideConfigPath, os.O_RDWR | os.O_CREAT, 0644),
"w") as fileObj:
with file_lock.ExclusiveFileLock(fileObj):
self.write(fileObj)
fileObj.flush()
fileObj.truncate()
def define(self, modelID, definition):
""" Define a new model in model checkpoint archive.
:param modelID: unique model ID hex string
:param definition: jsonifiable python object containing model definition
parameters.
raises:
ModelAlreadyExists if an entry already exists for the given modelID in the
checkpoint archive
"""
startTime = time.time()
modelEntryDirPath = self._getModelDir(modelID, mustExist=False)
if os.path.exists(modelEntryDirPath):
raise ModelAlreadyExists(
"Model archive entry already exists for model=%s at path=%s" %
(modelID, modelEntryDirPath,))
# Create the model entry subtree in a temp directory first, then rename it
# to its permanent location in the model archive for integrity
tempRoot = tempfile.mkdtemp(prefix=modelID, dir=self._scratchDir)
try:
tempModelEntryDirPath = os.path.join(tempRoot, modelID)
makeDirectoryFromAbsolutePath(tempModelEntryDirPath)
# Create the model entry version file
verFilePath = os.path.join(tempModelEntryDirPath,
self._MODEL_ENTRY_VERSION_FILE_NAME)
with open(verFilePath, "wb") as fileObj:
fileObj.write(self._MODEL_ENTRY_VERSION)
# Create the model definition file
definitionFilePath = os.path.join(tempModelEntryDirPath,
self._MODEL_DEFINITION_FILE_NAME)
with open(definitionFilePath, "wb") as fileObj:
json.dump(definition, fileObj)
# Get temp model entry tree in consistent state
self._fsyncDirectoryTreeRecursively(tempModelEntryDirPath)
# Atomically rename the temp model entry dir as the actual model entry dir
os.rename(tempModelEntryDirPath, modelEntryDirPath)
# Get checkpoint storage root directory into consistent state
self._fsyncDirectoryOnly(self._storageRoot)
finally:
# Clean up
shutil.rmtree(tempRoot)
self._logger.info(
"{TAG:MCKPT.DEFINE} "
"Created entry for model=%s: duration=%ss; directory=%s",
modelID, time.time() - startTime, modelEntryDirPath)
def __init__(self):
self._logger = _getLogger()
# Get the directory in which to save/load checkpoints
self._storageRoot = self._getStorageRoot()
self._logger.debug("Using storage root=%s", self._storageRoot)
if not os.path.exists(self._storageRoot):
makeDirectoryFromAbsolutePath(self._storageRoot)
self._scratchDir = os.path.join(self._storageRoot, self._SCRATCH_DIR_NAME)
if not os.path.exists(self._scratchDir):
makeDirectoryFromAbsolutePath(self._scratchDir)
def save(self):
""" Save the current override-only configuration to the override config
file
:raises TypeError: if mode != MODE_OVERRIDE_ONLY
"""
if self._mode != self.MODE_OVERRIDE_ONLY:
raise TypeError("Config=%s was not loaded in MODE_OVERRIDE_ONLY; mode=%s" % (self._configName, self._mode))
configOverrideDir = self._getConfigOverrideDir()
makeDirectoryFromAbsolutePath(configOverrideDir)
overrideConfigPath = os.path.join(configOverrideDir, self._configName)
# NOTE: we open with os.O_RDWR | os.O_CREAT so that we can acquire the
# file lock before altering contents of the file
with os.fdopen(os.open(overrideConfigPath, os.O_RDWR | os.O_CREAT, 0644), "w") as fileObj:
with file_lock.ExclusiveFileLock(fileObj):
self.write(fileObj)
fileObj.flush()
fileObj.truncate()
def initLogging(cls, loggingLevel=None, console="stderr",
logToFile=False):
""" A lower-level function to initialize python logging for the calling
process. Supports logging output to a console (stderr or stdout) and/or log
file.
See also higher-level functions initTool() and initService().
NOTE: by convention, logging should be initialized only by the main process.
modules that provide APIs should not initialize logging as it would
clobber the logging configuration desired by the application.
:param loggingLevel: logging level string for filtering in root logger and
output handlers; one of: "DEBUG", "INFO", "WARNING", "WARN", "ERROR",
"CRITICAL" or "FATAL" that correspond to logging.DEBUG, logging.INFO,
etc. Defaults to "INFO".
:param console: Console logging destination; either "stderr" or "stdout";
None to suppress output to console.
:param logToFile: True to output logs to a file. If enalbed, a log file
specific to the calling app instance will be created at file path
generated by our getApplicationLogFilePath method.
"""
validLoggingLevels = ["DEBUG", "INFO", "WARNING", "WARN", "ERROR",
"CRITICAL", "FATAL"]
if loggingLevel is not None and loggingLevel not in validLoggingLevels:
raise ValueError("loggingLevel %r not one of %s" %
(loggingLevel, validLoggingLevels))
consoleHandlerArgsMap = dict(
stderr="(sys.stderr, )",
stdout="(sys.stdout, )"
)
if console is not None and console not in consoleHandlerArgsMap:
raise ValueError("console %r not one of %s" %
(console, consoleHandlerArgsMap.keys()))
# Configure logging timestamp for UTC
logging.Formatter.converter = time.gmtime
# Load the config tempalte
config = ConfigParser()
with open(cls.getLoggingConfTemplatePath(), 'r') as fileObj:
config.readfp(fileObj)
# Customize the config template
handlers = []
if console is not None:
handlers.append("console")
if loggingLevel is not None:
config.set("handler_console", "level", loggingLevel)
config.set("handler_console", "args", consoleHandlerArgsMap[console])
if logToFile:
handlers.append("file")
# Get a log file path specific to the calling app
logFilePath = cls.getApplicationLogFilePath()
# Create the directory that will contain the log file
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
if loggingLevel is not None:
config.set("handler_file", "level", loggingLevel)
config.set("handler_file", "filename", logFilePath)
if not handlers:
print >> sys.stderr, (
"WARNING: logging_support is disabling logging output because all "
"output handlers are disabled")
handlers.append("null")
# Convert list of logging output handler names into comma-separated string
handlers = ",".join(handlers)
# Initialize the root logger
if loggingLevel is not None:
config.set("logger_root", "level", loggingLevel)
config.set("logger_root", "handlers", handlers)
# Initialize the list of all logging output handlers
config.set("handlers", "keys", handlers)
# Dump the customized config into a StringIO object for logging setup
customConfigFile = StringIO()
config.write(customConfigFile)
customConfigFile.seek(0)
# Initialize logging from StringIO file object
logging.config.fileConfig(customConfigFile, disable_existing_loggers=False)
def initLogging(cls, loggingLevel=None, console="stderr", logToFile=False):
""" A lower-level function to initialize python logging for the calling
process. Supports logging output to a console (stderr or stdout) and/or log
file.
See also higher-level functions initTool() and initService().
NOTE: by convention, logging should be initialized only by the main process.
modules that provide APIs should not initialize logging as it would
clobber the logging configuration desired by the application.
:param loggingLevel: logging level string for filtering in root logger and
output handlers; one of: "DEBUG", "INFO", "WARNING", "WARN", "ERROR",
"CRITICAL" or "FATAL" that correspond to logging.DEBUG, logging.INFO,
etc. Defaults to "INFO".
:param console: Console logging destination; either "stderr" or "stdout";
None to suppress output to console.
:param logToFile: True to output logs to a file. If enalbed, a log file
specific to the calling app instance will be created at file path
generated by our getApplicationLogFilePath method.
"""
validLoggingLevels = [
"DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"
]
if loggingLevel is not None and loggingLevel not in validLoggingLevels:
raise ValueError("loggingLevel %r not one of %s" %
(loggingLevel, validLoggingLevels))
consoleHandlerArgsMap = dict(stderr="(sys.stderr, )",
stdout="(sys.stdout, )")
if console is not None and console not in consoleHandlerArgsMap:
raise ValueError("console %r not one of %s" %
(console, consoleHandlerArgsMap.keys()))
# Configure logging timestamp for UTC
logging.Formatter.converter = time.gmtime
# Load the config tempalte
config = ConfigParser()
with open(cls.getLoggingConfTemplatePath(), 'r') as fileObj:
config.readfp(fileObj)
# Customize the config template
handlers = []
if console is not None:
handlers.append("console")
if loggingLevel is not None:
config.set("handler_console", "level", loggingLevel)
config.set("handler_console", "args",
consoleHandlerArgsMap[console])
if logToFile:
handlers.append("file")
# Get a log file path specific to the calling app
logFilePath = cls.getApplicationLogFilePath()
# Create the directory that will contain the log file
makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath))
if loggingLevel is not None:
config.set("handler_file", "level", loggingLevel)
config.set("handler_file", "filename", logFilePath)
if not handlers:
print >> sys.stderr, (
"WARNING: logging_support is disabling logging output because all "
"output handlers are disabled")
handlers.append("null")
# Convert list of logging output handler names into comma-separated string
handlers = ",".join(handlers)
# Initialize the root logger
if loggingLevel is not None:
config.set("logger_root", "level", loggingLevel)
config.set("logger_root", "handlers", handlers)
# Initialize the list of all logging output handlers
config.set("handlers", "keys", handlers)
# Dump the customized config into a StringIO object for logging setup
customConfigFile = StringIO()
config.write(customConfigFile)
customConfigFile.seek(0)
# Initialize logging from StringIO file object
logging.config.fileConfig(customConfigFile,
disable_existing_loggers=False)
def save(self, modelID, model, attributes):
""" Checkpoint a model instance.
:param modelID: unique model ID hex string
:param model: An OPF model instance object. This represents an instantiated
model that may have processed records and accumulated learning state.
:param attributes: checkpoint attributes; a JSONifiable object to save as an
integral component of the checkpoint. It may later be retrieved separately
via ModelCheckpointMgr.loadCheckpointAttributes()
:raises: ModelNotFound if model's entry doesn't exit in the checkpoint
archive
"""
startTime = time.time()
modelEntryDirPath = self._getModelDir(modelID, mustExist=True)
# Create the model checkpoint store in a temp directory first, then rename
# it to its location in the model entry for integrity
tempRoot = tempfile.mkdtemp(prefix=modelID, dir=self._scratchDir)
try:
tempCheckpointStoreDirPath = os.path.join(
tempRoot,
self._CHECKPOINT_STORE_DIR_NAME_BASE)
makeDirectoryFromAbsolutePath(tempCheckpointStoreDirPath)
# Save the checkpoint attributes
attributesFilePath = os.path.join(
tempCheckpointStoreDirPath,
self._CHECKPOINT_ATTRIBUTES_FILE_NAME)
with open(attributesFilePath, "wb") as fileObj:
json.dump(attributes, fileObj)
# Save the model
model.save(
saveModelDir=os.path.join(
tempCheckpointStoreDirPath,
self._CHECKPOINT_INSTANCE_DIR_NAME))
# Get temp checkpoint store tree in consistent state
self._fsyncDirectoryTreeRecursively(tempCheckpointStoreDirPath)
# Atomically rename the temp checkpoint store dir into model entry dir
newCheckpointStoreDirPath = os.path.join(
modelEntryDirPath,
"%s%f" % (self._CHECKPOINT_STORE_DIR_NAME_BASE, time.time()))
assert not os.path.exists(newCheckpointStoreDirPath), (
newCheckpointStoreDirPath)
os.rename(tempCheckpointStoreDirPath, newCheckpointStoreDirPath)
# Prepare to switch the current checkpoint link
currentStoreSymlinkPath = os.path.join(
modelEntryDirPath,
self._CHECKPOINT_LINK_NAME)
# But first, capture the real path of the old checkpoint store, so we can
# delete it later
if os.path.exists(currentStoreSymlinkPath):
oldCheckpointStoreDirPath = os.path.realpath(currentStoreSymlinkPath)
assert oldCheckpointStoreDirPath != currentStoreSymlinkPath, (
oldCheckpointStoreDirPath)
else:
oldCheckpointStoreDirPath = None
# Atomically point currentStoreSymlinkPath to
# newCheckpointStoreDirPath
tempCurrentStoreSymlinkPath = os.path.join(
tempRoot,
self._CHECKPOINT_LINK_NAME)
os.symlink(newCheckpointStoreDirPath, tempCurrentStoreSymlinkPath)
os.rename(tempCurrentStoreSymlinkPath, currentStoreSymlinkPath)
# Sync the model entry directory to ensure consistency
# NOTE: we do this before deleting the old checkpoint store to protect
# current checkpoint integrity in the event of failure while deleting the
# old one.
self._fsyncDirectoryOnly(modelEntryDirPath)
# Lastly, remove the old checkpoint store dir
if oldCheckpointStoreDirPath is not None:
shutil.rmtree(oldCheckpointStoreDirPath)
finally:
# Clean up
shutil.rmtree(tempRoot)
self._logger.info(
"{TAG:MCKPT.SAVE} Saved model=%s: duration=%ss; directory=%s",
modelID, time.time() - startTime, newCheckpointStoreDirPath)