def save(self): """ Save the current override-only configuration to the override config file :raises TypeError: if mode != MODE_OVERRIDE_ONLY """ if self._mode != self.MODE_OVERRIDE_ONLY: raise TypeError( "Config=%s was not loaded in MODE_OVERRIDE_ONLY; mode=%s" % (self._configName, self._mode)) configOverrideDir = self._getConfigOverrideDir() makeDirectoryFromAbsolutePath(configOverrideDir) overrideConfigPath = os.path.join(configOverrideDir, self._configName) # NOTE: we open with os.O_RDWR | os.O_CREAT so that we can acquire the # file lock before altering contents of the file with os.fdopen( os.open(overrideConfigPath, os.O_RDWR | os.O_CREAT, 0644), "w") as fileObj: with file_lock.ExclusiveFileLock(fileObj): self.write(fileObj) fileObj.flush() fileObj.truncate()
def define(self, modelID, definition): """ Define a new model in model checkpoint archive. :param modelID: unique model ID hex string :param definition: jsonifiable python object containing model definition parameters. raises: ModelAlreadyExists if an entry already exists for the given modelID in the checkpoint archive """ startTime = time.time() modelEntryDirPath = self._getModelDir(modelID, mustExist=False) if os.path.exists(modelEntryDirPath): raise ModelAlreadyExists( "Model archive entry already exists for model=%s at path=%s" % (modelID, modelEntryDirPath,)) # Create the model entry subtree in a temp directory first, then rename it # to its permanent location in the model archive for integrity tempRoot = tempfile.mkdtemp(prefix=modelID, dir=self._scratchDir) try: tempModelEntryDirPath = os.path.join(tempRoot, modelID) makeDirectoryFromAbsolutePath(tempModelEntryDirPath) # Create the model entry version file verFilePath = os.path.join(tempModelEntryDirPath, self._MODEL_ENTRY_VERSION_FILE_NAME) with open(verFilePath, "wb") as fileObj: fileObj.write(self._MODEL_ENTRY_VERSION) # Create the model definition file definitionFilePath = os.path.join(tempModelEntryDirPath, self._MODEL_DEFINITION_FILE_NAME) with open(definitionFilePath, "wb") as fileObj: json.dump(definition, fileObj) # Get temp model entry tree in consistent state self._fsyncDirectoryTreeRecursively(tempModelEntryDirPath) # Atomically rename the temp model entry dir as the actual model entry dir os.rename(tempModelEntryDirPath, modelEntryDirPath) # Get checkpoint storage root directory into consistent state self._fsyncDirectoryOnly(self._storageRoot) finally: # Clean up shutil.rmtree(tempRoot) self._logger.info( "{TAG:MCKPT.DEFINE} " "Created entry for model=%s: duration=%ss; directory=%s", modelID, time.time() - startTime, modelEntryDirPath)
def __init__(self): self._logger = _getLogger() # Get the directory in which to save/load checkpoints self._storageRoot = self._getStorageRoot() self._logger.debug("Using storage root=%s", self._storageRoot) if not os.path.exists(self._storageRoot): makeDirectoryFromAbsolutePath(self._storageRoot) self._scratchDir = os.path.join(self._storageRoot, self._SCRATCH_DIR_NAME) if not os.path.exists(self._scratchDir): makeDirectoryFromAbsolutePath(self._scratchDir)
def save(self): """ Save the current override-only configuration to the override config file :raises TypeError: if mode != MODE_OVERRIDE_ONLY """ if self._mode != self.MODE_OVERRIDE_ONLY: raise TypeError("Config=%s was not loaded in MODE_OVERRIDE_ONLY; mode=%s" % (self._configName, self._mode)) configOverrideDir = self._getConfigOverrideDir() makeDirectoryFromAbsolutePath(configOverrideDir) overrideConfigPath = os.path.join(configOverrideDir, self._configName) # NOTE: we open with os.O_RDWR | os.O_CREAT so that we can acquire the # file lock before altering contents of the file with os.fdopen(os.open(overrideConfigPath, os.O_RDWR | os.O_CREAT, 0644), "w") as fileObj: with file_lock.ExclusiveFileLock(fileObj): self.write(fileObj) fileObj.flush() fileObj.truncate()
def initLogging(cls, loggingLevel=None, console="stderr", logToFile=False): """ A lower-level function to initialize python logging for the calling process. Supports logging output to a console (stderr or stdout) and/or log file. See also higher-level functions initTool() and initService(). NOTE: by convention, logging should be initialized only by the main process. modules that provide APIs should not initialize logging as it would clobber the logging configuration desired by the application. :param loggingLevel: logging level string for filtering in root logger and output handlers; one of: "DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL" or "FATAL" that correspond to logging.DEBUG, logging.INFO, etc. Defaults to "INFO". :param console: Console logging destination; either "stderr" or "stdout"; None to suppress output to console. :param logToFile: True to output logs to a file. If enalbed, a log file specific to the calling app instance will be created at file path generated by our getApplicationLogFilePath method. """ validLoggingLevels = ["DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL"] if loggingLevel is not None and loggingLevel not in validLoggingLevels: raise ValueError("loggingLevel %r not one of %s" % (loggingLevel, validLoggingLevels)) consoleHandlerArgsMap = dict( stderr="(sys.stderr, )", stdout="(sys.stdout, )" ) if console is not None and console not in consoleHandlerArgsMap: raise ValueError("console %r not one of %s" % (console, consoleHandlerArgsMap.keys())) # Configure logging timestamp for UTC logging.Formatter.converter = time.gmtime # Load the config tempalte config = ConfigParser() with open(cls.getLoggingConfTemplatePath(), 'r') as fileObj: config.readfp(fileObj) # Customize the config template handlers = [] if console is not None: handlers.append("console") if loggingLevel is not None: config.set("handler_console", "level", loggingLevel) config.set("handler_console", "args", consoleHandlerArgsMap[console]) if logToFile: handlers.append("file") # Get a log file path specific to the calling app logFilePath = cls.getApplicationLogFilePath() # Create the directory that will contain the log file makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath)) if loggingLevel is not None: config.set("handler_file", "level", loggingLevel) config.set("handler_file", "filename", logFilePath) if not handlers: print >> sys.stderr, ( "WARNING: logging_support is disabling logging output because all " "output handlers are disabled") handlers.append("null") # Convert list of logging output handler names into comma-separated string handlers = ",".join(handlers) # Initialize the root logger if loggingLevel is not None: config.set("logger_root", "level", loggingLevel) config.set("logger_root", "handlers", handlers) # Initialize the list of all logging output handlers config.set("handlers", "keys", handlers) # Dump the customized config into a StringIO object for logging setup customConfigFile = StringIO() config.write(customConfigFile) customConfigFile.seek(0) # Initialize logging from StringIO file object logging.config.fileConfig(customConfigFile, disable_existing_loggers=False)
def initLogging(cls, loggingLevel=None, console="stderr", logToFile=False): """ A lower-level function to initialize python logging for the calling process. Supports logging output to a console (stderr or stdout) and/or log file. See also higher-level functions initTool() and initService(). NOTE: by convention, logging should be initialized only by the main process. modules that provide APIs should not initialize logging as it would clobber the logging configuration desired by the application. :param loggingLevel: logging level string for filtering in root logger and output handlers; one of: "DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL" or "FATAL" that correspond to logging.DEBUG, logging.INFO, etc. Defaults to "INFO". :param console: Console logging destination; either "stderr" or "stdout"; None to suppress output to console. :param logToFile: True to output logs to a file. If enalbed, a log file specific to the calling app instance will be created at file path generated by our getApplicationLogFilePath method. """ validLoggingLevels = [ "DEBUG", "INFO", "WARNING", "WARN", "ERROR", "CRITICAL", "FATAL" ] if loggingLevel is not None and loggingLevel not in validLoggingLevels: raise ValueError("loggingLevel %r not one of %s" % (loggingLevel, validLoggingLevels)) consoleHandlerArgsMap = dict(stderr="(sys.stderr, )", stdout="(sys.stdout, )") if console is not None and console not in consoleHandlerArgsMap: raise ValueError("console %r not one of %s" % (console, consoleHandlerArgsMap.keys())) # Configure logging timestamp for UTC logging.Formatter.converter = time.gmtime # Load the config tempalte config = ConfigParser() with open(cls.getLoggingConfTemplatePath(), 'r') as fileObj: config.readfp(fileObj) # Customize the config template handlers = [] if console is not None: handlers.append("console") if loggingLevel is not None: config.set("handler_console", "level", loggingLevel) config.set("handler_console", "args", consoleHandlerArgsMap[console]) if logToFile: handlers.append("file") # Get a log file path specific to the calling app logFilePath = cls.getApplicationLogFilePath() # Create the directory that will contain the log file makeDirectoryFromAbsolutePath(os.path.dirname(logFilePath)) if loggingLevel is not None: config.set("handler_file", "level", loggingLevel) config.set("handler_file", "filename", logFilePath) if not handlers: print >> sys.stderr, ( "WARNING: logging_support is disabling logging output because all " "output handlers are disabled") handlers.append("null") # Convert list of logging output handler names into comma-separated string handlers = ",".join(handlers) # Initialize the root logger if loggingLevel is not None: config.set("logger_root", "level", loggingLevel) config.set("logger_root", "handlers", handlers) # Initialize the list of all logging output handlers config.set("handlers", "keys", handlers) # Dump the customized config into a StringIO object for logging setup customConfigFile = StringIO() config.write(customConfigFile) customConfigFile.seek(0) # Initialize logging from StringIO file object logging.config.fileConfig(customConfigFile, disable_existing_loggers=False)
def save(self, modelID, model, attributes): """ Checkpoint a model instance. :param modelID: unique model ID hex string :param model: An OPF model instance object. This represents an instantiated model that may have processed records and accumulated learning state. :param attributes: checkpoint attributes; a JSONifiable object to save as an integral component of the checkpoint. It may later be retrieved separately via ModelCheckpointMgr.loadCheckpointAttributes() :raises: ModelNotFound if model's entry doesn't exit in the checkpoint archive """ startTime = time.time() modelEntryDirPath = self._getModelDir(modelID, mustExist=True) # Create the model checkpoint store in a temp directory first, then rename # it to its location in the model entry for integrity tempRoot = tempfile.mkdtemp(prefix=modelID, dir=self._scratchDir) try: tempCheckpointStoreDirPath = os.path.join( tempRoot, self._CHECKPOINT_STORE_DIR_NAME_BASE) makeDirectoryFromAbsolutePath(tempCheckpointStoreDirPath) # Save the checkpoint attributes attributesFilePath = os.path.join( tempCheckpointStoreDirPath, self._CHECKPOINT_ATTRIBUTES_FILE_NAME) with open(attributesFilePath, "wb") as fileObj: json.dump(attributes, fileObj) # Save the model model.save( saveModelDir=os.path.join( tempCheckpointStoreDirPath, self._CHECKPOINT_INSTANCE_DIR_NAME)) # Get temp checkpoint store tree in consistent state self._fsyncDirectoryTreeRecursively(tempCheckpointStoreDirPath) # Atomically rename the temp checkpoint store dir into model entry dir newCheckpointStoreDirPath = os.path.join( modelEntryDirPath, "%s%f" % (self._CHECKPOINT_STORE_DIR_NAME_BASE, time.time())) assert not os.path.exists(newCheckpointStoreDirPath), ( newCheckpointStoreDirPath) os.rename(tempCheckpointStoreDirPath, newCheckpointStoreDirPath) # Prepare to switch the current checkpoint link currentStoreSymlinkPath = os.path.join( modelEntryDirPath, self._CHECKPOINT_LINK_NAME) # But first, capture the real path of the old checkpoint store, so we can # delete it later if os.path.exists(currentStoreSymlinkPath): oldCheckpointStoreDirPath = os.path.realpath(currentStoreSymlinkPath) assert oldCheckpointStoreDirPath != currentStoreSymlinkPath, ( oldCheckpointStoreDirPath) else: oldCheckpointStoreDirPath = None # Atomically point currentStoreSymlinkPath to # newCheckpointStoreDirPath tempCurrentStoreSymlinkPath = os.path.join( tempRoot, self._CHECKPOINT_LINK_NAME) os.symlink(newCheckpointStoreDirPath, tempCurrentStoreSymlinkPath) os.rename(tempCurrentStoreSymlinkPath, currentStoreSymlinkPath) # Sync the model entry directory to ensure consistency # NOTE: we do this before deleting the old checkpoint store to protect # current checkpoint integrity in the event of failure while deleting the # old one. self._fsyncDirectoryOnly(modelEntryDirPath) # Lastly, remove the old checkpoint store dir if oldCheckpointStoreDirPath is not None: shutil.rmtree(oldCheckpointStoreDirPath) finally: # Clean up shutil.rmtree(tempRoot) self._logger.info( "{TAG:MCKPT.SAVE} Saved model=%s: duration=%ss; directory=%s", modelID, time.time() - startTime, newCheckpointStoreDirPath)