def _parseConfigTemplate(self, templatePath, cfg=None): """Parse the ConfigTemplate.cfg files. :param str templatePath: path to the folder containing a ConfigTemplate.cfg file :param CFG cfg: cfg to merge with the systems config :returns: CFG object """ cfg = CFG() if cfg is None else cfg system = os.path.split(templatePath.rstrip("/"))[1] if system.lower().endswith('system'): system = system[:-len('System')] if self.systems and system not in self.systems: return S_OK(cfg) templatePath = os.path.join(templatePath, 'ConfigTemplate.cfg') if not os.path.exists(templatePath): return S_ERROR("File not found: %s" % templatePath) loadCfg = CFG() loadCfg.loadFromFile(templatePath) newCfg = CFG() newCfg.createNewSection("/%s" % system, contents=loadCfg) cfg = cfg.mergeWith(newCfg) return S_OK(cfg)
def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal
def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal
def _loadWebAppCFGFiles(self): """ Load EiscatWeb/web.cfg definitions """ exts = [] for ext in CSGlobals.getCSExtensions(): if ext == "DIRAC": continue if ext[-5:] != "DIRAC": ext = "%sDIRAC" % ext if ext == "WebAppDIRAC": continue if ext != "EiscatWebDIRAC": exts.append(ext) exts.append("DIRAC") exts.append("EiscatWebDIRAC") print "exts in loadWebAppCFGFiles of App.py" print exts webCFG = CFG() for modName in reversed(exts): try: modPath = imp.find_module(modName)[1] except ImportError: continue gLogger.verbose("Found module %s at %s" % (modName, modPath)) cfgPath = os.path.join(modPath, "WebApp", "web.cfg") print "cfgPath" print cfgPath if not os.path.isfile(cfgPath): gLogger.verbose("Inexistant %s" % cfgPath) continue try: modCFG = CFG().loadFromFile(cfgPath) except Exception, excp: gLogger.error("Could not load %s: %s" % (cfgPath, excp)) continue gLogger.verbose("Loaded %s" % cfgPath) expl = [Conf.BASECS] while len(expl): current = expl.pop(0) if not modCFG.isSection(current): continue if modCFG.getOption("%s/AbsoluteDefinition" % current, False): gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) try: webCFG.deleteKey(current) except: pass modCFG.deleteKey("%s/AbsoluteDefinition" % current) else: for sec in modCFG[current].listSections(): expl.append("%s/%s" % (current, sec)) # Add the modCFG webCFG = webCFG.mergeWith(modCFG) print "al final webCFG" print webCFG
def updateCompleteDiracCFG(self): """Read the dirac.cfg and update the Systems sections from the ConfigTemplate.cfg files.""" compCfg = CFG() mainDiracCfgPath = self.config.cfg_baseFile if not os.path.exists(mainDiracCfgPath): LOG.error('Failed to find Main Dirac cfg at %r', mainDiracCfgPath) return 1 LOG.info('Extracting default configuration from %r', mainDiracCfgPath) loadCFG = CFG() loadCFG.loadFromFile(mainDiracCfgPath) compCfg = loadCFG.mergeWith(compCfg) cfg = self.getSystemsCFG() compCfg = compCfg.mergeWith(cfg) diracCfgOutput = self.config.cfg_targetFile LOG.info('Writing output to %r', diracCfgOutput) with open(diracCfgOutput, 'w') as rst: rst.write( textwrap.dedent(""" ========================== Full Configuration Example ========================== .. This file is created by docs/Tools/UpdateDiracCFG.py Below is a complete example configuration with anotations for some sections:: """)) # indent the cfg text cfgString = ''.join(' ' + line for line in str(compCfg).splitlines(True)) # fix the links, add back the # for targets # match .html with following character using positive look ahead htmlMatch = re.compile(r'\.html(?=[a-zA-Z0-9])') cfgString = re.sub(htmlMatch, '.html#', cfgString) rst.write(cfgString) return self.retVal
def loadWebAppCFGFiles(): """ Load WebApp/web.cfg definitions """ exts = [] for ext in CSGlobals.getCSExtensions(): if ext == "DIRAC": continue if ext[-5:] != "DIRAC": ext = "%sDIRAC" % ext if ext != "WebAppDIRAC": exts.append( ext ) exts.append( "DIRAC" ) exts.append( "WebAppDIRAC" ) webCFG = CFG() for modName in reversed( exts ): try: modPath = imp.find_module( modName )[1] except ImportError: continue gLogger.verbose( "Found module %s at %s" % ( modName, modPath ) ) cfgPath = os.path.join( modPath, "WebApp", "web.cfg" ) if not os.path.isfile( cfgPath ): gLogger.verbose( "Inexistant %s" % cfgPath ) continue try: modCFG = CFG().loadFromFile( cfgPath ) except Exception, excp: gLogger.error( "Could not load %s: %s" % ( cfgPath, excp ) ) continue gLogger.verbose( "Loaded %s" % cfgPath ) expl = [ BASECS ] while len( expl ): current = expl.pop( 0 ) if not modCFG.isSection( current ): continue if modCFG.getOption( "%s/AbsoluteDefinition" % current, False ): gLogger.verbose( "%s:%s is an absolute definition" % ( modName, current ) ) try: webCFG.deleteKey( current ) except: pass modCFG.deleteKey( "%s/AbsoluteDefinition" % current ) else: for sec in modCFG[ current ].listSections(): expl.append( "%s/%s" % ( current, sec ) ) #Add the modCFG webCFG = webCFG.mergeWith( modCFG )
def _loadWebAppCFGFiles(self, extension): """ Load WebApp/web.cfg definitions :param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC """ exts = [extension, "WebAppDIRAC"] webCFG = CFG() for modName in reversed(exts): cfgPath = os.path.join(self.__params.destination, "%s/WebApp" % modName, "web.cfg") if not os.path.isfile(cfgPath): gLogger.verbose("Web configuration file %s does not exists!" % cfgPath) continue try: modCFG = CFG().loadFromFile(cfgPath) except Exception, excp: gLogger.error("Could not load %s: %s" % (cfgPath, excp)) continue gLogger.verbose("Loaded %s" % cfgPath) expl = ["/WebApp"] while len(expl): current = expl.pop(0) if not modCFG.isSection(current): continue if modCFG.getOption("%s/AbsoluteDefinition" % current, False): gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) try: webCFG.deleteKey(current) except: pass modCFG.deleteKey("%s/AbsoluteDefinition" % current) else: for sec in modCFG[current].listSections(): expl.append("%s/%s" % (current, sec)) # Add the modCFG webCFG = webCFG.mergeWith(modCFG)
def _loadWebAppCFGFiles(self, extension): """ Load WebApp/web.cfg definitions :param str extension: the module name of the extension of WebAppDirac for example: LHCbWebDIRAC """ exts = [extension, "WebAppDIRAC"] webCFG = CFG() for modName in reversed(exts): cfgPath = os.path.join(self.__params.destination, "%s/WebApp" % modName, "web.cfg") if not os.path.isfile(cfgPath): gLogger.verbose("Web configuration file %s does not exists!" % cfgPath) continue try: modCFG = CFG().loadFromFile(cfgPath) except Exception, excp: gLogger.error("Could not load %s: %s" % (cfgPath, excp)) continue gLogger.verbose("Loaded %s" % cfgPath) expl = ["/WebApp"] while len(expl): current = expl.pop(0) if not modCFG.isSection(current): continue if modCFG.getOption("%s/AbsoluteDefinition" % current, False): gLogger.verbose("%s:%s is an absolute definition" % (modName, current)) try: webCFG.deleteKey(current) except: pass modCFG.deleteKey("%s/AbsoluteDefinition" % current) else: for sec in modCFG[current].listSections(): expl.append("%s/%s" % (current, sec)) # Add the modCFG webCFG = webCFG.mergeWith(modCFG)
class ConfigurationData: def __init__( self, loadDefaultCFG = True ): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.compressedConfigurationData = "" self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" ) self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" ) gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile ) retVal = self.loadFile( defaultCFGFile ) if not retVal[ 'OK' ]: gLogger.warn( "Can't load %s file" % defaultCFGFile ) self.sync() def getBackupDir( self ): return self.backupsDir def sync( self ): gLogger.debug( "Updating configuration internals" ) self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG ) self.remoteServerList = [] localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones = True ) if localServers: self.remoteServerList.extend( List.fromChar( localServers, "," ) ) remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones = True ) if remoteServers: self.remoteServerList.extend( List.fromChar( remoteServers, "," ) ) self.remoteServerList = List.uniqueElements( self.remoteServerList ) self.compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 ) def loadFile( self, fileName ): try: fileCFG = CFG() fileCFG.loadFromFile( fileName ) except IOError: self.localCFG = self.localCFG.mergeWith( fileCFG ) return S_ERROR( "Can't load a cfg file '%s'" % fileName ) return self.mergeWithLocal( fileCFG ) def mergeWithLocal( self, extraCFG ): self.lock() try: self.localCFG = self.localCFG.mergeWith( extraCFG ) self.unlock() gLogger.debug( "CFG merged" ) except Exception, e: self.unlock() return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) ) self.sync() return S_OK()
DIRAC.gLogger.initialize('test_gConfig', '/testSectionDebug') testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath cfg1 = CFG() cfg1.loadFromFile(testconfig) fd = file(testconfig) cfg1String = fd.read() fd.close() cfg2 = CFG() cfg2.loadFromBuffer(cfg1.serialize()) cfg3 = cfg1.mergeWith(cfg2) testList = [{ 'method': DIRAC.gConfig.loadFile, 'arguments': (testconfig, ), 'output': { 'OK': True, 'Value': '' } }, { 'method': DIRAC.gConfig.dumpLocalCFGToFile, 'arguments': (dumpconfig, ), 'output': { 'OK': True, 'Value': '' }
DIRAC.gLogger.initialize('test_gConfig','/testSectionDebug') testconfig = '%s/DIRAC/ConfigurationSystem/test/test.cfg' % DIRAC.rootPath dumpconfig = '%s/DIRAC/ConfigurationSystem/test/dump.cfg' % DIRAC.rootPath cfg1 = CFG() cfg1.loadFromFile( testconfig ) fd = file( testconfig ) cfg1String = fd.read() fd.close() cfg2 = CFG() cfg2.loadFromBuffer( cfg1.serialize() ) cfg3 = cfg1.mergeWith( cfg2 ) testList = [{ 'method' : DIRAC.gConfig.loadFile, 'arguments' : ( testconfig, ), 'output' : {'OK': True, 'Value': ''} }, { 'method' : DIRAC.gConfig.dumpLocalCFGToFile, 'arguments' : ( dumpconfig, ), 'output' : {'OK': True, 'Value': ''} }, { 'method' : cfg1.serialize, 'arguments' : ( ), 'output' : cfg1String }, { 'method' : cfg3.serialize, 'arguments' : ( ),
class Modificator: def __init__( self, rpcClient = False, commiterId = "unknown" ): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient( rpcClient ) def loadCredentials( self ): retVal = getProxyInfo() if retVal[ 'OK' ]: credDict = retVal[ 'Value' ] self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ], credDict[ 'group' ], Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) ) return retVal return retVal def setRPCClient( self, rpcClient ): self.rpcClient = rpcClient def loadFromRemote( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: self.cfgData = CFG() self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) return retVal def getCFG( self ): return self.cfgData def getSections( self, sectionPath ): return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData ) def getComment( self, sectionPath ): return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData ) def getOptions( self, sectionPath ): return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData ) def getOptionsDict(self, sectionPath): """Gives the options of a CS section in a Python dict with values as lists""" opts = self.getOptions(sectionPath) pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts ) return pathDict def getDictRootedAt(self, relpath = "", root = ""): """Gives the configuration rooted at path in a Python dict. The result is a Python dictionary that reflects the structure of the config file.""" def getDictRootedAt(path): retval = {} opts = self.getOptionsDict(path) secs = self.getSections(path) for k in opts: retval[k] = opts[k] for i in secs: retval[i] = getDictRootedAt(path + "/" + i) return retval return getDictRootedAt(root + "/" + relpath) def getValue( self, optionPath ): return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData ) def sortAlphabetically( self, path, ascending = True ): cfg = self.__getParentCFG( path, parentLevel = 0 ) if cfg: if cfg.sortAlphabetically( ascending ): self.__setCommiter( path ) def __getParentCFG( self, path, parentLevel = 1 ): sectionList = List.fromChar( path, "/" ) cfg = self.cfgData try: if parentLevel > 0: sectionList = sectionList[:-parentLevel] for section in sectionList: cfg = cfg[ section ] return cfg except: return False def __setCommiter( self, entryPath, cfg = False ): if not cfg: cfg = self.__getParentCFG( entryPath ) entry = List.fromChar( entryPath, "/" )[-1] comment = cfg.getComment( entry ) filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ] filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) ) cfg.setComment( entry, "\n".join( filteredComment ) ) def setOptionValue( self, optionPath, value ): levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ] parentPath = "/%s" % "/".join( levelList[:-1] ) optionName = List.fromChar( optionPath, "/" )[-1] self.createSection( parentPath ) cfg = self.__getParentCFG( optionPath ) if not cfg: return cfg.setOption( optionName, value ) self.__setCommiter( optionPath, cfg ) def createSection( self, sectionPath ): levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ] currentPath = "" cfg = self.cfgData createdSection = False for section in levelList: currentPath += "/%s" % section if section not in cfg.listSections(): cfg.createNewSection( section ) self.__setCommiter( currentPath ) createdSection = True cfg = cfg[ section ] return createdSection def setComment( self, entryPath, value ): cfg = self.__getParentCFG( entryPath ) entry = List.fromChar( entryPath, "/" )[-1] if cfg.setComment( entry, value ): self.__setCommiter( entryPath ) return True return False def existsSection( self, sectionPath ): sectionList = List.fromChar( sectionPath, "/" ) cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[ section ] return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections() except: return False def existsOption( self, optionPath ): sectionList = List.fromChar( optionPath, "/" ) cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[ section ] return sectionList[-1] in cfg.listOptions() except: return False def renameKey( self, path, newName ): parentCfg = self.cfgData.getRecursive( path, -1 ) if not parentCfg: return False pathList = List.fromChar( path, "/" ) oldName = pathList[-1] if parentCfg[ 'value' ].renameKey( oldName, newName ): pathList[-1] = newName self.__setCommiter( "/%s" % "/".join( pathList ) ) return True else: return False def copyKey( self, originalKeyPath, newKey ): parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 ) if not parentCfg: return False pathList = List.fromChar( originalKeyPath, "/" ) originalKey = pathList[-1] if parentCfg[ 'value' ].copyKey( originalKey, newKey ): self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) ) return True return False def removeOption( self, optionPath ): if not self.existsOption( optionPath ): return False cfg = self.__getParentCFG( optionPath ) optionName = List.fromChar( optionPath, "/" )[-1] return cfg.deleteKey( optionName ) def removeSection( self, sectionPath ): if not self.existsSection( sectionPath ): return False cfg = self.__getParentCFG( sectionPath ) sectionName = List.fromChar( sectionPath, "/" )[-1] return cfg.deleteKey( sectionName ) def loadFromBuffer( self, data ): self.cfgData = CFG() self.cfgData.loadFromBuffer( data ) def loadFromFile( self, filename ): self.cfgData = CFG() self.mergeFromFile( filename ) def dumpToFile( self, filename ): fd = file( filename, "w" ) fd.write( str( self.cfgData ) ) fd.close() def mergeFromFile( self, filename ): cfg = CFG() cfg.loadFromFile( filename ) self.cfgData = self.cfgData.mergeWith( cfg ) def mergeFromCFG( self, cfg ): self.cfgData = self.cfgData.mergeWith( cfg ) def mergeSectionFromCFG( self, sectionPath, cfg ): parentDict = self.cfgData.getRecursive( sectionPath, -1 ) parentCFG = parentDict[ 'value' ] secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1] secCFG = parentCFG[ secName ] if not secCFG: return False mergedCFG = secCFG.mergeWith( cfg ) parentCFG.deleteKey( secName ) parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG ) self.__setCommiter( sectionPath ) return True def __str__( self ): return str( self.cfgData ) def commit( self ): compressedData = zlib.compress( str( self.cfgData ), 9 ) return self.rpcClient.commitNewData( compressedData ) def getHistory( self, limit = 0 ): retVal = self.rpcClient.getCommitHistory( limit ) if retVal[ 'OK' ]: return retVal[ 'Value' ] return [] def showCurrentDiff( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines() localData = str( self.cfgData ).splitlines() return difflib.ndiff( remoteData, localData ) return [] def getVersionDiff( self, fromDate, toDate ): retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] ) if retVal[ 'OK' ]: fromData = zlib.decompress( retVal[ 'Value' ][0] ) toData = zlib.decompress( retVal[ 'Value' ][1] ) return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) ) return [] def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal def rollbackToVersion( self, version ): return self.rpcClient.rollbackToVersion( version ) def updateGConfigurationData( self ): gConfigurationData.setRemoteCFG( self.cfgData )
class Modificator( object ): def __init__( self, rpcClient = False, commiterId = "unknown" ): self.commiterTag = "@@-" self.commiterId = commiterId self.cfgData = CFG() self.rpcClient = None if rpcClient: self.setRPCClient( rpcClient ) def loadCredentials( self ): retVal = getProxyInfo() if retVal[ 'OK' ]: credDict = retVal[ 'Value' ] self.commiterId = "%s@%s - %s" % ( credDict[ 'username' ], credDict[ 'group' ], Time.dateTime().strftime( "%Y-%m-%d %H:%M:%S" ) ) return retVal return retVal def setRPCClient( self, rpcClient ): self.rpcClient = rpcClient def loadFromRemote( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: self.cfgData = CFG() self.cfgData.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) return retVal def getCFG( self ): return self.cfgData def getSections( self, sectionPath ): return gConfigurationData.getSectionsFromCFG( sectionPath, self.cfgData ) def getComment( self, sectionPath ): return gConfigurationData.getCommentFromCFG( sectionPath, self.cfgData ) def getOptions( self, sectionPath ): return gConfigurationData.getOptionsFromCFG( sectionPath, self.cfgData ) def getOptionsDict(self, sectionPath): """Gives the options of a CS section in a Python dict with values as lists""" opts = self.getOptions(sectionPath) pathDict = dict( ( o, self.getValue( "%s/%s" % ( sectionPath, o ) ) ) for o in opts ) return pathDict def getDictRootedAt(self, relpath = "", root = ""): """Gives the configuration rooted at path in a Python dict. The result is a Python dictionary that reflects the structure of the config file.""" def getDictRootedAt(path): retval = {} opts = self.getOptionsDict(path) secs = self.getSections(path) for k in opts: retval[k] = opts[k] for i in secs: retval[i] = getDictRootedAt(path + "/" + i) return retval return getDictRootedAt(root + "/" + relpath) def getValue( self, optionPath ): return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData ) def sortAlphabetically( self, path, ascending = True ): cfg = self.__getParentCFG( path, parentLevel = 0 ) if cfg: if cfg.sortAlphabetically( ascending ): self.__setCommiter( path ) def __getParentCFG( self, path, parentLevel = 1 ): sectionList = List.fromChar( path, "/" ) cfg = self.cfgData try: if parentLevel > 0: sectionList = sectionList[:-parentLevel] for section in sectionList: cfg = cfg[ section ] return cfg except: return False def __setCommiter( self, entryPath, cfg = False ): if not cfg: cfg = self.__getParentCFG( entryPath ) entry = List.fromChar( entryPath, "/" )[-1] comment = cfg.getComment( entry ) filteredComment = [ line.strip() for line in comment.split( "\n" ) if line.find( self.commiterTag ) != 0 ] filteredComment.append( "%s%s" % ( self.commiterTag, self.commiterId ) ) cfg.setComment( entry, "\n".join( filteredComment ) ) def setOptionValue( self, optionPath, value ): levelList = [ level.strip() for level in optionPath.split( "/" ) if level.strip() != "" ] parentPath = "/%s" % "/".join( levelList[:-1] ) optionName = List.fromChar( optionPath, "/" )[-1] self.createSection( parentPath ) cfg = self.__getParentCFG( optionPath ) if not cfg: return cfg.setOption( optionName, value ) self.__setCommiter( optionPath, cfg ) def createSection( self, sectionPath ): levelList = [ level.strip() for level in sectionPath.split( "/" ) if level.strip() != "" ] currentPath = "" cfg = self.cfgData createdSection = False for section in levelList: currentPath += "/%s" % section if section not in cfg.listSections(): cfg.createNewSection( section ) self.__setCommiter( currentPath ) createdSection = True cfg = cfg[ section ] return createdSection def setComment( self, entryPath, value ): cfg = self.__getParentCFG( entryPath ) entry = List.fromChar( entryPath, "/" )[-1] if cfg.setComment( entry, value ): self.__setCommiter( entryPath ) return True return False def existsSection( self, sectionPath ): sectionList = List.fromChar( sectionPath, "/" ) cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[ section ] return len( sectionList ) == 0 or sectionList[-1] in cfg.listSections() except: return False def existsOption( self, optionPath ): sectionList = List.fromChar( optionPath, "/" ) cfg = self.cfgData try: for section in sectionList[:-1]: cfg = cfg[ section ] return sectionList[-1] in cfg.listOptions() except: return False def renameKey( self, path, newName ): parentCfg = self.cfgData.getRecursive( path, -1 ) if not parentCfg: return False pathList = List.fromChar( path, "/" ) oldName = pathList[-1] if parentCfg[ 'value' ].renameKey( oldName, newName ): pathList[-1] = newName self.__setCommiter( "/%s" % "/".join( pathList ) ) return True else: return False def copyKey( self, originalKeyPath, newKey ): parentCfg = self.cfgData.getRecursive( originalKeyPath, -1 ) if not parentCfg: return False pathList = List.fromChar( originalKeyPath, "/" ) originalKey = pathList[-1] if parentCfg[ 'value' ].copyKey( originalKey, newKey ): self.__setCommiter( "/%s/%s" % ( "/".join( pathList[:-1] ), newKey ) ) return True return False def removeOption( self, optionPath ): if not self.existsOption( optionPath ): return False cfg = self.__getParentCFG( optionPath ) optionName = List.fromChar( optionPath, "/" )[-1] return cfg.deleteKey( optionName ) def removeSection( self, sectionPath ): if not self.existsSection( sectionPath ): return False cfg = self.__getParentCFG( sectionPath ) sectionName = List.fromChar( sectionPath, "/" )[-1] return cfg.deleteKey( sectionName ) def loadFromBuffer( self, data ): self.cfgData = CFG() self.cfgData.loadFromBuffer( data ) def loadFromFile( self, filename ): self.cfgData = CFG() self.mergeFromFile( filename ) def dumpToFile( self, filename ): fd = file( filename, "w" ) fd.write( str( self.cfgData ) ) fd.close() def mergeFromFile( self, filename ): cfg = CFG() cfg.loadFromFile( filename ) self.cfgData = self.cfgData.mergeWith( cfg ) def mergeFromCFG( self, cfg ): self.cfgData = self.cfgData.mergeWith( cfg ) def mergeSectionFromCFG( self, sectionPath, cfg ): parentDict = self.cfgData.getRecursive( sectionPath, -1 ) parentCFG = parentDict[ 'value' ] secName = [ lev.strip() for lev in sectionPath.split( "/" ) if lev.strip() ][-1] secCFG = parentCFG[ secName ] if not secCFG: return False mergedCFG = secCFG.mergeWith( cfg ) parentCFG.deleteKey( secName ) parentCFG.createNewSection( secName, parentDict[ 'comment' ], mergedCFG ) self.__setCommiter( sectionPath ) return True def __str__( self ): return str( self.cfgData ) def commit( self ): compressedData = zlib.compress( str( self.cfgData ), 9 ) return self.rpcClient.commitNewData( compressedData ) def getHistory( self, limit = 0 ): retVal = self.rpcClient.getCommitHistory( limit ) if retVal[ 'OK' ]: return retVal[ 'Value' ] return [] def showCurrentDiff( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteData = zlib.decompress( retVal[ 'Value' ] ).splitlines() localData = str( self.cfgData ).splitlines() return difflib.ndiff( remoteData, localData ) return [] def getVersionDiff( self, fromDate, toDate ): retVal = self.rpcClient.getVersionContents( [ fromDate, toDate ] ) if retVal[ 'OK' ]: fromData = zlib.decompress( retVal[ 'Value' ][0] ) toData = zlib.decompress( retVal[ 'Value' ][1] ) return difflib.ndiff( fromData.split( "\n" ), toData.split( "\n" ) ) return [] def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal def rollbackToVersion( self, version ): return self.rpcClient.rollbackToVersion( version ) def updateGConfigurationData( self ): gConfigurationData.setRemoteCFG( self.cfgData )
class ConfigurationData: def __init__(self, loadDefaultCFG=True): self.threadingEvent = threading.Event() self.threadingEvent.set() self.threadingLock = threading.Lock() self.runningThreadsNumber = 0 self.compressedConfigurationData = "" self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup") self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) retVal = self.loadFile(defaultCFGFile) if not retVal['OK']: gLogger.warn("Can't load %s file" % defaultCFGFile) self.sync() def getBackupDir(self): return self.backupsDir def sync(self): gLogger.debug("Updating configuration internals") self.mergedCFG = self.remoteCFG.mergeWith(self.localCFG) self.remoteServerList = [] localServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones=True) if localServers: self.remoteServerList.extend(List.fromChar(localServers, ",")) remoteServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones=True) if remoteServers: self.remoteServerList.extend(List.fromChar(remoteServers, ",")) self.remoteServerList = List.uniqueElements(self.remoteServerList) self.compressedConfigurationData = zlib.compress( str(self.remoteCFG), 9) def loadFile(self, fileName): try: fileCFG = CFG() fileCFG.loadFromFile(fileName) except IOError: self.localCFG = self.localCFG.mergeWith(fileCFG) return S_ERROR("Can't load a cfg file '%s'" % fileName) return self.mergeWithLocal(fileCFG) def mergeWithLocal(self, extraCFG): self.lock() try: self.localCFG = self.localCFG.mergeWith(extraCFG) self.unlock() gLogger.debug("CFG merged") except Exception, e: self.unlock() return S_ERROR("Cannot merge with new cfg: %s" % str(e)) self.sync() return S_OK()
class ConfigurationData(object): def __init__(self, loadDefaultCFG=True): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.__compressedConfigurationData = None self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join(DIRAC.rootPath, "etc", "csbackup") self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join(DIRAC.rootPath, "etc", "dirac.cfg") gLogger.debug("dirac.cfg should be at", "%s" % defaultCFGFile) retVal = self.loadFile(defaultCFGFile) if not retVal['OK']: gLogger.warn("Can't load %s file" % defaultCFGFile) self.sync() def getBackupDir(self): return self.backupsDir def sync(self): gLogger.debug("Updating configuration internals") self.mergedCFG = self.remoteCFG.mergeWith(self.localCFG) self.remoteServerList = [] localServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones=True) if localServers: self.remoteServerList.extend(List.fromChar(localServers, ",")) remoteServers = self.extractOptionFromCFG("%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones=True) if remoteServers: self.remoteServerList.extend(List.fromChar(remoteServers, ",")) self.remoteServerList = List.uniqueElements(self.remoteServerList) self.__compressedConfigurationData = None def loadFile(self, fileName): try: fileCFG = CFG() fileCFG.loadFromFile(fileName) except IOError: self.localCFG = self.localCFG.mergeWith(fileCFG) return S_ERROR("Can't load a cfg file '%s'" % fileName) return self.mergeWithLocal(fileCFG) def mergeWithLocal(self, extraCFG): self.lock() try: self.localCFG = self.localCFG.mergeWith(extraCFG) self.unlock() gLogger.debug("CFG merged") except Exception as e: self.unlock() return S_ERROR("Cannot merge with new cfg: %s" % str(e)) self.sync() return S_OK() def loadRemoteCFGFromCompressedMem(self, data): sUncompressedData = zlib.decompress(data) self.loadRemoteCFGFromMem(sUncompressedData) def loadRemoteCFGFromMem(self, data): self.lock() self.remoteCFG.loadFromBuffer(data) self.unlock() self.sync() def loadConfigurationData(self, fileName=False): name = self.getName() self.lock() try: if not fileName: fileName = "%s.cfg" % name if fileName[0] != "/": fileName = os.path.join(DIRAC.rootPath, "etc", fileName) self.remoteCFG.loadFromFile(fileName) except Exception as e: print e self.unlock() self.sync() def getCommentFromCFG(self, path, cfg=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[section] return self.dangerZoneEnd(cfg.getComment(levelList[-1])) except Exception: pass return self.dangerZoneEnd(None) def getSectionsFromCFG(self, path, cfg=False, ordered=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList: cfg = cfg[section] return self.dangerZoneEnd(cfg.listSections(ordered)) except Exception: pass return self.dangerZoneEnd(None) def getOptionsFromCFG(self, path, cfg=False, ordered=False): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList: cfg = cfg[section] return self.dangerZoneEnd(cfg.listOptions(ordered)) except Exception: pass return self.dangerZoneEnd(None) def extractOptionFromCFG(self, path, cfg=False, disableDangerZones=False): if not cfg: cfg = self.mergedCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[section] if levelList[-1] in cfg.listOptions(): return self.dangerZoneEnd(cfg[levelList[-1]]) except Exception: pass if not disableDangerZones: self.dangerZoneEnd() def setOptionInCFG(self, path, value, cfg=False, disableDangerZones=False): if not cfg: cfg = self.localCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): cfg.createNewSection(section) cfg = cfg[section] cfg.setOption(levelList[-1], value) finally: if not disableDangerZones: self.dangerZoneEnd() self.sync() def deleteOptionInCFG(self, path, cfg=False): if not cfg: cfg = self.localCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split("/") if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): return cfg = cfg[section] cfg.deleteKey(levelList[-1]) finally: self.dangerZoneEnd() self.sync() def generateNewVersion(self): self.setVersion(Time.toString()) self.sync() gLogger.info("Generated new version %s" % self.getVersion()) def setVersion(self, version, cfg=False): if not cfg: cfg = self.remoteCFG self.setOptionInCFG("%s/Version" % self.configurationPath, version, cfg) def getVersion(self, cfg=False): if not cfg: cfg = self.remoteCFG value = self.extractOptionFromCFG( "%s/Version" % self.configurationPath, cfg) if value: return value return "0" def getName(self): return self.extractOptionFromCFG("%s/Name" % self.configurationPath, self.mergedCFG) def exportName(self): return self.setOptionInCFG("%s/Name" % self.configurationPath, self.getName(), self.remoteCFG) def getRefreshTime(self): try: return int( self.extractOptionFromCFG( "%s/RefreshTime" % self.configurationPath, self.mergedCFG)) except: return 300 def getPropagationTime(self): try: return int( self.extractOptionFromCFG( "%s/PropagationTime" % self.configurationPath, self.mergedCFG)) except: return 300 def getSlavesGraceTime(self): try: return int( self.extractOptionFromCFG( "%s/SlavesGraceTime" % self.configurationPath, self.mergedCFG)) except: return 600 def mergingEnabled(self): try: val = self.extractOptionFromCFG( "%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG) return val.lower() in ("yes", "true", "y") except: return False def getAutoPublish(self): value = self.extractOptionFromCFG( "%s/AutoPublish" % self.configurationPath, self.localCFG) if value and value.lower() in ("no", "false", "n"): return False else: return True def getServers(self): return list(self.remoteServerList) def getConfigurationGateway(self): return self.extractOptionFromCFG("/DIRAC/Gateway", self.localCFG) def setServers(self, sServers): self.setOptionInCFG("%s/Servers" % self.configurationPath, sServers, self.remoteCFG) self.sync() def deleteLocalOption(self, optionPath): self.deleteOptionInCFG(optionPath, self.localCFG) def getMasterServer(self): return self.extractOptionFromCFG( "%s/MasterServer" % self.configurationPath, self.remoteCFG) def setMasterServer(self, sURL): self.setOptionInCFG("%s/MasterServer" % self.configurationPath, sURL, self.remoteCFG) self.sync() def getCompressedData(self): if self.__compressedConfigurationData is None: self.__compressedConfigurationData = zlib.compress( str(self.remoteCFG), 9) return self.__compressedConfigurationData def isMaster(self): value = self.extractOptionFromCFG("%s/Master" % self.configurationPath, self.localCFG) if value and value.lower() in ("yes", "true", "y"): return True else: return False def getServicesPath(self): return "/Services" def setAsService(self): self._isService = True def isService(self): return self._isService def useServerCertificate(self): value = self.extractOptionFromCFG( "/DIRAC/Security/UseServerCertificate") if value and value.lower() in ("y", "yes", "true"): return True return False def skipCACheck(self): value = self.extractOptionFromCFG("/DIRAC/Security/SkipCAChecks") if value and value.lower() in ("y", "yes", "true"): return True return False def dumpLocalCFGToFile(self, fileName): try: with open(fileName, "w") as fd: fd.write(str(self.localCFG)) gLogger.verbose("Configuration file dumped", "'%s'" % fileName) except IOError: gLogger.error("Can't dump cfg file", "'%s'" % fileName) return S_ERROR("Can't dump cfg file '%s'" % fileName) return S_OK() def getRemoteCFG(self): return self.remoteCFG def getMergedCFGAsString(self): return str(self.mergedCFG) def dumpRemoteCFGToFile(self, fileName): with open(fileName, "w") as fd: fd.write(str(self.remoteCFG)) def __backupCurrentConfiguration(self, backupName): configurationFilename = "%s.cfg" % self.getName() configurationFile = os.path.join(DIRAC.rootPath, "etc", configurationFilename) today = Time.date() backupPath = os.path.join(self.getBackupDir(), str(today.year), "%02d" % today.month) mkDir(backupPath) backupFile = os.path.join( backupPath, configurationFilename.replace(".cfg", ".%s.zip" % backupName)) if os.path.isfile(configurationFile): gLogger.info("Making a backup of configuration in %s" % backupFile) try: with zipfile.ZipFile(backupFile, "w", zipfile.ZIP_DEFLATED) as zf: zf.write( configurationFile, "%s.backup.%s" % (os.path.split(configurationFile)[1], backupName)) except Exception: gLogger.exception() gLogger.error("Cannot backup configuration data file", "file %s" % backupFile) else: gLogger.warn("CS data file does not exist", configurationFile) def writeRemoteConfigurationToDisk(self, backupName=False): configurationFile = os.path.join(DIRAC.rootPath, "etc", "%s.cfg" % self.getName()) try: with open(configurationFile, "w") as fd: fd.write(str(self.remoteCFG)) except Exception as e: gLogger.fatal("Cannot write new configuration to disk!", "file %s" % configurationFile) return S_ERROR("Can't write cs file %s!: %s" % (configurationFile, repr(e).replace(',)', ')'))) if backupName: self.__backupCurrentConfiguration(backupName) return S_OK() def setRemoteCFG(self, cfg, disableSync=False): self.remoteCFG = cfg.clone() if not disableSync: self.sync() def lock(self): """ Locks Event to prevent further threads from reading. Stops current thread until no other thread is accessing. PRIVATE USE """ self.threadingEvent.clear() while self.runningThreadsNumber > 0: time.sleep(0.1) def unlock(self): """ Unlocks Event. PRIVATE USE """ self.threadingEvent.set() def dangerZoneStart(self): """ Start of danger zone. This danger zone may be or may not be a mutual exclusion zone. Counter is maintained to know how many threads are inside and be able to enable and disable mutual exclusion. PRIVATE USE """ self.threadingEvent.wait() self.threadingLock.acquire() self.runningThreadsNumber += 1 try: self.threadingLock.release() except thread.error: pass def dangerZoneEnd(self, returnValue=None): """ End of danger zone. PRIVATE USE """ self.threadingLock.acquire() self.runningThreadsNumber -= 1 try: self.threadingLock.release() except thread.error: pass return returnValue
class ConfigurationData( object ): def __init__( self, loadDefaultCFG = True ): lr = LockRing() self.threadingEvent = lr.getEvent() self.threadingEvent.set() self.threadingLock = lr.getLock() self.runningThreadsNumber = 0 self.__compressedConfigurationData = None self.configurationPath = "/DIRAC/Configuration" self.backupsDir = os.path.join( DIRAC.rootPath, "etc", "csbackup" ) self._isService = False self.localCFG = CFG() self.remoteCFG = CFG() self.mergedCFG = CFG() self.remoteServerList = [] if loadDefaultCFG: defaultCFGFile = os.path.join( DIRAC.rootPath, "etc", "dirac.cfg" ) gLogger.debug( "dirac.cfg should be at", "%s" % defaultCFGFile ) retVal = self.loadFile( defaultCFGFile ) if not retVal[ 'OK' ]: gLogger.warn( "Can't load %s file" % defaultCFGFile ) self.sync() def getBackupDir( self ): return self.backupsDir def sync( self ): gLogger.debug( "Updating configuration internals" ) self.mergedCFG = self.remoteCFG.mergeWith( self.localCFG ) self.remoteServerList = [] localServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.localCFG, disableDangerZones = True ) if localServers: self.remoteServerList.extend( List.fromChar( localServers, "," ) ) remoteServers = self.extractOptionFromCFG( "%s/Servers" % self.configurationPath, self.remoteCFG, disableDangerZones = True ) if remoteServers: self.remoteServerList.extend( List.fromChar( remoteServers, "," ) ) self.remoteServerList = List.uniqueElements( self.remoteServerList ) self.__compressedConfigurationData = None def loadFile( self, fileName ): try: fileCFG = CFG() fileCFG.loadFromFile( fileName ) except IOError: self.localCFG = self.localCFG.mergeWith( fileCFG ) return S_ERROR( "Can't load a cfg file '%s'" % fileName ) return self.mergeWithLocal( fileCFG ) def mergeWithLocal( self, extraCFG ): self.lock() try: self.localCFG = self.localCFG.mergeWith( extraCFG ) self.unlock() gLogger.debug( "CFG merged" ) except Exception as e: self.unlock() return S_ERROR( "Cannot merge with new cfg: %s" % str( e ) ) self.sync() return S_OK() def loadRemoteCFGFromCompressedMem( self, data ): sUncompressedData = zlib.decompress( data ) self.loadRemoteCFGFromMem( sUncompressedData ) def loadRemoteCFGFromMem( self, data ): self.lock() self.remoteCFG.loadFromBuffer( data ) self.unlock() self.sync() def loadConfigurationData( self, fileName = False ): name = self.getName() self.lock() try: if not fileName: fileName = "%s.cfg" % name if fileName[0] != "/": fileName = os.path.join( DIRAC.rootPath, "etc", fileName ) self.remoteCFG.loadFromFile( fileName ) except Exception as e: print e pass self.unlock() self.sync() def getCommentFromCFG( self, path, cfg = False ): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[ section ] return self.dangerZoneEnd( cfg.getComment( levelList[-1] ) ) except Exception: pass return self.dangerZoneEnd( None ) def getSectionsFromCFG( self, path, cfg = False, ordered = False ): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList: cfg = cfg[ section ] return self.dangerZoneEnd( cfg.listSections( ordered ) ) except Exception: pass return self.dangerZoneEnd( None ) def getOptionsFromCFG( self, path, cfg = False, ordered = False ): if not cfg: cfg = self.mergedCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList: cfg = cfg[ section ] return self.dangerZoneEnd( cfg.listOptions( ordered ) ) except Exception: pass return self.dangerZoneEnd( None ) def extractOptionFromCFG( self, path, cfg = False, disableDangerZones = False ): if not cfg: cfg = self.mergedCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList[:-1]: cfg = cfg[ section ] if levelList[-1] in cfg.listOptions(): return self.dangerZoneEnd( cfg[ levelList[ -1 ] ] ) except Exception: pass if not disableDangerZones: self.dangerZoneEnd() def setOptionInCFG( self, path, value, cfg = False, disableDangerZones = False ): if not cfg: cfg = self.localCFG if not disableDangerZones: self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): cfg.createNewSection( section ) cfg = cfg[ section ] cfg.setOption( levelList[ -1 ], value ) finally: if not disableDangerZones: self.dangerZoneEnd() self.sync() def deleteOptionInCFG( self, path, cfg = False ): if not cfg: cfg = self.localCFG self.dangerZoneStart() try: levelList = [ level.strip() for level in path.split( "/" ) if level.strip() != "" ] for section in levelList[:-1]: if section not in cfg.listSections(): return cfg = cfg[ section ] cfg.deleteKey( levelList[ -1 ] ) finally: self.dangerZoneEnd() self.sync() def generateNewVersion( self ): self.setVersion( Time.toString() ) self.sync() gLogger.info( "Generated new version %s" % self.getVersion() ) def setVersion( self, version, cfg = False ): if not cfg: cfg = self.remoteCFG self.setOptionInCFG( "%s/Version" % self.configurationPath, version, cfg ) def getVersion( self, cfg = False ): if not cfg: cfg = self.remoteCFG value = self.extractOptionFromCFG( "%s/Version" % self.configurationPath, cfg ) if value: return value return "0" def getName( self ): return self.extractOptionFromCFG( "%s/Name" % self.configurationPath, self.mergedCFG ) def exportName( self ): return self.setOptionInCFG( "%s/Name" % self.configurationPath, self.getName(), self.remoteCFG ) def getRefreshTime( self ): try: return int( self.extractOptionFromCFG( "%s/RefreshTime" % self.configurationPath, self.mergedCFG ) ) except: return 300 def getPropagationTime( self ): try: return int( self.extractOptionFromCFG( "%s/PropagationTime" % self.configurationPath, self.mergedCFG ) ) except: return 300 def getSlavesGraceTime( self ): try: return int( self.extractOptionFromCFG( "%s/SlavesGraceTime" % self.configurationPath, self.mergedCFG ) ) except: return 600 def mergingEnabled( self ): try: val = self.extractOptionFromCFG( "%s/EnableAutoMerge" % self.configurationPath, self.mergedCFG ) return val.lower() in ( "yes", "true", "y" ) except: return False def getAutoPublish( self ): value = self.extractOptionFromCFG( "%s/AutoPublish" % self.configurationPath, self.localCFG ) if value and value.lower() in ( "no", "false", "n" ): return False else: return True def getServers( self ): return list( self.remoteServerList ) def getConfigurationGateway( self ): return self.extractOptionFromCFG( "/DIRAC/Gateway", self.localCFG ) def setServers( self, sServers ): self.setOptionInCFG( "%s/Servers" % self.configurationPath, sServers, self.remoteCFG ) self.sync() def deleteLocalOption( self, optionPath ): self.deleteOptionInCFG( optionPath, self.localCFG ) def getMasterServer( self ): return self.extractOptionFromCFG( "%s/MasterServer" % self.configurationPath, self.remoteCFG ) def setMasterServer( self, sURL ): self.setOptionInCFG( "%s/MasterServer" % self.configurationPath, sURL, self.remoteCFG ) self.sync() def getCompressedData( self ): if self.__compressedConfigurationData is None: self.__compressedConfigurationData = zlib.compress( str( self.remoteCFG ), 9 ) return self.__compressedConfigurationData def isMaster( self ): value = self.extractOptionFromCFG( "%s/Master" % self.configurationPath, self.localCFG ) if value and value.lower() in ( "yes", "true", "y" ): return True else: return False def getServicesPath( self ): return "/Services" def setAsService( self ): self._isService = True def isService( self ): return self._isService def useServerCertificate( self ): value = self.extractOptionFromCFG( "/DIRAC/Security/UseServerCertificate" ) if value and value.lower() in ( "y", "yes", "true" ): return True return False def skipCACheck( self ): value = self.extractOptionFromCFG( "/DIRAC/Security/SkipCAChecks" ) if value and value.lower() in ( "y", "yes", "true" ): return True return False def dumpLocalCFGToFile( self, fileName ): try: with open( fileName, "w" ) as fd: fd.write( str( self.localCFG ) ) gLogger.verbose( "Configuration file dumped", "'%s'" % fileName ) except IOError: gLogger.error( "Can't dump cfg file", "'%s'" % fileName ) return S_ERROR( "Can't dump cfg file '%s'" % fileName ) return S_OK() def getRemoteCFG( self ): return self.remoteCFG def getMergedCFGAsString( self ): return str( self.mergedCFG ) def dumpRemoteCFGToFile( self, fileName ): with open( fileName, "w" ) as fd: fd.write( str( self.remoteCFG ) ) def __backupCurrentConfiguration( self, backupName ): configurationFilename = "%s.cfg" % self.getName() configurationFile = os.path.join( DIRAC.rootPath, "etc", configurationFilename ) today = Time.date() backupPath = os.path.join( self.getBackupDir(), str( today.year ), "%02d" % today.month ) mkDir(backupPath) backupFile = os.path.join( backupPath, configurationFilename.replace( ".cfg", ".%s.zip" % backupName ) ) if os.path.isfile( configurationFile ): gLogger.info( "Making a backup of configuration in %s" % backupFile ) try: zf = zipfile.ZipFile( backupFile, "w", zipfile.ZIP_DEFLATED ) zf.write( configurationFile, "%s.backup.%s" % ( os.path.split( configurationFile )[1], backupName ) ) zf.close() except Exception: gLogger.exception() gLogger.error( "Cannot backup configuration data file", "file %s" % backupFile ) else: gLogger.warn( "CS data file does not exist", configurationFile ) def writeRemoteConfigurationToDisk( self, backupName = False ): configurationFile = os.path.join( DIRAC.rootPath, "etc", "%s.cfg" % self.getName() ) try: with open( configurationFile, "w" ) as fd: fd.write( str( self.remoteCFG ) ) except Exception as e: gLogger.fatal( "Cannot write new configuration to disk!", "file %s" % configurationFile ) return S_ERROR( "Can't write cs file %s!: %s" % ( configurationFile, repr( e ).replace( ',)', ')' ) ) ) if backupName: self.__backupCurrentConfiguration( backupName ) return S_OK() def setRemoteCFG( self, cfg, disableSync = False ): self.remoteCFG = cfg.clone() if not disableSync: self.sync() def lock( self ): """ Locks Event to prevent further threads from reading. Stops current thread until no other thread is accessing. PRIVATE USE """ self.threadingEvent.clear() while self.runningThreadsNumber > 0: time.sleep( 0.1 ) def unlock( self ): """ Unlocks Event. PRIVATE USE """ self.threadingEvent.set() def dangerZoneStart( self ): """ Start of danger zone. This danger zone may be or may not be a mutual exclusion zone. Counter is maintained to know how many threads are inside and be able to enable and disable mutual exclusion. PRIVATE USE """ self.threadingEvent.wait() self.threadingLock.acquire() self.runningThreadsNumber += 1 try: self.threadingLock.release() except thread.error: pass def dangerZoneEnd( self, returnValue = None ): """ End of danger zone. PRIVATE USE """ self.threadingLock.acquire() self.runningThreadsNumber -= 1 try: self.threadingLock.release() except thread.error: pass return returnValue