def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): """ Update the master configuration with the newly received changes :param str sBuffer: newly received configuration data :param str committer: the user name of the committer :param bool updateVersionOption: flag to update the current configuration version :return: S_OK/S_ERROR of the write-to-disk of the new configuration """ if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") # Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) # Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result['OK']: gLogger.warn("Could not AutoMerge!", result['Message']) return S_ERROR("AutoMerge failed: %s" % result['Message']) requestedRemoteCFG = result['Value'] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) # Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) # Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() # self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk") retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) gLogger.info("New version", gConfigurationData.getVersion()) # Attempt to update the configuration on currently registered slave services if gConfigurationData.getAutoSlaveSync(): result = self.forceSlavesUpdate() if not result['OK']: gLogger.warn('Failed to update slave servers') return retVal
def updateConfiguration(self, sBuffer, commiterDN="", updateVersionOption=False): if not gConfigurationData.isMaster(): return S_ERROR( "Configuration modification is not allowed in this server") #Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) #Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR( "Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result['OK']: gLogger.warn("Could not AutoMerge!", result['Message']) return S_ERROR("AutoMerge failed: %s" % result['Message']) requestedRemoteCFG = result['Value'] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) #Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) #Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() #self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk!") retVal = gConfigurationData.writeRemoteConfigurationToDisk( "%s@%s" % (commiterDN, gConfigurationData.getVersion())) gLogger.info("New version it is!") return retVal
def _updateFromRemoteLocation( serviceClient ): gLogger.debug( "", "Trying to refresh from %s" % serviceClient.serviceURL ) localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer( localVersion ) if retVal[ 'OK' ]: dataDict = retVal[ 'Value' ] if localVersion < dataDict[ 'newestVersion' ] : gLogger.debug( "New version available", "Updating to version %s..." % dataDict[ 'newestVersion' ] ) gConfigurationData.loadRemoteCFGFromCompressedMem( dataDict[ 'data' ] ) gLogger.debug( "Updated to version %s" % gConfigurationData.getVersion() ) gEventDispatcher.triggerEvent( "CSNewVersion", dataDict[ 'newestVersion' ], threaded = True ) return S_OK() return retVal
def _updateFromRemoteLocation(serviceClient): gLogger.debug("", "Trying to refresh from %s" % serviceClient.serviceURL) localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer(localVersion) if retVal['OK']: dataDict = retVal['Value'] if localVersion < dataDict['newestVersion']: gLogger.debug("New version available", "Updating to version %s..." % dataDict['newestVersion']) gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict['data']) gLogger.debug("Updated to version %s" % gConfigurationData.getVersion()) gEventDispatcher.triggerEvent("CSNewVersion", dataDict['newestVersion'], threaded=True) return S_OK() return retVal
def updateConfiguration(self, sBuffer, commiter="", updateVersionOption=False): if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") # Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) # Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR( "Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion) ) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result["OK"]: gLogger.warn("Could not AutoMerge!", result["Message"]) return S_ERROR("AutoMerge failed: %s" % result["Message"]) requestedRemoteCFG = result["Value"] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) # Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) # Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() # self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk!") retVal = gConfigurationData.writeRemoteConfigurationToDisk( "%s@%s" % (commiter, gConfigurationData.getVersion()) ) gLogger.info("New version it is!") return retVal
def __loadConfigurationData(self): try: os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup")) except: pass gConfigurationData.loadConfigurationData() if gConfigurationData.isMaster(): bBuiltNewConfiguration = False if not gConfigurationData.getName(): DIRAC.abort( 10, "Missing name for the configuration to be exported!") gConfigurationData.exportName() sVersion = gConfigurationData.getVersion() if sVersion == "0": gLogger.info("There's no version. Generating a new one") gConfigurationData.generateNewVersion() bBuiltNewConfiguration = True if self.sURL not in gConfigurationData.getServers(): gConfigurationData.setServers(self.sURL) bBuiltNewConfiguration = True gConfigurationData.setMasterServer(self.sURL) if bBuiltNewConfiguration: gConfigurationData.writeRemoteConfigurationToDisk()
def __getCache( self ): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = ( self.__vo, self.__setup ) if cacheKey in Operations.__cache: return Operations.__cache[ cacheKey ] mergedCFG = CFG.CFG() for path in self.__getSearchPaths(): pathCFG = gConfigurationData.mergedCFG[ path ] if pathCFG: mergedCFG = mergedCFG.mergeWith( pathCFG ) Operations.__cache[ cacheKey ] = mergedCFG return Operations.__cache[ cacheKey ] finally: try: Operations.__cacheLock.release() except thread.error: pass
def __loadConfigurationData( self ): try: os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) ) except: pass gConfigurationData.loadConfigurationData() if gConfigurationData.isMaster(): bBuiltNewConfiguration = False if not gConfigurationData.getName(): DIRAC.abort( 10, "Missing name for the configuration to be exported!" ) gConfigurationData.exportName() sVersion = gConfigurationData.getVersion() if sVersion == "0": gLogger.info( "There's no version. Generating a new one" ) gConfigurationData.generateNewVersion() bBuiltNewConfiguration = True if self.sURL not in gConfigurationData.getServers(): gConfigurationData.setServers( self.sURL ) bBuiltNewConfiguration = True gConfigurationData.setMasterServer( self.sURL ) if bBuiltNewConfiguration: gConfigurationData.writeRemoteConfigurationToDisk()
def __getCache(self): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = (self.__vo, self.__setup) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] mergedCFG = CFG() for path in self.__getSearchPaths(): pathCFG = gConfigurationData.mergedCFG[path] if pathCFG: mergedCFG = mergedCFG.mergeWith(pathCFG) Operations.__cache[cacheKey] = mergedCFG return Operations.__cache[cacheKey] finally: try: Operations.__cacheLock.release() except thread.error: pass
def web_conf(self): """ Configuration endpoint, used to: GET /conf/get?<options> -- get configuration information, with arguments * options: * fullCFG - to get dump of configuration * option - option path to get option value * options - section path to get list of options * section - section path to get dict of all options/values there * sections - section path to get list of sections there * version - version of configuration that request information(optional) GET /conf/<helper method>?<arguments> -- get some information by using helpers methods * helper method - helper method of configuration service * arguments - arguments specifecly for every helper method :return: json with requested data """ self.log.notice('Request configuration information') optns = self.overpath.strip('/').split('/') if not optns or len(optns) > 1: raise WErr(404, "Wrone way") if optns[0] == 'get': if 'version' in self.args and ( self.args.get('version') or '0') >= gConfigurationData.getVersion(): self.finish() result = {} if 'fullCFG' in self.args: remoteCFG = yield self.threadTask( gConfigurationData.getRemoteCFG) result['Value'] = str(remoteCFG) elif 'option' in self.args: result = yield self.threadTask(gConfig.getOption, self.args['option']) elif 'section' in self.args: result = yield self.threadTask(gConfig.getOptionsDict, self.args['section']) elif 'options' in self.args: result = yield self.threadTask(gConfig.getOptions, self.args['options']) elif 'sections' in self.args: result = yield self.threadTask(gConfig.getSections, self.args['sections']) else: raise WErr(500, 'Invalid argument') elif any([ optns[0] == m and re.match('^[a-z][A-z]+', m) for m in dir(Registry) ]) and self.isRegisteredUser(): result = yield self.threadTask(getattr(Registry, optns[0]), **self.args) if not result['OK']: raise WErr(404, result['Message']) self.finishJEncode(result['Value'])
def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal
def mergeWithServer( self ): retVal = self.rpcClient.getCompressedData() if retVal[ 'OK' ]: remoteCFG = CFG() remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) ) serverVersion = gConfigurationData.getVersion( remoteCFG ) self.cfgData = remoteCFG.mergeWith( self.cfgData ) gConfigurationData.setVersion( serverVersion, self.cfgData ) return retVal
def _updateFromRemoteLocation(serviceClient): """ Refresh the configuration """ gLogger.debug("", "Trying to refresh from %s" % serviceClient.serverURL) localVersion = gConfigurationData.getVersion() retVal = serviceClient.getCompressedDataIfNewer(localVersion) if retVal["OK"]: dataDict = retVal["Value"] newestVersion = dataDict["newestVersion"] if localVersion < newestVersion: gLogger.debug("New version available", "Updating to version %s..." % newestVersion) gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict["data"]) gLogger.debug("Updated to version %s" % gConfigurationData.getVersion()) gEventDispatcher.triggerEvent("CSNewVersion", newestVersion, threaded=True) return S_OK() return retVal
def mergeWithServer(self): retVal = self.rpcClient.getCompressedData() if retVal["OK"]: remoteCFG = CFG() data = retVal["Value"] if isinstance(data, str): data = data.encode(errors="surrogateescape") remoteCFG.loadFromBuffer(zlib.decompress(data).decode()) serverVersion = gConfigurationData.getVersion(remoteCFG) self.cfgData = remoteCFG.mergeWith(self.cfgData) gConfigurationData.setVersion(serverVersion, self.cfgData) return retVal
def __forwardRPCCall( self, targetService, clientInitArgs, method, params ): if targetService == "Configuration/Server": if method == "getCompressedDataIfNewer": #Relay CS data directly serviceVersion = gConfigurationData.getVersion() retDict = { 'newestVersion' : serviceVersion } clientVersion = params[0] if clientVersion < serviceVersion: retDict[ 'data' ] = gConfigurationData.getCompressedData() return S_OK( retDict ) #Default rpcClient = RPCClient( targetService, **clientInitArgs ) methodObj = getattr( rpcClient, method ) return methodObj( *params )
def __forwardRPCCall(self, targetService, clientInitArgs, method, params): if targetService == "Configuration/Server": if method == "getCompressedDataIfNewer": # Relay CS data directly serviceVersion = gConfigurationData.getVersion() retDict = {"newestVersion": serviceVersion} clientVersion = params[0] if clientVersion < serviceVersion: retDict["data"] = gConfigurationData.getCompressedData() return S_OK(retDict) # Default rpcClient = RPCClient(targetService, **clientInitArgs) methodObj = getattr(rpcClient, method) return methodObj(*params)
def __getCache(self): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = (self.__threadData.vo, self.__threadData.setup) if cacheKey in Operations.__cache: return Operations.__cache[cacheKey] mergedCFG = CFG.CFG() for path in (self.__getDefaultPath(), self.__getSetupPath()): pathCFG = gConfigurationData.mergedCFG[path] if pathCFG: mergedCFG = mergedCFG.mergeWith(pathCFG) Operations.__cache[cacheKey] = mergedCFG return Operations.__cache[cacheKey] finally: Operations.__cacheLock.release()
def __getCache( self ): Operations.__cacheLock.acquire() try: currentVersion = gConfigurationData.getVersion() if currentVersion != Operations.__cacheVersion: Operations.__cache = {} Operations.__cacheVersion = currentVersion cacheKey = ( self.__threadData.vo, self.__threadData.setup ) if cacheKey in Operations.__cache: return Operations.__cache[ cacheKey ] mergedCFG = CFG.CFG() for path in ( self.__getDefaultPath(), self.__getSetupPath() ): pathCFG = gConfigurationData.mergedCFG[ path ] if pathCFG: mergedCFG = mergedCFG.mergeWith( pathCFG ) Operations.__cache[ cacheKey ] = mergedCFG return Operations.__cache[ cacheKey ] finally: Operations.__cacheLock.release()
def getVersion( self ): return gConfigurationData.getVersion()
def getVersion(self): return gConfigurationData.getVersion()