def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False):
"""
Update the master configuration with the newly received changes
:param str sBuffer: newly received configuration data
:param str committer: the user name of the committer
:param bool updateVersionOption: flag to update the current configuration version
:return: S_OK/S_ERROR of the write-to-disk of the new configuration
"""
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk")
retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion()))
gLogger.info("New version", gConfigurationData.getVersion())
# Attempt to update the configuration on currently registered slave services
if gConfigurationData.getAutoSlaveSync():
result = self.forceSlavesUpdate()
if not result['OK']:
gLogger.warn('Failed to update slave servers')
return retVal
def updateConfiguration(self,
sBuffer,
commiterDN="",
updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR(
"Configuration modification is not allowed in this server")
#Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
#Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" %
(sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit."
% (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
#Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" %
(sLocalName, sRemoteName))
#Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
#self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiterDN, gConfigurationData.getVersion()))
gLogger.info("New version it is!")
return retVal
def _updateFromRemoteLocation( serviceClient ):
gLogger.debug( "", "Trying to refresh from %s" % serviceClient.serviceURL )
localVersion = gConfigurationData.getVersion()
retVal = serviceClient.getCompressedDataIfNewer( localVersion )
if retVal[ 'OK' ]:
dataDict = retVal[ 'Value' ]
if localVersion < dataDict[ 'newestVersion' ] :
gLogger.debug( "New version available", "Updating to version %s..." % dataDict[ 'newestVersion' ] )
gConfigurationData.loadRemoteCFGFromCompressedMem( dataDict[ 'data' ] )
gLogger.debug( "Updated to version %s" % gConfigurationData.getVersion() )
gEventDispatcher.triggerEvent( "CSNewVersion", dataDict[ 'newestVersion' ], threaded = True )
return S_OK()
return retVal
def _updateFromRemoteLocation(serviceClient):
gLogger.debug("", "Trying to refresh from %s" % serviceClient.serviceURL)
localVersion = gConfigurationData.getVersion()
retVal = serviceClient.getCompressedDataIfNewer(localVersion)
if retVal['OK']:
dataDict = retVal['Value']
if localVersion < dataDict['newestVersion']:
gLogger.debug("New version available", "Updating to version %s..." % dataDict['newestVersion'])
gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict['data'])
gLogger.debug("Updated to version %s" % gConfigurationData.getVersion())
gEventDispatcher.triggerEvent("CSNewVersion", dataDict['newestVersion'], threaded=True)
return S_OK()
return retVal
def updateConfiguration(self, sBuffer, commiter="", updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)
)
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result["OK"]:
gLogger.warn("Could not AutoMerge!", result["Message"])
return S_ERROR("AutoMerge failed: %s" % result["Message"])
requestedRemoteCFG = result["Value"]
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiter, gConfigurationData.getVersion())
)
gLogger.info("New version it is!")
return retVal
def __loadConfigurationData(self):
try:
os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup"))
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort(
10, "Missing name for the configuration to be exported!")
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info("There's no version. Generating a new one")
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers(self.sURL)
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer(self.sURL)
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __getCache( self ):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = ( self.__vo, self.__setup )
if cacheKey in Operations.__cache:
return Operations.__cache[ cacheKey ]
mergedCFG = CFG.CFG()
for path in self.__getSearchPaths():
pathCFG = gConfigurationData.mergedCFG[ path ]
if pathCFG:
mergedCFG = mergedCFG.mergeWith( pathCFG )
Operations.__cache[ cacheKey ] = mergedCFG
return Operations.__cache[ cacheKey ]
finally:
try:
Operations.__cacheLock.release()
except thread.error:
pass
def __loadConfigurationData( self ):
try:
os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) )
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort( 10, "Missing name for the configuration to be exported!" )
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info( "There's no version. Generating a new one" )
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers( self.sURL )
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer( self.sURL )
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __getCache(self):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = (self.__vo, self.__setup)
if cacheKey in Operations.__cache:
return Operations.__cache[cacheKey]
mergedCFG = CFG()
for path in self.__getSearchPaths():
pathCFG = gConfigurationData.mergedCFG[path]
if pathCFG:
mergedCFG = mergedCFG.mergeWith(pathCFG)
Operations.__cache[cacheKey] = mergedCFG
return Operations.__cache[cacheKey]
finally:
try:
Operations.__cacheLock.release()
except thread.error:
pass
def web_conf(self):
""" Configuration endpoint, used to:
GET /conf/get?<options> -- get configuration information, with arguments
* options:
* fullCFG - to get dump of configuration
* option - option path to get option value
* options - section path to get list of options
* section - section path to get dict of all options/values there
* sections - section path to get list of sections there
* version - version of configuration that request information(optional)
GET /conf/<helper method>?<arguments> -- get some information by using helpers methods
* helper method - helper method of configuration service
* arguments - arguments specifecly for every helper method
:return: json with requested data
"""
self.log.notice('Request configuration information')
optns = self.overpath.strip('/').split('/')
if not optns or len(optns) > 1:
raise WErr(404, "Wrone way")
if optns[0] == 'get':
if 'version' in self.args and (
self.args.get('version')
or '0') >= gConfigurationData.getVersion():
self.finish()
result = {}
if 'fullCFG' in self.args:
remoteCFG = yield self.threadTask(
gConfigurationData.getRemoteCFG)
result['Value'] = str(remoteCFG)
elif 'option' in self.args:
result = yield self.threadTask(gConfig.getOption,
self.args['option'])
elif 'section' in self.args:
result = yield self.threadTask(gConfig.getOptionsDict,
self.args['section'])
elif 'options' in self.args:
result = yield self.threadTask(gConfig.getOptions,
self.args['options'])
elif 'sections' in self.args:
result = yield self.threadTask(gConfig.getSections,
self.args['sections'])
else:
raise WErr(500, 'Invalid argument')
elif any([
optns[0] == m and re.match('^[a-z][A-z]+', m)
for m in dir(Registry)
]) and self.isRegisteredUser():
result = yield self.threadTask(getattr(Registry, optns[0]),
**self.args)
if not result['OK']:
raise WErr(404, result['Message'])
self.finishJEncode(result['Value'])
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def mergeWithServer( self ):
retVal = self.rpcClient.getCompressedData()
if retVal[ 'OK' ]:
remoteCFG = CFG()
remoteCFG.loadFromBuffer( zlib.decompress( retVal[ 'Value' ] ) )
serverVersion = gConfigurationData.getVersion( remoteCFG )
self.cfgData = remoteCFG.mergeWith( self.cfgData )
gConfigurationData.setVersion( serverVersion, self.cfgData )
return retVal
def _updateFromRemoteLocation(serviceClient):
"""
Refresh the configuration
"""
gLogger.debug("", "Trying to refresh from %s" % serviceClient.serverURL)
localVersion = gConfigurationData.getVersion()
retVal = serviceClient.getCompressedDataIfNewer(localVersion)
if retVal["OK"]:
dataDict = retVal["Value"]
newestVersion = dataDict["newestVersion"]
if localVersion < newestVersion:
gLogger.debug("New version available",
"Updating to version %s..." % newestVersion)
gConfigurationData.loadRemoteCFGFromCompressedMem(dataDict["data"])
gLogger.debug("Updated to version %s" %
gConfigurationData.getVersion())
gEventDispatcher.triggerEvent("CSNewVersion",
newestVersion,
threaded=True)
return S_OK()
return retVal
def mergeWithServer(self):
retVal = self.rpcClient.getCompressedData()
if retVal["OK"]:
remoteCFG = CFG()
data = retVal["Value"]
if isinstance(data, str):
data = data.encode(errors="surrogateescape")
remoteCFG.loadFromBuffer(zlib.decompress(data).decode())
serverVersion = gConfigurationData.getVersion(remoteCFG)
self.cfgData = remoteCFG.mergeWith(self.cfgData)
gConfigurationData.setVersion(serverVersion, self.cfgData)
return retVal
def __forwardRPCCall( self, targetService, clientInitArgs, method, params ):
if targetService == "Configuration/Server":
if method == "getCompressedDataIfNewer":
#Relay CS data directly
serviceVersion = gConfigurationData.getVersion()
retDict = { 'newestVersion' : serviceVersion }
clientVersion = params[0]
if clientVersion < serviceVersion:
retDict[ 'data' ] = gConfigurationData.getCompressedData()
return S_OK( retDict )
#Default
rpcClient = RPCClient( targetService, **clientInitArgs )
methodObj = getattr( rpcClient, method )
return methodObj( *params )
def __forwardRPCCall(self, targetService, clientInitArgs, method, params):
if targetService == "Configuration/Server":
if method == "getCompressedDataIfNewer":
# Relay CS data directly
serviceVersion = gConfigurationData.getVersion()
retDict = {"newestVersion": serviceVersion}
clientVersion = params[0]
if clientVersion < serviceVersion:
retDict["data"] = gConfigurationData.getCompressedData()
return S_OK(retDict)
# Default
rpcClient = RPCClient(targetService, **clientInitArgs)
methodObj = getattr(rpcClient, method)
return methodObj(*params)
def __getCache(self):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = (self.__threadData.vo, self.__threadData.setup)
if cacheKey in Operations.__cache:
return Operations.__cache[cacheKey]
mergedCFG = CFG.CFG()
for path in (self.__getDefaultPath(), self.__getSetupPath()):
pathCFG = gConfigurationData.mergedCFG[path]
if pathCFG:
mergedCFG = mergedCFG.mergeWith(pathCFG)
Operations.__cache[cacheKey] = mergedCFG
return Operations.__cache[cacheKey]
finally:
Operations.__cacheLock.release()
def __getCache( self ):
Operations.__cacheLock.acquire()
try:
currentVersion = gConfigurationData.getVersion()
if currentVersion != Operations.__cacheVersion:
Operations.__cache = {}
Operations.__cacheVersion = currentVersion
cacheKey = ( self.__threadData.vo, self.__threadData.setup )
if cacheKey in Operations.__cache:
return Operations.__cache[ cacheKey ]
mergedCFG = CFG.CFG()
for path in ( self.__getDefaultPath(), self.__getSetupPath() ):
pathCFG = gConfigurationData.mergedCFG[ path ]
if pathCFG:
mergedCFG = mergedCFG.mergeWith( pathCFG )
Operations.__cache[ cacheKey ] = mergedCFG
return Operations.__cache[ cacheKey ]
finally:
Operations.__cacheLock.release()
def getVersion( self ): return gConfigurationData.getVersion()
def getVersion(self):
return gConfigurationData.getVersion()
