def publishSlaveServer(self, sSlaveURL):
"""
Called by the slave server via service, it register a new slave server
:param sSlaveURL: url of slave server
"""
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
gLogger.info("Pinging slave %s" % sSlaveURL)
rpcClient = ConfigurationClient(url=sSlaveURL, timeout=10, useCertificates=True)
retVal = rpcClient.ping()
if not retVal["OK"]:
gLogger.info("Slave %s didn't reply" % sSlaveURL)
return
if retVal["Value"]["name"] != "Configuration/Server":
gLogger.info("Slave %s is not a CS serveR" % sSlaveURL)
return
bNewSlave = False
if sSlaveURL not in self.dAliveSlaveServers:
bNewSlave = True
gLogger.info("New slave registered", sSlaveURL)
self.dAliveSlaveServers[sSlaveURL] = time.time()
if bNewSlave:
gConfigurationData.setServers(", ".join(self.dAliveSlaveServers))
self.__generateNewVersion()
def __loadConfigurationData(self):
try:
os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup"))
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort(
10, "Missing name for the configuration to be exported!")
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info("There's no version. Generating a new one")
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers(self.sURL)
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer(self.sURL)
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def __generateNewVersion(self):
"""
After changing configuration, we use this method to save them
"""
if gConfigurationData.isMaster():
gConfigurationData.generateNewVersion()
gConfigurationData.writeRemoteConfigurationToDisk()
def __loadConfigurationData( self ):
try:
os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) )
except:
pass
gConfigurationData.loadConfigurationData()
if gConfigurationData.isMaster():
bBuiltNewConfiguration = False
if not gConfigurationData.getName():
DIRAC.abort( 10, "Missing name for the configuration to be exported!" )
gConfigurationData.exportName()
sVersion = gConfigurationData.getVersion()
if sVersion == "0":
gLogger.info( "There's no version. Generating a new one" )
gConfigurationData.generateNewVersion()
bBuiltNewConfiguration = True
if self.sURL not in gConfigurationData.getServers():
gConfigurationData.setServers( self.sURL )
bBuiltNewConfiguration = True
gConfigurationData.setMasterServer( self.sURL )
if bBuiltNewConfiguration:
gConfigurationData.writeRemoteConfigurationToDisk()
def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False):
"""
Update the master configuration with the newly received changes
:param str sBuffer: newly received configuration data
:param str committer: the user name of the committer
:param bool updateVersionOption: flag to update the current configuration version
:return: S_OK/S_ERROR of the write-to-disk of the new configuration
"""
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk")
retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion()))
gLogger.info("New version", gConfigurationData.getVersion())
# Attempt to update the configuration on currently registered slave services
if gConfigurationData.getAutoSlaveSync():
result = self.forceSlavesUpdate()
if not result['OK']:
gLogger.warn('Failed to update slave servers')
return retVal
def updateConfiguration(self,
sBuffer,
commiterDN="",
updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR(
"Configuration modification is not allowed in this server")
#Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
#Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" %
(sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit."
% (sLocalVersion, sRemoteVersion))
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result['OK']:
gLogger.warn("Could not AutoMerge!", result['Message'])
return S_ERROR("AutoMerge failed: %s" % result['Message'])
requestedRemoteCFG = result['Value']
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
#Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" %
(sLocalName, sRemoteName))
#Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
#self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiterDN, gConfigurationData.getVersion()))
gLogger.info("New version it is!")
return retVal
def __init__(self, sURL):
self.sURL = sURL
gLogger.info("Initializing Configuration Service", "URL is %s" % sURL)
self.__modificationsIgnoreMask = ["/DIRAC/Configuration/Servers", "/DIRAC/Configuration/Version"]
gConfigurationData.setAsService()
if not gConfigurationData.isMaster():
gLogger.info("Starting configuration service as slave")
gRefresher.autoRefreshAndPublish(self.sURL)
else:
gLogger.info("Starting configuration service as master")
gRefresher.disable()
self.__loadConfigurationData()
self.dAliveSlaveServers = {}
self._launchCheckSlaves()
def __init__( self, sURL ):
threading.Thread.__init__( self )
self.sURL = sURL
gLogger.info( "Initializing Configuration Service", "URL is %s" % sURL )
self.__modificationsIgnoreMask = [ '/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version' ]
gConfigurationData.setAsService()
if not gConfigurationData.isMaster():
gLogger.info( "Starting configuration service as slave" )
gRefresher.autoRefreshAndPublish( self.sURL )
else:
gLogger.info( "Starting configuration service as master" )
gRefresher.disable()
self.__loadConfigurationData()
self.dAliveSlaveServers = {}
self.__launchCheckSlaves()
def updateConfiguration(self, sBuffer, commiter="", updateVersionOption=False):
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
# Load the data in a ConfigurationData object
oRemoteConfData = ConfigurationData(False)
oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer)
if updateVersionOption:
oRemoteConfData.setVersion(gConfigurationData.getVersion())
# Test that remote and new versions are the same
sRemoteVersion = oRemoteConfData.getVersion()
sLocalVersion = gConfigurationData.getVersion()
gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion))
if sRemoteVersion != sLocalVersion:
if not gConfigurationData.mergingEnabled():
return S_ERROR(
"Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)
)
else:
gLogger.info("AutoMerging new data!")
if updateVersionOption:
return S_ERROR("Cannot AutoMerge! version was overwritten")
result = self.__mergeIndependentUpdates(oRemoteConfData)
if not result["OK"]:
gLogger.warn("Could not AutoMerge!", result["Message"])
return S_ERROR("AutoMerge failed: %s" % result["Message"])
requestedRemoteCFG = result["Value"]
gLogger.info("AutoMerge successful!")
oRemoteConfData.setRemoteCFG(requestedRemoteCFG)
# Test that configuration names are the same
sRemoteName = oRemoteConfData.getName()
sLocalName = gConfigurationData.getName()
if sRemoteName != sLocalName:
return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName))
# Update and generate a new version
gLogger.info("Committing new data...")
gConfigurationData.lock()
gLogger.info("Setting the new CFG")
gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG())
gConfigurationData.unlock()
gLogger.info("Generating new version")
gConfigurationData.generateNewVersion()
# self.__checkSlavesStatus( forceWriteConfiguration = True )
gLogger.info("Writing new version to disk!")
retVal = gConfigurationData.writeRemoteConfigurationToDisk(
"%s@%s" % (commiter, gConfigurationData.getVersion())
)
gLogger.info("New version it is!")
return retVal
def syncRemoteConfiguration(self, strict=False):
"""
Force a Resync with Configuration Server
Under normal conditions this is triggered by an access to any configuration data
"""
if self.componentName == "Configuration/Server":
if gConfigurationData.isMaster():
gLogger.info("Starting Master Configuration Server")
gRefresher.disable()
return S_OK()
retDict = gRefresher.forceRefresh()
if not retDict['OK']:
gLogger.error("Can't update from any server", retDict['Message'])
if strict:
return retDict
return S_OK()
def syncRemoteConfiguration( self, strict = False ):
"""
Force a Resync with Configuration Server
Under normal conditions this is triggered by an access to any configuration data
"""
if self.componentName == "Configuration/Server" :
if gConfigurationData.isMaster():
gLogger.info( "Starting Master Configuration Server" )
gRefresher.disable()
return S_OK()
retDict = gRefresher.forceRefresh()
if not retDict['OK']:
gLogger.error( "Can't update from any server", retDict[ 'Message' ] )
if strict:
return retDict
return S_OK()
def main():
if os.environ.get("DIRAC_USE_TORNADO_IOLOOP",
"false").lower() not in ("yes", "true"):
raise RuntimeError(
"DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." +
"\n" + "It is necessary to run with Tornado." + "\n" +
"https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html"
)
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.private.Refresher import gRefresher
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.FrameworkSystem.Client.Logger import gLogger
if gConfigurationData.isMaster():
gRefresher.disable()
localCfg = Script.localCfg
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict["OK"]:
gLogger.initialize("Tornado-CS", "/")
gLogger.error("There were errors when loading configuration",
resultDict["Message"])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize("Tornado-CS", "/")
# get the specific master CS port
try:
csPort = int(
gConfigurationData.extractOptionFromCFG(
"%s/Port" % getServiceSection("Configuration/Server")))
except TypeError:
csPort = None
serverToLaunch = TornadoServer(services="Configuration/Server",
port=csPort)
serverToLaunch.startTornado()
def __init__(self, sURL):
threading.Thread.__init__(self)
self.sURL = sURL
gLogger.info("Initializing Configuration Service", "URL is %s" % sURL)
self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version']
gConfigurationData.setAsService()
if not gConfigurationData.isMaster():
gLogger.info("Starting configuration service as slave")
gRefresher.autoRefreshAndPublish(self.sURL)
else:
gLogger.info("Starting configuration service as master")
gRefresher.disable()
self.__loadConfigurationData()
self.dAliveSlaveServers = {}
self.__launchCheckSlaves()
self.__updateResultDict = {"Successful": {}, "Failed": {}}
def main():
if os.environ.get("DIRAC_USE_TORNADO_IOLOOP",
"false").lower() not in ("yes", "true"):
raise RuntimeError(
"DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." +
"\n" + "It is necessary to run with Tornado." + "\n" +
"https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html"
)
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.FrameworkSystem.Client.Logger import gLogger
localCfg = Script.localCfg
localCfg.setConfigurationForServer("Tornado/Tornado")
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict["OK"]:
gLogger.initialize("Tornado", "/")
gLogger.error("There were errors when loading configuration",
resultDict["Message"])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize("Tornado", "/")
# We check if there is no configuration server started as master
# If you want to start a master CS you should use Configuration_Server.cfg and
# use tornado-start-CS.py
key = "/Systems/Configuration/%s/Services/Server/Protocol" % PathFinder.getSystemInstance(
"Configuration")
if gConfigurationData.isMaster() and gConfig.getValue(
key, "dips").lower() == "https":
gLogger.fatal("You can't run the CS and services in the same server!")
sys.exit(0)
serverToLaunch = TornadoServer(endpoints=True)
serverToLaunch.startTornado()
def main():
# Must be defined BEFORE any dirac import
os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True"
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration
from DIRAC.ConfigurationSystem.private.Refresher import gRefresher
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.FrameworkSystem.Client.Logger import gLogger
if gConfigurationData.isMaster():
gRefresher.disable()
localCfg = LocalConfiguration()
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict['OK']:
gLogger.initialize("Tornado-CS", "/")
gLogger.error("There were errors when loading configuration",
resultDict['Message'])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize('Tornado-CS', "/")
# get the specific master CS port
try:
csPort = int(
gConfigurationData.extractOptionFromCFG(
'%s/Port' % getServiceSection('Configuration/Server')))
except TypeError:
csPort = None
serverToLaunch = TornadoServer(services='Configuration/Server',
port=csPort)
serverToLaunch.startTornado()
def publishSlaveServer(self, sSlaveURL):
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
gLogger.info("Pinging slave %s" % sSlaveURL)
rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True)
retVal = rpcClient.ping()
if not retVal["OK"]:
gLogger.info("Slave %s didn't reply" % sSlaveURL)
return
if retVal["Value"]["name"] != "Configuration/Server":
gLogger.info("Slave %s is not a CS serveR" % sSlaveURL)
return
bNewSlave = False
if not sSlaveURL in self.dAliveSlaveServers.keys():
bNewSlave = True
gLogger.info("New slave registered", sSlaveURL)
self.dAliveSlaveServers[sSlaveURL] = time.time()
if bNewSlave:
gConfigurationData.setServers("%s, %s" % (self.sURL, ", ".join(self.dAliveSlaveServers.keys())))
self.__generateNewVersion()
def publishSlaveServer(self, sSlaveURL):
if not gConfigurationData.isMaster():
return S_ERROR("Configuration modification is not allowed in this server")
gLogger.info("Pinging slave %s" % sSlaveURL)
rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True)
retVal = rpcClient.ping()
if not retVal['OK']:
gLogger.info("Slave %s didn't reply" % sSlaveURL)
return
if retVal['Value']['name'] != 'Configuration/Server':
gLogger.info("Slave %s is not a CS serveR" % sSlaveURL)
return
bNewSlave = False
if sSlaveURL not in self.dAliveSlaveServers:
bNewSlave = True
gLogger.info("New slave registered", sSlaveURL)
self.dAliveSlaveServers[sSlaveURL] = time.time()
if bNewSlave:
gConfigurationData.setServers("%s, %s" % (self.sURL,
", ".join(self.dAliveSlaveServers.keys())))
self.__generateNewVersion()
def main():
# Must be defined BEFORE any dirac import
os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True"
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client import PathFinder
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.FrameworkSystem.Client.Logger import gLogger
# We check if there is no configuration server started as master
# If you want to start a master CS you should use Configuration_Server.cfg and
# use tornado-start-CS.py
key = '/Systems/Configuration/%s/Services/Server/Protocol' % PathFinder.getSystemInstance('Configuration')
if gConfigurationData.isMaster() and gConfig.getValue(key, 'dips').lower() == 'https':
gLogger.fatal("You can't run the CS and services in the same server!")
sys.exit(0)
localCfg = LocalConfiguration()
localCfg.setConfigurationForServer('Tornado/Tornado')
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict['OK']:
gLogger.initialize("Tornado", "/")
gLogger.error("There were errors when loading configuration", resultDict['Message'])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize('Tornado', "/")
serverToLaunch = TornadoServer()
serverToLaunch.startTornado()
def __generateNewVersion( self ):
if gConfigurationData.isMaster():
gConfigurationData.generateNewVersion()
gConfigurationData.writeRemoteConfigurationToDisk()
def isMaster( self ): return gConfigurationData.isMaster()
def __generateNewVersion(self):
if gConfigurationData.isMaster():
gConfigurationData.generateNewVersion()
gConfigurationData.writeRemoteConfigurationToDisk()
def isMaster(self):
return gConfigurationData.isMaster()