def publishSlaveServer(self, sSlaveURL): """ Called by the slave server via service, it register a new slave server :param sSlaveURL: url of slave server """ if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") gLogger.info("Pinging slave %s" % sSlaveURL) rpcClient = ConfigurationClient(url=sSlaveURL, timeout=10, useCertificates=True) retVal = rpcClient.ping() if not retVal["OK"]: gLogger.info("Slave %s didn't reply" % sSlaveURL) return if retVal["Value"]["name"] != "Configuration/Server": gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) return bNewSlave = False if sSlaveURL not in self.dAliveSlaveServers: bNewSlave = True gLogger.info("New slave registered", sSlaveURL) self.dAliveSlaveServers[sSlaveURL] = time.time() if bNewSlave: gConfigurationData.setServers(", ".join(self.dAliveSlaveServers)) self.__generateNewVersion()
def __loadConfigurationData(self): try: os.makedirs(os.path.join(DIRAC.rootPath, "etc", "csbackup")) except: pass gConfigurationData.loadConfigurationData() if gConfigurationData.isMaster(): bBuiltNewConfiguration = False if not gConfigurationData.getName(): DIRAC.abort( 10, "Missing name for the configuration to be exported!") gConfigurationData.exportName() sVersion = gConfigurationData.getVersion() if sVersion == "0": gLogger.info("There's no version. Generating a new one") gConfigurationData.generateNewVersion() bBuiltNewConfiguration = True if self.sURL not in gConfigurationData.getServers(): gConfigurationData.setServers(self.sURL) bBuiltNewConfiguration = True gConfigurationData.setMasterServer(self.sURL) if bBuiltNewConfiguration: gConfigurationData.writeRemoteConfigurationToDisk()
def __generateNewVersion(self): """ After changing configuration, we use this method to save them """ if gConfigurationData.isMaster(): gConfigurationData.generateNewVersion() gConfigurationData.writeRemoteConfigurationToDisk()
def __loadConfigurationData( self ): try: os.makedirs( os.path.join( DIRAC.rootPath, "etc", "csbackup" ) ) except: pass gConfigurationData.loadConfigurationData() if gConfigurationData.isMaster(): bBuiltNewConfiguration = False if not gConfigurationData.getName(): DIRAC.abort( 10, "Missing name for the configuration to be exported!" ) gConfigurationData.exportName() sVersion = gConfigurationData.getVersion() if sVersion == "0": gLogger.info( "There's no version. Generating a new one" ) gConfigurationData.generateNewVersion() bBuiltNewConfiguration = True if self.sURL not in gConfigurationData.getServers(): gConfigurationData.setServers( self.sURL ) bBuiltNewConfiguration = True gConfigurationData.setMasterServer( self.sURL ) if bBuiltNewConfiguration: gConfigurationData.writeRemoteConfigurationToDisk()
def updateConfiguration(self, sBuffer, committer="", updateVersionOption=False): """ Update the master configuration with the newly received changes :param str sBuffer: newly received configuration data :param str committer: the user name of the committer :param bool updateVersionOption: flag to update the current configuration version :return: S_OK/S_ERROR of the write-to-disk of the new configuration """ if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") # Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) # Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR("Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result['OK']: gLogger.warn("Could not AutoMerge!", result['Message']) return S_ERROR("AutoMerge failed: %s" % result['Message']) requestedRemoteCFG = result['Value'] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) # Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) # Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() # self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk") retVal = gConfigurationData.writeRemoteConfigurationToDisk("%s@%s" % (committer, gConfigurationData.getVersion())) gLogger.info("New version", gConfigurationData.getVersion()) # Attempt to update the configuration on currently registered slave services if gConfigurationData.getAutoSlaveSync(): result = self.forceSlavesUpdate() if not result['OK']: gLogger.warn('Failed to update slave servers') return retVal
def updateConfiguration(self, sBuffer, commiterDN="", updateVersionOption=False): if not gConfigurationData.isMaster(): return S_ERROR( "Configuration modification is not allowed in this server") #Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) #Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR( "Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion)) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result['OK']: gLogger.warn("Could not AutoMerge!", result['Message']) return S_ERROR("AutoMerge failed: %s" % result['Message']) requestedRemoteCFG = result['Value'] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) #Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) #Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() #self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk!") retVal = gConfigurationData.writeRemoteConfigurationToDisk( "%s@%s" % (commiterDN, gConfigurationData.getVersion())) gLogger.info("New version it is!") return retVal
def __init__(self, sURL): self.sURL = sURL gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) self.__modificationsIgnoreMask = ["/DIRAC/Configuration/Servers", "/DIRAC/Configuration/Version"] gConfigurationData.setAsService() if not gConfigurationData.isMaster(): gLogger.info("Starting configuration service as slave") gRefresher.autoRefreshAndPublish(self.sURL) else: gLogger.info("Starting configuration service as master") gRefresher.disable() self.__loadConfigurationData() self.dAliveSlaveServers = {} self._launchCheckSlaves()
def __init__( self, sURL ): threading.Thread.__init__( self ) self.sURL = sURL gLogger.info( "Initializing Configuration Service", "URL is %s" % sURL ) self.__modificationsIgnoreMask = [ '/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version' ] gConfigurationData.setAsService() if not gConfigurationData.isMaster(): gLogger.info( "Starting configuration service as slave" ) gRefresher.autoRefreshAndPublish( self.sURL ) else: gLogger.info( "Starting configuration service as master" ) gRefresher.disable() self.__loadConfigurationData() self.dAliveSlaveServers = {} self.__launchCheckSlaves()
def updateConfiguration(self, sBuffer, commiter="", updateVersionOption=False): if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") # Load the data in a ConfigurationData object oRemoteConfData = ConfigurationData(False) oRemoteConfData.loadRemoteCFGFromCompressedMem(sBuffer) if updateVersionOption: oRemoteConfData.setVersion(gConfigurationData.getVersion()) # Test that remote and new versions are the same sRemoteVersion = oRemoteConfData.getVersion() sLocalVersion = gConfigurationData.getVersion() gLogger.info("Checking versions\nremote: %s\nlocal: %s" % (sRemoteVersion, sLocalVersion)) if sRemoteVersion != sLocalVersion: if not gConfigurationData.mergingEnabled(): return S_ERROR( "Local and remote versions differ (%s vs %s). Cannot commit." % (sLocalVersion, sRemoteVersion) ) else: gLogger.info("AutoMerging new data!") if updateVersionOption: return S_ERROR("Cannot AutoMerge! version was overwritten") result = self.__mergeIndependentUpdates(oRemoteConfData) if not result["OK"]: gLogger.warn("Could not AutoMerge!", result["Message"]) return S_ERROR("AutoMerge failed: %s" % result["Message"]) requestedRemoteCFG = result["Value"] gLogger.info("AutoMerge successful!") oRemoteConfData.setRemoteCFG(requestedRemoteCFG) # Test that configuration names are the same sRemoteName = oRemoteConfData.getName() sLocalName = gConfigurationData.getName() if sRemoteName != sLocalName: return S_ERROR("Names differ: Server is %s and remote is %s" % (sLocalName, sRemoteName)) # Update and generate a new version gLogger.info("Committing new data...") gConfigurationData.lock() gLogger.info("Setting the new CFG") gConfigurationData.setRemoteCFG(oRemoteConfData.getRemoteCFG()) gConfigurationData.unlock() gLogger.info("Generating new version") gConfigurationData.generateNewVersion() # self.__checkSlavesStatus( forceWriteConfiguration = True ) gLogger.info("Writing new version to disk!") retVal = gConfigurationData.writeRemoteConfigurationToDisk( "%s@%s" % (commiter, gConfigurationData.getVersion()) ) gLogger.info("New version it is!") return retVal
def syncRemoteConfiguration(self, strict=False): """ Force a Resync with Configuration Server Under normal conditions this is triggered by an access to any configuration data """ if self.componentName == "Configuration/Server": if gConfigurationData.isMaster(): gLogger.info("Starting Master Configuration Server") gRefresher.disable() return S_OK() retDict = gRefresher.forceRefresh() if not retDict['OK']: gLogger.error("Can't update from any server", retDict['Message']) if strict: return retDict return S_OK()
def syncRemoteConfiguration( self, strict = False ): """ Force a Resync with Configuration Server Under normal conditions this is triggered by an access to any configuration data """ if self.componentName == "Configuration/Server" : if gConfigurationData.isMaster(): gLogger.info( "Starting Master Configuration Server" ) gRefresher.disable() return S_OK() retDict = gRefresher.forceRefresh() if not retDict['OK']: gLogger.error( "Can't update from any server", retDict[ 'Message' ] ) if strict: return retDict return S_OK()
def main(): if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." + "\n" + "It is necessary to run with Tornado." + "\n" + "https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html" ) from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.FrameworkSystem.Client.Logger import gLogger if gConfigurationData.isMaster(): gRefresher.disable() localCfg = Script.localCfg localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: gLogger.initialize("Tornado-CS", "/") gLogger.error("There were errors when loading configuration", resultDict["Message"]) sys.exit(1) includeExtensionErrors() gLogger.initialize("Tornado-CS", "/") # get the specific master CS port try: csPort = int( gConfigurationData.extractOptionFromCFG( "%s/Port" % getServiceSection("Configuration/Server"))) except TypeError: csPort = None serverToLaunch = TornadoServer(services="Configuration/Server", port=csPort) serverToLaunch.startTornado()
def __init__(self, sURL): threading.Thread.__init__(self) self.sURL = sURL gLogger.info("Initializing Configuration Service", "URL is %s" % sURL) self.__modificationsIgnoreMask = ['/DIRAC/Configuration/Servers', '/DIRAC/Configuration/Version'] gConfigurationData.setAsService() if not gConfigurationData.isMaster(): gLogger.info("Starting configuration service as slave") gRefresher.autoRefreshAndPublish(self.sURL) else: gLogger.info("Starting configuration service as master") gRefresher.disable() self.__loadConfigurationData() self.dAliveSlaveServers = {} self.__launchCheckSlaves() self.__updateResultDict = {"Successful": {}, "Failed": {}}
def main(): if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." + "\n" + "It is necessary to run with Tornado." + "\n" + "https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html" ) from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.FrameworkSystem.Client.Logger import gLogger localCfg = Script.localCfg localCfg.setConfigurationForServer("Tornado/Tornado") localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: gLogger.initialize("Tornado", "/") gLogger.error("There were errors when loading configuration", resultDict["Message"]) sys.exit(1) includeExtensionErrors() gLogger.initialize("Tornado", "/") # We check if there is no configuration server started as master # If you want to start a master CS you should use Configuration_Server.cfg and # use tornado-start-CS.py key = "/Systems/Configuration/%s/Services/Server/Protocol" % PathFinder.getSystemInstance( "Configuration") if gConfigurationData.isMaster() and gConfig.getValue( key, "dips").lower() == "https": gLogger.fatal("You can't run the CS and services in the same server!") sys.exit(0) serverToLaunch = TornadoServer(endpoints=True) serverToLaunch.startTornado()
def main(): # Must be defined BEFORE any dirac import os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.FrameworkSystem.Client.Logger import gLogger if gConfigurationData.isMaster(): gRefresher.disable() localCfg = LocalConfiguration() localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict['OK']: gLogger.initialize("Tornado-CS", "/") gLogger.error("There were errors when loading configuration", resultDict['Message']) sys.exit(1) includeExtensionErrors() gLogger.initialize('Tornado-CS', "/") # get the specific master CS port try: csPort = int( gConfigurationData.extractOptionFromCFG( '%s/Port' % getServiceSection('Configuration/Server'))) except TypeError: csPort = None serverToLaunch = TornadoServer(services='Configuration/Server', port=csPort) serverToLaunch.startTornado()
def publishSlaveServer(self, sSlaveURL): if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") gLogger.info("Pinging slave %s" % sSlaveURL) rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) retVal = rpcClient.ping() if not retVal["OK"]: gLogger.info("Slave %s didn't reply" % sSlaveURL) return if retVal["Value"]["name"] != "Configuration/Server": gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) return bNewSlave = False if not sSlaveURL in self.dAliveSlaveServers.keys(): bNewSlave = True gLogger.info("New slave registered", sSlaveURL) self.dAliveSlaveServers[sSlaveURL] = time.time() if bNewSlave: gConfigurationData.setServers("%s, %s" % (self.sURL, ", ".join(self.dAliveSlaveServers.keys()))) self.__generateNewVersion()
def publishSlaveServer(self, sSlaveURL): if not gConfigurationData.isMaster(): return S_ERROR("Configuration modification is not allowed in this server") gLogger.info("Pinging slave %s" % sSlaveURL) rpcClient = RPCClient(sSlaveURL, timeout=10, useCertificates=True) retVal = rpcClient.ping() if not retVal['OK']: gLogger.info("Slave %s didn't reply" % sSlaveURL) return if retVal['Value']['name'] != 'Configuration/Server': gLogger.info("Slave %s is not a CS serveR" % sSlaveURL) return bNewSlave = False if sSlaveURL not in self.dAliveSlaveServers: bNewSlave = True gLogger.info("New slave registered", sSlaveURL) self.dAliveSlaveServers[sSlaveURL] = time.time() if bNewSlave: gConfigurationData.setServers("%s, %s" % (self.sURL, ", ".join(self.dAliveSlaveServers.keys()))) self.__generateNewVersion()
def main(): # Must be defined BEFORE any dirac import os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client import PathFinder from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.FrameworkSystem.Client.Logger import gLogger # We check if there is no configuration server started as master # If you want to start a master CS you should use Configuration_Server.cfg and # use tornado-start-CS.py key = '/Systems/Configuration/%s/Services/Server/Protocol' % PathFinder.getSystemInstance('Configuration') if gConfigurationData.isMaster() and gConfig.getValue(key, 'dips').lower() == 'https': gLogger.fatal("You can't run the CS and services in the same server!") sys.exit(0) localCfg = LocalConfiguration() localCfg.setConfigurationForServer('Tornado/Tornado') localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict['OK']: gLogger.initialize("Tornado", "/") gLogger.error("There were errors when loading configuration", resultDict['Message']) sys.exit(1) includeExtensionErrors() gLogger.initialize('Tornado', "/") serverToLaunch = TornadoServer() serverToLaunch.startTornado()
def __generateNewVersion( self ): if gConfigurationData.isMaster(): gConfigurationData.generateNewVersion() gConfigurationData.writeRemoteConfigurationToDisk()
def isMaster( self ): return gConfigurationData.isMaster()
def __generateNewVersion(self): if gConfigurationData.isMaster(): gConfigurationData.generateNewVersion() gConfigurationData.writeRemoteConfigurationToDisk()
def isMaster(self): return gConfigurationData.isMaster()