def getOption( self, optionName ):
if optionName[0] == "/":
return gConfigurationData.extractOptionFromCFG( optionName )
for path in self.pathList:
value = gConfigurationData.extractOptionFromCFG( "%s/%s" % ( path, optionName ) )
if value:
return value
return None
def getOption(self, optionName):
if optionName[0] == "/":
return gConfigurationData.extractOptionFromCFG(optionName)
for path in self.pathList:
value = gConfigurationData.extractOptionFromCFG("%s/%s" % (path, optionName))
if value:
return value
return None
def getSystemInstance(systemName, setup=False):
if not setup:
setup = gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")
optionPath = "/DIRAC/Setups/%s/%s" % (setup, systemName)
instance = gConfigurationData.extractOptionFromCFG(optionPath)
if instance:
return instance
else:
raise RuntimeError("Option %s is not defined" % optionPath)
def getSystemInstance( systemName, setup = False ):
if not setup:
setup = gConfigurationData.extractOptionFromCFG( "/DIRAC/Setup" )
optionPath = "/DIRAC/Setups/%s/%s" % ( setup, systemName )
instance = gConfigurationData.extractOptionFromCFG( optionPath )
if instance:
return instance
else:
raise RuntimeError( "Option %s is not defined" % optionPath )
def getGatewayURLs( serviceName = "" ):
siteName = gConfigurationData.extractOptionFromCFG( "/LocalSite/Site" )
if not siteName:
return False
gatewayList = gConfigurationData.extractOptionFromCFG( "/DIRAC/Gateways/%s" % siteName )
if not gatewayList:
return False
if serviceName:
gatewayList = [ "%s/%s" % ( "/".join( gw.split( "/" )[:3] ), serviceName ) for gw in List.fromChar( gatewayList, "," ) ]
return List.randomize( gatewayList )
def getGatewayURLs( serviceName = "" ):
siteName = gConfigurationData.extractOptionFromCFG( "/LocalSite/Site" )
if not siteName:
return False
gatewayList = gConfigurationData.extractOptionFromCFG( "/DIRAC/Gateways/%s" % siteName )
if not gatewayList:
return False
if serviceName:
gatewayList = [ "%s/%s" % ( "/".join( gw.split( "/" )[:3] ), serviceName ) for gw in List.fromChar( gatewayList, "," ) ]
return List.randomize( gatewayList )
def loadUserData(self):
"""
This is the magic method that reads the command line and processes it
It is used by the Script Base class and the dirac-service and dirac-agent scripts
Before being called:
- any additional switches to be processed
- mandatory and default configuration configuration options must be defined.
"""
if self.initialized:
return S_OK()
self.initialized = True
try:
retVal = self.__addUserDataToConfiguration()
for optionTuple in self.optionalEntryList:
optionPath = self.__getAbsolutePath(optionTuple[0])
if not gConfigurationData.extractOptionFromCFG(optionPath):
gConfigurationData.setOptionInCFG(optionPath,
optionTuple[1])
self.__initLogger(self.componentName, self.loggingSection)
if not retVal['OK']:
return retVal
retVal = self.__checkMandatoryOptions()
if not retVal['OK']:
return retVal
except Exception as e:
gLogger.exception()
return S_ERROR(str(e))
return S_OK()
def getOption( self, optionPath, typeValue = None ):
gRefresher.refreshConfigurationIfNeeded()
optionValue = gConfigurationData.extractOptionFromCFG( optionPath )
if optionValue == None:
return S_ERROR( "Path %s does not exist or it's not an option" % optionPath )
# Value has been returned from the configuration
if typeValue == None:
return S_OK( optionValue )
# Casting to typeValue's type
requestedType = typeValue
if not type( typeValue ) == types.TypeType:
requestedType = type( typeValue )
if requestedType == types.ListType:
try:
return S_OK( List.fromChar( optionValue, ',' ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to comma separated list" % str( optionValue ) )
elif requestedType == types.BooleanType:
try:
return S_OK( optionValue.lower() in ( "y", "yes", "true", "1" ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to Boolean" % str( optionValue ) )
else:
try:
return S_OK( requestedType( optionValue ) )
except:
return S_ERROR( "Type mismatch between default (%s) and configured value (%s) " % ( str( typeValue ), optionValue ) )
def loadUserData( self ):
"""
This is the magic method that reads the command line and processes it
It is used by the Script Base class and the dirac-service and dirac-agent scripts
Before being called:
- any additional switches to be processed
- mandatory and default configuration configuration options must be defined.
"""
if self.initialized:
return S_OK()
self.initialized = True
try:
retVal = self.__addUserDataToConfiguration()
for optionTuple in self.optionalEntryList:
optionPath = self.__getAbsolutePath( optionTuple[0] )
if not gConfigurationData.extractOptionFromCFG( optionPath ):
gConfigurationData.setOptionInCFG( optionPath, optionTuple[1] )
self.__initLogger( self.componentName, self.loggingSection )
if not retVal[ 'OK' ]:
return retVal
retVal = self.__checkMandatoryOptions()
if not retVal[ 'OK' ]:
return retVal
except Exception as e:
gLogger.exception()
return S_ERROR( str( e ) )
return S_OK()
def getOption( self, optionPath, typeValue = None ):
gRefresher.refreshConfigurationIfNeeded()
optionValue = gConfigurationData.extractOptionFromCFG( optionPath )
if optionValue == None:
return S_ERROR( "Path %s does not exist or it's not an option" % optionPath )
#Value has been returned from the configuration
if typeValue == None:
return S_OK( optionValue )
#Casting to typeValue's type
requestedType = typeValue
if not type( typeValue ) == types.TypeType:
requestedType = type( typeValue )
if requestedType == types.ListType:
try:
return S_OK( List.fromChar( optionValue, ',' ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to comma separated list" % str( optionValue ) )
elif requestedType == types.BooleanType:
try:
return S_OK( optionValue.lower() in ( "y", "yes", "true", "1" ) )
except Exception:
return S_ERROR( "Can't convert value (%s) to Boolean" % str( optionValue ) )
else:
try:
return S_OK( requestedType( optionValue ) )
except:
return S_ERROR( "Type mismatch between default (%s) and configured value (%s) " % ( str( typeValue ), optionValue ) )
def getOption(self, optionPath, typeValue=None):
""" Get configuration option
:param str optionPath: option path
:param typeValue: type of value
:return: S_OK()/S_ERROR()
"""
gRefresher.refreshConfigurationIfNeeded()
optionValue = gConfigurationData.extractOptionFromCFG(optionPath)
if optionValue is None:
return S_ERROR("Path %s does not exist or it's not an option" %
optionPath)
# Value has been returned from the configuration
if typeValue is None:
return S_OK(optionValue)
# Casting to typeValue's type
if not isinstance(typeValue, type):
# typeValue is not a type but a default object
requestedType = type(typeValue)
else:
requestedType = typeValue
if requestedType in (list, tuple, set):
try:
return S_OK(requestedType(List.fromChar(optionValue, ',')))
except Exception as e:
return S_ERROR(
"Can't convert value (%s) to comma separated list \n%s" %
(str(optionValue), repr(e)))
elif requestedType == bool:
try:
return S_OK(optionValue.lower() in ("y", "yes", "true", "1"))
except Exception as e:
return S_ERROR("Can't convert value (%s) to Boolean \n%s" %
(str(optionValue), repr(e)))
elif requestedType == dict:
try:
splitOption = List.fromChar(optionValue, ',')
value = {}
for opt in splitOption:
keyVal = [x.strip() for x in opt.split(':')]
if len(keyVal) == 1:
keyVal.append(True)
value[keyVal[0]] = keyVal[1]
return S_OK(value)
except Exception as e:
return S_ERROR("Can't convert value (%s) to Dict \n%s" %
(str(optionValue), repr(e)))
else:
try:
return S_OK(requestedType(optionValue))
except Exception as e:
return S_ERROR(
"Type mismatch between default (%s) and configured value (%s) \n%s"
% (str(typeValue), optionValue, repr(e)))
def addDefaultEntry(self, optionPath, value):
"""
Define a default value for a Configuration data option
"""
if optionPath[0] == "/":
if not gConfigurationData.extractOptionFromCFG(optionPath):
self.__setOptionValue(optionPath, value)
else:
self.optionalEntryList.append((optionPath, str(value)))
def getGatewayURLs(system="", service=None):
"""Get gateway URLs for service
:param str system: system name or full name, like 'Framework/Service'.
:param str service: service name, like 'ProxyManager'.
:return: list or False
"""
if system:
system, service = divideFullName(system, service)
siteName = gConfigurationData.extractOptionFromCFG("/LocalSite/Site")
if not siteName:
return False
gateways = gConfigurationData.extractOptionFromCFG("/DIRAC/Gateways/%s" % siteName)
if not gateways:
return False
gateways = List.randomize(List.fromChar(gateways, ","))
return [checkComponentURL(u, system, service) for u in gateways if u] if system and service else gateways
def addDefaultEntry( self, optionPath, value ):
"""
Define a default value for a Configuration data option
"""
if optionPath[0] == "/":
if not gConfigurationData.extractOptionFromCFG( optionPath ):
self.__setOptionValue( optionPath, value )
else:
self.optionalEntryList.append( ( optionPath,
str( value ) ) )
def getOptionsDict(self, sectionPath):
gRefresher.refreshConfigurationIfNeeded()
optionsDict = {}
optionList = gConfigurationData.getOptionsFromCFG(sectionPath)
if type(optionList) == types.ListType:
for option in optionList:
optionsDict[option] = gConfigurationData.extractOptionFromCFG("%s/%s" % (sectionPath, option))
return S_OK(optionsDict)
else:
return S_ERROR("Path %s does not exist or it's not a section" % sectionPath)
def getServiceURL( serviceName, serviceTuple = False, setup = False ):
if not serviceTuple:
serviceTuple = divideFullName( serviceName )
systemSection = getSystemSection( serviceName, serviceTuple, setup = setup )
url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) )
if not url:
return ""
if len( url.split( "/" ) ) < 5:
url = "%s/%s" % ( url, serviceName )
return url
def getServiceURL( serviceName, serviceTuple = False, setup = False ):
if not serviceTuple:
serviceTuple = divideFullName( serviceName )
systemSection = getSystemSection( serviceName, serviceTuple, setup = setup )
url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) )
if not url:
return ""
if len( url.split( "/" ) ) < 5:
url = "%s/%s" % ( url, serviceName )
return url
def getServiceURLs(system, service=None, setup=False, failover=False):
"""Generate url.
:param str system: system name or full name e.g.: Framework/ProxyManager
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:param bool failover: to add failover URLs to end of result list
:return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service]
"""
system, service = divideFullName(system, service)
resList = []
mainServers = None
systemSection = getSystemSection(system, setup=setup)
# Add failover URLs at the end of the list
failover = "Failover" if failover else ""
for fURLs in ["", "Failover"] if failover else [""]:
urlList = []
urls = List.fromChar(
gConfigurationData.extractOptionFromCFG(
"%s/%sURLs/%s" % (systemSection, fURLs, service)))
# Be sure that urls not None
for url in urls or []:
# Trying if we are refering to the list of main servers
# which would be like dips://$MAINSERVERS$:1234/System/Component
if "$MAINSERVERS$" in url:
if not mainServers:
# Operations cannot be imported at the beginning because of a bootstrap problem
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
mainServers = Operations(setup=setup).getValue(
"MainServers", [])
if not mainServers:
raise Exception("No Main servers defined")
for srv in mainServers:
_url = checkComponentURL(url.replace("$MAINSERVERS$", srv),
system,
service,
pathMandatory=True)
if _url not in urlList:
urlList.append(_url)
continue
_url = checkComponentURL(url, system, service, pathMandatory=True)
if _url not in urlList:
urlList.append(_url)
# Randomize list if needed
resList.extend(List.randomize(urlList))
return resList
def getOptionsDict( self, sectionPath ):
gRefresher.refreshConfigurationIfNeeded()
optionsDict = {}
optionList = gConfigurationData.getOptionsFromCFG( sectionPath )
if type( optionList ) == types.ListType:
for option in optionList:
optionsDict[ option ] = gConfigurationData.extractOptionFromCFG( "%s/%s" %
( sectionPath, option ) )
return S_OK( optionsDict )
else:
return S_ERROR( "Path %s does not exist or it's not a section" % sectionPath )
def getSystemInstance(system, setup=False):
"""Find system instance name
:param str system: system name
:param str setup: setup name
:return: str
"""
optionPath = Path.cfgPath("/DIRAC/Setups", setup or getDIRACSetup(), system)
instance = gConfigurationData.extractOptionFromCFG(optionPath)
if not instance:
raise RuntimeError("Option %s is not defined" % optionPath)
return instance
def __checkMandatoryOptions(self):
try:
isMandatoryMissing = False
for optionPath in self.mandatoryEntryList:
optionPath = self.__getAbsolutePath(optionPath)
if not gConfigurationData.extractOptionFromCFG(optionPath):
gLogger.fatal("Missing mandatory local configuration option", optionPath)
isMandatoryMissing = True
if isMandatoryMissing:
return S_ERROR()
return S_OK()
except Exception as e:
gLogger.exception()
return S_ERROR(str(e))
def __checkMandatoryOptions( self ):
try:
isMandatoryMissing = False
for optionPath in self.mandatoryEntryList:
optionPath = self.__getAbsolutePath( optionPath )
if not gConfigurationData.extractOptionFromCFG( optionPath ):
gLogger.fatal( "Missing mandatory local configuration option", optionPath )
isMandatoryMissing = True
if isMandatoryMissing:
return S_ERROR()
return S_OK()
except Exception as e:
gLogger.exception()
return S_ERROR( str( e ) )
def getServiceFailoverURL(system, service=None, setup=False):
"""Get failover URLs for service
:param str system: system name or full name, like 'Framework/Service'.
:param str service: service name, like 'ProxyManager'.
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: str -- complete list of urls
"""
system, service = divideFullName(system, service)
systemSection = getSystemSection(system, setup=setup)
failovers = gConfigurationData.extractOptionFromCFG("%s/FailoverURLs/%s" % (systemSection, service))
if not failovers:
return ""
return ",".join([checkComponentURL(u, system, service) for u in List.fromChar(failovers, ",") if u])
def getServiceURL(serviceName, serviceTuple=False, setup=False):
"""
Generate url.
:param serviceName: Name of service, like 'Framework/Service'.
:param serviceTuple: (optional) also name of service but look like ('Framework', 'Service').
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: complete url. e.g. dips://some-domain:3424/Framework/Service
"""
if not serviceTuple:
serviceTuple = divideFullName(serviceName)
systemSection = getSystemSection(serviceName, serviceTuple, setup=setup)
url = gConfigurationData.extractOptionFromCFG(
"%s/URLs/%s" % (systemSection, serviceTuple[1]))
if not url:
return ""
# Trying if we are refering to the list of main servers
# which would be like dips://$MAINSERVERS$:1234/System/Component
# This can only happen if there is only one server defined
if ',' not in url:
urlParse = urlparse.urlparse(url)
server, port = urlParse.netloc.split(':')
mainUrlsList = []
if server == '$MAINSERVERS$':
# Operations cannot be imported at the beginning because of a bootstrap problem
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
mainServers = Operations().getValue('MainServers', [])
if not mainServers:
raise Exception("No Main servers defined")
for srv in mainServers:
mainUrlsList.append(
urlparse.ParseResult(scheme=urlParse.scheme,
netloc=':'.join([srv, port]),
path=urlParse.path,
params='',
query='',
fragment='').geturl())
return ','.join(mainUrlsList)
if len(url.split("/")) < 5:
url = "%s/%s" % (url, serviceName)
return url
def main():
if os.environ.get("DIRAC_USE_TORNADO_IOLOOP",
"false").lower() not in ("yes", "true"):
raise RuntimeError(
"DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." +
"\n" + "It is necessary to run with Tornado." + "\n" +
"https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html"
)
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.private.Refresher import gRefresher
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.FrameworkSystem.Client.Logger import gLogger
if gConfigurationData.isMaster():
gRefresher.disable()
localCfg = Script.localCfg
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict["OK"]:
gLogger.initialize("Tornado-CS", "/")
gLogger.error("There were errors when loading configuration",
resultDict["Message"])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize("Tornado-CS", "/")
# get the specific master CS port
try:
csPort = int(
gConfigurationData.extractOptionFromCFG(
"%s/Port" % getServiceSection("Configuration/Server")))
except TypeError:
csPort = None
serverToLaunch = TornadoServer(services="Configuration/Server",
port=csPort)
serverToLaunch.startTornado()
def getServiceURL( serviceName, serviceTuple = False, setup = False ):
"""
Generate url.
:param serviceName: Name of service, like 'Framework/Service'.
:param serviceTuple: (optional) also name of service but look like ('Framework', 'Service').
:param str setup: DIRAC setup name, can be defined in dirac.cfg
:return: complete url. e.g. dips://some-domain:3424/Framework/Service
"""
if not serviceTuple:
serviceTuple = divideFullName( serviceName )
systemSection = getSystemSection( serviceName, serviceTuple, setup = setup )
url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) )
if not url:
return ""
# Trying if we are refering to the list of main servers
# which would be like dips://$MAINSERVERS$:1234/System/Component
# This can only happen if there is only one server defined
if ',' not in url:
urlParse = urlparse.urlparse(url)
server, port = urlParse.netloc.split(':')
mainUrlsList = []
if server == '$MAINSERVERS$':
# Operations cannot be imported at the beginning because of a bootstrap problem
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
mainServers = Operations().getValue('MainServers',[])
if not mainServers:
raise Exception("No Main servers defined")
for srv in mainServers:
mainUrlsList.append(urlparse.ParseResult( scheme = urlParse.scheme, netloc = ':'.join([srv, port]),
path = urlParse.path, params = '', query = '', fragment = '').geturl())
return ','.join(mainUrlsList)
if len( url.split( "/" ) ) < 5:
url = "%s/%s" % ( url, serviceName )
return url
def getOptionsDict(self, sectionPath):
""" Get configuration options in dictionary
:param str sectionPath: section path
:return: S_OK(dict)/S_ERROR()
"""
gRefresher.refreshConfigurationIfNeeded()
optionsDict = {}
optionList = gConfigurationData.getOptionsFromCFG(sectionPath)
if isinstance(optionList, list):
for option in optionList:
optionsDict[option] = gConfigurationData.extractOptionFromCFG(
"%s/%s" % (sectionPath, option))
return S_OK(optionsDict)
else:
return S_ERROR("Path %s does not exist or it's not a section" %
sectionPath)
def main():
# Must be defined BEFORE any dirac import
os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True"
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration
from DIRAC.ConfigurationSystem.private.Refresher import gRefresher
from DIRAC.Core.Utilities.DErrno import includeExtensionErrors
from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer
from DIRAC.FrameworkSystem.Client.Logger import gLogger
if gConfigurationData.isMaster():
gRefresher.disable()
localCfg = LocalConfiguration()
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict['OK']:
gLogger.initialize("Tornado-CS", "/")
gLogger.error("There were errors when loading configuration",
resultDict['Message'])
sys.exit(1)
includeExtensionErrors()
gLogger.initialize('Tornado-CS', "/")
# get the specific master CS port
try:
csPort = int(
gConfigurationData.extractOptionFromCFG(
'%s/Port' % getServiceSection('Configuration/Server')))
except TypeError:
csPort = None
serverToLaunch = TornadoServer(services='Configuration/Server',
port=csPort)
serverToLaunch.startTornado()
def getDIRACSetup(): return gConfigurationData.extractOptionFromCFG( "/DIRAC/Setup" )
def getServiceURL( serviceName, serviceTuple = False, setup = False ):
if not serviceTuple:
serviceTuple = divideFullName( serviceName )
systemSection = getSystemSection( serviceName, serviceTuple, setup = setup )
return gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) )
def getDIRACSetup():
"""Get DIRAC default setup name
:return: str
"""
return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")
def getServiceURL(serviceName, serviceTuple=False, setup=False):
if not serviceTuple:
serviceTuple = divideFullName(serviceName)
systemSection = getSystemSection(serviceName, serviceTuple, setup=setup)
return gConfigurationData.extractOptionFromCFG(
"%s/URLs/%s" % (systemSection, serviceTuple[1]))
def getDIRACSetup():
return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")
def getValue(self, optionPath):
return gConfigurationData.extractOptionFromCFG(optionPath,
self.cfgData)
def getValue( self, optionPath ): return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def getOption( self, optionName ):
if optionName[0] != "/":
optionName = "%s/%s" % ( self.serviceSectionPath, optionName )
return gConfigurationData.extractOptionFromCFG( optionName )