def getOption( self, optionName ): if optionName[0] == "/": return gConfigurationData.extractOptionFromCFG( optionName ) for path in self.pathList: value = gConfigurationData.extractOptionFromCFG( "%s/%s" % ( path, optionName ) ) if value: return value return None
def getOption(self, optionName): if optionName[0] == "/": return gConfigurationData.extractOptionFromCFG(optionName) for path in self.pathList: value = gConfigurationData.extractOptionFromCFG("%s/%s" % (path, optionName)) if value: return value return None
def getSystemInstance(systemName, setup=False): if not setup: setup = gConfigurationData.extractOptionFromCFG("/DIRAC/Setup") optionPath = "/DIRAC/Setups/%s/%s" % (setup, systemName) instance = gConfigurationData.extractOptionFromCFG(optionPath) if instance: return instance else: raise RuntimeError("Option %s is not defined" % optionPath)
def getSystemInstance( systemName, setup = False ): if not setup: setup = gConfigurationData.extractOptionFromCFG( "/DIRAC/Setup" ) optionPath = "/DIRAC/Setups/%s/%s" % ( setup, systemName ) instance = gConfigurationData.extractOptionFromCFG( optionPath ) if instance: return instance else: raise RuntimeError( "Option %s is not defined" % optionPath )
def getGatewayURLs( serviceName = "" ): siteName = gConfigurationData.extractOptionFromCFG( "/LocalSite/Site" ) if not siteName: return False gatewayList = gConfigurationData.extractOptionFromCFG( "/DIRAC/Gateways/%s" % siteName ) if not gatewayList: return False if serviceName: gatewayList = [ "%s/%s" % ( "/".join( gw.split( "/" )[:3] ), serviceName ) for gw in List.fromChar( gatewayList, "," ) ] return List.randomize( gatewayList )
def loadUserData(self): """ This is the magic method that reads the command line and processes it It is used by the Script Base class and the dirac-service and dirac-agent scripts Before being called: - any additional switches to be processed - mandatory and default configuration configuration options must be defined. """ if self.initialized: return S_OK() self.initialized = True try: retVal = self.__addUserDataToConfiguration() for optionTuple in self.optionalEntryList: optionPath = self.__getAbsolutePath(optionTuple[0]) if not gConfigurationData.extractOptionFromCFG(optionPath): gConfigurationData.setOptionInCFG(optionPath, optionTuple[1]) self.__initLogger(self.componentName, self.loggingSection) if not retVal['OK']: return retVal retVal = self.__checkMandatoryOptions() if not retVal['OK']: return retVal except Exception as e: gLogger.exception() return S_ERROR(str(e)) return S_OK()
def getOption( self, optionPath, typeValue = None ): gRefresher.refreshConfigurationIfNeeded() optionValue = gConfigurationData.extractOptionFromCFG( optionPath ) if optionValue == None: return S_ERROR( "Path %s does not exist or it's not an option" % optionPath ) # Value has been returned from the configuration if typeValue == None: return S_OK( optionValue ) # Casting to typeValue's type requestedType = typeValue if not type( typeValue ) == types.TypeType: requestedType = type( typeValue ) if requestedType == types.ListType: try: return S_OK( List.fromChar( optionValue, ',' ) ) except Exception: return S_ERROR( "Can't convert value (%s) to comma separated list" % str( optionValue ) ) elif requestedType == types.BooleanType: try: return S_OK( optionValue.lower() in ( "y", "yes", "true", "1" ) ) except Exception: return S_ERROR( "Can't convert value (%s) to Boolean" % str( optionValue ) ) else: try: return S_OK( requestedType( optionValue ) ) except: return S_ERROR( "Type mismatch between default (%s) and configured value (%s) " % ( str( typeValue ), optionValue ) )
def loadUserData( self ): """ This is the magic method that reads the command line and processes it It is used by the Script Base class and the dirac-service and dirac-agent scripts Before being called: - any additional switches to be processed - mandatory and default configuration configuration options must be defined. """ if self.initialized: return S_OK() self.initialized = True try: retVal = self.__addUserDataToConfiguration() for optionTuple in self.optionalEntryList: optionPath = self.__getAbsolutePath( optionTuple[0] ) if not gConfigurationData.extractOptionFromCFG( optionPath ): gConfigurationData.setOptionInCFG( optionPath, optionTuple[1] ) self.__initLogger( self.componentName, self.loggingSection ) if not retVal[ 'OK' ]: return retVal retVal = self.__checkMandatoryOptions() if not retVal[ 'OK' ]: return retVal except Exception as e: gLogger.exception() return S_ERROR( str( e ) ) return S_OK()
def getOption( self, optionPath, typeValue = None ): gRefresher.refreshConfigurationIfNeeded() optionValue = gConfigurationData.extractOptionFromCFG( optionPath ) if optionValue == None: return S_ERROR( "Path %s does not exist or it's not an option" % optionPath ) #Value has been returned from the configuration if typeValue == None: return S_OK( optionValue ) #Casting to typeValue's type requestedType = typeValue if not type( typeValue ) == types.TypeType: requestedType = type( typeValue ) if requestedType == types.ListType: try: return S_OK( List.fromChar( optionValue, ',' ) ) except Exception: return S_ERROR( "Can't convert value (%s) to comma separated list" % str( optionValue ) ) elif requestedType == types.BooleanType: try: return S_OK( optionValue.lower() in ( "y", "yes", "true", "1" ) ) except Exception: return S_ERROR( "Can't convert value (%s) to Boolean" % str( optionValue ) ) else: try: return S_OK( requestedType( optionValue ) ) except: return S_ERROR( "Type mismatch between default (%s) and configured value (%s) " % ( str( typeValue ), optionValue ) )
def getOption(self, optionPath, typeValue=None): """ Get configuration option :param str optionPath: option path :param typeValue: type of value :return: S_OK()/S_ERROR() """ gRefresher.refreshConfigurationIfNeeded() optionValue = gConfigurationData.extractOptionFromCFG(optionPath) if optionValue is None: return S_ERROR("Path %s does not exist or it's not an option" % optionPath) # Value has been returned from the configuration if typeValue is None: return S_OK(optionValue) # Casting to typeValue's type if not isinstance(typeValue, type): # typeValue is not a type but a default object requestedType = type(typeValue) else: requestedType = typeValue if requestedType in (list, tuple, set): try: return S_OK(requestedType(List.fromChar(optionValue, ','))) except Exception as e: return S_ERROR( "Can't convert value (%s) to comma separated list \n%s" % (str(optionValue), repr(e))) elif requestedType == bool: try: return S_OK(optionValue.lower() in ("y", "yes", "true", "1")) except Exception as e: return S_ERROR("Can't convert value (%s) to Boolean \n%s" % (str(optionValue), repr(e))) elif requestedType == dict: try: splitOption = List.fromChar(optionValue, ',') value = {} for opt in splitOption: keyVal = [x.strip() for x in opt.split(':')] if len(keyVal) == 1: keyVal.append(True) value[keyVal[0]] = keyVal[1] return S_OK(value) except Exception as e: return S_ERROR("Can't convert value (%s) to Dict \n%s" % (str(optionValue), repr(e))) else: try: return S_OK(requestedType(optionValue)) except Exception as e: return S_ERROR( "Type mismatch between default (%s) and configured value (%s) \n%s" % (str(typeValue), optionValue, repr(e)))
def addDefaultEntry(self, optionPath, value): """ Define a default value for a Configuration data option """ if optionPath[0] == "/": if not gConfigurationData.extractOptionFromCFG(optionPath): self.__setOptionValue(optionPath, value) else: self.optionalEntryList.append((optionPath, str(value)))
def getGatewayURLs(system="", service=None): """Get gateway URLs for service :param str system: system name or full name, like 'Framework/Service'. :param str service: service name, like 'ProxyManager'. :return: list or False """ if system: system, service = divideFullName(system, service) siteName = gConfigurationData.extractOptionFromCFG("/LocalSite/Site") if not siteName: return False gateways = gConfigurationData.extractOptionFromCFG("/DIRAC/Gateways/%s" % siteName) if not gateways: return False gateways = List.randomize(List.fromChar(gateways, ",")) return [checkComponentURL(u, system, service) for u in gateways if u] if system and service else gateways
def addDefaultEntry( self, optionPath, value ): """ Define a default value for a Configuration data option """ if optionPath[0] == "/": if not gConfigurationData.extractOptionFromCFG( optionPath ): self.__setOptionValue( optionPath, value ) else: self.optionalEntryList.append( ( optionPath, str( value ) ) )
def getOptionsDict(self, sectionPath): gRefresher.refreshConfigurationIfNeeded() optionsDict = {} optionList = gConfigurationData.getOptionsFromCFG(sectionPath) if type(optionList) == types.ListType: for option in optionList: optionsDict[option] = gConfigurationData.extractOptionFromCFG("%s/%s" % (sectionPath, option)) return S_OK(optionsDict) else: return S_ERROR("Path %s does not exist or it's not a section" % sectionPath)
def getServiceURL( serviceName, serviceTuple = False, setup = False ): if not serviceTuple: serviceTuple = divideFullName( serviceName ) systemSection = getSystemSection( serviceName, serviceTuple, setup = setup ) url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) ) if not url: return "" if len( url.split( "/" ) ) < 5: url = "%s/%s" % ( url, serviceName ) return url
def getServiceURLs(system, service=None, setup=False, failover=False): """Generate url. :param str system: system name or full name e.g.: Framework/ProxyManager :param str service: service name, like 'ProxyManager'. :param str setup: DIRAC setup name, can be defined in dirac.cfg :param bool failover: to add failover URLs to end of result list :return: list -- complete urls. e.g. [dips://some-domain:3424/Framework/Service] """ system, service = divideFullName(system, service) resList = [] mainServers = None systemSection = getSystemSection(system, setup=setup) # Add failover URLs at the end of the list failover = "Failover" if failover else "" for fURLs in ["", "Failover"] if failover else [""]: urlList = [] urls = List.fromChar( gConfigurationData.extractOptionFromCFG( "%s/%sURLs/%s" % (systemSection, fURLs, service))) # Be sure that urls not None for url in urls or []: # Trying if we are refering to the list of main servers # which would be like dips://$MAINSERVERS$:1234/System/Component if "$MAINSERVERS$" in url: if not mainServers: # Operations cannot be imported at the beginning because of a bootstrap problem from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations mainServers = Operations(setup=setup).getValue( "MainServers", []) if not mainServers: raise Exception("No Main servers defined") for srv in mainServers: _url = checkComponentURL(url.replace("$MAINSERVERS$", srv), system, service, pathMandatory=True) if _url not in urlList: urlList.append(_url) continue _url = checkComponentURL(url, system, service, pathMandatory=True) if _url not in urlList: urlList.append(_url) # Randomize list if needed resList.extend(List.randomize(urlList)) return resList
def getOptionsDict( self, sectionPath ): gRefresher.refreshConfigurationIfNeeded() optionsDict = {} optionList = gConfigurationData.getOptionsFromCFG( sectionPath ) if type( optionList ) == types.ListType: for option in optionList: optionsDict[ option ] = gConfigurationData.extractOptionFromCFG( "%s/%s" % ( sectionPath, option ) ) return S_OK( optionsDict ) else: return S_ERROR( "Path %s does not exist or it's not a section" % sectionPath )
def getSystemInstance(system, setup=False): """Find system instance name :param str system: system name :param str setup: setup name :return: str """ optionPath = Path.cfgPath("/DIRAC/Setups", setup or getDIRACSetup(), system) instance = gConfigurationData.extractOptionFromCFG(optionPath) if not instance: raise RuntimeError("Option %s is not defined" % optionPath) return instance
def __checkMandatoryOptions(self): try: isMandatoryMissing = False for optionPath in self.mandatoryEntryList: optionPath = self.__getAbsolutePath(optionPath) if not gConfigurationData.extractOptionFromCFG(optionPath): gLogger.fatal("Missing mandatory local configuration option", optionPath) isMandatoryMissing = True if isMandatoryMissing: return S_ERROR() return S_OK() except Exception as e: gLogger.exception() return S_ERROR(str(e))
def __checkMandatoryOptions( self ): try: isMandatoryMissing = False for optionPath in self.mandatoryEntryList: optionPath = self.__getAbsolutePath( optionPath ) if not gConfigurationData.extractOptionFromCFG( optionPath ): gLogger.fatal( "Missing mandatory local configuration option", optionPath ) isMandatoryMissing = True if isMandatoryMissing: return S_ERROR() return S_OK() except Exception as e: gLogger.exception() return S_ERROR( str( e ) )
def getServiceFailoverURL(system, service=None, setup=False): """Get failover URLs for service :param str system: system name or full name, like 'Framework/Service'. :param str service: service name, like 'ProxyManager'. :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: str -- complete list of urls """ system, service = divideFullName(system, service) systemSection = getSystemSection(system, setup=setup) failovers = gConfigurationData.extractOptionFromCFG("%s/FailoverURLs/%s" % (systemSection, service)) if not failovers: return "" return ",".join([checkComponentURL(u, system, service) for u in List.fromChar(failovers, ",") if u])
def getServiceURL(serviceName, serviceTuple=False, setup=False): """ Generate url. :param serviceName: Name of service, like 'Framework/Service'. :param serviceTuple: (optional) also name of service but look like ('Framework', 'Service'). :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: complete url. e.g. dips://some-domain:3424/Framework/Service """ if not serviceTuple: serviceTuple = divideFullName(serviceName) systemSection = getSystemSection(serviceName, serviceTuple, setup=setup) url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % (systemSection, serviceTuple[1])) if not url: return "" # Trying if we are refering to the list of main servers # which would be like dips://$MAINSERVERS$:1234/System/Component # This can only happen if there is only one server defined if ',' not in url: urlParse = urlparse.urlparse(url) server, port = urlParse.netloc.split(':') mainUrlsList = [] if server == '$MAINSERVERS$': # Operations cannot be imported at the beginning because of a bootstrap problem from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations mainServers = Operations().getValue('MainServers', []) if not mainServers: raise Exception("No Main servers defined") for srv in mainServers: mainUrlsList.append( urlparse.ParseResult(scheme=urlParse.scheme, netloc=':'.join([srv, port]), path=urlParse.path, params='', query='', fragment='').geturl()) return ','.join(mainUrlsList) if len(url.split("/")) < 5: url = "%s/%s" % (url, serviceName) return url
def main(): if os.environ.get("DIRAC_USE_TORNADO_IOLOOP", "false").lower() not in ("yes", "true"): raise RuntimeError( "DIRAC_USE_TORNADO_IOLOOP is not defined in the environment." + "\n" + "It is necessary to run with Tornado." + "\n" + "https://dirac.readthedocs.io/en/latest/DeveloperGuide/TornadoServices/index.html" ) from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.FrameworkSystem.Client.Logger import gLogger if gConfigurationData.isMaster(): gRefresher.disable() localCfg = Script.localCfg localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: gLogger.initialize("Tornado-CS", "/") gLogger.error("There were errors when loading configuration", resultDict["Message"]) sys.exit(1) includeExtensionErrors() gLogger.initialize("Tornado-CS", "/") # get the specific master CS port try: csPort = int( gConfigurationData.extractOptionFromCFG( "%s/Port" % getServiceSection("Configuration/Server"))) except TypeError: csPort = None serverToLaunch = TornadoServer(services="Configuration/Server", port=csPort) serverToLaunch.startTornado()
def getServiceURL( serviceName, serviceTuple = False, setup = False ): """ Generate url. :param serviceName: Name of service, like 'Framework/Service'. :param serviceTuple: (optional) also name of service but look like ('Framework', 'Service'). :param str setup: DIRAC setup name, can be defined in dirac.cfg :return: complete url. e.g. dips://some-domain:3424/Framework/Service """ if not serviceTuple: serviceTuple = divideFullName( serviceName ) systemSection = getSystemSection( serviceName, serviceTuple, setup = setup ) url = gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) ) if not url: return "" # Trying if we are refering to the list of main servers # which would be like dips://$MAINSERVERS$:1234/System/Component # This can only happen if there is only one server defined if ',' not in url: urlParse = urlparse.urlparse(url) server, port = urlParse.netloc.split(':') mainUrlsList = [] if server == '$MAINSERVERS$': # Operations cannot be imported at the beginning because of a bootstrap problem from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations mainServers = Operations().getValue('MainServers',[]) if not mainServers: raise Exception("No Main servers defined") for srv in mainServers: mainUrlsList.append(urlparse.ParseResult( scheme = urlParse.scheme, netloc = ':'.join([srv, port]), path = urlParse.path, params = '', query = '', fragment = '').geturl()) return ','.join(mainUrlsList) if len( url.split( "/" ) ) < 5: url = "%s/%s" % ( url, serviceName ) return url
def getOptionsDict(self, sectionPath): """ Get configuration options in dictionary :param str sectionPath: section path :return: S_OK(dict)/S_ERROR() """ gRefresher.refreshConfigurationIfNeeded() optionsDict = {} optionList = gConfigurationData.getOptionsFromCFG(sectionPath) if isinstance(optionList, list): for option in optionList: optionsDict[option] = gConfigurationData.extractOptionFromCFG( "%s/%s" % (sectionPath, option)) return S_OK(optionsDict) else: return S_ERROR("Path %s does not exist or it's not a section" % sectionPath)
def main(): # Must be defined BEFORE any dirac import os.environ['DIRAC_USE_TORNADO_IOLOOP'] = "True" from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData from DIRAC.ConfigurationSystem.Client.LocalConfiguration import LocalConfiguration from DIRAC.ConfigurationSystem.private.Refresher import gRefresher from DIRAC.Core.Utilities.DErrno import includeExtensionErrors from DIRAC.Core.Tornado.Server.TornadoServer import TornadoServer from DIRAC.FrameworkSystem.Client.Logger import gLogger if gConfigurationData.isMaster(): gRefresher.disable() localCfg = LocalConfiguration() localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict['OK']: gLogger.initialize("Tornado-CS", "/") gLogger.error("There were errors when loading configuration", resultDict['Message']) sys.exit(1) includeExtensionErrors() gLogger.initialize('Tornado-CS', "/") # get the specific master CS port try: csPort = int( gConfigurationData.extractOptionFromCFG( '%s/Port' % getServiceSection('Configuration/Server'))) except TypeError: csPort = None serverToLaunch = TornadoServer(services='Configuration/Server', port=csPort) serverToLaunch.startTornado()
def getDIRACSetup(): return gConfigurationData.extractOptionFromCFG( "/DIRAC/Setup" )
def getServiceURL( serviceName, serviceTuple = False, setup = False ): if not serviceTuple: serviceTuple = divideFullName( serviceName ) systemSection = getSystemSection( serviceName, serviceTuple, setup = setup ) return gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % ( systemSection, serviceTuple[1] ) )
def getDIRACSetup(): """Get DIRAC default setup name :return: str """ return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")
def getServiceURL(serviceName, serviceTuple=False, setup=False): if not serviceTuple: serviceTuple = divideFullName(serviceName) systemSection = getSystemSection(serviceName, serviceTuple, setup=setup) return gConfigurationData.extractOptionFromCFG( "%s/URLs/%s" % (systemSection, serviceTuple[1]))
def getDIRACSetup(): return gConfigurationData.extractOptionFromCFG("/DIRAC/Setup")
def getValue(self, optionPath): return gConfigurationData.extractOptionFromCFG(optionPath, self.cfgData)
def getValue( self, optionPath ): return gConfigurationData.extractOptionFromCFG( optionPath, self.cfgData )
def getOption( self, optionName ): if optionName[0] != "/": optionName = "%s/%s" % ( self.serviceSectionPath, optionName ) return gConfigurationData.extractOptionFromCFG( optionName )