def __getBackendOptionsFromCFG(self, cfgPath, backend): """ Get backend options from the configuration. :params cfgPath: string of the configuration path :params backend: string representing a backend identifier: stdout, file, f04 """ # We have to put the import lines here to avoid a dependancy loop from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getBackendConfig backendOptions = {} # Search backends config in the resources section retDictRessources = getBackendConfig(backend) if retDictRessources['OK']: backendOptions = retDictRessources['Value'] # Search backends config in the component to update some options retDictConfig = gConfig.getOptionsDict( "%s/%s/%s" % (cfgPath, 'LogBackendsConfig', backend)) if retDictConfig['OK']: backendOptions.update(retDictConfig['Value']) else: # Search backends config in the component with the old option 'BackendsOptions' retDictOptions = gConfig.getOptionsDict("%s/BackendsOptions" % cfgPath) if retDictOptions['OK']: # We have to write the deprecated message with the print method because we are changing # the backends, so we can not be sure of the display using a log. print "WARNING: Use of a deprecated cfg section: BackendsOptions. Please replace it by BackendConfig." backendOptions.update(retDictOptions['Value']) return backendOptions
def getComponentsStatus( self, conditionDict = {} ): """ Get the status of the defined components in the CS compared to the ones that are known in the DB """ result = self.__getComponents( conditionDict ) if not result[ 'OK' ]: return result statusSet = result[ 'Value' ] requiredComponents = {} result = gConfig.getSections( "/DIRAC/Setups" ) if not result[ 'OK' ]: return result for setup in result[ 'Value' ]: if not self.__checkCondition( conditionDict, "Setup", setup ): continue #Iterate through systems result = gConfig.getOptionsDict( "/DIRAC/Setups/%s" % setup ) if not result[ 'OK' ]: return result systems = result[ 'Value' ] for system in systems: instance = systems[ system ] #Check defined agents and serviecs for type in ( 'agent' , 'service' ): #Get entries for the instance of a system result = gConfig.getSections( "/Systems/%s/%s/%s" % ( system, instance, "%ss" % type.capitalize() ) ) if not result[ 'OK' ]: self.log.warn( "Opps, sytem seems to be defined wrong\n", "System %s at %s: %s" % ( system, instance, result[ 'Message' ] ) ) continue components = result[ 'Value' ] for component in components: componentName = "%s/%s" % ( system, component ) compDict = self.__getComponentDefinitionFromCS( system, setup, instance, type, component ) if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ): statusSet.addUniqueToSet( requiredComponents, compDict ) #Walk the URLs result = gConfig.getOptionsDict( "/Systems/%s/%s/URLs" % ( system, instance ) ) if not result[ 'OK' ]: self.log.warn ( "There doesn't to be defined the URLs section for %s in %s instance" % ( system, instance ) ) else: serviceURLs = result[ 'Value' ] for service in serviceURLs: for url in List.fromChar( serviceURLs[ service ] ): loc = url[ url.find( "://" ) + 3: ] iS = loc.find( "/" ) componentName = loc[ iS + 1: ] loc = loc[ :iS ] hostname, port = loc.split( ":" ) compDict = { 'ComponentName' : componentName, 'Type' : 'service', 'Setup' : setup, 'Host' : hostname, 'Port' : int( port ) } if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ): statusSet.addUniqueToSet( requiredComponents, compDict ) #WALK THE DICT statusSet.setComponentsAsRequired( requiredComponents ) return S_OK( ( statusSet.getRequiredComponents(), self.__mainFields[1:] + self.__versionFields + ( 'Status', 'Message' ) ) )
def getQueue(site, ce, queue): """ Get parameters of the specified queue """ grid = site.split('.')[0] result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce)) if not result['OK']: return result resultDict = result['Value'] # Get queue defaults result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % (grid, site, ce, queue)) if not result['OK']: return result resultDict.update(result['Value']) # Handle tag lists for the queue for tagFieldName in ('Tag', 'RequiredTag'): tags = [] ceTags = resultDict.get(tagFieldName) if ceTags: tags = fromChar(ceTags) queueTags = resultDict.get(tagFieldName) if queueTags: queueTags = fromChar(queueTags) tags = list(set(tags + queueTags)) if tags: resultDict[tagFieldName] = tags resultDict['Queue'] = queue return S_OK(resultDict)
def __getBackendOptionsFromCFG(self, cfgPath, backend): """ Get backend options from the configuration. :params cfgPath: string of the configuration path :params backend: string representing a backend identifier: stdout, file, f04 """ # We have to put the import lines here to avoid a dependancy loop from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getBackendConfig backendOptions = {} # Search backends config in the resources section retDictRessources = getBackendConfig(backend) if retDictRessources['OK']: backendOptions = retDictRessources['Value'] # Search backends config in the component to update some options retDictConfig = gConfig.getOptionsDict( "%s/%s/%s" % (cfgPath, 'LogBackendsConfig', backend)) if retDictConfig['OK']: backendOptions.update(retDictConfig['Value']) else: # Search backends config in the component with the old option # 'BackendsOptions' retDictOptions = gConfig.getOptionsDict("%s/BackendsOptions" % cfgPath) if retDictOptions['OK']: # We have to write the deprecated message with the print method because we are changing # the backends, so we can not be sure of the display using a log. print "WARNING: Use of a deprecated cfg section: BackendsOptions. Please replace it by BackendConfig." backendOptions.update(retDictOptions['Value']) return backendOptions
def getRunningEnPointDict( self, runningEndPointName ): """ Return from CS a Dictionary with getRunningEnPointDict definition """ runningEndPointCSPath = '/Resources/BigDataEndPoints' definedRunningEndPoints = gConfig.getSections( runningEndPointCSPath ) if not definedRunningEndPoints[ 'OK' ]: return S_ERROR( 'BigData section not defined' ) runningBDCSPath = '%s/%s' % ( runningEndPointCSPath, runningEndPointName ) runningEndPointBDDict = {} if not runningBDCSPath: return S_ERROR( 'Missing BigDataEndpoint "%s"' % runningEndPointName ) for option, value in gConfig.getOptionsDict( runningBDCSPath )['Value'].items(): runningEndPointBDDict[option] = value runningHighLevelLanguajeDict = gConfig.getOptionsDict( '%s/HighLevelLanguage' % runningBDCSPath ) if not runningHighLevelLanguajeDict[ 'OK' ]: return S_ERROR( 'Missing HighLevelLang in "%s"' % runningBDCSPath ) runningEndPointBDDict['HighLevelLanguage'] = runningHighLevelLanguajeDict['Value'] runningReqDict = gConfig.getOptionsDict( '%s/Requirements' % runningBDCSPath ) if not runningReqDict[ 'OK' ]: return S_ERROR( 'Missing Requirements in "%s"' % runningBDCSPath ) runningEndPointBDDict['Requirements'] = runningReqDict['Value'] return S_OK( runningEndPointBDDict )
def getStorageElementOptions( seName ): """ Get the CS StorageElementOptions """ storageConfigPath = '/Resources/StorageElements/%s' % seName result = gConfig.getOptionsDict( storageConfigPath ) if not result['OK']: return result options = result['Value'] # If the SE is an baseSE or an alias, derefence it if 'BaseSE' in options or 'Alias' in options: storageConfigPath = '/Resources/StorageElements/%s' % options.get( 'BaseSE', options.get( 'Alias' ) ) result = gConfig.getOptionsDict( storageConfigPath ) if not result['OK']: return result result['Value'].update( options ) options = result['Value'] # Help distinguishing storage type diskSE = True tapeSE = False if 'SEType' in options: # Type should follow the convention TXDY seType = options['SEType'] diskSE = re.search( 'D[1-9]', seType ) != None tapeSE = re.search( 'T[1-9]', seType ) != None options['DiskSE'] = diskSE options['TapeSE'] = tapeSE return S_OK( options )
def getVMImageConfig( site, ce, image = '' ): """ Get parameters of the specified queue """ Tags = [] grid = site.split( '.' )[0] result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s' % ( grid, site, ce ) ) if not result['OK']: return result resultDict = result['Value'] ceTags = resultDict.get( 'Tag' ) if ceTags: Tags = fromChar( ceTags ) if image: result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s' % ( grid, site, ce, image ) ) if not result['OK']: return result resultDict.update( result['Value'] ) queueTags = resultDict.get( 'Tag' ) if queueTags: queueTags = fromChar( queueTags ) Tags = list( set( Tags + queueTags ) ) if Tags: resultDict['Tag'] = Tags resultDict['Image'] = image resultDict['Site'] = site return S_OK( resultDict )
def getQueue(site, ce, queue): """ Get parameters of the specified queue """ Tags = [] grid = site.split('.')[0] result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce)) if not result['OK']: return result resultDict = result['Value'] ceTags = resultDict.get('Tag') if ceTags: Tags = fromChar(ceTags) result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % (grid, site, ce, queue)) if not result['OK']: return result resultDict.update(result['Value']) queueTags = resultDict.get('Tag') if queueTags: queueTags = fromChar(queueTags) Tags = list(set(Tags + queueTags)) if Tags: resultDict['Tag'] = Tags resultDict['Queue'] = queue return S_OK(resultDict)
def getImages( siteList = None, ceList = None, imageList = None, vo = None ): """ Get CE/image options according to the specified selection """ result = gConfig.getSections( '/Resources/Sites' ) if not result['OK']: return result resultDict = {} grids = result['Value'] for grid in grids: result = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not result['OK']: continue sites = result['Value'] for site in sites: if siteList is not None and not site in siteList: continue if vo: voList = gConfig.getValue( '/Resources/Sites/%s/%s/VO' % ( grid, site ), [] ) if voList and not vo in voList: continue result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud' % ( grid, site ) ) if not result['OK']: continue ces = result['Value'] for ce in ces: if ceList is not None and not ce in ceList: continue if vo: voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/VO' % ( grid, site, ce ), [] ) if voList and not vo in voList: continue result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s' % ( grid, site, ce ) ) if not result['OK']: continue ceOptionsDict = result['Value'] result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud/%s/Images' % ( grid, site, ce ) ) if not result['OK']: continue images = result['Value'] for image in images: if imageList is not None and not image in imageList: continue if vo: voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s/VO' % ( grid, site, ce, image ), [] ) if voList and not vo in voList: continue resultDict.setdefault( site, {} ) resultDict[site].setdefault( ce, ceOptionsDict ) resultDict[site][ce].setdefault( 'Images', {} ) result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s' % ( grid, site, ce, image ) ) if not result['OK']: continue imageOptionsDict = result['Value'] resultDict[site][ce]['Images'][image] = imageOptionsDict return S_OK( resultDict )
def getOptionsDict(self, optionPath, optionVOPath): comOptDict = {} if optionVOPath is not None: result = gConfig.getOptionsDict(optionVOPath) if result['OK']: comOptDict = result['Value'] result = gConfig.getOptionsDict(optionPath) if not result['OK']: return result optDict = result['Value'] optDict.update(comOptDict) return S_OK(optDict)
def getOptionsDict( self, optionPath, optionVOPath ): comOptDict = {} if optionVOPath is not None: result = gConfig.getOptionsDict( optionVOPath ) if result['OK']: comOptDict = result['Value'] result = gConfig.getOptionsDict( optionPath ) if not result['OK']: return result optDict = result['Value'] optDict.update( comOptDict ) return S_OK( optDict )
def getVMTypeConfig(site, ce="", vmtype=""): """Get the VM image type parameters of the specified queue""" tags = [] reqtags = [] grid = site.split(".")[0] if not ce: result = gConfig.getSections("/Resources/Sites/%s/%s/Cloud" % (grid, site)) if not result["OK"]: return result ceList = result["Value"] if len(ceList) == 1: ce = ceList[0] else: return S_ERROR("No cloud endpoint specified") result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/Cloud/%s" % (grid, site, ce)) if not result["OK"]: return result resultDict = result["Value"] ceTags = resultDict.get("Tag") if ceTags: tags = fromChar(ceTags) ceTags = resultDict.get("RequiredTag") if ceTags: reqtags = fromChar(ceTags) resultDict["CEName"] = ce if vmtype: result = gConfig.getOptionsDict( "/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s" % (grid, site, ce, vmtype)) if not result["OK"]: return result resultDict.update(result["Value"]) queueTags = resultDict.get("Tag") if queueTags: queueTags = fromChar(queueTags) tags = list(set(tags + queueTags)) queueTags = resultDict.get("RequiredTag") if queueTags: queueTags = fromChar(queueTags) reqtags = list(set(reqtags + queueTags)) if tags: resultDict["Tag"] = tags if reqtags: resultDict["RequiredTag"] = reqtags resultDict["VMType"] = vmtype resultDict["Site"] = site return S_OK(resultDict)
def __init__( self, imageName, endPointName ): self.log = gLogger.getSubLogger( 'ImageConfiguration' ) imageOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s' % imageName ) if not imageOptions[ 'OK' ]: self.log.error( imageOptions[ 'Message' ] ) imageOptions = {} else: imageOptions = imageOptions[ 'Value' ] self.__ic_DIRACImageName = imageName endPointName = endPointName.strip() # A DIRAC image can have different boot image names in the cloud endPoints bootImageOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s/BootImages' % imageName ) if not bootImageOptions[ 'OK' ]: self.log.error( bootImageOptions[ 'Message' ] ) return bootImageOptions = bootImageOptions[ 'Value' ] bootImageName = None for bootEndpoint, bootImage in bootImageOptions.items(): if endPointName == bootEndpoint: bootImageName = bootImage break if bootImageName is None: self.log.error( 'Missing mandatory boot image of the endPoint %s in BootImages section, image %s' % (endPointName, imageName) ) self.__ic_bootImageName = bootImageName # A DIRAC image can have different flavor names names in the cloud endPoints flavorOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s/Flavors' % imageName ) if not flavorOptions[ 'OK' ]: self.log.error( flavorOptions[ 'Message' ] ) return flavorOptions = flavorOptions[ 'Value' ] flavorName = None for bootEndpoint, flavor in flavorOptions.items(): if endPointName == bootEndpoint: flavorName = flavor break if flavorName is None: self.log.error( 'Missing mandatory flavor of the endPoint %s in BootImages section, image %s' % (endPointName, imageName) ) self.__ic_flavorName = flavorName self.__ic_contextMethod = imageOptions.get( 'contextMethod' , None ) #optional: self.__ic_maxAllowedPrice = imageOptions.get( 'maxAllowedPrice' , None ) self.__ic_keyName = imageOptions.get( 'keyName' , None ) #self.__ic_contextConfig = ContextConfig( self.__ic_bootImageName, self.__ic_contextMethod ) self.__ic_contextConfig = ContextConfig( imageName, self.__ic_contextMethod )
def getMQParamsFromCS(mqURI): """Function gets parameters of a MQ destination (queue/topic) from the CS. Args: mqURI(str):Pseudo URI identifing the MQ service. It has the following format: mqConnection::DestinationType::DestinationName e.g. blabla.cern.ch::Queues::MyQueue1 mType(str): 'consumer' or 'producer' Returns: S_OK(param_dicts) or S_ERROR """ # API initialization is required to get an up-to-date configuration from the CS csAPI = CSAPI() csAPI.initialize() try: mqService, mqType, mqName = mqURI.split("::") except ValueError: return S_ERROR("Bad format of mqURI address:%s" % (mqURI)) result = gConfig.getConfigurationTree("/Resources/MQServices", mqService, mqType, mqName) if not result["OK"] or not result["Value"]: return S_ERROR( "Requested destination not found in the CS: %s::%s::%s" % (mqService, mqType, mqName)) mqDestinationPath = None for path, value in result["Value"].items(): if not value and path.endswith(mqName): mqDestinationPath = path # set-up internal parameter depending on the destination type tmp = mqDestinationPath.split("Queues")[0].split("Topics") servicePath = tmp[0] serviceDict = {} if len(tmp) > 1: serviceDict["Topic"] = mqName else: serviceDict["Queue"] = mqName result = gConfig.getOptionsDict(servicePath) if not result["OK"]: return result serviceDict.update(result["Value"]) result = gConfig.getOptionsDict(mqDestinationPath) if not result["OK"]: return result serviceDict.update(result["Value"]) return S_OK(serviceDict)
def getQueue( site, ce, queue ): """ Get parameters of the specified queue """ grid = site.split( '.' )[0] result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) ) if not result['OK']: return result resultDict = result['Value'] result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) ) if not result['OK']: return result resultDict.update( result['Value'] ) resultDict['Queue'] = queue return S_OK( resultDict )
def getQueue(site, ce, queue): """ Get parameters of the specified queue """ grid = site.split(".")[0] result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/CEs/%s" % (grid, site, ce)) if not result["OK"]: return result resultDict = result["Value"] result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/CEs/%s/Queues/%s" % (grid, site, ce, queue)) if not result["OK"]: return result resultDict.update(result["Value"]) resultDict["Queue"] = queue return S_OK(resultDict)
def createCatalog(self, catalogName, useProxy=False): """ Create a file catalog object from its name and CS description """ catalogPath = getCatalogPath(catalogName) catalogType = gConfig.getValue(catalogPath + '/CatalogType', catalogName) catalogURL = gConfig.getValue(catalogPath + '/CatalogURL', "DataManagement/" + catalogType) optionsDict = {} result = gConfig.getOptionsDict(catalogPath) if result['OK']: optionsDict = result['Value'] if useProxy: result = self.__getCatalogClass(catalogType) if not result['OK']: return result catalogClass = result['Value'] methods = catalogClass.getInterfaceMethods() catalog = FileCatalogProxyClient(catalogName) catalog.setInterfaceMethods(methods) return S_OK(catalog) return self.__createCatalog(catalogName, catalogType, catalogURL, optionsDict)
def _getCurrentConfig(self): """Return the current system configuration.""" from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData gConfig.forceRefresh() fullCfg = CFG() setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict for system, setup in serviceSetups.items(): if self.systems and system not in self.systems: continue systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup)) for section in systemCfg.listSections(): if section not in ('Agents', 'Services', 'Executors'): systemCfg.deleteKey(section) fullCfg.createNewSection("/%s" % system, contents=systemCfg) return S_OK(fullCfg)
def getDefinedTransferPaths(self): result = gConfig.getSections(self.__transfersCSPath) if not result['OK']: self.log.info('No Input/Output Pair defined in CS') return S_OK() pathList = result['Value'] tPaths = {} for name in pathList: csPath = self.__transfersCSPath + '/%s' % name result = gConfig.getOptionsDict(csPath) if not result['OK']: continue transferDict = result['Value'] ok = True for i in self.__requiredCSOptions: if i not in transferDict: self.log.error('Missing Option %s in %s' % (i, csPath)) ok = False break if not ok: continue tPaths[name] = transferDict return S_OK(tPaths)
def __generateDBs(self): self.__log.notice("Creating default AccountingDB...") self.__allDBs = {self.__defaultDB: AccountingDB(readOnly=self.__readOnly)} types = self.__allDBs[self.__defaultDB].getRegisteredTypes() result = gConfig.getOptionsDict(self.__csPath) if not result['OK']: gLogger.verbose("No extra databases defined", "in %s" % self.__csPath) return validTypes = TypeLoader().getTypes() opts = result['Value'] for acType in opts: if acType not in validTypes: msg = "(%s defined in %s)" % (acType, self.__csPath) self.__log.fatal("Not a known accounting type", msg) raise RuntimeError(msg) dbName = opts[acType] gLogger.notice("Type will be assigned", "(%s to %s)" % (acType, dbName)) if dbName not in self.__allDBs: fields = dbName.split("/") if len(fields) == 1: dbName = "Accounting/%s" % dbName gLogger.notice("Creating DB", '%s' % dbName) self.__allDBs[dbName] = AccountingDB(dbName, readOnly=self.__readOnly) self.__dbByType[acType] = dbName
def getDefinedTransferPaths( self ): result = gConfig.getSections( self.__transfersCSPath ) if not result['OK']: self.log.info( 'No Input/Output Pair defined in CS' ) return S_OK() pathList = result['Value'] tPaths = {} for name in pathList: csPath = self.__transfersCSPath + '/%s' % name result = gConfig.getOptionsDict( csPath ) if not result['OK']: continue transferDict = result['Value'] ok = True for i in self.__requiredCSOptions: if i not in transferDict: self.log.error( 'Missing Option %s in %s' % ( i, csPath ) ) ok = False break if not ok: continue tPaths[ name ] = transferDict return S_OK( tPaths )
def __init__( self, stratuslabEndpoint ): """ Constructor directly reads the named stratuslabEndpoint section in the configuration. :Parameters: **stratuslabEndpoint** - `string` name of the element containing the configuration for a StratusLab cloud """ super( StratusLabConfiguration, self ).__init__() options = gConfig.getOptionsDict( '%s/%s' % ( self.ENDPOINT_PATH, stratuslabEndpoint ) ) if not options[ 'OK' ]: self.log.error(options[ 'Message' ] ) options = {} else: options = options[ 'Value' ] # Save a shallow copy of the given dictionary for safety. self._options = copy.copy(options) # Remove any 'None' mappings from the dictionary. for key, value in self._options.items(): if value is None: del self._options[ key ]
def __init__( self, amazonEndpoint ): """ Constructor :Parameters: **amazonEndpoint** - `string` string with the name of the CloudEndpoint defined on the CS """ super( AmazonConfiguration, self ).__init__() amazonOptions = gConfig.getOptionsDict( '%s/%s' % ( self.ENDPOINT_PATH, amazonEndpoint ) ) if not amazonOptions[ 'OK' ]: self.log.error( amazonOptions[ 'Message' ] ) amazonOptions = {} else: amazonOptions = amazonOptions[ 'Value' ] # FIXME: make it generic ! # Purely endpoint configuration ............................................ # This two are passed as arguments, not keyword arguments self.__accessKey = amazonOptions.get( 'accessKey' , None ) self.__secretKey = amazonOptions.get( 'secretKey' , None ) self.__cloudDriver = amazonOptions.get( 'cloudDriver' , None ) self.__vmStopPolicy = amazonOptions.get( 'vmStopPolicy' , None ) self.__vmPolicy = amazonOptions.get( 'vmPolicy' , None ) self.__siteName = amazonOptions.get( 'siteName' , None ) self.__endpointURL = amazonOptions.get( 'endpointURL' , None ) self.__regionName = amazonOptions.get( 'regionName' , None ) self.__maxEndpointInstances = amazonOptions.get( 'maxEndpointInstances' , None ) self.__maxOportunisticEndpointInstances = amazonOptions.get( 'maxOportunisticEndpointInstances' , 0 ) self.__cvmfs_http_proxy = amazonOptions.get( 'cvmfs_http_proxy' , None ) self.__auth = amazonOptions.get( 'auth' , None )
def __instantiateSEs(self): """ Get storage config for all the SE in the dirac config. """ SEDict = {} configPath = 'Resources/StorageElements' res = gConfig.getSections(configPath) if not res['OK']: return S_ERROR('Not able to get StorageConfig: %s') % res['Message'] SEList = res['Value'] for SE in SEList: # se = StorageElement(SE, protocols='GFAL2_HTTP') # se.getParameters() seConfigPath = os.path.join(configPath, SE) res = gConfig.getSections(seConfigPath) if not res['OK']: continue # contains 'AccessProtocol.x' accessProtocols = res['Value'] for entry in accessProtocols: protocolConfigPath = os.path.join(seConfigPath, entry) res = gConfig.getOptionsDict(protocolConfigPath) if not res['OK']: continue res = res['Value'] # there should only be one GFAL2_HTTP plugin defined if res.get('PluginName', None) == 'GFAL2_HTTP': if not SEDict.get(res['Host'], None): SEDict[res['Host']] = {} SEDict[res['Host']][SE] = res return S_OK(SEDict)
def convertTransfers(): global csapi gLogger.notice('Converting Transfer services') result = gConfig.getOptionsDict('/Resources/FTSEndpoints') if not result['OK']: gLogger.error(result['Message']) return result ftsDict = result['Value'] for site in ftsDict: result = getSiteName(site) siteName = result['Value'] gLogger.notice('Processing FTS endpoint at site %s' % siteName) csapi.createSection('%s/Sites/%s/Transfer/FTS' % (RESOURCES_NEW_SECTION, siteName)) csapi.setOptionComment( '%s/Sites/%s/Transfer/FTS' % (RESOURCES_NEW_SECTION, siteName), 'File Transfer Service') csapi.setOption( '%s/Sites/%s/Transfer/FTS/URL' % (RESOURCES_NEW_SECTION, siteName), ftsDict[site]) csapi.setOptionComment( '%s/Sites/%s/Transfer' % (RESOURCES_NEW_SECTION, siteName), 'Data Transfer Service resources')
def __getOptionsFromCS(self, path="/Website/Launchpad/Options", delimiter=","): gLogger.info("start __getOptionsFromCS") result = gConfig.getOptionsDict(path) gLogger.always(result) if not result["OK"]: return [] options = result["Value"] for i in options.keys(): options[i] = options[i].split(delimiter) result = gConfig.getSections(path) if result["OK"]: sections = result["Value"] if len(sections) > 0: for i in sections: options[i] = self.__getOptionsFromCS(path + '/' + i, delimiter) gLogger.always("options: %s" % options) gLogger.info("end __getOptionsFromCS") return options
def __init__(self, stratuslabEndpoint): """ Constructor directly reads the named stratuslabEndpoint section in the configuration. :Parameters: **stratuslabEndpoint** - `string` name of the element containing the configuration for a StratusLab cloud """ super(StratusLabEndpointConfiguration, self).__init__() options = gConfig.getOptionsDict('%s/%s' % (self.ENDPOINT_PATH, stratuslabEndpoint)) if not options['OK']: self.log.error(options['Message']) options = {} else: options = options['Value'] # Save a shallow copy of the given dictionary for safety. self._options = copy.copy(options) # Remove any 'None' mappings from the dictionary. for key, value in self._options.items(): if value is None: del self._options[key]
def getCompatiblePlatforms(originalPlatforms): """ Get a list of platforms compatible with the given list """ if type(originalPlatforms) == type(' '): platforms = [originalPlatforms] else: platforms = list(originalPlatforms) platformDict = {} result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility') if result['OK'] and result['Value']: platformDict = result['Value'] for platform in platformDict: platformDict[platform] = [ x.strip() for x in platformDict[platform].split(',') ] else: return S_ERROR('OS compatibility info not found') resultList = list(platforms) for p in platforms: tmpList = platformDict.get(p, []) for pp in platformDict: if p in platformDict[pp]: tmpList.append(pp) tmpList += platformDict[pp] if tmpList: resultList += tmpList return S_OK(uniqueElements(resultList))
def getRegistryUsers(): ''' Gets all users from /Registry/Users ''' _basePath = 'Registry/Users' registryUsers = {} userNames = gConfig.getSections( _basePath ) if not userNames[ 'OK' ]: return userNames userNames = userNames[ 'Value' ] for userName in userNames: # returns { 'Email' : x, 'DN': y, 'CA' : z } userDetails = gConfig.getOptionsDict( '%s/%s' % ( _basePath, userName ) ) if not userDetails[ 'OK' ]: return userDetails registryUsers[ userName ] = userDetails[ 'Value' ] return S_OK( registryUsers ) ################################################################################ #EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def _getCatalogConfigDetails(self, catalogName): # First obtain the options that are available catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName) result = gConfig.getOptionsDict(catalogConfigPath) if not result["OK"]: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." self.log.error(errStr, catalogName) return S_ERROR(errStr) catalogConfig = result["Value"] result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName) if result["OK"]: catalogConfig.update(result["Value"]) # The 'Status' option should be defined (default = 'Active') if "Status" not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." self.log.warn(warnStr, catalogName) catalogConfig["Status"] = "Active" # The 'AccessType' option must be defined if "AccessType" not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." self.log.error(errStr, catalogName) return S_ERROR(errStr) # Anything other than 'True' in the 'Master' option means it is not catalogConfig["Master"] = catalogConfig.setdefault("Master", False) == "True" return S_OK(catalogConfig)
def getDIRACPlatforms(): """ just returns list of platforms defined in the CS """ result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility') if not (result['OK'] and result['Value']): return S_ERROR("OS compatibility info not found") return S_OK(result['Value'].keys())
def _getCatalogConfigDetails( self, catalogName ): # First obtain the options that are available catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName ) result = gConfig.getOptionsDict( catalogConfigPath ) if not result['OK']: errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options." self.log.error( errStr, catalogName ) return S_ERROR( errStr ) catalogConfig = result['Value'] result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName ) if result['OK']: catalogConfig.update( result['Value'] ) # The 'Status' option should be defined (default = 'Active') if 'Status' not in catalogConfig: warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined." self.log.warn( warnStr, catalogName ) catalogConfig['Status'] = 'Active' # The 'AccessType' option must be defined if 'AccessType' not in catalogConfig: errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined." self.log.error( errStr, catalogName ) return S_ERROR( errStr ) # Anything other than 'True' in the 'Master' option means it is not catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' ) return S_OK( catalogConfig )
def getDIRACPlatform( OS ): """ Get standard DIRAC platform(s) compatible with the argument. NB: The returned value is a list! ordered, in reverse, using distutils.version.LooseVersion In practice the "highest" version (which should be the most "desirable" one is returned first) """ result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' ) if not ( result['OK'] and result['Value'] ): return S_ERROR( "OS compatibility info not found" ) platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] ) for k, v in platformsDict.iteritems(): if k not in v: v.append( k ) # making an OS -> platforms dict os2PlatformDict = dict() for platform, osItems in platformsDict.iteritems(): for osItem in osItems: if os2PlatformDict.get( osItem ): os2PlatformDict[osItem].append( platform ) else: os2PlatformDict[osItem] = [platform] if OS not in os2PlatformDict: return S_ERROR( 'No compatible DIRAC platform found for %s' % OS ) platforms = os2PlatformDict[OS] platforms.sort( key = LooseVersion, reverse = True ) return S_OK( platforms )
def getCompatiblePlatforms( originalPlatforms ): """ Get a list of platforms compatible with the given list """ if type( originalPlatforms ) == type( ' ' ): platforms = [originalPlatforms] else: platforms = list( originalPlatforms ) platforms = list( platform.replace( ' ', '' ) for platform in platforms ) result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' ) if not ( result['OK'] and result['Value'] ): return S_ERROR( "OS compatibility info not found" ) platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] ) for k, v in platformsDict.iteritems(): if k not in v: v.append( k ) resultList = list( platforms ) for p in platforms: tmpList = platformsDict.get( p, [] ) for pp in platformsDict: if p in platformsDict[pp]: tmpList.append( pp ) tmpList += platformsDict[pp] if tmpList: resultList += tmpList return S_OK( uniqueElements( resultList ) )
def getCompatiblePlatforms(originalPlatforms): """ Get a list of platforms compatible with the given list """ if isinstance(originalPlatforms, six.string_types): platforms = [originalPlatforms] else: platforms = list(originalPlatforms) platforms = list(platform.replace(' ', '') for platform in platforms) result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility') if not (result['OK'] and result['Value']): return S_ERROR("OS compatibility info not found") platformsDict = dict( (k, v.replace(' ', '').split(',')) for k, v in result['Value'].items()) # can be an iterator for k, v in platformsDict.items(): # can be an iterator if k not in v: v.append(k) resultList = list(platforms) for p in platforms: tmpList = platformsDict.get(p, []) for pp in platformsDict: if p in platformsDict[pp]: tmpList.append(pp) tmpList += platformsDict[pp] if tmpList: resultList += tmpList return S_OK(uniqueElements(resultList))
def _getCurrentConfig(self): """Return the current system configuration.""" from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData gConfig.forceRefresh() fullCfg = CFG() setup = gConfig.getValue('/DIRAC/Setup', '') setupList = gConfig.getSections('/DIRAC/Setups', []) if not setupList['OK']: return S_ERROR('Could not get /DIRAC/Setups sections') setupList = setupList['Value'] if setup not in setupList: return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList))) serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup) if not serviceSetups['OK']: return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup) serviceSetups = serviceSetups['Value'] # dict for system, setup in serviceSetups.items(): if self.systems and system not in self.systems: continue systemCfg = gConfigurationData.remoteCFG.getAsCFG( "/Systems/%s/%s" % (system, setup)) for section in systemCfg.listSections(): if section not in ('Agents', 'Services', 'Executors'): systemCfg.deleteKey(section) fullCfg.createNewSection("/%s" % system, contents=systemCfg) return S_OK(fullCfg)
def getDIRACPlatforms(): """ just returns list of platforms defined in the CS """ result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' ) if not ( result['OK'] and result['Value'] ): return S_ERROR( "OS compatibility info not found" ) return S_OK( result['Value'].keys() )
def getCompatiblePlatforms( originalPlatforms ): """ Get a list of platforms compatible with the given list """ if type( originalPlatforms ) == type( ' ' ): platforms = [originalPlatforms] else: platforms = list( originalPlatforms ) platformDict = {} result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' ) if result['OK'] and result['Value']: platformDict = result['Value'] for platform in platformDict: platformDict[platform] = [ x.strip() for x in platformDict[platform].split( ',' ) ] else: return S_ERROR( 'OS compatibility info not found' ) resultList = list( platforms ) for p in platforms: tmpList = platformDict.get( p, [] ) for pp in platformDict: if p in platformDict[pp]: tmpList.append( pp ) tmpList += platformDict[pp] if tmpList: resultList += tmpList return S_OK( uniqueElements( resultList ) )
def _getBrokerParamsFromCS(mqService: str) -> dict: """Return the configuration of the broker for a given MQService The Sections ``Topics`` and ``Queues`` are returned as the "destinations" key. """ # Compatibility layer in case the full ``qualified`` name is given if "::" in mqService: if os.environ.get("DIRAC_DEPRECATED_FAIL", None): raise NotImplementedError( f"ERROR: deprecated do not give the full mqURI, just the service name: {mqService}" ) print( f"WARNING: deprecated do not give the full mqURI, just the service name: {mqService}" ) mqService = mqService.split("::")[0] brokerParams = returnValueOrRaise( gConfig.getOptionsDict(f"/Resources/MQServices/{mqService}")) # This is for compatibility reasons with the existing configuration definition # Although there are no reasons to separate queues and topics for stomp topics = [ f"/topic/{dest}" for dest in gConfig.getSections( f"/Resources/MQServices/{mqService}/Topics").get("Value", []) ] queues = [ f"/queue/{dest}" for dest in gConfig.getSections( f"/Resources/MQServices/{mqService}/Queues").get("Value", []) ] brokerParams["destinations"] = topics + queues return brokerParams
def __generateDBs(self): self.__log.notice("Creating default AccountingDB...") self.__allDBs = { self.__defaultDB: AccountingDB(readOnly=self.__readOnly) } types = self.__allDBs[self.__defaultDB].getRegisteredTypes() result = gConfig.getOptionsDict(self.__csPath) if not result['OK']: gLogger.verbose("No extra databases defined in %s" % self.__csPath) return validTypes = TypeLoader().getTypes() opts = result['Value'] for acType in opts: if acType not in validTypes: msg = "Oops... %s defined in %s is not a known accounting type" % ( acType, self.__csPath) self.__log.fatal(msg) raise RuntimeError(msg) dbName = opts[acType] gLogger.notice("%s type will be assigned to %s" % (acType, dbName)) if dbName not in self.__allDBs: fields = dbName.split("/") if len(fields) == 1: dbName = "Accounting/%s" % dbName gLogger.notice("Creating DB %s" % dbName) self.__allDBs[dbName] = AccountingDB(dbName, readOnly=self.__readOnly) self.__dbByType[acType] = dbName
def __init__(self, group, lifetime=3600 * 12, voms=False, aToken=None, proxyPath=None): """ C'r :param str group: requested group :param int lifetime: requested proxy lifetime :param bool voms: requested voms extension :param str aToken: access token or path :param str proxyPath: proxy path """ self.log = gLogger.getSubLogger(__name__) # Defaulf location for proxy is /tmp/x509up_uXXXX self.pPath = proxyPath or '/tmp/x509up_u%s' % os.getuid() self.group = group self.lifetime = lifetime self.voms = voms # Default access token path for notebook: /var/run/secrets/egi.eu/access_token self.accessToken = aToken or '/var/run/secrets/egi.eu/access_token' # Load client metadata result = gConfig.getOptionsDict( "/LocalInstallation/AuthorizationClient") if not result['OK']: raise Exception("Can't load client settings.") self.metadata = result['Value'] # For this open client we don't verify ssl certs urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def __site_parameters(sites, wlcg): """Function that returns the sites parameters. :param list sites: List of sites or single site with same site name (e.g ['LCG.CERN.cern'] or [LCG.Manchester.uk, VAC.Manchester.uk]) :param dict wlcg: It's a dictionary with the WLCG parameters from all sites grabbed from https://wlcg-rebus.cern.ch/apps/topology/all/json Keys: 'WlcgName', 'Coordinates', 'Description', 'Mail', 'DiracName', 'Tier', 'Sub-Tier', 'SE', 'Country':, 'Federation', 'FederationAccountingName', 'Infrastructure', 'Institute Name', 'Grid' If the site is not listed in WLCG or have no MoU Tier Level the function will return False """ grid_dict = {} grid, real_site_name, country = sites[0].split(".") site_opts = gConfig.getOptionsDict( 'Resources/Sites/%s/%s' % (grid, sites[0])) site_opts = site_opts.get('Value') site_name = site_opts.get('Name') site_tier = site_opts.get('MoUTierLevel', 'None') if site_tier != 'None': wlcg_params = [s for s in wlcg if site_name in s.get('Site')] wlcg_params = wlcg_params[0] if wlcg_params else {} if not wlcg_params: return False if len(sites) > 1: for i in sites: grid = i.split(".")[0] CE = gConfig.getSections('Resources/Sites/%s/%s/CEs' % (grid, i)) CE = CE['Value'] if CE['OK'] else None grid_dict.update({grid: {'SiteName': i, 'CEs': CE}}) else: CE = gConfig.getSections( 'Resources/Sites/%s/%s/CEs' % (grid, sites[0])) CE = CE['Value'] if CE['OK'] else None grid_dict.update({grid: {'SiteName': sites[0], 'CEs': CE}}) site_subtier = site_opts.get('SubTier', None) ses = site_opts.get('SE', None) site_params = {'WlcgName': site_opts.get('Name'), 'Coordinates': site_opts.get('Coordinates'), 'Description': site_opts.get('Description'), 'Mail': site_opts.get('Mail'), 'DiracName': ('LCG.' + real_site_name + "." + country), 'Tier': site_tier, 'Sub-Tier': site_subtier, 'SE': ses, 'Country': wlcg_params.get('Country'), 'Federation': wlcg_params.get('Federation'), 'FederationAccountingName': wlcg_params.get('FederationAccountingName'), 'Infrastructure': wlcg_params.get('Infrastructure'), 'Institute Name': wlcg_params.get('Institute Name'), 'Grid': grid_dict } return site_params else: return False
def getMQParamsFromCS( mqURI ): """ Function gets parameters of a MQ destination (queue/topic) from the CS. Args: mqURI(str):Pseudo URI identifing the MQ service. It has the following format: mqConnection::DestinationType::DestinationName e.g. blabla.cern.ch::Queue::MyQueue1 mType(str): 'consumer' or 'producer' Returns: S_OK(param_dicts)/S_ERROR: """ # API initialization is required to get an up-to-date configuration from the CS csAPI = CSAPI() csAPI.initialize() try : mqService, mqType, mqName = mqURI.split( "::" ) except ValueError: return S_ERROR( 'Bad format of mqURI address:%s' % ( mqURI ) ) result = gConfig.getConfigurationTree( '/Resources/MQServices', mqService, mqType, mqName ) if not result['OK'] or len( result['Value'] ) == 0: return S_ERROR( 'Requested destination not found in the CS: %s::%s::%s' % ( mqService, mqType, mqName ) ) mqDestinationPath = None for path, value in result['Value'].iteritems(): if not value and path.endswith( mqName ): mqDestinationPath = path # set-up internal parameter depending on the destination type tmp = mqDestinationPath.split( 'Queue' )[0].split( 'Topic' ) servicePath = tmp[0] serviceDict = {} if len( tmp ) > 1: serviceDict['Topic'] = mqName else: serviceDict['Queue'] = mqName result = gConfig.getOptionsDict( servicePath ) if not result['OK']: return result serviceDict.update( result['Value'] ) result = gConfig.getOptionsDict( mqDestinationPath ) if not result['OK']: return result serviceDict.update( result['Value'] ) return S_OK( serviceDict )
def get_se_sites(): # Read the data from CSV file and returns dictionary coordinates_data = {} try: csv_file = open('sites.tmp', "r") except IOError: sys.exit("There are no coordinates info for SE. Get first the computing sites with get_ce_sites()") lines = csv_file.read().split("\n") for row in lines: if row != "": site = row.split(',') if site[1].replace('.','',1).replace('-','',1).isdigit() and \ site[2].replace('.','',1).replace('-','',1).isdigit(): coordinates_data[site[0]] = [float(site[1]),float(site[2])] se_sites = {} manager = Manager() se_result = manager.listSEs()['Value'] se_health = service.getHealthyProductionSEs() #se_health = service.getAllStorageElementStatus() if not se_health['OK']: print "[WARNING]: No health info for SE sites" for se in sorted(se_result['Active']): se_site = {} result = gConfig.getOptionsDict('/Resources/StorageElements/' + se) result2 = gConfig.getOptionsDict('/Resources/StorageElements/' + se \ + '/AccessProtocol.1') if result['OK'] and result2['OK']: result = result['Value'] result2 = result2['Value'] se_site = result2 se_site['Read'] = result['ReadAccess'] se_site['Write'] = result['WriteAccess'] if se.split('-')[0] in coordinates_data: se_site['Coordinates'] = coordinates_data[se.split('-')[0]] else: print "[WARNING]: No location info for " + se se_sites[se] = se_site if se_health['OK'] and se in se_health['Value']: se_site['Health'] = se_health['Value'][se][0] return se_sites
def getServicePorts( self, setup = '', printOutput = False ): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} @return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue( '/DIRAC/Setup', '' ) setupList = gConfig.getSections( '/DIRAC/Setups', [] ) if not setupList['OK']: return S_ERROR( 'Could not get /DIRAC/Setups sections' ) setupList = setupList['Value'] if not setup in setupList: return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) ) serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup ) if not serviceSetups['OK']: return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup ) serviceSetups = serviceSetups['Value'] #dict systemList = gConfig.getSections( '/Systems' ) if not systemList['OK']: return S_ERROR( 'Could not get Systems sections' ) systemList = systemList['Value'] result = {} for system in systemList: if serviceSetups.has_key( system ): path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] ) servicesList = gConfig.getSections( path ) if not servicesList['OK']: self.log.warn( 'Could not get sections in %s' % path ) else: servicesList = servicesList['Value'] if not servicesList: servicesList = [] self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) ) for service in servicesList: spath = '%s/%s/Port' % ( path, service ) servicePort = gConfig.getValue( spath, 0 ) if servicePort: self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) ) result['%s/%s' % ( system, service )] = servicePort else: self.log.warn( 'No port found for %s' % spath ) else: self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) ) if printOutput: print self.pPrint.pformat( result ) return S_OK( result )
def getServicePorts(self, setup="", printOutput=False): """Checks the service ports for the specified setup. If not given this is taken from the current installation (/DIRAC/Setup) Example usage: >>> print diracAdmin.getServicePorts() {'OK': True, 'Value':''} :return: S_OK,S_ERROR """ if not setup: setup = gConfig.getValue("/DIRAC/Setup", "") setupList = gConfig.getSections("/DIRAC/Setups", []) if not setupList["OK"]: return S_ERROR("Could not get /DIRAC/Setups sections") setupList = setupList["Value"] if not setup in setupList: return S_ERROR("Setup %s is not in allowed list: %s" % (setup, ", ".join(setupList))) serviceSetups = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup) if not serviceSetups["OK"]: return S_ERROR("Could not get /DIRAC/Setups/%s options" % setup) serviceSetups = serviceSetups["Value"] # dict systemList = gConfig.getSections("/Systems") if not systemList["OK"]: return S_ERROR("Could not get Systems sections") systemList = systemList["Value"] result = {} for system in systemList: if serviceSetups.has_key(system): path = "/Systems/%s/%s/Services" % (system, serviceSetups[system]) servicesList = gConfig.getSections(path) if not servicesList["OK"]: self.log.warn("Could not get sections in %s" % path) else: servicesList = servicesList["Value"] if not servicesList: servicesList = [] self.log.verbose("System: %s ServicesList: %s" % (system, ", ".join(servicesList))) for service in servicesList: spath = "%s/%s/Port" % (path, service) servicePort = gConfig.getValue(spath, 0) if servicePort: self.log.verbose("Found port for %s/%s = %s" % (system, service, servicePort)) result["%s/%s" % (system, service)] = servicePort else: self.log.warn("No port found for %s" % spath) else: self.log.warn("%s is not defined in /DIRAC/Setups/%s" % (system, setup)) if printOutput: print self.pPrint.pformat(result) return S_OK(result)
def getOptionsDict(self, path): """ Mock the getOptionsDict call of gConfig It reads from dict_cs """ if 'StorageElements' not in path: return gConfig.getOptionsDict(path) csSection = self.crawlCS(path) options = dict((opt, val) for opt, val in csSection.iteritems() if not isinstance(val, dict)) return S_OK(options)
def getExtraConditions(self, site): """ Get extra conditions allowing site throttling """ # Find Site job limits grid, siteName, country = site.split(".") siteSection = "/Resources/Sites/%s/%s" % (grid, site) result = gConfig.getSections(siteSection) if not result["OK"]: return result if not "JobLimits" in result["Value"]: return S_OK({}) result = gConfig.getSections("%s/JobLimits" % siteSection) if not result["OK"]: return result sections = result["Value"] limitDict = {} resultDict = {} if sections: for section in sections: result = gConfig.getOptionsDict("%s/JobLimits/%s" % (siteSection, section)) if not result["OK"]: return result optionDict = result["Value"] if optionDict: limitDict[section] = [] for k, v in optionDict.items(): limitDict[section].append((k, int(v))) if not limitDict: return S_OK({}) # Check if the site exceeding the given limits fields = limitDict.keys() for field in fields: for key, value in limitDict[field]: if int(value) > 0: result = jobDB.getCounters("Jobs", ["Status"], {"Site": site, field: key}) if not result["OK"]: return result count = 0 if result["Value"]: for countDict, number in result["Value"]: if countDict["Status"] in ["Running", "Matched"]: count += number if count > value: if not resultDict.has_key(field): resultDict[field] = [] resultDict[field].append(key) gLogger.verbose( "Job Limit imposed at %s on %s/%s/%d, %d jobs already deployed" % (site, field, key, value, count) ) else: if not resultDict.has_key(field): resultDict[field] = [] resultDict[field].append(key) gLogger.verbose("Jobs prohibited at %s for %s/%s" % (site, field, key)) return S_OK(resultDict)
def _getConfigStorageProtocolDetails(self, storageName, protocol): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName, protocol) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = { 'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'ProtocolName': '', 'SpaceToken': '', 'WSUrl': '' } for option in options: configPath = '%s/%s' % (protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict( cfgPath(protocolConfigPath, 'VOPath')) voPath = '' if result['OK']: voPath = result['Value'].get(self.vo, '') if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'] == 'remote': self.remoteProtocols.append(protocolDict['ProtocolName']) elif protocolDict['Access'] == 'local': self.localProtocols.append(protocolDict['ProtocolName']) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocol) gLogger.warn(errStr) # The ProtocolName option must be defined if not protocolDict['ProtocolName']: errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined." gLogger.error(errStr, "%s: %s" % (storageName, protocol)) return S_ERROR(errStr) return S_OK(protocolDict)
def _getDefaultComponentStatus(self): """Get the configured status of the components.""" host = socket.gethostname() defaultStatus = {'Down': set(), 'Run': set(), 'All': set()} resRunning = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Running')) resStopped = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Stopped')) if not resRunning['OK']: return resRunning if not resStopped['OK']: return resStopped defaultStatus['Run'] = set(resRunning['Value'].keys()) defaultStatus['Down'] = set(resStopped['Value'].keys()) defaultStatus['All'] = defaultStatus['Run'] | defaultStatus['Down'] if defaultStatus['Run'].intersection(defaultStatus['Down']): self.logError("Overlap in configuration", str(defaultStatus['Run'].intersection(defaultStatus['Down']))) return S_ERROR("Bad host configuration") return S_OK(defaultStatus)
def __getExtraOptions(currentSectionPath): from DIRAC import gConfig optionsDict = {} if not currentSectionPath: return optionsDict result = gConfig.getOptionsDict(currentSectionPath) if not result['OK']: return optionsDict print result return result['Value']
def __getCSStorageElementStatus( self, elementName, statusType, default ): ''' Gets from the CS the StorageElements status ''' cs_path = "/Resources/StorageElements" if not isinstance( elementName, list ): elementName = [ elementName ] statuses = self.rssConfig.getConfigStatusType( 'StorageElement' ) #statuses = self.__opHelper.getOptionsDict( 'RSSConfiguration/GeneralConfig/Resources/StorageElement' ) #statuses = gConfig.getOptionsDict( '/Operations/RSSConfiguration/GeneralConfig/Resources/StorageElement' ) result = {} for element in elementName: if statusType is not None: # Added Active by default #res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Allowed' ) res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' ) if res[ 'OK' ] and res[ 'Value' ]: result[ element ] = { statusType : res[ 'Value' ] } else: res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) ) if res[ 'OK' ] and res[ 'Value' ]: elementStatuses = {} for elementStatusType, value in res[ 'Value' ].items(): #k = k.replace( 'Access', '' ) if elementStatusType in statuses: elementStatuses[ elementStatusType ] = value # If there is no status defined in the CS, we add by default Read and # Write as Active. if elementStatuses == {}: #elementStatuses = { 'ReadAccess' : 'Allowed', 'WriteAccess' : 'Allowed' } elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' } result[ element ] = elementStatuses if result: return S_OK( result ) if default is not None: # sec check if statusType is None: statusType = 'none' defList = [ [ el, statusType, default ] for el in elementName ] return S_OK( getDictFromList( defList ) ) _msg = "StorageElement '%s', with statusType '%s' is unknown for CS." return S_ERROR( _msg % ( elementName, statusType ) )
def getExtraConditions( self, site ): """ Get extra conditions allowing site throttling """ # Find Site job limits grid = site.split( '.' )[0] siteSection = '/Resources/Sites/%s/%s' % ( grid, site ) result = gConfig.getSections( siteSection ) if not result['OK']: return result if not 'JobLimits' in result['Value']: return S_OK( {} ) result = gConfig.getSections( '%s/JobLimits' % siteSection ) if not result['OK']: return result sections = result['Value'] limitDict = {} resultDict = {} if sections: for section in sections: result = gConfig.getOptionsDict( '%s/JobLimits/%s' % ( siteSection, section ) ) if not result['OK']: return result optionDict = result['Value'] if optionDict: limitDict[section] = [] for key, value in optionDict.items(): limitDict[section].append( ( key, int( value ) ) ) if not limitDict: return S_OK( {} ) # Check if the site exceeding the given limits fields = limitDict.keys() for field in fields: for key, value in limitDict[field]: if int( value ) > 0: result = gJobDB.getCounters( 'Jobs', ['Status'], {'Site':site, field:key} ) if not result['OK']: return result count = 0 if result['Value']: for countDict, number in result['Value']: if countDict['Status'] in ["Running", "Matched"]: count += number if count > value: if not resultDict.has_key( field ): resultDict[field] = [] resultDict[field].append( key ) gLogger.verbose( 'Job Limit imposed at %s on %s/%s/%d,' ' %d jobs already deployed' % ( site, field, key, value, count ) ) else: if not resultDict.has_key( field ): resultDict[field] = [] resultDict[field].append( key ) gLogger.verbose( 'Jobs prohibited at %s for %s/%s' % ( site, field, key ) ) return S_OK( resultDict )