def addCEtoSite(self, siteName, ceName, optionsDict=None): """ Adds a new CE to a site definition in the CS. A CE normally has queues, so addQueueToCE should be called after this one. :param str siteName: FQN of the site (e.g. LCG.CERN.ch) :param str ceName: FQN of the CE (e.g. ce503.cern.ch) :param dict optionsDict: optional dictionary of options :returns: S_OK/S_ERROR structure """ res = getSites() if not res['OK']: return res if siteName not in res['Value']: res = self.addSite(siteName) if not res['OK']: return res # CSAPI.createSection() always returns S_OK even if the section already exists self.__csMod.createSection( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName, 'CEs')) self.__csMod.createSection( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName, 'CEs', ceName)) # add options if requested if optionsDict is not None: for option, optionValue in optionsDict.items( ): # can be an iterator self.__csMod.setOptionValue( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName, 'CEs', ceName, option), optionValue) self.csModified = True return S_OK(True)
def addSite(self, siteName, optionsDict=None): """ Adds a new site to the CS. A site is a container for services, so after calling this function, at least addCEtoSite() should be called. :param str siteName: FQN of the site (e.g. LCG.CERN.ch) :param dict optionsDict: optional dictionary of options :returns: S_OK/S_ERROR structure """ if not self.__initialized['OK']: return self.__initialized self.__csMod.createSection(cfgPath(self.__baseResources, 'Sites')) self.__csMod.createSection( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0])) self.__csMod.createSection( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName)) # add options if requested if optionsDict is not None: for option, optionValue in optionsDict.items( ): # can be an iterator self.__csMod.setOptionValue( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], option), optionValue) self.csModified = True return S_OK(True)
def configure(self, csSection, submitPool): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection(csSection) self.reloadConfiguration(csSection, submitPool) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0] self.installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject) self.installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation) self.pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", self.pilotExtensionsList) self.log.info("===============================================") self.log.info("Configuration:") self.log.info("") self.log.info(" Target Grids: ", ", ".join(self.targetGrids)) self.log.info(" Install script: ", self.install) self.log.info(" Pilot script: ", self.pilot) self.log.info(" Pilot modules", self.extraModules) self.log.info(" Install Ver: ", self.installVersion) if self.installProject: self.log.info(" Project: ", self.installProject) if self.installation: self.log.info(" Installation: ", self.installation) if self.extraPilotOptions: self.log.info(" Extra Options: ", " ".join(self.extraPilotOptions)) self.log.info(" ListMatch: ", self.enableListMatch) self.log.info(" Private %: ", self.privatePilotFraction * 100) if self.enableListMatch: self.log.info(" ListMatch Delay:", self.listMatchDelay) self.listMatchCache.purgeExpired()
def addQueueToCE(self, ceName, queueName, optionsDict=None): """ Adds a new queue to a CE definition in the CS. :param str ceName: FQN of the CE (e.g. ce503.cern.ch) :param str queueName: name of the queue (e.g. ce503.cern.ch-condor) :param dict optionsDict: optional dictionary of options :returns: S_OK/S_ERROR structure """ res = getCESiteMapping(ceName) if not res['OK']: return res if ceName not in res['Value']: return S_ERROR("CE does not exist") siteName = res['Value'][ceName] # CSAPI.createSection() always returns S_OK even if the section already exists self.__csMod.createSection( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName, 'CEs', ceName, 'Queues', queueName)) # add options if requested if optionsDict is not None: for option, optionValue in optionsDict.items( ): # can be an iterator self.__csMod.setOptionValue( cfgPath(self.__baseResources, 'Sites', siteName.split('.')[0], siteName, 'CEs', ceName, 'Queues', queueName, option), optionValue) self.csModified = True return S_OK(True)
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) setup = gConfig.getValue( '/DIRAC/Setup', '' ) section = cfgPath( 'Operations', self.virtualOrganization, setup, 'Versions' ) self.installVersion = gConfig.getValue( cfgPath( section, 'PilotVersion' ), self.installVersion ) self.installInstallation = gConfig.getValue( cfgPath( section, 'PilotInstallation' ), self.installInstallation ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installInstallation: self.log.info( ' Installation: ', self.installInstallation ) if self.extraPilotOptions: self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def __filterByPlatform(self, jobPlatform, userSites): """ Filters out sites that have no CE with a matching platform. """ basePath = "/Resources/Sites" filteredSites = set() for site in userSites: if "." not in site: # Invalid site name: Doesn't contain a dot! self.jobLog.info("Skipped invalid site name: %s" % site) continue grid = site.split('.')[0] sitePath = cfgPath(basePath, grid, site, "CEs") result = gConfig.getSections(sitePath) if not result['OK']: self.jobLog.info("Failed to get CEs at site %s." % site) continue siteCEs = result['Value'] for CEName in siteCEs: CEPlatform = gConfig.getValue(cfgPath(sitePath, CEName, "OS")) if jobPlatform == CEPlatform: # Site has a CE with a matchin platform filteredSites.add(site) return S_OK(list(filteredSites))
def updateCS(changeSet): global vo, dry, ceBdiiDict changeList = sorted(changeSet) if dry: gLogger.notice('The following needed changes are detected:\n') else: gLogger.notice('We are about to make the following changes to CS:\n') for entry in changeList: gLogger.notice("%s/%s %s -> %s" % entry) if not dry: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error('Failed to initialize CSAPI object', result['Message']) DIRACExit(-1) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: csAPI.setOption(cfgPath(section, option), new_value) else: csAPI.modifyValue(cfgPath(section, option), new_value) yn = six.moves.input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ') if yn == '' or yn.lower().startswith('y'): result = csAPI.commit() if not result['OK']: gLogger.error("Error while commit to CS", result['Message']) else: gLogger.notice("Successfully committed %d changes to CS" % len(changeSet))
def __init__(self, database=False): super(PolicyBasedSecurityManager, self).__init__(database) from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath serviceSection = getServiceSection('DataManagement/FileCatalog') pluginPath = gConfig.getValue(cfgPath(serviceSection, 'SecurityPolicy')) if not pluginPath: raise Exception("SecurityPolicy not defined in service options") pluginCls = self.__loadPlugin(pluginPath) self.policyObj = pluginCls(database=database) # For the old clients to work with the new policy (since getPathPermissions is meant to disappear...) # we fetch the old SecurityManager, and we call it if needed in the plugin. oldSecurityManagerName = gConfig.getValue( cfgPath(serviceSection, 'OldSecurityManager'), '') self.policyObj.oldSecurityManager = None if oldSecurityManagerName: self.policyObj.oldSecurityManager = eval("%s(self.db)" % oldSecurityManagerName)
def getSEsFromCS(protocol='srm'): """ Get all the SEs defined in the CS """ knownSEs = {} result = gConfig.getSections('/Resources/StorageElements') if not result['OK']: return result ses = result['Value'] for se in ses: seSection = '/Resources/StorageElements/%s' % se result = gConfig.getSections(seSection) if not result['OK']: continue accesses = result['Value'] for access in accesses: seProtocol = gConfig.getValue( cfgPath(seSection, access, 'Protocol'), '') if seProtocol.lower() == protocol.lower() or protocol == 'any': host = gConfig.getValue(cfgPath(seSection, access, 'Host'), '') knownSEs.setdefault(host, []) knownSEs[host].append(se) else: continue return S_OK(knownSEs)
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0] self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject ) self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installProject: self.log.info( ' Project: ', self.installProject ) if self.installation: self.log.info( ' Installation: ', self.installation ) if self.extraPilotOptions: self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) # Get the defaults for the Setup where the Director is running opsHelper = Operations() self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0] self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject ) self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation ) self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Pilot modules', self.extraModules ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installProject: self.log.info( ' Project: ', self.installProject ) if self.installation: self.log.info( ' Installation: ', self.installation ) if self.extraPilotOptions: self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def _getConfigStorageOptions( self, storageName, derivedStorageName = None ): """ Get the options associated to the StorageElement as defined in the CS """ optionsDict = {} # We first get the options of the baseSE, and then overwrite with the derivedSE for seName in ( storageName, derivedStorageName ) if derivedStorageName else ( storageName, ): storageConfigPath = cfgPath( self.rootConfigPath, seName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) ) return S_ERROR( errStr ) for option in set( res['Value'] ) - set( ( 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess' ) ): optionConfigPath = cfgPath( storageConfigPath, option ) default = [] if option in [ 'VO' ] else '' optionsDict[option] = gConfig.getValue( optionConfigPath, default ) # The status is that of the derived SE only seName = derivedStorageName if derivedStorageName else storageName res = self.resourceStatus.getStorageElementStatus( seName ) if not res[ 'OK' ]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) ) return S_ERROR( errStr ) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update( res[ 'Value' ][ seName ] ) return S_OK( optionsDict )
def add(self, section, option, new_value): """ Add a value into the configuration system. This method will overwrite any existing option's value. Args: section (str): The section option (str): The option to be created/modified new_value: The value to be assigned Example: >>> cs = ConfigurationSystem() >>> cs.add('/Registry', 'DefaultGroup', 'dteam_user') """ if isinstance(new_value, (tuple, list, set, GeneratorType)): new_value = ', '.join(sorted(map(str, new_value))) else: new_value = str(new_value) old_value = gConfig.getValue(cfgPath(section, option), None) if old_value == new_value: return if old_value is None: gLogger.notice("Setting %s/%s: -> %s" % (section, option, new_value)) self.setOption(cfgPath(section, option), new_value) else: gLogger.notice("Modifying %s/%s: %s -> %s" % (section, option, old_value, new_value)) self.modifyValue(cfgPath(section, option), new_value) self._num_changes += 1
def updateCS( changeSet ): global vo, dry, ceBdiiDict changeList = list( changeSet ) changeList.sort() if dry: gLogger.notice( 'The following needed changes are detected:\n' ) else: gLogger.notice( 'We are about to make the following changes to CS:\n' ) for entry in changeList: gLogger.notice( "%s/%s %s -> %s" % entry ) if not dry: csAPI = CSAPI() csAPI.initialize() result = csAPI.downloadCSData() if not result['OK']: gLogger.error( 'Failed to initialize CSAPI object', result['Message'] ) DIRACExit( -1 ) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: csAPI.setOption( cfgPath( section, option ), new_value ) else: csAPI.modifyValue( cfgPath( section, option ), new_value ) yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' ) if yn == '' or yn.lower().startswith( 'y' ): result = csAPI.commit() if not result['OK']: gLogger.error( "Error while commit to CS", result['Message'] ) else: gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def remove(self, section, option=None, value=None): """ Remove a section/option from the configuration system. This method will remove the specified section if the option argument is None (default). If the option argument is given but value is None then that option (formed of section/option) is removed. If both option and value are given then that value is removed from the comma seperated values associated with that option. Args: section (str): The section option (str): [optional] The option to remove value (str): [optional] The value to remove Example: >>> ConfigurationSystem().remove('/Registry', 'DefaultGroup') """ if option is None: gLogger.notice("Removing section %s" % section) self.delSection(section) self._num_changes += 1 elif value is None: gLogger.notice("Removing option %s/%s" % (section, option)) self.delOption(cfgPath(section, option)) self._num_changes += 1 else: if isinstance(value, str): value = [value] gLogger.notice("Removing value(s) %s from option %s/%s" % (list(value), section, option)) old_values = (v.strip() for v in gConfig.getValue(cfgPath(section, option), '').split(',')) new_values = [v for v in old_values if v and v not in value] self.add(section, option, new_values)
def __filterByPlatform(self, jobPlatform, userSites): """ Filters out sites that have no CE with a matching platform. """ basePath = "/Resources/Sites" filteredSites = set() for site in userSites: if "." not in site: # Invalid site name: Doesn't contain a dot! self.jobLog.info("Skipped invalid site name: %s" % site) continue grid = site.split('.')[0] sitePath = cfgPath(basePath, grid, site, "CEs") result = gConfig.getSections(sitePath) if not result['OK']: self.jobLog.info("Failed to get CEs at site %s." % site) continue siteCEs = result['Value'] for CEName in siteCEs: CEPlatform = gConfig.getValue(cfgPath(sitePath, CEName, "OS")) if jobPlatform == CEPlatform: # Site has a CE with a matchin platform filteredSites.add(site) return S_OK(list(filteredSites))
def configure( self, csSection, submitPool ): """ Here goes common configuration for all PilotDirectors """ self.configureFromSection( csSection ) self.reloadConfiguration( csSection, submitPool ) setup = gConfig.getValue( '/DIRAC/Setup', '' ) section = cfgPath( 'Operations', self.virtualOrganization, setup, 'Versions' ) self.installVersion = gConfig.getValue( cfgPath( section, 'PilotVersion' ), self.installVersion ) self.installInstallation = gConfig.getValue( cfgPath( section, 'PilotInstallation' ), self.installInstallation ) self.log.info( '===============================================' ) self.log.info( 'Configuration:' ) self.log.info( '' ) self.log.info( ' Install script: ', self.install ) self.log.info( ' Pilot script: ', self.pilot ) self.log.info( ' Install Ver: ', self.installVersion ) if self.installInstallation: self.log.info( ' Installation: ', self.installInstallation ) if self.extraPilotOptions: self.log.info( ' Exta Options: ', ' '.join( self.extraPilotOptions ) ) self.log.info( ' ListMatch: ', self.enableListMatch ) self.log.info( ' Private %: ', self.privatePilotFraction * 100 ) if self.enableListMatch: self.log.info( ' ListMatch Delay:', self.listMatchDelay ) self.listMatchCache.purgeExpired()
def _getConfigStorageOptions( self, storageName ): """ Get the options associated to the StorageElement as defined in the CS """ storageConfigPath = cfgPath( self.rootConfigPath, storageName ) res = gConfig.getOptions( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) options = res['Value'] optionsDict = {} for option in options: if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']: continue optionConfigPath = cfgPath( storageConfigPath, option ) if option in [ 'VO' ]: optionsDict[option] = gConfig.getValue( optionConfigPath, [] ) else: optionsDict[option] = gConfig.getValue( optionConfigPath, '' ) res = self.resourceStatus.getStorageElementStatus( storageName ) if not res[ 'OK' ]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update( res[ 'Value' ][ storageName ] ) return S_OK( optionsDict )
def __updateCS(self, bdiiChangeSet): queueVODict = {} changeSet = set() for entry in bdiiChangeSet: section, option, _value, new_value = entry if option == "VO": queueVODict.setdefault(section, set()) queueVODict[section] = queueVODict[section].union( set(new_value.split(','))) else: changeSet.add(entry) for section, VOs in queueVODict.items(): changeSet.add((section, 'VO', '', ','.join(VOs))) if changeSet: changeList = list(changeSet) changeList.sort() body = '\n'.join( ["%s/%s %s -> %s" % entry for entry in changeList]) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail(self.addressTo, self.subject, body, self.addressFrom, localAttempt=False) if body: self.log.info( 'The following configuration changes were detected:') self.log.info(body) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: self.csAPI.setOption(cfgPath(section, option), new_value) else: self.csAPI.modifyValue(cfgPath(section, option), new_value) if self.dryRun: self.log.info("Dry Run: CS won't be updated") self.csAPI.showDiff() else: result = self.csAPI.commit() if not result['OK']: self.log.error("Error while committing to CS", result['Message']) else: self.log.info("Successfully committed %d changes to CS" % len(changeList)) return result else: self.log.info("No changes found") return S_OK()
def _getConfigStorageOptions(self, storageName, derivedStorageName=None, seConfigPath=SE_CONFIG_PATH): """ Get the options associated to the StorageElement as defined in the CS :param storageName: is the storage section to check in the CS :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param derivedStorageName: is the storage section of a derived storage if it inherits from a base :return: options associated to the StorageElement as defined in the CS """ optionsDict = {} # We first get the options of the baseSE, and then overwrite with the derivedSE for seName in (storageName, derivedStorageName) if derivedStorageName else ( storageName, ): storageConfigPath = cfgPath(seConfigPath, seName) res = gConfig.getOptions(storageConfigPath) if not res["OK"]: errStr = "StorageFactory._getStorageOptions: Failed to get storage options." gLogger.error(errStr, "%s: %s" % (seName, res["Message"])) return S_ERROR(errStr) for option in set(res["Value"]) - set( ("ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess")): optionConfigPath = cfgPath(storageConfigPath, option) default = [] if option in [ "VO", "AccessProtocols", "WriteProtocols" ] else "" optionsDict[option] = gConfig.getValue(optionConfigPath, default) # We update the seConfigPath in order to find option in derivedSE now seConfigPath = SE_CONFIG_PATH # The status is that of the derived SE only seName = derivedStorageName if derivedStorageName else storageName res = self.resourceStatus.getElementStatus(seName, "StorageElement") if not res["OK"]: errStr = "StorageFactory._getStorageOptions: Failed to get storage status" gLogger.error(errStr, "%s: %s" % (seName, res["Message"])) return S_ERROR(errStr) # For safety, we did not add the ${statusType}Access keys # this requires modifications in the StorageElement class # We add the dictionary with the statusTypes and values # { 'statusType1' : 'status1', 'statusType2' : 'status2' ... } optionsDict.update(res["Value"][seName]) return S_OK(optionsDict)
def getSiteSEMapping(gridName=''): """ Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} If gridName is specified, result is restricted to that Grid type. """ siteSEMapping = {} gridTypes = gConfig.getSections('Resources/Sites/') if not gridTypes['OK']: gLogger.warn('Problem retrieving sections in /Resources/Sites') return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR('Could not get sections for /Resources/Sites/%s' % gridName) gridTypes = [gridName] gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes))) for grid in gridTypes: sites = gConfig.getSections('/Resources/Sites/%s' % grid) if not sites['OK']: gLogger.warn('Problem retrieving /Resources/Sites/%s section' % grid) return sites for candidate in sites['Value']: candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % (grid, candidate), []) if candidateSEs: siteSEMapping[candidate] = candidateSEs else: gLogger.debug('No SEs defined for site %s' % candidate) # Add Sites from the SiteLocalSEMapping in the CS cfgLocalSEPath = cfgPath('SiteLocalSEMapping') opsHelper = Operations() result = opsHelper.getOptionsDict(cfgLocalSEPath) if result['OK']: mapping = result['Value'] for site in mapping: ses = opsHelper.getValue(cfgPath(cfgLocalSEPath, site), []) if not ses: continue if gridName: if gridName != site.split('.')[0]: continue if site not in siteSEMapping: siteSEMapping[site] = [] for se in ses: if se not in siteSEMapping[site]: siteSEMapping[site].append(se) return S_OK(siteSEMapping)
def _getConfigStorageProtocolDetails(self, storageName, protocolSection, seConfigPath=SE_CONFIG_PATH, checkAccess=True): """ Parse the contents of the protocol block :param storageName: is the storage section to check in the CS :param protocolSection: name of the protocol section to find information :param seConfigPath: the path of the storage section. It can be /Resources/StorageElements or StorageElementBases :param checkAccess: if not set, don't complain if "Access" is not in the section :return: dictionary of the protocol options """ # First obtain the options that are available protocolConfigPath = cfgPath(seConfigPath, storageName, protocolSection) res = gConfig.getOptions(protocolConfigPath) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error(errStr, "%s: %s" % (storageName, protocolSection)) return S_ERROR(errStr) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': ''} for option in options: configPath = cfgPath(protocolConfigPath, option) optionValue = gConfig.getValue(configPath, '') protocolDict[option] = optionValue # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict(cfgPath(protocolConfigPath, 'VOPath')) voPath = '' if result['OK']: voPath = result['Value'].get(self.vo, '') if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set and the plugin is not already in remote or local. plugin = protocolDict.get('PluginName', protocolSection) if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append(plugin) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append(plugin) # If it is a derived SE, this is normal, no warning elif checkAccess and protocolSection not in self.protocols: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option \ for %s:%s is neither 'local' or 'remote'." % (storageName, protocolSection) gLogger.warn(errStr) return S_OK(protocolDict)
def getFTS3Servers(): """ get FTSServers for sites """ csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" ) # We do it in two times to keep the order ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] ) ftsServers = [] for name in ftsServerNames: ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) ) return S_OK( ftsServers )
def getFTS3Servers(): """ get FTSServers for sites """ csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" ) # We do it in two times to keep the order ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] ) ftsServers = [] for name in ftsServerNames: ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) ) return S_OK( ftsServers )
def _getConfigStorageProtocolDetails( self, storageName, protocolSection ): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection ) res = gConfig.getOptions( protocolConfigPath ) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'PluginName':'', 'SpaceToken':'', 'WSUrl':''} for option in options: configPath = cfgPath( protocolConfigPath, option ) optionValue = gConfig.getValue( configPath, '' ) protocolDict[option] = optionValue # This is a temporary for backward compatibility if "ProtocolName" in protocolDict and not protocolDict['PluginName']: protocolDict['PluginName'] = protocolDict['ProtocolName'] protocolDict.pop( 'ProtocolName', None ) # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) ) voPath = '' if result['OK']: voPath = result['Value'].get( self.vo, '' ) if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append( protocolDict['PluginName'] ) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append( protocolDict['PluginName'] ) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection ) gLogger.warn( errStr ) # The PluginName option must be defined if not protocolDict['PluginName']: errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) return S_OK( protocolDict )
def remove_old(self, removal_threshold=5): """Remove old hosts/sites.""" cfg_system = ConfigurationSystem() result = cfg_system.getCurrentCFG() if not result['OK']: self.log.error('Could not get current config from the CS') raise RuntimeError("Error removing old Resources/Registry.") today = date.today() removal_threshold = timedelta(days=removal_threshold) old_ces = set() base_path = '/Resources/Sites' for site_type in ('VAC', 'CLOUD'): site_type_path = cfgPath(base_path, site_type) for site, site_info in result['Value'].getAsDict(base_path).iteritems(): site_path = cfgPath(site_type_path, site) for ce, ce_info in site_info.get('CEs', {}).iteritems(): ce_path = cfgPath(site_path, 'CEs', ce) if 'LastSeen' not in ce_info: self.log.warn("No LastSeen info for CE: %s at site: %s" % (ce, site)) continue last_seen = datetime.strptime(ce_info['LastSeen'], '%d/%m/%Y').date() delta = today - last_seen if delta > removal_threshold: self.log.warn("Last seen %s:%s %s days ago...removing" % (site, ce, delta.days)) cfg_system.remove(section=ce_path) old_ces.add(ce) if old_ces: cfg_system.remove(section=site_path, option='CE', value=old_ces) old_ces.clear() host_base = '/Registry/Hosts' for host, host_info in result['Value'].getAsDict(host_base).iteritems(): host_path = cfgPath(host_base, host) if 'LastSeen' not in host_info: self.log.warn("No LastSeen info for host: %s" % host) continue last_seen = datetime.strptime(host_info['LastSeen'], '%d/%m/%Y').date() delta = today - last_seen if delta > removal_threshold: self.log.warn("Last seen host %s %s days ago...removing" % (host, delta.days)) cfg_system.remove(section=host_path) cfg_system.commit() return S_OK()
def _getConfigStorageProtocolDetails( self, storageName, protocolSection, checkAccess = True ): """ Parse the contents of the protocol block """ # First obtain the options that are available protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection ) res = gConfig.getOptions( protocolConfigPath ) if not res['OK']: errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) options = res['Value'] # We must have certain values internally even if not supplied in CS protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'SpaceToken':'', 'WSUrl':''} for option in options: configPath = cfgPath( protocolConfigPath, option ) optionValue = gConfig.getValue( configPath, '' ) protocolDict[option] = optionValue # This is a temporary for backward compatibility: move ProtocolName to PluginName protocolDict.setdefault( 'PluginName', protocolDict.pop( 'ProtocolName', None ) ) # Evaluate the base path taking into account possible VO specific setting if self.vo: result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) ) voPath = '' if result['OK']: voPath = result['Value'].get( self.vo, '' ) if voPath: protocolDict['Path'] = voPath # Now update the local and remote protocol lists. # A warning will be given if the Access option is not set. if checkAccess: if protocolDict['Access'].lower() == 'remote': self.remotePlugins.append( protocolDict['PluginName'] ) elif protocolDict['Access'].lower() == 'local': self.localPlugins.append( protocolDict['PluginName'] ) else: errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection ) gLogger.warn( errStr ) # The PluginName option must be defined if not protocolDict['PluginName']: errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined." gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) ) return S_ERROR( errStr ) return S_OK( protocolDict )
def getSiteSEMapping( gridName = '' ): """ Returns a dictionary of all sites and their localSEs as a list, e.g. {'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]} If gridName is specified, result is restricted to that Grid type. """ siteSEMapping = {} gridTypes = gConfig.getSections( 'Resources/Sites/' ) if not gridTypes['OK']: gLogger.warn( 'Problem retrieving sections in /Resources/Sites' ) return gridTypes gridTypes = gridTypes['Value'] if gridName: if not gridName in gridTypes: return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName ) gridTypes = [gridName] gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) ) for grid in gridTypes: sites = gConfig.getSections( '/Resources/Sites/%s' % grid ) if not sites['OK']: gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid ) return sites for candidate in sites['Value']: candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] ) if candidateSEs: siteSEMapping[candidate] = candidateSEs else: gLogger.debug( 'No SEs defined for site %s' % candidate ) # Add Sites from the SiteLocalSEMapping in the CS vo = getVO() setup = getDIRACSetup() cfgLocalSEPath = cfgPath( 'Operations', vo, setup, 'SiteLocalSEMapping' ) result = gConfig.getOptionsDict( cfgLocalSEPath ) if result['OK']: mapping = result['Value'] for site in mapping: ses = gConfig.getValue( cfgPath( cfgLocalSEPath, site ), [] ) if not ses: continue if gridName and site not in sites: continue if site not in siteSEMapping: siteSEMapping[site] = [] for se in ses: if se not in siteSEMapping[site]: siteSEMapping[site].append( se ) return S_OK( siteSEMapping )
def getSites(): """ Get the list of all the sites defined in the CS """ result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites")) if not result["OK"]: return result grids = result["Value"] sites = [] for grid in grids: result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites", grid)) if not result["OK"]: return result sites += result["Value"] return S_OK(sites)
def getSites(): """ Get the list of all the sites defined in the CS """ result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) ) if not result['OK']: return result grids = result['Value'] sites = [] for grid in grids: result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites', grid ) ) if not result['OK']: return result sites += result['Value'] return S_OK( sites )
def getSites(): """Get the list of all the sites defined in the CS""" result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites")) if not result["OK"]: return result grids = result["Value"] sites = [] for grid in grids: result = gConfig.getSections( cfgPath(gBaseResourcesSection, "Sites", grid)) if not result["OK"]: return result sites += result["Value"] return S_OK(sites)
def getSites(): """ Get the list of all the sites defined in the CS """ result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) ) if not result['OK']: return result grids = result['Value'] sites = [] for grid in grids: result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites', grid ) ) if not result['OK']: return result sites += result['Value'] return S_OK( sites )
def _getConfigStorageName(self, storageName, referenceType): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. 'storageName' is the storage section to check in the CS """ configPath = '%s/%s' % (self.rootConfigPath, storageName) res = gConfig.getOptions(configPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error(errStr, res['Message']) return S_ERROR(errStr) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error(errStr, configPath) return S_ERROR(errStr) if referenceType in res['Value']: configPath = cfgPath(self.rootConfigPath, storageName, referenceType) referenceName = gConfig.getValue(configPath) result = self._getConfigStorageName(referenceName, 'Alias') if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK(resolvedName)
def getSites( siteList = [], fullName = False ): """ Get the list of site names possibly limited by a given list of sites names in arbitrary form """ if not siteList: result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) ) if not result['OK']: return result sList = result['Value'] else: if not type( siteList ) == ListType: siteList = [ siteList ] sList = [] for s in siteList: result = getSiteName( s ) if not result['OK']: continue sList.append( result['Value'] ) resultList = [] if fullName: for site in sList: result = getSiteFullNames( site ) if not result['OK']: continue resultList += result['Value'] else: for site in sList: result = getSiteName( site ) if not result['OK']: continue resultList.append( result['Value'] ) return S_OK( resultList )
def getRunningInstances(self, instanceType='Agents', runitStatus='Run'): """Return a dict of running agents, executors or services. Key is agent's name, value contains dict with PollingTime, PID, Port, Module, RunitStatus, LogFileLocation :param str instanceType: 'Agents', 'Executors', 'Services' :param str runitStatus: Return only those instances with given RunitStatus or 'All' :returns: Dictionary of running instances """ res = self.sysAdminClient.getOverallStatus() if not res["OK"]: self.logError("Failure to get %s from system administrator client" % instanceType, res["Message"]) return res val = res['Value'][instanceType] runningAgents = defaultdict(dict) for system, agents in val.iteritems(): for agentName, agentInfo in agents.iteritems(): if agentInfo['Setup'] and agentInfo['Installed']: if runitStatus != 'All' and agentInfo['RunitStatus'] != runitStatus: continue confPath = cfgPath('/Systems/' + system + '/' + self.setup + '/%s/' % instanceType + agentName) for option, default in (('PollingTime', HOUR), ('Port', None)): optPath = os.path.join(confPath, option) runningAgents[agentName][option] = gConfig.getValue(optPath, default) runningAgents[agentName]["LogFileLocation"] = \ os.path.join(self.diracLocation, 'runit', system, agentName, 'log', 'current') runningAgents[agentName]["PID"] = agentInfo["PID"] runningAgents[agentName]['Module'] = agentInfo['Module'] runningAgents[agentName]['RunitStatus'] = agentInfo['RunitStatus'] runningAgents[agentName]['System'] = system return S_OK(runningAgents)
def __getNodePath(self, nodeName, resourceType): """ Get path to the resource CS section """ site, resource, node = nodeName.split(RESOURCE_NAME_SEPARATOR) return cfgPath(gBaseResourcesSection, 'Sites', site, resourceType, resource, RESOURCE_NODE_MAPPING[resourceType] + 's', node)
def getSites(siteList=[], fullName=False): """ Get the list of site names possibly limited by a given list of sites names in arbitrary form """ if not siteList: result = gConfig.getSections(cfgPath(gBaseResourcesSection, 'Sites')) if not result['OK']: return result sList = result['Value'] else: if not type(siteList) == ListType: siteList = [siteList] sList = [] for s in siteList: result = getSiteName(s) if not result['OK']: continue sList.append(result['Value']) resultList = [] if fullName: for site in sList: result = getSiteFullNames(site) if not result['OK']: continue resultList += result['Value'] else: for site in sList: result = getSiteName(site) if not result['OK']: continue resultList.append(result['Value']) return S_OK(resultList)
def getMultiHopMatrix(self): """ Returns the multi-hop matrix described in DataManagement/MultiHopMatrixOfShame. .. code-block :: python 'Default': { 'Default': 'MultiHopSEUsedForAllTransfer', 'Dst3' : 'MultiHopFromAnySourceToDst3', } 'Src1' : { 'Default' : 'DefaultMultiHopSEFromSrc1', 'Dst1': 'MultiHopSEFromSrc1ToDst1}, 'Src2' : { 'Default' : 'DefaultMultiHopSEFromSrc2', 'Dst2': 'MultiHopSEFromSrc1ToDst2} :returns: dict of dict for all the source se / dest SE defined. We user defaultdict to allow for the use of non existing source/dest. """ matrixBasePath = "DataManagement/MultiHopMatrixOfShame" multiHopMatrix = defaultdict(lambda: defaultdict(lambda: None)) allSrcSEs = self.__opsHelper.getSections(matrixBasePath).get( "Value", []) for src in allSrcSEs: srcDst = self.__opsHelper.getOptionsDict( cfgPath(matrixBasePath, src)).get("Value") if srcDst: multiHopMatrix[src].update(srcDst) return multiHopMatrix
def getDIRACSiteName(gocSiteName): """ Get DIRAC site name, given the GOC DB site name, as it stored in the CS :params str gocSiteName: GOC DB site name (e.g. 'CERN-PROD') :returns: S_OK/S_ERROR structure """ res = getSites() if not res['OK']: return res sitesList = res['Value'] tmpList = [(site, gConfig.getValue( cfgPath(gBaseResourcesSection, 'Sites', site.split('.')[0], site, 'Name'))) for site in sitesList] diracSites = [dirac for (dirac, goc) in tmpList if goc == gocSiteName] if diracSites: return S_OK(diracSites) return S_ERROR("There's no site with GOCDB name = %s in DIRAC CS" % gocSiteName)
def _getConfigStorageName( self, storageName, referenceType ): """ This gets the name of the storage the configuration service. If the storage is a reference to another SE the resolution is performed. 'storageName' is the storage section to check in the CS """ configPath = '%s/%s' % ( self.rootConfigPath, storageName ) res = gConfig.getOptions( configPath ) if not res['OK']: errStr = "StorageFactory._getConfigStorageName: Failed to get storage options" gLogger.error( errStr, res['Message'] ) return S_ERROR( errStr ) if not res['Value']: errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist." gLogger.error( errStr, configPath ) return S_ERROR( errStr ) if referenceType in res['Value']: configPath = cfgPath( self.rootConfigPath, storageName, referenceType ) referenceName = gConfig.getValue( configPath ) result = self._getConfigStorageName( referenceName, 'Alias' ) if not result['OK']: return result resolvedName = result['Value'] else: resolvedName = storageName return S_OK( resolvedName )
def append_unique(self, section, option, new_value): """ Append a value onto the end of an existing CS option. This method is like append except that it ensures that the final list of values for the given option only contains unique entries. Args: section (str): The section option (str): The option to be modified new_value: The value to be appended """ if isinstance(new_value, (tuple, list, set, GeneratorType)): self._append_unique_dict[cfgPath(section, option)].update(new_value) else: self._append_unique_dict[cfgPath(section, option)].add(new_value)
def append(self, section, option, new_value): """ Append a value onto the end of an existing CS option. This method is like add with the exception that the new value is appended on to the end of the list of values associated with that option. Args: section (str): The section option (str): The option to be modified new_value: The value to be appended """ if isinstance(new_value, (tuple, list, set, GeneratorType)): self._append_dict[cfgPath(section, option)].extend(new_value) else: self._append_dict[cfgPath(section, option)].append(new_value)
def getSitePath( site ): """ Return path to the Site section on CS """ result = getSiteName( site ) if not result['OK']: return result siteName = result['Value'] return S_OK( cfgPath( gBaseResourcesSection, 'Sites', siteName ) )
def getSitePath(site): """ Return path to the Site section on CS """ result = getSiteName(site) if not result['OK']: return result siteName = result['Value'] return S_OK(cfgPath(gBaseResourcesSection, 'Sites', siteName))
def __updateCS( self, bdiiChangeSet ): queueVODict = {} changeSet = set() for entry in bdiiChangeSet: section, option , _value, new_value = entry if option == "VO": queueVODict.setdefault( section, set() ) queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) ) else: changeSet.add( entry ) for section, VOs in queueVODict.items(): changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) ) if changeSet: changeList = list( changeSet ) changeList.sort() body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] ) if body and self.addressTo and self.addressFrom: notification = NotificationClient() result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False ) if body: self.log.info( 'The following configuration changes were detected:' ) self.log.info( body ) for section, option, value, new_value in changeSet: if value == 'Unknown' or not value: self.csAPI.setOption( cfgPath( section, option ), new_value ) else: self.csAPI.modifyValue( cfgPath( section, option ), new_value ) if self.dryRun: self.log.info( "Dry Run: CS won't be updated" ) self.csAPI.showDiff() else: result = self.csAPI.commit() if not result['OK']: self.log.error( "Error while committing to CS", result['Message'] ) else: self.log.info( "Successfully committed %d changes to CS" % len( changeList ) ) return result else: self.log.info( "No changes found" ) return S_OK()
def getSiteTier(site): """ Return Tier level of the given Site """ result = getSitePath(site) if not result["OK"]: return result sitePath = result["Value"] return S_OK(gConfig.getValue(cfgPath(sitePath, "MoUTierLevel"), 2))
def getFTSServersForSites( self, siteList = None ): """ get FTSServers for sites :param list siteList: list of sites """ siteList = siteList if siteList else None if not siteList: siteList = getSites() if not siteList["OK"]: return siteList siteList = siteList["Value"] ftsServers = dict() defaultServ = gConfig.getValue( cfgPath( gBaseResourcesSection, 'FTSEndpoints/Default', 'FTSEndpoint' ), '' ) for site in siteList: serv = gConfig.getValue( cfgPath( gBaseResourcesSection, "FTSEndpoints", site ), defaultServ ) if serv: ftsServers[site] = serv return S_OK( ftsServers )
def getSiteTier(site): """ Return Tier level of the given Site """ result = getSitePath(site) if not result['OK']: return result sitePath = result['Value'] return S_OK(gConfig.getValue(cfgPath(sitePath, 'MoUTierLevel'), 2))
def getFTS3Servers(hostOnly=False): """ get list of FTS3 servers that are in CS :param bool hostOnly: flag for stripping down the protocol and ports """ csPath = cfgPath(gBaseResourcesSection, "FTSEndpoints/FTS3") # We do it in two times to keep the order ftsServerNames = gConfig.getOptions(csPath).get('Value', []) ftsServers = [] for name in ftsServerNames: serverPath = gConfig.getValue(cfgPath(csPath, name)) if hostOnly: serverPath = urlparse.urlparse(serverPath).hostname ftsServers.append(serverPath) return S_OK(ftsServers)
def getSitePath(site): """ Return path to the Site section on CS """ result = getSiteGrid(site) if not result["OK"]: return result grid = result["Value"] return S_OK(cfgPath(gBaseResourcesSection, "Sites", grid, site))
def getSiteTier(site): """ Return Tier level of the given Site """ result = getSitePath(site) if not result["OK"]: return result sitePath = result["Value"] return S_OK(gConfig.getValue(cfgPath(sitePath, "MoUTierLevel"), 2))
def __getProtocolsSections( self, storageName ): storageConfigPath = cfgPath( self.rootConfigPath, storageName ) res = gConfig.getSections( storageConfigPath ) if not res['OK']: errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections" gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) ) return S_ERROR( errStr ) protocolSections = res['Value'] return S_OK( protocolSections )
def getSitePath(site): """ Return path to the Site section on CS """ result = getSiteGrid(site) if not result["OK"]: return result grid = result["Value"] return S_OK(cfgPath(gBaseResourcesSection, "Sites", grid, site))
def getSiteTier( site ): """ Return Tier level of the given Site """ result = getSitePath( site ) if not result['OK']: return result sitePath = result['Value'] return S_OK( gConfig.getValue( cfgPath( sitePath, 'MoUTierLevel' ), 2 ) )
def getFTSServersForSites( self, siteList = None ): """ get FTSServers for sites :param list siteList: list of sites """ siteList = siteList if siteList else None if not siteList: siteList = getSites() if not siteList["OK"]: return siteList siteList = siteList["Value"] ftsServers = dict() defaultServ = gConfig.getValue( cfgPath( gBaseResourcesSection, 'FTSEndpoints/Default', 'FTSEndpoint' ), '' ) for site in siteList: serv = gConfig.getValue( cfgPath( gBaseResourcesSection, "FTSEndpoints", site ), defaultServ ) if serv: ftsServers[site] = serv return S_OK( ftsServers )
def __getProtocolsSections(self, storageName): storageConfigPath = cfgPath(self.rootConfigPath, storageName) res = gConfig.getSections(storageConfigPath) if not res['OK']: errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections" gLogger.error(errStr, "%s: %s" % (storageName, res['Message'])) return S_ERROR(errStr) protocolSections = res['Value'] return S_OK(protocolSections)
def getDIRACSesForSRM( srmService ): result = gConfig.getSections( "/Resources/StorageElements" ) if not result['OK']: return result diracSEs = result['Value'] resultDIRACSEs = [] for se in diracSEs: seSection = "/Resources/StorageElements/%s" % se result = gConfig.getSections( seSection ) if not result['OK']: continue accesses = result['Value'] for access in accesses: protocol = gConfig.getValue( cfgPath( seSection, access, 'Protocol' ), 'Unknown' ) if protocol == 'srm': seHost = gConfig.getValue( cfgPath( seSection, access, 'Host' ), 'Unknown' ) if seHost == srmService: resultDIRACSEs.append( se ) return S_OK( resultDIRACSEs )
def __checkElementCommunity( self, elementPath, community ): """ Check recursively the given element for eligibility with respect to the given community """ communities = gConfig.getValue( cfgPath( elementPath, 'Communities'), [] ) if community in communities: return True if communities: return False parentPath = os.path.dirname( os.path.dirname( elementPath ) ) if parentPath == gBaseResourcesSection: return True else: return self.__checkElementCommunity( parentPath, community )
def __init__(self, database=False): super(PolicyBasedSecurityManager, self).__init__(database) from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection from DIRAC import gConfig from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath serviceSection = getServiceSection("DataManagement/FileCatalog") pluginPath = gConfig.getValue(cfgPath(serviceSection, "SecurityPolicy")) if not pluginPath: raise Exception("SecurityPolicy not defined in service options") pluginCls = self.__loadPlugin(pluginPath) self.policyObj = pluginCls(database=database) # For the old clients to work with the new policy (since getPathPermissions is meant to disappear...) # we fetch the old SecurityManager, and we call it if needed in the plugin. oldSecurityManagerName = gConfig.getValue(cfgPath(serviceSection, "OldSecurityManager"), "") self.policyObj.oldSecurityManager = None if oldSecurityManagerName: self.policyObj.oldSecurityManager = eval("%s(self.db)" % oldSecurityManagerName)