def addCEtoSite(self, siteName, ceName, optionsDict=None):
""" Adds a new CE to a site definition in the CS.
A CE normally has queues, so addQueueToCE should be called after this one.
:param str siteName: FQN of the site (e.g. LCG.CERN.ch)
:param str ceName: FQN of the CE (e.g. ce503.cern.ch)
:param dict optionsDict: optional dictionary of options
:returns: S_OK/S_ERROR structure
"""
res = getSites()
if not res['OK']:
return res
if siteName not in res['Value']:
res = self.addSite(siteName)
if not res['OK']:
return res
# CSAPI.createSection() always returns S_OK even if the section already exists
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs'))
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName))
# add options if requested
if optionsDict is not None:
for option, optionValue in optionsDict.items(
): # can be an iterator
self.__csMod.setOptionValue(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName,
option), optionValue)
self.csModified = True
return S_OK(True)
def addSite(self, siteName, optionsDict=None):
""" Adds a new site to the CS.
A site is a container for services, so after calling this function,
at least addCEtoSite() should be called.
:param str siteName: FQN of the site (e.g. LCG.CERN.ch)
:param dict optionsDict: optional dictionary of options
:returns: S_OK/S_ERROR structure
"""
if not self.__initialized['OK']:
return self.__initialized
self.__csMod.createSection(cfgPath(self.__baseResources, 'Sites'))
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0]))
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName))
# add options if requested
if optionsDict is not None:
for option, optionValue in optionsDict.items(
): # can be an iterator
self.__csMod.setOptionValue(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], option), optionValue)
self.csModified = True
return S_OK(True)
def configure(self, csSection, submitPool):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection(csSection)
self.reloadConfiguration(csSection, submitPool)
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue(cfgPath("Pilot", "Version"), [self.installVersion])[0]
self.installProject = opsHelper.getValue(cfgPath("Pilot", "Project"), self.installProject)
self.installation = opsHelper.getValue(cfgPath("Pilot", "Installation"), self.installation)
self.pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", self.pilotExtensionsList)
self.log.info("===============================================")
self.log.info("Configuration:")
self.log.info("")
self.log.info(" Target Grids: ", ", ".join(self.targetGrids))
self.log.info(" Install script: ", self.install)
self.log.info(" Pilot script: ", self.pilot)
self.log.info(" Pilot modules", self.extraModules)
self.log.info(" Install Ver: ", self.installVersion)
if self.installProject:
self.log.info(" Project: ", self.installProject)
if self.installation:
self.log.info(" Installation: ", self.installation)
if self.extraPilotOptions:
self.log.info(" Extra Options: ", " ".join(self.extraPilotOptions))
self.log.info(" ListMatch: ", self.enableListMatch)
self.log.info(" Private %: ", self.privatePilotFraction * 100)
if self.enableListMatch:
self.log.info(" ListMatch Delay:", self.listMatchDelay)
self.listMatchCache.purgeExpired()
def addQueueToCE(self, ceName, queueName, optionsDict=None):
""" Adds a new queue to a CE definition in the CS.
:param str ceName: FQN of the CE (e.g. ce503.cern.ch)
:param str queueName: name of the queue (e.g. ce503.cern.ch-condor)
:param dict optionsDict: optional dictionary of options
:returns: S_OK/S_ERROR structure
"""
res = getCESiteMapping(ceName)
if not res['OK']:
return res
if ceName not in res['Value']:
return S_ERROR("CE does not exist")
siteName = res['Value'][ceName]
# CSAPI.createSection() always returns S_OK even if the section already exists
self.__csMod.createSection(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName, 'Queues',
queueName))
# add options if requested
if optionsDict is not None:
for option, optionValue in optionsDict.items(
): # can be an iterator
self.__csMod.setOptionValue(
cfgPath(self.__baseResources, 'Sites',
siteName.split('.')[0], siteName, 'CEs', ceName,
'Queues', queueName, option), optionValue)
self.csModified = True
return S_OK(True)
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
setup = gConfig.getValue( '/DIRAC/Setup', '' )
section = cfgPath( 'Operations', self.virtualOrganization, setup, 'Versions' )
self.installVersion = gConfig.getValue( cfgPath( section, 'PilotVersion' ),
self.installVersion )
self.installInstallation = gConfig.getValue( cfgPath( section, 'PilotInstallation' ),
self.installInstallation )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installInstallation:
self.log.info( ' Installation: ', self.installInstallation )
if self.extraPilotOptions:
self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def __filterByPlatform(self, jobPlatform, userSites):
""" Filters out sites that have no CE with a matching platform.
"""
basePath = "/Resources/Sites"
filteredSites = set()
for site in userSites:
if "." not in site:
# Invalid site name: Doesn't contain a dot!
self.jobLog.info("Skipped invalid site name: %s" % site)
continue
grid = site.split('.')[0]
sitePath = cfgPath(basePath, grid, site, "CEs")
result = gConfig.getSections(sitePath)
if not result['OK']:
self.jobLog.info("Failed to get CEs at site %s." % site)
continue
siteCEs = result['Value']
for CEName in siteCEs:
CEPlatform = gConfig.getValue(cfgPath(sitePath, CEName, "OS"))
if jobPlatform == CEPlatform:
# Site has a CE with a matchin platform
filteredSites.add(site)
return S_OK(list(filteredSites))
def updateCS(changeSet):
global vo, dry, ceBdiiDict
changeList = sorted(changeSet)
if dry:
gLogger.notice('The following needed changes are detected:\n')
else:
gLogger.notice('We are about to make the following changes to CS:\n')
for entry in changeList:
gLogger.notice("%s/%s %s -> %s" % entry)
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object',
result['Message'])
DIRACExit(-1)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption(cfgPath(section, option), new_value)
else:
csAPI.modifyValue(cfgPath(section, option), new_value)
yn = six.moves.input(
'Do you want to commit changes to CS ? [default yes] [yes|no]: ')
if yn == '' or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" %
len(changeSet))
def __init__(self, database=False):
super(PolicyBasedSecurityManager, self).__init__(database)
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
serviceSection = getServiceSection('DataManagement/FileCatalog')
pluginPath = gConfig.getValue(cfgPath(serviceSection,
'SecurityPolicy'))
if not pluginPath:
raise Exception("SecurityPolicy not defined in service options")
pluginCls = self.__loadPlugin(pluginPath)
self.policyObj = pluginCls(database=database)
# For the old clients to work with the new policy (since getPathPermissions is meant to disappear...)
# we fetch the old SecurityManager, and we call it if needed in the plugin.
oldSecurityManagerName = gConfig.getValue(
cfgPath(serviceSection, 'OldSecurityManager'), '')
self.policyObj.oldSecurityManager = None
if oldSecurityManagerName:
self.policyObj.oldSecurityManager = eval("%s(self.db)" %
oldSecurityManagerName)
def getSEsFromCS(protocol='srm'):
""" Get all the SEs defined in the CS
"""
knownSEs = {}
result = gConfig.getSections('/Resources/StorageElements')
if not result['OK']:
return result
ses = result['Value']
for se in ses:
seSection = '/Resources/StorageElements/%s' % se
result = gConfig.getSections(seSection)
if not result['OK']:
continue
accesses = result['Value']
for access in accesses:
seProtocol = gConfig.getValue(
cfgPath(seSection, access, 'Protocol'), '')
if seProtocol.lower() == protocol.lower() or protocol == 'any':
host = gConfig.getValue(cfgPath(seSection, access, 'Host'), '')
knownSEs.setdefault(host, [])
knownSEs[host].append(se)
else:
continue
return S_OK(knownSEs)
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0]
self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject )
self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installProject:
self.log.info( ' Project: ', self.installProject )
if self.installation:
self.log.info( ' Installation: ', self.installation )
if self.extraPilotOptions:
self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
# Get the defaults for the Setup where the Director is running
opsHelper = Operations()
self.installVersion = opsHelper.getValue( cfgPath( 'Pilot', 'Version' ), [ self.installVersion ] )[0]
self.installProject = opsHelper.getValue( cfgPath( 'Pilot', 'Project' ), self.installProject )
self.installation = opsHelper.getValue( cfgPath( 'Pilot', 'Installation' ), self.installation )
self.pilotExtensionsList = opsHelper.getValue( "Pilot/Extensions", self.pilotExtensionsList )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Target Grids: ', ', '.join( self.targetGrids ) )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Pilot modules', self.extraModules )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installProject:
self.log.info( ' Project: ', self.installProject )
if self.installation:
self.log.info( ' Installation: ', self.installation )
if self.extraPilotOptions:
self.log.info( ' Extra Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def _getConfigStorageOptions( self, storageName, derivedStorageName = None ):
""" Get the options associated to the StorageElement as defined in the CS
"""
optionsDict = {}
# We first get the options of the baseSE, and then overwrite with the derivedSE
for seName in ( storageName, derivedStorageName ) if derivedStorageName else ( storageName, ):
storageConfigPath = cfgPath( self.rootConfigPath, seName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) )
return S_ERROR( errStr )
for option in set( res['Value'] ) - set( ( 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess' ) ):
optionConfigPath = cfgPath( storageConfigPath, option )
default = [] if option in [ 'VO' ] else ''
optionsDict[option] = gConfig.getValue( optionConfigPath, default )
# The status is that of the derived SE only
seName = derivedStorageName if derivedStorageName else storageName
res = self.resourceStatus.getStorageElementStatus( seName )
if not res[ 'OK' ]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error( errStr, "%s: %s" % ( seName, res['Message'] ) )
return S_ERROR( errStr )
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update( res[ 'Value' ][ seName ] )
return S_OK( optionsDict )
def add(self, section, option, new_value):
"""
Add a value into the configuration system.
This method will overwrite any existing option's value.
Args:
section (str): The section
option (str): The option to be created/modified
new_value: The value to be assigned
Example:
>>> cs = ConfigurationSystem()
>>> cs.add('/Registry', 'DefaultGroup', 'dteam_user')
"""
if isinstance(new_value, (tuple, list, set, GeneratorType)):
new_value = ', '.join(sorted(map(str, new_value)))
else:
new_value = str(new_value)
old_value = gConfig.getValue(cfgPath(section, option), None)
if old_value == new_value:
return
if old_value is None:
gLogger.notice("Setting %s/%s: -> %s"
% (section, option, new_value))
self.setOption(cfgPath(section, option), new_value)
else:
gLogger.notice("Modifying %s/%s: %s -> %s"
% (section, option, old_value, new_value))
self.modifyValue(cfgPath(section, option), new_value)
self._num_changes += 1
def updateCS( changeSet ):
global vo, dry, ceBdiiDict
changeList = list( changeSet )
changeList.sort()
if dry:
gLogger.notice( 'The following needed changes are detected:\n' )
else:
gLogger.notice( 'We are about to make the following changes to CS:\n' )
for entry in changeList:
gLogger.notice( "%s/%s %s -> %s" % entry )
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption( cfgPath( section, option ), new_value )
else:
csAPI.modifyValue( cfgPath( section, option ), new_value )
yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
if yn == '' or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def remove(self, section, option=None, value=None):
"""
Remove a section/option from the configuration system.
This method will remove the specified section if the option argument
is None (default). If the option argument is given but value is None
then that option (formed of section/option) is removed. If both option
and value are given then that value is removed from the comma seperated
values associated with that option.
Args:
section (str): The section
option (str): [optional] The option to remove
value (str): [optional] The value to remove
Example:
>>> ConfigurationSystem().remove('/Registry', 'DefaultGroup')
"""
if option is None:
gLogger.notice("Removing section %s" % section)
self.delSection(section)
self._num_changes += 1
elif value is None:
gLogger.notice("Removing option %s/%s" % (section, option))
self.delOption(cfgPath(section, option))
self._num_changes += 1
else:
if isinstance(value, str):
value = [value]
gLogger.notice("Removing value(s) %s from option %s/%s"
% (list(value), section, option))
old_values = (v.strip() for v in gConfig.getValue(cfgPath(section, option), '').split(','))
new_values = [v for v in old_values if v and v not in value]
self.add(section, option, new_values)
def __filterByPlatform(self, jobPlatform, userSites):
""" Filters out sites that have no CE with a matching platform.
"""
basePath = "/Resources/Sites"
filteredSites = set()
for site in userSites:
if "." not in site:
# Invalid site name: Doesn't contain a dot!
self.jobLog.info("Skipped invalid site name: %s" % site)
continue
grid = site.split('.')[0]
sitePath = cfgPath(basePath, grid, site, "CEs")
result = gConfig.getSections(sitePath)
if not result['OK']:
self.jobLog.info("Failed to get CEs at site %s." % site)
continue
siteCEs = result['Value']
for CEName in siteCEs:
CEPlatform = gConfig.getValue(cfgPath(sitePath, CEName, "OS"))
if jobPlatform == CEPlatform:
# Site has a CE with a matchin platform
filteredSites.add(site)
return S_OK(list(filteredSites))
def configure( self, csSection, submitPool ):
"""
Here goes common configuration for all PilotDirectors
"""
self.configureFromSection( csSection )
self.reloadConfiguration( csSection, submitPool )
setup = gConfig.getValue( '/DIRAC/Setup', '' )
section = cfgPath( 'Operations', self.virtualOrganization, setup, 'Versions' )
self.installVersion = gConfig.getValue( cfgPath( section, 'PilotVersion' ),
self.installVersion )
self.installInstallation = gConfig.getValue( cfgPath( section, 'PilotInstallation' ),
self.installInstallation )
self.log.info( '===============================================' )
self.log.info( 'Configuration:' )
self.log.info( '' )
self.log.info( ' Install script: ', self.install )
self.log.info( ' Pilot script: ', self.pilot )
self.log.info( ' Install Ver: ', self.installVersion )
if self.installInstallation:
self.log.info( ' Installation: ', self.installInstallation )
if self.extraPilotOptions:
self.log.info( ' Exta Options: ', ' '.join( self.extraPilotOptions ) )
self.log.info( ' ListMatch: ', self.enableListMatch )
self.log.info( ' Private %: ', self.privatePilotFraction * 100 )
if self.enableListMatch:
self.log.info( ' ListMatch Delay:', self.listMatchDelay )
self.listMatchCache.purgeExpired()
def _getConfigStorageOptions( self, storageName ):
""" Get the options associated to the StorageElement as defined in the CS
"""
storageConfigPath = cfgPath( self.rootConfigPath, storageName )
res = gConfig.getOptions( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
options = res['Value']
optionsDict = {}
for option in options:
if option in [ 'ReadAccess', 'WriteAccess', 'CheckAccess', 'RemoveAccess']:
continue
optionConfigPath = cfgPath( storageConfigPath, option )
if option in [ 'VO' ]:
optionsDict[option] = gConfig.getValue( optionConfigPath, [] )
else:
optionsDict[option] = gConfig.getValue( optionConfigPath, '' )
res = self.resourceStatus.getStorageElementStatus( storageName )
if not res[ 'OK' ]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update( res[ 'Value' ][ storageName ] )
return S_OK( optionsDict )
def __updateCS(self, bdiiChangeSet):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option, _value, new_value = entry
if option == "VO":
queueVODict.setdefault(section, set())
queueVODict[section] = queueVODict[section].union(
set(new_value.split(',')))
else:
changeSet.add(entry)
for section, VOs in queueVODict.items():
changeSet.add((section, 'VO', '', ','.join(VOs)))
if changeSet:
changeList = list(changeSet)
changeList.sort()
body = '\n'.join(
["%s/%s %s -> %s" % entry for entry in changeList])
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
if body:
self.log.info(
'The following configuration changes were detected:')
self.log.info(body)
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption(cfgPath(section, option), new_value)
else:
self.csAPI.modifyValue(cfgPath(section, option), new_value)
if self.dryRun:
self.log.info("Dry Run: CS won't be updated")
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error("Error while committing to CS",
result['Message'])
else:
self.log.info("Successfully committed %d changes to CS" %
len(changeList))
return result
else:
self.log.info("No changes found")
return S_OK()
def _getConfigStorageOptions(self,
storageName,
derivedStorageName=None,
seConfigPath=SE_CONFIG_PATH):
"""
Get the options associated to the StorageElement as defined in the CS
:param storageName: is the storage section to check in the CS
:param seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:param derivedStorageName: is the storage section of a derived storage if it inherits from a base
:return: options associated to the StorageElement as defined in the CS
"""
optionsDict = {}
# We first get the options of the baseSE, and then overwrite with the derivedSE
for seName in (storageName,
derivedStorageName) if derivedStorageName else (
storageName, ):
storageConfigPath = cfgPath(seConfigPath, seName)
res = gConfig.getOptions(storageConfigPath)
if not res["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage options."
gLogger.error(errStr, "%s: %s" % (seName, res["Message"]))
return S_ERROR(errStr)
for option in set(res["Value"]) - set(
("ReadAccess", "WriteAccess", "CheckAccess", "RemoveAccess")):
optionConfigPath = cfgPath(storageConfigPath, option)
default = [] if option in [
"VO", "AccessProtocols", "WriteProtocols"
] else ""
optionsDict[option] = gConfig.getValue(optionConfigPath,
default)
# We update the seConfigPath in order to find option in derivedSE now
seConfigPath = SE_CONFIG_PATH
# The status is that of the derived SE only
seName = derivedStorageName if derivedStorageName else storageName
res = self.resourceStatus.getElementStatus(seName, "StorageElement")
if not res["OK"]:
errStr = "StorageFactory._getStorageOptions: Failed to get storage status"
gLogger.error(errStr, "%s: %s" % (seName, res["Message"]))
return S_ERROR(errStr)
# For safety, we did not add the ${statusType}Access keys
# this requires modifications in the StorageElement class
# We add the dictionary with the statusTypes and values
# { 'statusType1' : 'status1', 'statusType2' : 'status2' ... }
optionsDict.update(res["Value"][seName])
return S_OK(optionsDict)
def getSiteSEMapping(gridName=''):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections('Resources/Sites/')
if not gridTypes['OK']:
gLogger.warn('Problem retrieving sections in /Resources/Sites')
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR('Could not get sections for /Resources/Sites/%s' %
gridName)
gridTypes = [gridName]
gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes)))
for grid in gridTypes:
sites = gConfig.getSections('/Resources/Sites/%s' % grid)
if not sites['OK']:
gLogger.warn('Problem retrieving /Resources/Sites/%s section' %
grid)
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue(
'/Resources/Sites/%s/%s/SE' % (grid, candidate), [])
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug('No SEs defined for site %s' % candidate)
# Add Sites from the SiteLocalSEMapping in the CS
cfgLocalSEPath = cfgPath('SiteLocalSEMapping')
opsHelper = Operations()
result = opsHelper.getOptionsDict(cfgLocalSEPath)
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = opsHelper.getValue(cfgPath(cfgLocalSEPath, site), [])
if not ses:
continue
if gridName:
if gridName != site.split('.')[0]:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append(se)
return S_OK(siteSEMapping)
def _getConfigStorageProtocolDetails(self, storageName, protocolSection,
seConfigPath=SE_CONFIG_PATH, checkAccess=True):
"""
Parse the contents of the protocol block
:param storageName: is the storage section to check in the CS
:param protocolSection: name of the protocol section to find information
:param seConfigPath: the path of the storage section.
It can be /Resources/StorageElements or StorageElementBases
:param checkAccess: if not set, don't complain if "Access" is not in the section
:return: dictionary of the protocol options
"""
# First obtain the options that are available
protocolConfigPath = cfgPath(seConfigPath, storageName, protocolSection)
res = gConfig.getOptions(protocolConfigPath)
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocolSection))
return S_ERROR(errStr)
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access': '', 'Host': '', 'Path': '', 'Port': '', 'Protocol': '', 'SpaceToken': '', 'WSUrl': ''}
for option in options:
configPath = cfgPath(protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, '')
protocolDict[option] = optionValue
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict(cfgPath(protocolConfigPath, 'VOPath'))
voPath = ''
if result['OK']:
voPath = result['Value'].get(self.vo, '')
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set and the plugin is not already in remote or local.
plugin = protocolDict.get('PluginName', protocolSection)
if protocolDict['Access'].lower() == 'remote':
self.remotePlugins.append(plugin)
elif protocolDict['Access'].lower() == 'local':
self.localPlugins.append(plugin)
# If it is a derived SE, this is normal, no warning
elif checkAccess and protocolSection not in self.protocols:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option \
for %s:%s is neither 'local' or 'remote'." % (storageName, protocolSection)
gLogger.warn(errStr)
return S_OK(protocolDict)
def getFTS3Servers():
""" get FTSServers for sites
"""
csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" )
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] )
ftsServers = []
for name in ftsServerNames:
ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) )
return S_OK( ftsServers )
def getFTS3Servers():
""" get FTSServers for sites
"""
csPath = cfgPath( gBaseResourcesSection, "FTSEndpoints/FTS3" )
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions( csPath ).get( 'Value', [] )
ftsServers = []
for name in ftsServerNames:
ftsServers.append( gConfig.getValue( cfgPath( csPath, name ) ) )
return S_OK( ftsServers )
def _getConfigStorageProtocolDetails( self, storageName, protocolSection ):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection )
res = gConfig.getOptions( protocolConfigPath )
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'PluginName':'', 'SpaceToken':'', 'WSUrl':''}
for option in options:
configPath = cfgPath( protocolConfigPath, option )
optionValue = gConfig.getValue( configPath, '' )
protocolDict[option] = optionValue
# This is a temporary for backward compatibility
if "ProtocolName" in protocolDict and not protocolDict['PluginName']:
protocolDict['PluginName'] = protocolDict['ProtocolName']
protocolDict.pop( 'ProtocolName', None )
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) )
voPath = ''
if result['OK']:
voPath = result['Value'].get( self.vo, '' )
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'].lower() == 'remote':
self.remotePlugins.append( protocolDict['PluginName'] )
elif protocolDict['Access'].lower() == 'local':
self.localPlugins.append( protocolDict['PluginName'] )
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection )
gLogger.warn( errStr )
# The PluginName option must be defined
if not protocolDict['PluginName']:
errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
return S_OK( protocolDict )
def remove_old(self, removal_threshold=5):
"""Remove old hosts/sites."""
cfg_system = ConfigurationSystem()
result = cfg_system.getCurrentCFG()
if not result['OK']:
self.log.error('Could not get current config from the CS')
raise RuntimeError("Error removing old Resources/Registry.")
today = date.today()
removal_threshold = timedelta(days=removal_threshold)
old_ces = set()
base_path = '/Resources/Sites'
for site_type in ('VAC', 'CLOUD'):
site_type_path = cfgPath(base_path, site_type)
for site, site_info in result['Value'].getAsDict(base_path).iteritems():
site_path = cfgPath(site_type_path, site)
for ce, ce_info in site_info.get('CEs', {}).iteritems():
ce_path = cfgPath(site_path, 'CEs', ce)
if 'LastSeen' not in ce_info:
self.log.warn("No LastSeen info for CE: %s at site: %s" % (ce, site))
continue
last_seen = datetime.strptime(ce_info['LastSeen'], '%d/%m/%Y').date()
delta = today - last_seen
if delta > removal_threshold:
self.log.warn("Last seen %s:%s %s days ago...removing"
% (site, ce, delta.days))
cfg_system.remove(section=ce_path)
old_ces.add(ce)
if old_ces:
cfg_system.remove(section=site_path, option='CE', value=old_ces)
old_ces.clear()
host_base = '/Registry/Hosts'
for host, host_info in result['Value'].getAsDict(host_base).iteritems():
host_path = cfgPath(host_base, host)
if 'LastSeen' not in host_info:
self.log.warn("No LastSeen info for host: %s" % host)
continue
last_seen = datetime.strptime(host_info['LastSeen'], '%d/%m/%Y').date()
delta = today - last_seen
if delta > removal_threshold:
self.log.warn("Last seen host %s %s days ago...removing"
% (host, delta.days))
cfg_system.remove(section=host_path)
cfg_system.commit()
return S_OK()
def _getConfigStorageProtocolDetails( self, storageName, protocolSection, checkAccess = True ):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = cfgPath( self.rootConfigPath, storageName, protocolSection )
res = gConfig.getOptions( protocolConfigPath )
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {'Access':'', 'Host':'', 'Path':'', 'Port':'', 'Protocol':'', 'SpaceToken':'', 'WSUrl':''}
for option in options:
configPath = cfgPath( protocolConfigPath, option )
optionValue = gConfig.getValue( configPath, '' )
protocolDict[option] = optionValue
# This is a temporary for backward compatibility: move ProtocolName to PluginName
protocolDict.setdefault( 'PluginName', protocolDict.pop( 'ProtocolName', None ) )
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict( cfgPath( protocolConfigPath, 'VOPath' ) )
voPath = ''
if result['OK']:
voPath = result['Value'].get( self.vo, '' )
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if checkAccess:
if protocolDict['Access'].lower() == 'remote':
self.remotePlugins.append( protocolDict['PluginName'] )
elif protocolDict['Access'].lower() == 'local':
self.localPlugins.append( protocolDict['PluginName'] )
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % ( storageName, protocolSection )
gLogger.warn( errStr )
# The PluginName option must be defined
if not protocolDict['PluginName']:
errStr = "StorageFactory.__getProtocolDetails: 'PluginName' option is not defined."
gLogger.error( errStr, "%s: %s" % ( storageName, protocolSection ) )
return S_ERROR( errStr )
return S_OK( protocolDict )
def getSiteSEMapping( gridName = '' ):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections( 'Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug( 'No SEs defined for site %s' % candidate )
# Add Sites from the SiteLocalSEMapping in the CS
vo = getVO()
setup = getDIRACSetup()
cfgLocalSEPath = cfgPath( 'Operations', vo, setup, 'SiteLocalSEMapping' )
result = gConfig.getOptionsDict( cfgLocalSEPath )
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = gConfig.getValue( cfgPath( cfgLocalSEPath, site ), [] )
if not ses:
continue
if gridName and site not in sites:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append( se )
return S_OK( siteSEMapping )
def getSites():
""" Get the list of all the sites defined in the CS
"""
result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites"))
if not result["OK"]:
return result
grids = result["Value"]
sites = []
for grid in grids:
result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites", grid))
if not result["OK"]:
return result
sites += result["Value"]
return S_OK(sites)
def getSites():
""" Get the list of all the sites defined in the CS
"""
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) )
if not result['OK']:
return result
grids = result['Value']
sites = []
for grid in grids:
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites', grid ) )
if not result['OK']:
return result
sites += result['Value']
return S_OK( sites )
def getSites():
"""Get the list of all the sites defined in the CS"""
result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites"))
if not result["OK"]:
return result
grids = result["Value"]
sites = []
for grid in grids:
result = gConfig.getSections(
cfgPath(gBaseResourcesSection, "Sites", grid))
if not result["OK"]:
return result
sites += result["Value"]
return S_OK(sites)
def getSites():
""" Get the list of all the sites defined in the CS
"""
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) )
if not result['OK']:
return result
grids = result['Value']
sites = []
for grid in grids:
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites', grid ) )
if not result['OK']:
return result
sites += result['Value']
return S_OK( sites )
def _getConfigStorageName(self, storageName, referenceType):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
'storageName' is the storage section to check in the CS
"""
configPath = '%s/%s' % (self.rootConfigPath, storageName)
res = gConfig.getOptions(configPath)
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error(errStr, res['Message'])
return S_ERROR(errStr)
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error(errStr, configPath)
return S_ERROR(errStr)
if referenceType in res['Value']:
configPath = cfgPath(self.rootConfigPath, storageName,
referenceType)
referenceName = gConfig.getValue(configPath)
result = self._getConfigStorageName(referenceName, 'Alias')
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK(resolvedName)
def getSites( siteList = [], fullName = False ):
""" Get the list of site names possibly limited by a given list of sites names
in arbitrary form
"""
if not siteList:
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) )
if not result['OK']:
return result
sList = result['Value']
else:
if not type( siteList ) == ListType:
siteList = [ siteList ]
sList = []
for s in siteList:
result = getSiteName( s )
if not result['OK']:
continue
sList.append( result['Value'] )
resultList = []
if fullName:
for site in sList:
result = getSiteFullNames( site )
if not result['OK']:
continue
resultList += result['Value']
else:
for site in sList:
result = getSiteName( site )
if not result['OK']:
continue
resultList.append( result['Value'] )
return S_OK( resultList )
def getRunningInstances(self, instanceType='Agents', runitStatus='Run'):
"""Return a dict of running agents, executors or services.
Key is agent's name, value contains dict with PollingTime, PID, Port, Module, RunitStatus, LogFileLocation
:param str instanceType: 'Agents', 'Executors', 'Services'
:param str runitStatus: Return only those instances with given RunitStatus or 'All'
:returns: Dictionary of running instances
"""
res = self.sysAdminClient.getOverallStatus()
if not res["OK"]:
self.logError("Failure to get %s from system administrator client" % instanceType, res["Message"])
return res
val = res['Value'][instanceType]
runningAgents = defaultdict(dict)
for system, agents in val.iteritems():
for agentName, agentInfo in agents.iteritems():
if agentInfo['Setup'] and agentInfo['Installed']:
if runitStatus != 'All' and agentInfo['RunitStatus'] != runitStatus:
continue
confPath = cfgPath('/Systems/' + system + '/' + self.setup + '/%s/' % instanceType + agentName)
for option, default in (('PollingTime', HOUR), ('Port', None)):
optPath = os.path.join(confPath, option)
runningAgents[agentName][option] = gConfig.getValue(optPath, default)
runningAgents[agentName]["LogFileLocation"] = \
os.path.join(self.diracLocation, 'runit', system, agentName, 'log', 'current')
runningAgents[agentName]["PID"] = agentInfo["PID"]
runningAgents[agentName]['Module'] = agentInfo['Module']
runningAgents[agentName]['RunitStatus'] = agentInfo['RunitStatus']
runningAgents[agentName]['System'] = system
return S_OK(runningAgents)
def __getNodePath(self, nodeName, resourceType):
""" Get path to the resource CS section
"""
site, resource, node = nodeName.split(RESOURCE_NAME_SEPARATOR)
return cfgPath(gBaseResourcesSection, 'Sites', site, resourceType,
resource, RESOURCE_NODE_MAPPING[resourceType] + 's',
node)
def getSites(siteList=[], fullName=False):
""" Get the list of site names possibly limited by a given list of sites names
in arbitrary form
"""
if not siteList:
result = gConfig.getSections(cfgPath(gBaseResourcesSection, 'Sites'))
if not result['OK']:
return result
sList = result['Value']
else:
if not type(siteList) == ListType:
siteList = [siteList]
sList = []
for s in siteList:
result = getSiteName(s)
if not result['OK']:
continue
sList.append(result['Value'])
resultList = []
if fullName:
for site in sList:
result = getSiteFullNames(site)
if not result['OK']:
continue
resultList += result['Value']
else:
for site in sList:
result = getSiteName(site)
if not result['OK']:
continue
resultList.append(result['Value'])
return S_OK(resultList)
def getMultiHopMatrix(self):
"""
Returns the multi-hop matrix described in DataManagement/MultiHopMatrixOfShame.
.. code-block :: python
'Default': { 'Default': 'MultiHopSEUsedForAllTransfer',
'Dst3' : 'MultiHopFromAnySourceToDst3',
}
'Src1' : { 'Default' : 'DefaultMultiHopSEFromSrc1',
'Dst1': 'MultiHopSEFromSrc1ToDst1},
'Src2' : { 'Default' : 'DefaultMultiHopSEFromSrc2',
'Dst2': 'MultiHopSEFromSrc1ToDst2}
:returns: dict of dict for all the source se / dest SE defined. We user defaultdict
to allow for the use of non existing source/dest.
"""
matrixBasePath = "DataManagement/MultiHopMatrixOfShame"
multiHopMatrix = defaultdict(lambda: defaultdict(lambda: None))
allSrcSEs = self.__opsHelper.getSections(matrixBasePath).get(
"Value", [])
for src in allSrcSEs:
srcDst = self.__opsHelper.getOptionsDict(
cfgPath(matrixBasePath, src)).get("Value")
if srcDst:
multiHopMatrix[src].update(srcDst)
return multiHopMatrix
def getDIRACSiteName(gocSiteName):
"""
Get DIRAC site name, given the GOC DB site name, as it stored in the CS
:params str gocSiteName: GOC DB site name (e.g. 'CERN-PROD')
:returns: S_OK/S_ERROR structure
"""
res = getSites()
if not res['OK']:
return res
sitesList = res['Value']
tmpList = [(site,
gConfig.getValue(
cfgPath(gBaseResourcesSection, 'Sites',
site.split('.')[0], site, 'Name')))
for site in sitesList]
diracSites = [dirac for (dirac, goc) in tmpList if goc == gocSiteName]
if diracSites:
return S_OK(diracSites)
return S_ERROR("There's no site with GOCDB name = %s in DIRAC CS" %
gocSiteName)
def _getConfigStorageName( self, storageName, referenceType ):
"""
This gets the name of the storage the configuration service.
If the storage is a reference to another SE the resolution is performed.
'storageName' is the storage section to check in the CS
"""
configPath = '%s/%s' % ( self.rootConfigPath, storageName )
res = gConfig.getOptions( configPath )
if not res['OK']:
errStr = "StorageFactory._getConfigStorageName: Failed to get storage options"
gLogger.error( errStr, res['Message'] )
return S_ERROR( errStr )
if not res['Value']:
errStr = "StorageFactory._getConfigStorageName: Supplied storage doesn't exist."
gLogger.error( errStr, configPath )
return S_ERROR( errStr )
if referenceType in res['Value']:
configPath = cfgPath( self.rootConfigPath, storageName, referenceType )
referenceName = gConfig.getValue( configPath )
result = self._getConfigStorageName( referenceName, 'Alias' )
if not result['OK']:
return result
resolvedName = result['Value']
else:
resolvedName = storageName
return S_OK( resolvedName )
def append_unique(self, section, option, new_value):
"""
Append a value onto the end of an existing CS option.
This method is like append except that it ensures that the final list
of values for the given option only contains unique entries.
Args:
section (str): The section
option (str): The option to be modified
new_value: The value to be appended
"""
if isinstance(new_value, (tuple, list, set, GeneratorType)):
self._append_unique_dict[cfgPath(section, option)].update(new_value)
else:
self._append_unique_dict[cfgPath(section, option)].add(new_value)
def append(self, section, option, new_value):
"""
Append a value onto the end of an existing CS option.
This method is like add with the exception that the new value
is appended on to the end of the list of values associated
with that option.
Args:
section (str): The section
option (str): The option to be modified
new_value: The value to be appended
"""
if isinstance(new_value, (tuple, list, set, GeneratorType)):
self._append_dict[cfgPath(section, option)].extend(new_value)
else:
self._append_dict[cfgPath(section, option)].append(new_value)
def getSitePath( site ):
""" Return path to the Site section on CS
"""
result = getSiteName( site )
if not result['OK']:
return result
siteName = result['Value']
return S_OK( cfgPath( gBaseResourcesSection, 'Sites', siteName ) )
def getSitePath(site):
""" Return path to the Site section on CS
"""
result = getSiteName(site)
if not result['OK']:
return result
siteName = result['Value']
return S_OK(cfgPath(gBaseResourcesSection, 'Sites', siteName))
def __updateCS( self, bdiiChangeSet ):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option , _value, new_value = entry
if option == "VO":
queueVODict.setdefault( section, set() )
queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) )
else:
changeSet.add( entry )
for section, VOs in queueVODict.items():
changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) )
if changeSet:
changeList = list( changeSet )
changeList.sort()
body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if body:
self.log.info( 'The following configuration changes were detected:' )
self.log.info( body )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption( cfgPath( section, option ), new_value )
else:
self.csAPI.modifyValue( cfgPath( section, option ), new_value )
if self.dryRun:
self.log.info( "Dry Run: CS won't be updated" )
self.csAPI.showDiff()
else:
result = self.csAPI.commit()
if not result['OK']:
self.log.error( "Error while committing to CS", result['Message'] )
else:
self.log.info( "Successfully committed %d changes to CS" % len( changeList ) )
return result
else:
self.log.info( "No changes found" )
return S_OK()
def getSiteTier(site):
"""
Return Tier level of the given Site
"""
result = getSitePath(site)
if not result["OK"]:
return result
sitePath = result["Value"]
return S_OK(gConfig.getValue(cfgPath(sitePath, "MoUTierLevel"), 2))
def getFTSServersForSites( self, siteList = None ):
""" get FTSServers for sites
:param list siteList: list of sites
"""
siteList = siteList if siteList else None
if not siteList:
siteList = getSites()
if not siteList["OK"]:
return siteList
siteList = siteList["Value"]
ftsServers = dict()
defaultServ = gConfig.getValue( cfgPath( gBaseResourcesSection, 'FTSEndpoints/Default', 'FTSEndpoint' ), '' )
for site in siteList:
serv = gConfig.getValue( cfgPath( gBaseResourcesSection, "FTSEndpoints", site ), defaultServ )
if serv:
ftsServers[site] = serv
return S_OK( ftsServers )
def getSiteTier(site):
"""
Return Tier level of the given Site
"""
result = getSitePath(site)
if not result['OK']:
return result
sitePath = result['Value']
return S_OK(gConfig.getValue(cfgPath(sitePath, 'MoUTierLevel'), 2))
def getFTS3Servers(hostOnly=False):
""" get list of FTS3 servers that are in CS
:param bool hostOnly: flag for stripping down the protocol and ports
"""
csPath = cfgPath(gBaseResourcesSection, "FTSEndpoints/FTS3")
# We do it in two times to keep the order
ftsServerNames = gConfig.getOptions(csPath).get('Value', [])
ftsServers = []
for name in ftsServerNames:
serverPath = gConfig.getValue(cfgPath(csPath, name))
if hostOnly:
serverPath = urlparse.urlparse(serverPath).hostname
ftsServers.append(serverPath)
return S_OK(ftsServers)
def getSitePath(site):
"""
Return path to the Site section on CS
"""
result = getSiteGrid(site)
if not result["OK"]:
return result
grid = result["Value"]
return S_OK(cfgPath(gBaseResourcesSection, "Sites", grid, site))
def getSiteTier(site):
"""
Return Tier level of the given Site
"""
result = getSitePath(site)
if not result["OK"]:
return result
sitePath = result["Value"]
return S_OK(gConfig.getValue(cfgPath(sitePath, "MoUTierLevel"), 2))
def __getProtocolsSections( self, storageName ):
storageConfigPath = cfgPath( self.rootConfigPath, storageName )
res = gConfig.getSections( storageConfigPath )
if not res['OK']:
errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections"
gLogger.error( errStr, "%s: %s" % ( storageName, res['Message'] ) )
return S_ERROR( errStr )
protocolSections = res['Value']
return S_OK( protocolSections )
def getSitePath(site):
"""
Return path to the Site section on CS
"""
result = getSiteGrid(site)
if not result["OK"]:
return result
grid = result["Value"]
return S_OK(cfgPath(gBaseResourcesSection, "Sites", grid, site))
def getSiteTier( site ):
"""
Return Tier level of the given Site
"""
result = getSitePath( site )
if not result['OK']:
return result
sitePath = result['Value']
return S_OK( gConfig.getValue( cfgPath( sitePath, 'MoUTierLevel' ), 2 ) )
def getFTSServersForSites( self, siteList = None ):
""" get FTSServers for sites
:param list siteList: list of sites
"""
siteList = siteList if siteList else None
if not siteList:
siteList = getSites()
if not siteList["OK"]:
return siteList
siteList = siteList["Value"]
ftsServers = dict()
defaultServ = gConfig.getValue( cfgPath( gBaseResourcesSection, 'FTSEndpoints/Default', 'FTSEndpoint' ), '' )
for site in siteList:
serv = gConfig.getValue( cfgPath( gBaseResourcesSection, "FTSEndpoints", site ), defaultServ )
if serv:
ftsServers[site] = serv
return S_OK( ftsServers )
def __getProtocolsSections(self, storageName):
storageConfigPath = cfgPath(self.rootConfigPath, storageName)
res = gConfig.getSections(storageConfigPath)
if not res['OK']:
errStr = "StorageFactory._getConfigStorageProtocols: Failed to get storage sections"
gLogger.error(errStr, "%s: %s" % (storageName, res['Message']))
return S_ERROR(errStr)
protocolSections = res['Value']
return S_OK(protocolSections)
def getDIRACSesForSRM( srmService ):
result = gConfig.getSections( "/Resources/StorageElements" )
if not result['OK']:
return result
diracSEs = result['Value']
resultDIRACSEs = []
for se in diracSEs:
seSection = "/Resources/StorageElements/%s" % se
result = gConfig.getSections( seSection )
if not result['OK']:
continue
accesses = result['Value']
for access in accesses:
protocol = gConfig.getValue( cfgPath( seSection, access, 'Protocol' ), 'Unknown' )
if protocol == 'srm':
seHost = gConfig.getValue( cfgPath( seSection, access, 'Host' ), 'Unknown' )
if seHost == srmService:
resultDIRACSEs.append( se )
return S_OK( resultDIRACSEs )
def __checkElementCommunity( self, elementPath, community ):
""" Check recursively the given element for eligibility with respect to the given community
"""
communities = gConfig.getValue( cfgPath( elementPath, 'Communities'), [] )
if community in communities:
return True
if communities:
return False
parentPath = os.path.dirname( os.path.dirname( elementPath ) )
if parentPath == gBaseResourcesSection:
return True
else:
return self.__checkElementCommunity( parentPath, community )
def __init__(self, database=False):
super(PolicyBasedSecurityManager, self).__init__(database)
from DIRAC.ConfigurationSystem.Client.PathFinder import getServiceSection
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Path import cfgPath
serviceSection = getServiceSection("DataManagement/FileCatalog")
pluginPath = gConfig.getValue(cfgPath(serviceSection, "SecurityPolicy"))
if not pluginPath:
raise Exception("SecurityPolicy not defined in service options")
pluginCls = self.__loadPlugin(pluginPath)
self.policyObj = pluginCls(database=database)
# For the old clients to work with the new policy (since getPathPermissions is meant to disappear...)
# we fetch the old SecurityManager, and we call it if needed in the plugin.
oldSecurityManagerName = gConfig.getValue(cfgPath(serviceSection, "OldSecurityManager"), "")
self.policyObj.oldSecurityManager = None
if oldSecurityManagerName:
self.policyObj.oldSecurityManager = eval("%s(self.db)" % oldSecurityManagerName)
