def __getBackendOptionsFromCFG(self, cfgPath, backend):
"""
Get backend options from the configuration.
:params cfgPath: string of the configuration path
:params backend: string representing a backend identifier: stdout, file, f04
"""
# We have to put the import lines here to avoid a dependancy loop
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getBackendConfig
backendOptions = {}
# Search backends config in the resources section
retDictRessources = getBackendConfig(backend)
if retDictRessources['OK']:
backendOptions = retDictRessources['Value']
# Search backends config in the component to update some options
retDictConfig = gConfig.getOptionsDict(
"%s/%s/%s" % (cfgPath, 'LogBackendsConfig', backend))
if retDictConfig['OK']:
backendOptions.update(retDictConfig['Value'])
else:
# Search backends config in the component with the old option 'BackendsOptions'
retDictOptions = gConfig.getOptionsDict("%s/BackendsOptions" %
cfgPath)
if retDictOptions['OK']:
# We have to write the deprecated message with the print method because we are changing
# the backends, so we can not be sure of the display using a log.
print "WARNING: Use of a deprecated cfg section: BackendsOptions. Please replace it by BackendConfig."
backendOptions.update(retDictOptions['Value'])
return backendOptions
def getComponentsStatus( self, conditionDict = {} ):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
"""
result = self.__getComponents( conditionDict )
if not result[ 'OK' ]:
return result
statusSet = result[ 'Value' ]
requiredComponents = {}
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
for setup in result[ 'Value' ]:
if not self.__checkCondition( conditionDict, "Setup", setup ):
continue
#Iterate through systems
result = gConfig.getOptionsDict( "/DIRAC/Setups/%s" % setup )
if not result[ 'OK' ]:
return result
systems = result[ 'Value' ]
for system in systems:
instance = systems[ system ]
#Check defined agents and serviecs
for type in ( 'agent' , 'service' ):
#Get entries for the instance of a system
result = gConfig.getSections( "/Systems/%s/%s/%s" % ( system, instance, "%ss" % type.capitalize() ) )
if not result[ 'OK' ]:
self.log.warn( "Opps, sytem seems to be defined wrong\n", "System %s at %s: %s" % ( system, instance, result[ 'Message' ] ) )
continue
components = result[ 'Value' ]
for component in components:
componentName = "%s/%s" % ( system, component )
compDict = self.__getComponentDefinitionFromCS( system, setup, instance, type, component )
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#Walk the URLs
result = gConfig.getOptionsDict( "/Systems/%s/%s/URLs" % ( system, instance ) )
if not result[ 'OK' ]:
self.log.warn ( "There doesn't to be defined the URLs section for %s in %s instance" % ( system, instance ) )
else:
serviceURLs = result[ 'Value' ]
for service in serviceURLs:
for url in List.fromChar( serviceURLs[ service ] ):
loc = url[ url.find( "://" ) + 3: ]
iS = loc.find( "/" )
componentName = loc[ iS + 1: ]
loc = loc[ :iS ]
hostname, port = loc.split( ":" )
compDict = { 'ComponentName' : componentName,
'Type' : 'service',
'Setup' : setup,
'Host' : hostname,
'Port' : int( port )
}
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#WALK THE DICT
statusSet.setComponentsAsRequired( requiredComponents )
return S_OK( ( statusSet.getRequiredComponents(),
self.__mainFields[1:] + self.__versionFields + ( 'Status', 'Message' ) ) )
def getQueue(site, ce, queue):
""" Get parameters of the specified queue
"""
grid = site.split('.')[0]
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' %
(grid, site, ce))
if not result['OK']:
return result
resultDict = result['Value']
# Get queue defaults
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' %
(grid, site, ce, queue))
if not result['OK']:
return result
resultDict.update(result['Value'])
# Handle tag lists for the queue
for tagFieldName in ('Tag', 'RequiredTag'):
tags = []
ceTags = resultDict.get(tagFieldName)
if ceTags:
tags = fromChar(ceTags)
queueTags = resultDict.get(tagFieldName)
if queueTags:
queueTags = fromChar(queueTags)
tags = list(set(tags + queueTags))
if tags:
resultDict[tagFieldName] = tags
resultDict['Queue'] = queue
return S_OK(resultDict)
def __getBackendOptionsFromCFG(self, cfgPath, backend):
"""
Get backend options from the configuration.
:params cfgPath: string of the configuration path
:params backend: string representing a backend identifier: stdout, file, f04
"""
# We have to put the import lines here to avoid a dependancy loop
from DIRAC import gConfig
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getBackendConfig
backendOptions = {}
# Search backends config in the resources section
retDictRessources = getBackendConfig(backend)
if retDictRessources['OK']:
backendOptions = retDictRessources['Value']
# Search backends config in the component to update some options
retDictConfig = gConfig.getOptionsDict(
"%s/%s/%s" % (cfgPath, 'LogBackendsConfig', backend))
if retDictConfig['OK']:
backendOptions.update(retDictConfig['Value'])
else:
# Search backends config in the component with the old option
# 'BackendsOptions'
retDictOptions = gConfig.getOptionsDict("%s/BackendsOptions" % cfgPath)
if retDictOptions['OK']:
# We have to write the deprecated message with the print method because we are changing
# the backends, so we can not be sure of the display using a log.
print "WARNING: Use of a deprecated cfg section: BackendsOptions. Please replace it by BackendConfig."
backendOptions.update(retDictOptions['Value'])
return backendOptions
def getComponentsStatus( self, conditionDict = {} ):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
"""
result = self.__getComponents( conditionDict )
if not result[ 'OK' ]:
return result
statusSet = result[ 'Value' ]
requiredComponents = {}
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
for setup in result[ 'Value' ]:
if not self.__checkCondition( conditionDict, "Setup", setup ):
continue
#Iterate through systems
result = gConfig.getOptionsDict( "/DIRAC/Setups/%s" % setup )
if not result[ 'OK' ]:
return result
systems = result[ 'Value' ]
for system in systems:
instance = systems[ system ]
#Check defined agents and serviecs
for type in ( 'agent' , 'service' ):
#Get entries for the instance of a system
result = gConfig.getSections( "/Systems/%s/%s/%s" % ( system, instance, "%ss" % type.capitalize() ) )
if not result[ 'OK' ]:
self.log.warn( "Opps, sytem seems to be defined wrong\n", "System %s at %s: %s" % ( system, instance, result[ 'Message' ] ) )
continue
components = result[ 'Value' ]
for component in components:
componentName = "%s/%s" % ( system, component )
compDict = self.__getComponentDefinitionFromCS( system, setup, instance, type, component )
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#Walk the URLs
result = gConfig.getOptionsDict( "/Systems/%s/%s/URLs" % ( system, instance ) )
if not result[ 'OK' ]:
self.log.warn ( "There doesn't to be defined the URLs section for %s in %s instance" % ( system, instance ) )
else:
serviceURLs = result[ 'Value' ]
for service in serviceURLs:
for url in List.fromChar( serviceURLs[ service ] ):
loc = url[ url.find( "://" ) + 3: ]
iS = loc.find( "/" )
componentName = loc[ iS + 1: ]
loc = loc[ :iS ]
hostname, port = loc.split( ":" )
compDict = { 'ComponentName' : componentName,
'Type' : 'service',
'Setup' : setup,
'Host' : hostname,
'Port' : int( port )
}
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#WALK THE DICT
statusSet.setComponentsAsRequired( requiredComponents )
return S_OK( ( statusSet.getRequiredComponents(),
self.__mainFields[1:] + self.__versionFields + ( 'Status', 'Message' ) ) )
def getRunningEnPointDict( self, runningEndPointName ):
"""
Return from CS a Dictionary with getRunningEnPointDict definition
"""
runningEndPointCSPath = '/Resources/BigDataEndPoints'
definedRunningEndPoints = gConfig.getSections( runningEndPointCSPath )
if not definedRunningEndPoints[ 'OK' ]:
return S_ERROR( 'BigData section not defined' )
runningBDCSPath = '%s/%s' % ( runningEndPointCSPath, runningEndPointName )
runningEndPointBDDict = {}
if not runningBDCSPath:
return S_ERROR( 'Missing BigDataEndpoint "%s"' % runningEndPointName )
for option, value in gConfig.getOptionsDict( runningBDCSPath )['Value'].items():
runningEndPointBDDict[option] = value
runningHighLevelLanguajeDict = gConfig.getOptionsDict( '%s/HighLevelLanguage' % runningBDCSPath )
if not runningHighLevelLanguajeDict[ 'OK' ]:
return S_ERROR( 'Missing HighLevelLang in "%s"' % runningBDCSPath )
runningEndPointBDDict['HighLevelLanguage'] = runningHighLevelLanguajeDict['Value']
runningReqDict = gConfig.getOptionsDict( '%s/Requirements' % runningBDCSPath )
if not runningReqDict[ 'OK' ]:
return S_ERROR( 'Missing Requirements in "%s"' % runningBDCSPath )
runningEndPointBDDict['Requirements'] = runningReqDict['Value']
return S_OK( runningEndPointBDDict )
def getStorageElementOptions( seName ):
""" Get the CS StorageElementOptions
"""
storageConfigPath = '/Resources/StorageElements/%s' % seName
result = gConfig.getOptionsDict( storageConfigPath )
if not result['OK']:
return result
options = result['Value']
# If the SE is an baseSE or an alias, derefence it
if 'BaseSE' in options or 'Alias' in options:
storageConfigPath = '/Resources/StorageElements/%s' % options.get( 'BaseSE', options.get( 'Alias' ) )
result = gConfig.getOptionsDict( storageConfigPath )
if not result['OK']:
return result
result['Value'].update( options )
options = result['Value']
# Help distinguishing storage type
diskSE = True
tapeSE = False
if 'SEType' in options:
# Type should follow the convention TXDY
seType = options['SEType']
diskSE = re.search( 'D[1-9]', seType ) != None
tapeSE = re.search( 'T[1-9]', seType ) != None
options['DiskSE'] = diskSE
options['TapeSE'] = tapeSE
return S_OK( options )
def getQueue(site, ce, queue):
""" Get parameters of the specified queue
"""
grid = site.split('.')[0]
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce))
if not result['OK']:
return result
resultDict = result['Value']
# Get queue defaults
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % (grid, site, ce, queue))
if not result['OK']:
return result
resultDict.update(result['Value'])
# Handle tag lists for the queue
for tagFieldName in ('Tag', 'RequiredTag'):
tags = []
ceTags = resultDict.get(tagFieldName)
if ceTags:
tags = fromChar(ceTags)
queueTags = resultDict.get(tagFieldName)
if queueTags:
queueTags = fromChar(queueTags)
tags = list(set(tags + queueTags))
if tags:
resultDict[tagFieldName] = tags
resultDict['Queue'] = queue
return S_OK(resultDict)
def getVMImageConfig( site, ce, image = '' ):
""" Get parameters of the specified queue
"""
Tags = []
grid = site.split( '.' )[0]
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s' % ( grid, site, ce ) )
if not result['OK']:
return result
resultDict = result['Value']
ceTags = resultDict.get( 'Tag' )
if ceTags:
Tags = fromChar( ceTags )
if image:
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s' % ( grid, site, ce, image ) )
if not result['OK']:
return result
resultDict.update( result['Value'] )
queueTags = resultDict.get( 'Tag' )
if queueTags:
queueTags = fromChar( queueTags )
Tags = list( set( Tags + queueTags ) )
if Tags:
resultDict['Tag'] = Tags
resultDict['Image'] = image
resultDict['Site'] = site
return S_OK( resultDict )
def getStorageElementOptions( seName ):
""" Get the CS StorageElementOptions
"""
storageConfigPath = '/Resources/StorageElements/%s' % seName
result = gConfig.getOptionsDict( storageConfigPath )
if not result['OK']:
return result
options = result['Value']
# If the SE is an baseSE or an alias, derefence it
if 'BaseSE' in options or 'Alias' in options:
storageConfigPath = '/Resources/StorageElements/%s' % options.get( 'BaseSE', options.get( 'Alias' ) )
result = gConfig.getOptionsDict( storageConfigPath )
if not result['OK']:
return result
result['Value'].update( options )
options = result['Value']
# Help distinguishing storage type
diskSE = True
tapeSE = False
if 'SEType' in options:
# Type should follow the convention TXDY
seType = options['SEType']
diskSE = re.search( 'D[1-9]', seType ) != None
tapeSE = re.search( 'T[1-9]', seType ) != None
options['DiskSE'] = diskSE
options['TapeSE'] = tapeSE
return S_OK( options )
def getQueue(site, ce, queue):
""" Get parameters of the specified queue
"""
Tags = []
grid = site.split('.')[0]
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s' %
(grid, site, ce))
if not result['OK']:
return result
resultDict = result['Value']
ceTags = resultDict.get('Tag')
if ceTags:
Tags = fromChar(ceTags)
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs/%s/Queues/%s' %
(grid, site, ce, queue))
if not result['OK']:
return result
resultDict.update(result['Value'])
queueTags = resultDict.get('Tag')
if queueTags:
queueTags = fromChar(queueTags)
Tags = list(set(Tags + queueTags))
if Tags:
resultDict['Tag'] = Tags
resultDict['Queue'] = queue
return S_OK(resultDict)
def getImages( siteList = None, ceList = None, imageList = None, vo = None ):
""" Get CE/image options according to the specified selection
"""
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
resultDict = {}
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
continue
sites = result['Value']
for site in sites:
if siteList is not None and not site in siteList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/VO' % ( grid, site ), [] )
if voList and not vo in voList:
continue
result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud' % ( grid, site ) )
if not result['OK']:
continue
ces = result['Value']
for ce in ces:
if ceList is not None and not ce in ceList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/VO' % ( grid, site, ce ), [] )
if voList and not vo in voList:
continue
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s' % ( grid, site, ce ) )
if not result['OK']:
continue
ceOptionsDict = result['Value']
result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud/%s/Images' % ( grid, site, ce ) )
if not result['OK']:
continue
images = result['Value']
for image in images:
if imageList is not None and not image in imageList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s/VO' % ( grid, site, ce, image ), [] )
if voList and not vo in voList:
continue
resultDict.setdefault( site, {} )
resultDict[site].setdefault( ce, ceOptionsDict )
resultDict[site][ce].setdefault( 'Images', {} )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s' % ( grid, site, ce, image ) )
if not result['OK']:
continue
imageOptionsDict = result['Value']
resultDict[site][ce]['Images'][image] = imageOptionsDict
return S_OK( resultDict )
def getOptionsDict(self, optionPath, optionVOPath):
comOptDict = {}
if optionVOPath is not None:
result = gConfig.getOptionsDict(optionVOPath)
if result['OK']:
comOptDict = result['Value']
result = gConfig.getOptionsDict(optionPath)
if not result['OK']:
return result
optDict = result['Value']
optDict.update(comOptDict)
return S_OK(optDict)
def getOptionsDict( self, optionPath, optionVOPath ):
comOptDict = {}
if optionVOPath is not None:
result = gConfig.getOptionsDict( optionVOPath )
if result['OK']:
comOptDict = result['Value']
result = gConfig.getOptionsDict( optionPath )
if not result['OK']:
return result
optDict = result['Value']
optDict.update( comOptDict )
return S_OK( optDict )
def getVMTypeConfig(site, ce="", vmtype=""):
"""Get the VM image type parameters of the specified queue"""
tags = []
reqtags = []
grid = site.split(".")[0]
if not ce:
result = gConfig.getSections("/Resources/Sites/%s/%s/Cloud" %
(grid, site))
if not result["OK"]:
return result
ceList = result["Value"]
if len(ceList) == 1:
ce = ceList[0]
else:
return S_ERROR("No cloud endpoint specified")
result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/Cloud/%s" %
(grid, site, ce))
if not result["OK"]:
return result
resultDict = result["Value"]
ceTags = resultDict.get("Tag")
if ceTags:
tags = fromChar(ceTags)
ceTags = resultDict.get("RequiredTag")
if ceTags:
reqtags = fromChar(ceTags)
resultDict["CEName"] = ce
if vmtype:
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s" %
(grid, site, ce, vmtype))
if not result["OK"]:
return result
resultDict.update(result["Value"])
queueTags = resultDict.get("Tag")
if queueTags:
queueTags = fromChar(queueTags)
tags = list(set(tags + queueTags))
queueTags = resultDict.get("RequiredTag")
if queueTags:
queueTags = fromChar(queueTags)
reqtags = list(set(reqtags + queueTags))
if tags:
resultDict["Tag"] = tags
if reqtags:
resultDict["RequiredTag"] = reqtags
resultDict["VMType"] = vmtype
resultDict["Site"] = site
return S_OK(resultDict)
def __init__( self, imageName, endPointName ):
self.log = gLogger.getSubLogger( 'ImageConfiguration' )
imageOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s' % imageName )
if not imageOptions[ 'OK' ]:
self.log.error( imageOptions[ 'Message' ] )
imageOptions = {}
else:
imageOptions = imageOptions[ 'Value' ]
self.__ic_DIRACImageName = imageName
endPointName = endPointName.strip()
# A DIRAC image can have different boot image names in the cloud endPoints
bootImageOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s/BootImages' % imageName )
if not bootImageOptions[ 'OK' ]:
self.log.error( bootImageOptions[ 'Message' ] )
return
bootImageOptions = bootImageOptions[ 'Value' ]
bootImageName = None
for bootEndpoint, bootImage in bootImageOptions.items():
if endPointName == bootEndpoint:
bootImageName = bootImage
break
if bootImageName is None:
self.log.error( 'Missing mandatory boot image of the endPoint %s in BootImages section, image %s' % (endPointName, imageName) )
self.__ic_bootImageName = bootImageName
# A DIRAC image can have different flavor names names in the cloud endPoints
flavorOptions = gConfig.getOptionsDict( '/Resources/VirtualMachines/Images/%s/Flavors' % imageName )
if not flavorOptions[ 'OK' ]:
self.log.error( flavorOptions[ 'Message' ] )
return
flavorOptions = flavorOptions[ 'Value' ]
flavorName = None
for bootEndpoint, flavor in flavorOptions.items():
if endPointName == bootEndpoint:
flavorName = flavor
break
if flavorName is None:
self.log.error( 'Missing mandatory flavor of the endPoint %s in BootImages section, image %s' % (endPointName, imageName) )
self.__ic_flavorName = flavorName
self.__ic_contextMethod = imageOptions.get( 'contextMethod' , None )
#optional:
self.__ic_maxAllowedPrice = imageOptions.get( 'maxAllowedPrice' , None )
self.__ic_keyName = imageOptions.get( 'keyName' , None )
#self.__ic_contextConfig = ContextConfig( self.__ic_bootImageName, self.__ic_contextMethod )
self.__ic_contextConfig = ContextConfig( imageName, self.__ic_contextMethod )
def getMQParamsFromCS(mqURI):
"""Function gets parameters of a MQ destination (queue/topic) from the CS.
Args:
mqURI(str):Pseudo URI identifing the MQ service. It has the following format:
mqConnection::DestinationType::DestinationName
e.g. blabla.cern.ch::Queues::MyQueue1
mType(str): 'consumer' or 'producer'
Returns:
S_OK(param_dicts) or S_ERROR
"""
# API initialization is required to get an up-to-date configuration from the CS
csAPI = CSAPI()
csAPI.initialize()
try:
mqService, mqType, mqName = mqURI.split("::")
except ValueError:
return S_ERROR("Bad format of mqURI address:%s" % (mqURI))
result = gConfig.getConfigurationTree("/Resources/MQServices", mqService,
mqType, mqName)
if not result["OK"] or not result["Value"]:
return S_ERROR(
"Requested destination not found in the CS: %s::%s::%s" %
(mqService, mqType, mqName))
mqDestinationPath = None
for path, value in result["Value"].items():
if not value and path.endswith(mqName):
mqDestinationPath = path
# set-up internal parameter depending on the destination type
tmp = mqDestinationPath.split("Queues")[0].split("Topics")
servicePath = tmp[0]
serviceDict = {}
if len(tmp) > 1:
serviceDict["Topic"] = mqName
else:
serviceDict["Queue"] = mqName
result = gConfig.getOptionsDict(servicePath)
if not result["OK"]:
return result
serviceDict.update(result["Value"])
result = gConfig.getOptionsDict(mqDestinationPath)
if not result["OK"]:
return result
serviceDict.update(result["Value"])
return S_OK(serviceDict)
def getQueue( site, ce, queue ):
""" Get parameters of the specified queue
"""
grid = site.split( '.' )[0]
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) )
if not result['OK']:
return result
resultDict = result['Value']
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) )
if not result['OK']:
return result
resultDict.update( result['Value'] )
resultDict['Queue'] = queue
return S_OK( resultDict )
def getQueue(site, ce, queue):
""" Get parameters of the specified queue
"""
grid = site.split(".")[0]
result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/CEs/%s" % (grid, site, ce))
if not result["OK"]:
return result
resultDict = result["Value"]
result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/CEs/%s/Queues/%s" % (grid, site, ce, queue))
if not result["OK"]:
return result
resultDict.update(result["Value"])
resultDict["Queue"] = queue
return S_OK(resultDict)
def getQueue( site, ce, queue ):
""" Get parameters of the specified queue
"""
grid = site.split( '.' )[0]
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s' % ( grid, site, ce ) )
if not result['OK']:
return result
resultDict = result['Value']
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/CEs/%s/Queues/%s' % ( grid, site, ce, queue ) )
if not result['OK']:
return result
resultDict.update( result['Value'] )
resultDict['Queue'] = queue
return S_OK( resultDict )
def createCatalog(self, catalogName, useProxy=False):
""" Create a file catalog object from its name and CS description
"""
catalogPath = getCatalogPath(catalogName)
catalogType = gConfig.getValue(catalogPath + '/CatalogType',
catalogName)
catalogURL = gConfig.getValue(catalogPath + '/CatalogURL',
"DataManagement/" + catalogType)
optionsDict = {}
result = gConfig.getOptionsDict(catalogPath)
if result['OK']:
optionsDict = result['Value']
if useProxy:
result = self.__getCatalogClass(catalogType)
if not result['OK']:
return result
catalogClass = result['Value']
methods = catalogClass.getInterfaceMethods()
catalog = FileCatalogProxyClient(catalogName)
catalog.setInterfaceMethods(methods)
return S_OK(catalog)
return self.__createCatalog(catalogName, catalogType, catalogURL,
optionsDict)
def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' % (setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG("/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def getDefinedTransferPaths(self):
result = gConfig.getSections(self.__transfersCSPath)
if not result['OK']:
self.log.info('No Input/Output Pair defined in CS')
return S_OK()
pathList = result['Value']
tPaths = {}
for name in pathList:
csPath = self.__transfersCSPath + '/%s' % name
result = gConfig.getOptionsDict(csPath)
if not result['OK']:
continue
transferDict = result['Value']
ok = True
for i in self.__requiredCSOptions:
if i not in transferDict:
self.log.error('Missing Option %s in %s' % (i, csPath))
ok = False
break
if not ok:
continue
tPaths[name] = transferDict
return S_OK(tPaths)
def __generateDBs(self):
self.__log.notice("Creating default AccountingDB...")
self.__allDBs = {self.__defaultDB: AccountingDB(readOnly=self.__readOnly)}
types = self.__allDBs[self.__defaultDB].getRegisteredTypes()
result = gConfig.getOptionsDict(self.__csPath)
if not result['OK']:
gLogger.verbose("No extra databases defined",
"in %s" % self.__csPath)
return
validTypes = TypeLoader().getTypes()
opts = result['Value']
for acType in opts:
if acType not in validTypes:
msg = "(%s defined in %s)" % (acType, self.__csPath)
self.__log.fatal("Not a known accounting type", msg)
raise RuntimeError(msg)
dbName = opts[acType]
gLogger.notice("Type will be assigned", "(%s to %s)" % (acType, dbName))
if dbName not in self.__allDBs:
fields = dbName.split("/")
if len(fields) == 1:
dbName = "Accounting/%s" % dbName
gLogger.notice("Creating DB", '%s' % dbName)
self.__allDBs[dbName] = AccountingDB(dbName, readOnly=self.__readOnly)
self.__dbByType[acType] = dbName
def getDefinedTransferPaths( self ):
result = gConfig.getSections( self.__transfersCSPath )
if not result['OK']:
self.log.info( 'No Input/Output Pair defined in CS' )
return S_OK()
pathList = result['Value']
tPaths = {}
for name in pathList:
csPath = self.__transfersCSPath + '/%s' % name
result = gConfig.getOptionsDict( csPath )
if not result['OK']:
continue
transferDict = result['Value']
ok = True
for i in self.__requiredCSOptions:
if i not in transferDict:
self.log.error( 'Missing Option %s in %s' % ( i, csPath ) )
ok = False
break
if not ok:
continue
tPaths[ name ] = transferDict
return S_OK( tPaths )
def __init__( self, stratuslabEndpoint ):
"""
Constructor directly reads the named stratuslabEndpoint section
in the configuration.
:Parameters:
**stratuslabEndpoint** - `string`
name of the element containing the configuration for a StratusLab cloud
"""
super( StratusLabConfiguration, self ).__init__()
options = gConfig.getOptionsDict( '%s/%s' % ( self.ENDPOINT_PATH, stratuslabEndpoint ) )
if not options[ 'OK' ]:
self.log.error(options[ 'Message' ] )
options = {}
else:
options = options[ 'Value' ]
# Save a shallow copy of the given dictionary for safety.
self._options = copy.copy(options)
# Remove any 'None' mappings from the dictionary.
for key, value in self._options.items():
if value is None:
del self._options[ key ]
def __init__( self, amazonEndpoint ):
"""
Constructor
:Parameters:
**amazonEndpoint** - `string`
string with the name of the CloudEndpoint defined on the CS
"""
super( AmazonConfiguration, self ).__init__()
amazonOptions = gConfig.getOptionsDict( '%s/%s' % ( self.ENDPOINT_PATH, amazonEndpoint ) )
if not amazonOptions[ 'OK' ]:
self.log.error( amazonOptions[ 'Message' ] )
amazonOptions = {}
else:
amazonOptions = amazonOptions[ 'Value' ]
# FIXME: make it generic !
# Purely endpoint configuration ............................................
# This two are passed as arguments, not keyword arguments
self.__accessKey = amazonOptions.get( 'accessKey' , None )
self.__secretKey = amazonOptions.get( 'secretKey' , None )
self.__cloudDriver = amazonOptions.get( 'cloudDriver' , None )
self.__vmStopPolicy = amazonOptions.get( 'vmStopPolicy' , None )
self.__vmPolicy = amazonOptions.get( 'vmPolicy' , None )
self.__siteName = amazonOptions.get( 'siteName' , None )
self.__endpointURL = amazonOptions.get( 'endpointURL' , None )
self.__regionName = amazonOptions.get( 'regionName' , None )
self.__maxEndpointInstances = amazonOptions.get( 'maxEndpointInstances' , None )
self.__maxOportunisticEndpointInstances = amazonOptions.get( 'maxOportunisticEndpointInstances' , 0 )
self.__cvmfs_http_proxy = amazonOptions.get( 'cvmfs_http_proxy' , None )
self.__auth = amazonOptions.get( 'auth' , None )
def __instantiateSEs(self):
""" Get storage config for all the SE in the dirac config. """
SEDict = {}
configPath = 'Resources/StorageElements'
res = gConfig.getSections(configPath)
if not res['OK']:
return S_ERROR('Not able to get StorageConfig: %s') % res['Message']
SEList = res['Value']
for SE in SEList:
# se = StorageElement(SE, protocols='GFAL2_HTTP')
# se.getParameters()
seConfigPath = os.path.join(configPath, SE)
res = gConfig.getSections(seConfigPath)
if not res['OK']:
continue
# contains 'AccessProtocol.x'
accessProtocols = res['Value']
for entry in accessProtocols:
protocolConfigPath = os.path.join(seConfigPath, entry)
res = gConfig.getOptionsDict(protocolConfigPath)
if not res['OK']:
continue
res = res['Value']
# there should only be one GFAL2_HTTP plugin defined
if res.get('PluginName', None) == 'GFAL2_HTTP':
if not SEDict.get(res['Host'], None):
SEDict[res['Host']] = {}
SEDict[res['Host']][SE] = res
return S_OK(SEDict)
def convertTransfers():
global csapi
gLogger.notice('Converting Transfer services')
result = gConfig.getOptionsDict('/Resources/FTSEndpoints')
if not result['OK']:
gLogger.error(result['Message'])
return result
ftsDict = result['Value']
for site in ftsDict:
result = getSiteName(site)
siteName = result['Value']
gLogger.notice('Processing FTS endpoint at site %s' % siteName)
csapi.createSection('%s/Sites/%s/Transfer/FTS' %
(RESOURCES_NEW_SECTION, siteName))
csapi.setOptionComment(
'%s/Sites/%s/Transfer/FTS' % (RESOURCES_NEW_SECTION, siteName),
'File Transfer Service')
csapi.setOption(
'%s/Sites/%s/Transfer/FTS/URL' % (RESOURCES_NEW_SECTION, siteName),
ftsDict[site])
csapi.setOptionComment(
'%s/Sites/%s/Transfer' % (RESOURCES_NEW_SECTION, siteName),
'Data Transfer Service resources')
def __getOptionsFromCS(self,
path="/Website/Launchpad/Options",
delimiter=","):
gLogger.info("start __getOptionsFromCS")
result = gConfig.getOptionsDict(path)
gLogger.always(result)
if not result["OK"]:
return []
options = result["Value"]
for i in options.keys():
options[i] = options[i].split(delimiter)
result = gConfig.getSections(path)
if result["OK"]:
sections = result["Value"]
if len(sections) > 0:
for i in sections:
options[i] = self.__getOptionsFromCS(path + '/' + i, delimiter)
gLogger.always("options: %s" % options)
gLogger.info("end __getOptionsFromCS")
return options
def __init__(self, stratuslabEndpoint):
"""
Constructor directly reads the named stratuslabEndpoint section
in the configuration.
:Parameters:
**stratuslabEndpoint** - `string`
name of the element containing the configuration for a StratusLab cloud
"""
super(StratusLabEndpointConfiguration, self).__init__()
options = gConfig.getOptionsDict('%s/%s' % (self.ENDPOINT_PATH, stratuslabEndpoint))
if not options['OK']:
self.log.error(options['Message'])
options = {}
else:
options = options['Value']
# Save a shallow copy of the given dictionary for safety.
self._options = copy.copy(options)
# Remove any 'None' mappings from the dictionary.
for key, value in self._options.items():
if value is None:
del self._options[key]
def getCompatiblePlatforms(originalPlatforms):
""" Get a list of platforms compatible with the given list
"""
if type(originalPlatforms) == type(' '):
platforms = [originalPlatforms]
else:
platforms = list(originalPlatforms)
platformDict = {}
result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility')
if result['OK'] and result['Value']:
platformDict = result['Value']
for platform in platformDict:
platformDict[platform] = [
x.strip() for x in platformDict[platform].split(',')
]
else:
return S_ERROR('OS compatibility info not found')
resultList = list(platforms)
for p in platforms:
tmpList = platformDict.get(p, [])
for pp in platformDict:
if p in platformDict[pp]:
tmpList.append(pp)
tmpList += platformDict[pp]
if tmpList:
resultList += tmpList
return S_OK(uniqueElements(resultList))
def getRegistryUsers():
'''
Gets all users from /Registry/Users
'''
_basePath = 'Registry/Users'
registryUsers = {}
userNames = gConfig.getSections( _basePath )
if not userNames[ 'OK' ]:
return userNames
userNames = userNames[ 'Value' ]
for userName in userNames:
# returns { 'Email' : x, 'DN': y, 'CA' : z }
userDetails = gConfig.getOptionsDict( '%s/%s' % ( _basePath, userName ) )
if not userDetails[ 'OK' ]:
return userDetails
registryUsers[ userName ] = userDetails[ 'Value' ]
return S_OK( registryUsers )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def _getCatalogConfigDetails(self, catalogName):
# First obtain the options that are available
catalogConfigPath = "%s/%s" % (self.rootConfigPath, catalogName)
result = gConfig.getOptionsDict(catalogConfigPath)
if not result["OK"]:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
self.log.error(errStr, catalogName)
return S_ERROR(errStr)
catalogConfig = result["Value"]
result = self.opHelper.getOptionsDict("/Services/Catalogs/%s" % catalogName)
if result["OK"]:
catalogConfig.update(result["Value"])
# The 'Status' option should be defined (default = 'Active')
if "Status" not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
self.log.warn(warnStr, catalogName)
catalogConfig["Status"] = "Active"
# The 'AccessType' option must be defined
if "AccessType" not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
self.log.error(errStr, catalogName)
return S_ERROR(errStr)
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig["Master"] = catalogConfig.setdefault("Master", False) == "True"
return S_OK(catalogConfig)
def getDIRACPlatforms():
""" just returns list of platforms defined in the CS
"""
result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility')
if not (result['OK'] and result['Value']):
return S_ERROR("OS compatibility info not found")
return S_OK(result['Value'].keys())
def _getCatalogConfigDetails( self, catalogName ):
# First obtain the options that are available
catalogConfigPath = '%s/%s' % ( self.rootConfigPath, catalogName )
result = gConfig.getOptionsDict( catalogConfigPath )
if not result['OK']:
errStr = "FileCatalog._getCatalogConfigDetails: Failed to get catalog options."
self.log.error( errStr, catalogName )
return S_ERROR( errStr )
catalogConfig = result['Value']
result = self.opHelper.getOptionsDict( '/Services/Catalogs/%s' % catalogName )
if result['OK']:
catalogConfig.update( result['Value'] )
# The 'Status' option should be defined (default = 'Active')
if 'Status' not in catalogConfig:
warnStr = "FileCatalog._getCatalogConfigDetails: 'Status' option not defined."
self.log.warn( warnStr, catalogName )
catalogConfig['Status'] = 'Active'
# The 'AccessType' option must be defined
if 'AccessType' not in catalogConfig:
errStr = "FileCatalog._getCatalogConfigDetails: Required option 'AccessType' not defined."
self.log.error( errStr, catalogName )
return S_ERROR( errStr )
# Anything other than 'True' in the 'Master' option means it is not
catalogConfig['Master'] = ( catalogConfig.setdefault( 'Master', False ) == 'True' )
return S_OK( catalogConfig )
def getDIRACPlatform( OS ):
""" Get standard DIRAC platform(s) compatible with the argument.
NB: The returned value is a list! ordered, in reverse, using distutils.version.LooseVersion
In practice the "highest" version (which should be the most "desirable" one is returned first)
"""
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if not ( result['OK'] and result['Value'] ):
return S_ERROR( "OS compatibility info not found" )
platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] )
for k, v in platformsDict.iteritems():
if k not in v:
v.append( k )
# making an OS -> platforms dict
os2PlatformDict = dict()
for platform, osItems in platformsDict.iteritems():
for osItem in osItems:
if os2PlatformDict.get( osItem ):
os2PlatformDict[osItem].append( platform )
else:
os2PlatformDict[osItem] = [platform]
if OS not in os2PlatformDict:
return S_ERROR( 'No compatible DIRAC platform found for %s' % OS )
platforms = os2PlatformDict[OS]
platforms.sort( key = LooseVersion, reverse = True )
return S_OK( platforms )
def getCompatiblePlatforms( originalPlatforms ):
""" Get a list of platforms compatible with the given list
"""
if type( originalPlatforms ) == type( ' ' ):
platforms = [originalPlatforms]
else:
platforms = list( originalPlatforms )
platforms = list( platform.replace( ' ', '' ) for platform in platforms )
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if not ( result['OK'] and result['Value'] ):
return S_ERROR( "OS compatibility info not found" )
platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] )
for k, v in platformsDict.iteritems():
if k not in v:
v.append( k )
resultList = list( platforms )
for p in platforms:
tmpList = platformsDict.get( p, [] )
for pp in platformsDict:
if p in platformsDict[pp]:
tmpList.append( pp )
tmpList += platformsDict[pp]
if tmpList:
resultList += tmpList
return S_OK( uniqueElements( resultList ) )
def getCompatiblePlatforms(originalPlatforms):
""" Get a list of platforms compatible with the given list
"""
if isinstance(originalPlatforms, six.string_types):
platforms = [originalPlatforms]
else:
platforms = list(originalPlatforms)
platforms = list(platform.replace(' ', '') for platform in platforms)
result = gConfig.getOptionsDict('/Resources/Computing/OSCompatibility')
if not (result['OK'] and result['Value']):
return S_ERROR("OS compatibility info not found")
platformsDict = dict(
(k, v.replace(' ', '').split(','))
for k, v in result['Value'].items()) # can be an iterator
for k, v in platformsDict.items(): # can be an iterator
if k not in v:
v.append(k)
resultList = list(platforms)
for p in platforms:
tmpList = platformsDict.get(p, [])
for pp in platformsDict:
if p in platformsDict[pp]:
tmpList.append(pp)
tmpList += platformsDict[pp]
if tmpList:
resultList += tmpList
return S_OK(uniqueElements(resultList))
def _getCurrentConfig(self):
"""Return the current system configuration."""
from DIRAC.ConfigurationSystem.Client.ConfigurationData import gConfigurationData
gConfig.forceRefresh()
fullCfg = CFG()
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if setup not in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, ', '.join(setupList)))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] # dict
for system, setup in serviceSetups.items():
if self.systems and system not in self.systems:
continue
systemCfg = gConfigurationData.remoteCFG.getAsCFG(
"/Systems/%s/%s" % (system, setup))
for section in systemCfg.listSections():
if section not in ('Agents', 'Services', 'Executors'):
systemCfg.deleteKey(section)
fullCfg.createNewSection("/%s" % system, contents=systemCfg)
return S_OK(fullCfg)
def getDIRACPlatforms():
""" just returns list of platforms defined in the CS
"""
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if not ( result['OK'] and result['Value'] ):
return S_ERROR( "OS compatibility info not found" )
return S_OK( result['Value'].keys() )
def getRegistryUsers():
'''
Gets all users from /Registry/Users
'''
_basePath = 'Registry/Users'
registryUsers = {}
userNames = gConfig.getSections( _basePath )
if not userNames[ 'OK' ]:
return userNames
userNames = userNames[ 'Value' ]
for userName in userNames:
# returns { 'Email' : x, 'DN': y, 'CA' : z }
userDetails = gConfig.getOptionsDict( '%s/%s' % ( _basePath, userName ) )
if not userDetails[ 'OK' ]:
return userDetails
registryUsers[ userName ] = userDetails[ 'Value' ]
return S_OK( registryUsers )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def getCompatiblePlatforms( originalPlatforms ):
""" Get a list of platforms compatible with the given list
"""
if type( originalPlatforms ) == type( ' ' ):
platforms = [originalPlatforms]
else:
platforms = list( originalPlatforms )
platformDict = {}
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if result['OK'] and result['Value']:
platformDict = result['Value']
for platform in platformDict:
platformDict[platform] = [ x.strip() for x in platformDict[platform].split( ',' ) ]
else:
return S_ERROR( 'OS compatibility info not found' )
resultList = list( platforms )
for p in platforms:
tmpList = platformDict.get( p, [] )
for pp in platformDict:
if p in platformDict[pp]:
tmpList.append( pp )
tmpList += platformDict[pp]
if tmpList:
resultList += tmpList
return S_OK( uniqueElements( resultList ) )
def _getBrokerParamsFromCS(mqService: str) -> dict:
"""Return the configuration of the broker for a given MQService
The Sections ``Topics`` and ``Queues`` are returned as the "destinations" key.
"""
# Compatibility layer in case the full ``qualified`` name is given
if "::" in mqService:
if os.environ.get("DIRAC_DEPRECATED_FAIL", None):
raise NotImplementedError(
f"ERROR: deprecated do not give the full mqURI, just the service name: {mqService}"
)
print(
f"WARNING: deprecated do not give the full mqURI, just the service name: {mqService}"
)
mqService = mqService.split("::")[0]
brokerParams = returnValueOrRaise(
gConfig.getOptionsDict(f"/Resources/MQServices/{mqService}"))
# This is for compatibility reasons with the existing configuration definition
# Although there are no reasons to separate queues and topics for stomp
topics = [
f"/topic/{dest}" for dest in gConfig.getSections(
f"/Resources/MQServices/{mqService}/Topics").get("Value", [])
]
queues = [
f"/queue/{dest}" for dest in gConfig.getSections(
f"/Resources/MQServices/{mqService}/Queues").get("Value", [])
]
brokerParams["destinations"] = topics + queues
return brokerParams
def __generateDBs(self):
self.__log.notice("Creating default AccountingDB...")
self.__allDBs = {
self.__defaultDB: AccountingDB(readOnly=self.__readOnly)
}
types = self.__allDBs[self.__defaultDB].getRegisteredTypes()
result = gConfig.getOptionsDict(self.__csPath)
if not result['OK']:
gLogger.verbose("No extra databases defined in %s" % self.__csPath)
return
validTypes = TypeLoader().getTypes()
opts = result['Value']
for acType in opts:
if acType not in validTypes:
msg = "Oops... %s defined in %s is not a known accounting type" % (
acType, self.__csPath)
self.__log.fatal(msg)
raise RuntimeError(msg)
dbName = opts[acType]
gLogger.notice("%s type will be assigned to %s" % (acType, dbName))
if dbName not in self.__allDBs:
fields = dbName.split("/")
if len(fields) == 1:
dbName = "Accounting/%s" % dbName
gLogger.notice("Creating DB %s" % dbName)
self.__allDBs[dbName] = AccountingDB(dbName,
readOnly=self.__readOnly)
self.__dbByType[acType] = dbName
def getDIRACPlatform( OS ):
""" Get standard DIRAC platform(s) compatible with the argument.
NB: The returned value is a list! ordered, in reverse, using distutils.version.LooseVersion
In practice the "highest" version (which should be the most "desirable" one is returned first)
"""
result = gConfig.getOptionsDict( '/Resources/Computing/OSCompatibility' )
if not ( result['OK'] and result['Value'] ):
return S_ERROR( "OS compatibility info not found" )
platformsDict = dict( [( k, v.replace( ' ', '' ).split( ',' ) ) for k, v in result['Value'].iteritems()] )
for k, v in platformsDict.iteritems():
if k not in v:
v.append( k )
# making an OS -> platforms dict
os2PlatformDict = dict()
for platform, osItems in platformsDict.iteritems():
for osItem in osItems:
if os2PlatformDict.get( osItem ):
os2PlatformDict[osItem].append( platform )
else:
os2PlatformDict[osItem] = [platform]
if OS not in os2PlatformDict:
return S_ERROR( 'No compatible DIRAC platform found for %s' % OS )
platforms = os2PlatformDict[OS]
platforms.sort( key = LooseVersion, reverse = True )
return S_OK( platforms )
def __init__(self,
group,
lifetime=3600 * 12,
voms=False,
aToken=None,
proxyPath=None):
""" C'r
:param str group: requested group
:param int lifetime: requested proxy lifetime
:param bool voms: requested voms extension
:param str aToken: access token or path
:param str proxyPath: proxy path
"""
self.log = gLogger.getSubLogger(__name__)
# Defaulf location for proxy is /tmp/x509up_uXXXX
self.pPath = proxyPath or '/tmp/x509up_u%s' % os.getuid()
self.group = group
self.lifetime = lifetime
self.voms = voms
# Default access token path for notebook: /var/run/secrets/egi.eu/access_token
self.accessToken = aToken or '/var/run/secrets/egi.eu/access_token'
# Load client metadata
result = gConfig.getOptionsDict(
"/LocalInstallation/AuthorizationClient")
if not result['OK']:
raise Exception("Can't load client settings.")
self.metadata = result['Value']
# For this open client we don't verify ssl certs
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
def __site_parameters(sites, wlcg):
"""Function that returns the sites parameters.
:param list sites:
List of sites or single site with same site name (e.g ['LCG.CERN.cern'] or
[LCG.Manchester.uk, VAC.Manchester.uk])
:param dict wlcg:
It's a dictionary with the WLCG parameters from all sites grabbed from
https://wlcg-rebus.cern.ch/apps/topology/all/json
Keys:
'WlcgName', 'Coordinates', 'Description', 'Mail', 'DiracName', 'Tier', 'Sub-Tier',
'SE', 'Country':, 'Federation', 'FederationAccountingName', 'Infrastructure',
'Institute Name', 'Grid'
If the site is not listed in WLCG or have no MoU Tier Level the function will return False
"""
grid_dict = {}
grid, real_site_name, country = sites[0].split(".")
site_opts = gConfig.getOptionsDict(
'Resources/Sites/%s/%s' % (grid, sites[0]))
site_opts = site_opts.get('Value')
site_name = site_opts.get('Name')
site_tier = site_opts.get('MoUTierLevel', 'None')
if site_tier != 'None':
wlcg_params = [s for s in wlcg if site_name in s.get('Site')]
wlcg_params = wlcg_params[0] if wlcg_params else {}
if not wlcg_params:
return False
if len(sites) > 1:
for i in sites:
grid = i.split(".")[0]
CE = gConfig.getSections('Resources/Sites/%s/%s/CEs' % (grid, i))
CE = CE['Value'] if CE['OK'] else None
grid_dict.update({grid: {'SiteName': i, 'CEs': CE}})
else:
CE = gConfig.getSections(
'Resources/Sites/%s/%s/CEs' % (grid, sites[0]))
CE = CE['Value'] if CE['OK'] else None
grid_dict.update({grid: {'SiteName': sites[0], 'CEs': CE}})
site_subtier = site_opts.get('SubTier', None)
ses = site_opts.get('SE', None)
site_params = {'WlcgName': site_opts.get('Name'), 'Coordinates': site_opts.get('Coordinates'),
'Description': site_opts.get('Description'), 'Mail': site_opts.get('Mail'),
'DiracName': ('LCG.' + real_site_name + "." + country),
'Tier': site_tier, 'Sub-Tier': site_subtier, 'SE': ses,
'Country': wlcg_params.get('Country'), 'Federation': wlcg_params.get('Federation'),
'FederationAccountingName': wlcg_params.get('FederationAccountingName'),
'Infrastructure': wlcg_params.get('Infrastructure'),
'Institute Name': wlcg_params.get('Institute Name'), 'Grid': grid_dict
}
return site_params
else:
return False
def getMQParamsFromCS( mqURI ):
""" Function gets parameters of a MQ destination (queue/topic) from the CS.
Args:
mqURI(str):Pseudo URI identifing the MQ service. It has the following format:
mqConnection::DestinationType::DestinationName
e.g. blabla.cern.ch::Queue::MyQueue1
mType(str): 'consumer' or 'producer'
Returns:
S_OK(param_dicts)/S_ERROR:
"""
# API initialization is required to get an up-to-date configuration from the CS
csAPI = CSAPI()
csAPI.initialize()
try :
mqService, mqType, mqName = mqURI.split( "::" )
except ValueError:
return S_ERROR( 'Bad format of mqURI address:%s' % ( mqURI ) )
result = gConfig.getConfigurationTree( '/Resources/MQServices', mqService, mqType, mqName )
if not result['OK'] or len( result['Value'] ) == 0:
return S_ERROR( 'Requested destination not found in the CS: %s::%s::%s' % ( mqService, mqType, mqName ) )
mqDestinationPath = None
for path, value in result['Value'].iteritems():
if not value and path.endswith( mqName ):
mqDestinationPath = path
# set-up internal parameter depending on the destination type
tmp = mqDestinationPath.split( 'Queue' )[0].split( 'Topic' )
servicePath = tmp[0]
serviceDict = {}
if len( tmp ) > 1:
serviceDict['Topic'] = mqName
else:
serviceDict['Queue'] = mqName
result = gConfig.getOptionsDict( servicePath )
if not result['OK']:
return result
serviceDict.update( result['Value'] )
result = gConfig.getOptionsDict( mqDestinationPath )
if not result['OK']:
return result
serviceDict.update( result['Value'] )
return S_OK( serviceDict )
def get_se_sites():
# Read the data from CSV file and returns dictionary
coordinates_data = {}
try:
csv_file = open('sites.tmp', "r")
except IOError:
sys.exit("There are no coordinates info for SE. Get first the computing sites with get_ce_sites()")
lines = csv_file.read().split("\n")
for row in lines:
if row != "":
site = row.split(',')
if site[1].replace('.','',1).replace('-','',1).isdigit() and \
site[2].replace('.','',1).replace('-','',1).isdigit():
coordinates_data[site[0]] = [float(site[1]),float(site[2])]
se_sites = {}
manager = Manager()
se_result = manager.listSEs()['Value']
se_health = service.getHealthyProductionSEs()
#se_health = service.getAllStorageElementStatus()
if not se_health['OK']:
print "[WARNING]: No health info for SE sites"
for se in sorted(se_result['Active']):
se_site = {}
result = gConfig.getOptionsDict('/Resources/StorageElements/' + se)
result2 = gConfig.getOptionsDict('/Resources/StorageElements/' + se \
+ '/AccessProtocol.1')
if result['OK'] and result2['OK']:
result = result['Value']
result2 = result2['Value']
se_site = result2
se_site['Read'] = result['ReadAccess']
se_site['Write'] = result['WriteAccess']
if se.split('-')[0] in coordinates_data:
se_site['Coordinates'] = coordinates_data[se.split('-')[0]]
else:
print "[WARNING]: No location info for " + se
se_sites[se] = se_site
if se_health['OK'] and se in se_health['Value']:
se_site['Health'] = se_health['Value'][se][0]
return se_sites
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
@return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] #dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
@return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] #dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
def getServicePorts(self, setup="", printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "")
setupList = gConfig.getSections("/DIRAC/Setups", [])
if not setupList["OK"]:
return S_ERROR("Could not get /DIRAC/Setups sections")
setupList = setupList["Value"]
if not setup in setupList:
return S_ERROR("Setup %s is not in allowed list: %s" % (setup, ", ".join(setupList)))
serviceSetups = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup)
if not serviceSetups["OK"]:
return S_ERROR("Could not get /DIRAC/Setups/%s options" % setup)
serviceSetups = serviceSetups["Value"] # dict
systemList = gConfig.getSections("/Systems")
if not systemList["OK"]:
return S_ERROR("Could not get Systems sections")
systemList = systemList["Value"]
result = {}
for system in systemList:
if serviceSetups.has_key(system):
path = "/Systems/%s/%s/Services" % (system, serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList["OK"]:
self.log.warn("Could not get sections in %s" % path)
else:
servicesList = servicesList["Value"]
if not servicesList:
servicesList = []
self.log.verbose("System: %s ServicesList: %s" % (system, ", ".join(servicesList)))
for service in servicesList:
spath = "%s/%s/Port" % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose("Found port for %s/%s = %s" % (system, service, servicePort))
result["%s/%s" % (system, service)] = servicePort
else:
self.log.warn("No port found for %s" % spath)
else:
self.log.warn("%s is not defined in /DIRAC/Setups/%s" % (system, setup))
if printOutput:
print self.pPrint.pformat(result)
return S_OK(result)
def getOptionsDict(self, path):
""" Mock the getOptionsDict call of gConfig
It reads from dict_cs
"""
if 'StorageElements' not in path:
return gConfig.getOptionsDict(path)
csSection = self.crawlCS(path)
options = dict((opt, val) for opt, val in csSection.iteritems() if not isinstance(val, dict))
return S_OK(options)
def getExtraConditions(self, site):
""" Get extra conditions allowing site throttling
"""
# Find Site job limits
grid, siteName, country = site.split(".")
siteSection = "/Resources/Sites/%s/%s" % (grid, site)
result = gConfig.getSections(siteSection)
if not result["OK"]:
return result
if not "JobLimits" in result["Value"]:
return S_OK({})
result = gConfig.getSections("%s/JobLimits" % siteSection)
if not result["OK"]:
return result
sections = result["Value"]
limitDict = {}
resultDict = {}
if sections:
for section in sections:
result = gConfig.getOptionsDict("%s/JobLimits/%s" % (siteSection, section))
if not result["OK"]:
return result
optionDict = result["Value"]
if optionDict:
limitDict[section] = []
for k, v in optionDict.items():
limitDict[section].append((k, int(v)))
if not limitDict:
return S_OK({})
# Check if the site exceeding the given limits
fields = limitDict.keys()
for field in fields:
for key, value in limitDict[field]:
if int(value) > 0:
result = jobDB.getCounters("Jobs", ["Status"], {"Site": site, field: key})
if not result["OK"]:
return result
count = 0
if result["Value"]:
for countDict, number in result["Value"]:
if countDict["Status"] in ["Running", "Matched"]:
count += number
if count > value:
if not resultDict.has_key(field):
resultDict[field] = []
resultDict[field].append(key)
gLogger.verbose(
"Job Limit imposed at %s on %s/%s/%d, %d jobs already deployed"
% (site, field, key, value, count)
)
else:
if not resultDict.has_key(field):
resultDict[field] = []
resultDict[field].append(key)
gLogger.verbose("Jobs prohibited at %s for %s/%s" % (site, field, key))
return S_OK(resultDict)
def _getConfigStorageProtocolDetails(self, storageName, protocol):
"""
Parse the contents of the protocol block
"""
# First obtain the options that are available
protocolConfigPath = '%s/%s/%s' % (self.rootConfigPath, storageName,
protocol)
res = gConfig.getOptions(protocolConfigPath)
if not res['OK']:
errStr = "StorageFactory.__getProtocolDetails: Failed to get protocol options."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
options = res['Value']
# We must have certain values internally even if not supplied in CS
protocolDict = {
'Access': '',
'Host': '',
'Path': '',
'Port': '',
'Protocol': '',
'ProtocolName': '',
'SpaceToken': '',
'WSUrl': ''
}
for option in options:
configPath = '%s/%s' % (protocolConfigPath, option)
optionValue = gConfig.getValue(configPath, '')
protocolDict[option] = optionValue
# Evaluate the base path taking into account possible VO specific setting
if self.vo:
result = gConfig.getOptionsDict(
cfgPath(protocolConfigPath, 'VOPath'))
voPath = ''
if result['OK']:
voPath = result['Value'].get(self.vo, '')
if voPath:
protocolDict['Path'] = voPath
# Now update the local and remote protocol lists.
# A warning will be given if the Access option is not set.
if protocolDict['Access'] == 'remote':
self.remoteProtocols.append(protocolDict['ProtocolName'])
elif protocolDict['Access'] == 'local':
self.localProtocols.append(protocolDict['ProtocolName'])
else:
errStr = "StorageFactory.__getProtocolDetails: The 'Access' option for %s:%s is neither 'local' or 'remote'." % (
storageName, protocol)
gLogger.warn(errStr)
# The ProtocolName option must be defined
if not protocolDict['ProtocolName']:
errStr = "StorageFactory.__getProtocolDetails: 'ProtocolName' option is not defined."
gLogger.error(errStr, "%s: %s" % (storageName, protocol))
return S_ERROR(errStr)
return S_OK(protocolDict)
def _getDefaultComponentStatus(self):
"""Get the configured status of the components."""
host = socket.gethostname()
defaultStatus = {'Down': set(), 'Run': set(), 'All': set()}
resRunning = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Running'))
resStopped = gConfig.getOptionsDict(os.path.join('/Registry/Hosts/', host, 'Stopped'))
if not resRunning['OK']:
return resRunning
if not resStopped['OK']:
return resStopped
defaultStatus['Run'] = set(resRunning['Value'].keys())
defaultStatus['Down'] = set(resStopped['Value'].keys())
defaultStatus['All'] = defaultStatus['Run'] | defaultStatus['Down']
if defaultStatus['Run'].intersection(defaultStatus['Down']):
self.logError("Overlap in configuration", str(defaultStatus['Run'].intersection(defaultStatus['Down'])))
return S_ERROR("Bad host configuration")
return S_OK(defaultStatus)
def __getExtraOptions(currentSectionPath):
from DIRAC import gConfig
optionsDict = {}
if not currentSectionPath:
return optionsDict
result = gConfig.getOptionsDict(currentSectionPath)
if not result['OK']:
return optionsDict
print result
return result['Value']
def __getCSStorageElementStatus( self, elementName, statusType, default ):
'''
Gets from the CS the StorageElements status
'''
cs_path = "/Resources/StorageElements"
if not isinstance( elementName, list ):
elementName = [ elementName ]
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
#statuses = self.__opHelper.getOptionsDict( 'RSSConfiguration/GeneralConfig/Resources/StorageElement' )
#statuses = gConfig.getOptionsDict( '/Operations/RSSConfiguration/GeneralConfig/Resources/StorageElement' )
result = {}
for element in elementName:
if statusType is not None:
# Added Active by default
#res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Allowed' )
res = gConfig.getOption( "%s/%s/%s" % ( cs_path, element, statusType ), 'Active' )
if res[ 'OK' ] and res[ 'Value' ]:
result[ element ] = { statusType : res[ 'Value' ] }
else:
res = gConfig.getOptionsDict( "%s/%s" % ( cs_path, element ) )
if res[ 'OK' ] and res[ 'Value' ]:
elementStatuses = {}
for elementStatusType, value in res[ 'Value' ].items():
#k = k.replace( 'Access', '' )
if elementStatusType in statuses:
elementStatuses[ elementStatusType ] = value
# If there is no status defined in the CS, we add by default Read and
# Write as Active.
if elementStatuses == {}:
#elementStatuses = { 'ReadAccess' : 'Allowed', 'WriteAccess' : 'Allowed' }
elementStatuses = { 'ReadAccess' : 'Active', 'WriteAccess' : 'Active' }
result[ element ] = elementStatuses
if result:
return S_OK( result )
if default is not None:
# sec check
if statusType is None:
statusType = 'none'
defList = [ [ el, statusType, default ] for el in elementName ]
return S_OK( getDictFromList( defList ) )
_msg = "StorageElement '%s', with statusType '%s' is unknown for CS."
return S_ERROR( _msg % ( elementName, statusType ) )
def getExtraConditions( self, site ):
""" Get extra conditions allowing site throttling
"""
# Find Site job limits
grid = site.split( '.' )[0]
siteSection = '/Resources/Sites/%s/%s' % ( grid, site )
result = gConfig.getSections( siteSection )
if not result['OK']:
return result
if not 'JobLimits' in result['Value']:
return S_OK( {} )
result = gConfig.getSections( '%s/JobLimits' % siteSection )
if not result['OK']:
return result
sections = result['Value']
limitDict = {}
resultDict = {}
if sections:
for section in sections:
result = gConfig.getOptionsDict( '%s/JobLimits/%s' % ( siteSection, section ) )
if not result['OK']:
return result
optionDict = result['Value']
if optionDict:
limitDict[section] = []
for key, value in optionDict.items():
limitDict[section].append( ( key, int( value ) ) )
if not limitDict:
return S_OK( {} )
# Check if the site exceeding the given limits
fields = limitDict.keys()
for field in fields:
for key, value in limitDict[field]:
if int( value ) > 0:
result = gJobDB.getCounters( 'Jobs', ['Status'], {'Site':site, field:key} )
if not result['OK']:
return result
count = 0
if result['Value']:
for countDict, number in result['Value']:
if countDict['Status'] in ["Running", "Matched"]:
count += number
if count > value:
if not resultDict.has_key( field ):
resultDict[field] = []
resultDict[field].append( key )
gLogger.verbose( 'Job Limit imposed at %s on %s/%s/%d,'
' %d jobs already deployed' % ( site, field, key, value, count ) )
else:
if not resultDict.has_key( field ):
resultDict[field] = []
resultDict[field].append( key )
gLogger.verbose( 'Jobs prohibited at %s for %s/%s' % ( site, field, key ) )
return S_OK( resultDict )
