def getSiteMapping(self, resourceType, mapping=''):
""" Get site mapping to resources of a given type
"""
resultDict = {}
if mapping:
result = self.getOptions("SiteTo%sMapping/%s" %
(resourceType, mapping))
if not result['OK']:
return result
for site in result['Value']:
if site != "UseLocalResources":
resultDict[site] = self.getValue(
"SiteTo%sMapping/%s/%s" %
(resourceType, mapping, site), [])
useLocalResources = self.getValue(
"SiteTo%sMapping/%s/UseLocalResources" % (resourceType, mapping),
False)
if useLocalResources:
reHelper = Resources.Resources(vo=self.__vo)
result = reHelper.getEligibleResources(resourceType)
if result['OK']:
for site in result['Value']:
resultDict.setdefault(site, [])
resultDict[site].extend(result['Value'][site])
resultDict[site] = List.uniqueElements(resultDict[site])
return S_OK(resultDict)
def export_getSitesResources( self, siteNames ):
resources = Resources.Resources()
if siteNames is None:
siteNames = Resources.getSites()
if not siteNames[ 'OK' ]:
return siteNames
siteNames = siteNames[ 'Value' ]
if isinstance( siteNames, str ):
siteNames = [ siteNames ]
sitesRes = {}
for siteName in siteNames:
res = {}
res[ 'ces' ] = resources.getEligibleResources( 'Computing', { 'Site': siteName } )
ses = resources.getEligibleStorageElements( { 'Site': siteName } )
sesHosts = CSHelpers.getStorageElementsHosts( ses )
if not sesHosts[ 'OK' ]:
return sesHosts
res[ 'ses' ] = list( set( sesHosts[ 'Value' ] ) )
sitesRes[ siteName ] = res
return S_OK( sitesRes )
def getStorageElementEndpoint(storageElement):
resources = Resources.Resources()
result = resources.getAccessProtocols(storageElement)
if not result['OK']:
return result
# FIXME: There can be several access protocols for the same SE !
try:
ap = result['Value'][0]
except IndexError:
return S_ERROR('No AccessProtocol associated to %s' % storageElement)
result = resources.getAccessProtocolOptionsDict(ap)
#result = resources.getAccessProtocols( storageElement )
if not result['OK']:
return result
host = result['Value'].get('Host', '')
port = result['Value'].get('Port', '')
wsurl = result['Value'].get('WSUrl', '')
# MAYBE wusrl is not defined
#if host and port and wsurl:
if host and port:
url = 'httpg://%s:%s%s' % (host, port, wsurl)
url = url.replace('?SFN=', '')
return S_OK(url)
return S_ERROR((host, port, wsurl))
def export_getTree( self, element, elementType, elementName ):
tree = {}
resources = Resources.Resources()
#site = self.getSite( element, elementType, elementName )
result = getSiteForResource( elementName )
if not result['OK']:
return S_ERROR( 'Can not get site name: %s' % result[ 'Message' ] )
site = result['Value']
if not site:
return S_ERROR( 'No site' )
siteStatus = rsClient.selectStatusElement( 'Site', 'Status', name = site,
meta = { 'columns' : [ 'StatusType', 'Status' ] } )
if not siteStatus[ 'OK' ]:
return siteStatus
tree[ site ] = { 'statusTypes' : dict( siteStatus[ 'Value' ] ) }
ces = resources.getEligibleResources( 'Computing', { 'Site': site } )
cesStatus = rsClient.selectStatusElement( 'Resource', 'Status', name = ces,
meta = { 'columns' : [ 'Name', 'StatusType', 'Status'] } )
if not cesStatus[ 'OK' ]:
return cesStatus
tree[ site ][ 'ces' ] = {}
for ceTuple in cesStatus[ 'Value' ]:
name, statusType, status = ceTuple
if not name in tree[ site ][ 'ces' ]:
tree[ site ][ 'ces' ][ name ] = {}
tree[ site ][ 'ces' ][ name ][ statusType ] = status
ses = resources.getEligibleStorageElements( { 'Site': site } )
sesStatus = rsClient.selectStatusElement( 'Resource', 'Status', name = ses,
meta = { 'columns' : [ 'Name', 'StatusType', 'Status'] } )
if not sesStatus[ 'OK' ]:
return sesStatus
tree[ site ][ 'ses' ] = {}
for seTuple in sesStatus[ 'Value' ]:
name, statusType, status = seTuple
if not name in tree[ site ][ 'ses' ]:
tree[ site ][ 'ses' ][ name ] = {}
tree[ site ][ 'ses' ][ name ][ statusType ] = status
return S_OK( tree )
def getSEProtocolOption(se, optionName):
"""
Get option of the Storage Element access protocol
"""
resources = Resources.Resources()
result = resources.getAccessProtocols(se)
if not result['OK']:
return S_ERROR("Acces Protocol for SE %s not found: %s" %
(se, result['Message']))
try:
ap = result['Value'][0]
except IndexError:
return S_ERROR('No AccessProtocol associated to %s' % se)
return resources.getAccessProtocolOption(ap, optionName)
def __init__(self, args=None, clients=None):
super(SuccessfullPilotsByCESplittedCommand,
self).__init__(args, clients)
self.resources = Resources.Resources()
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
def getStorageElementsHosts(seNames=None):
seHosts = []
resources = Resources.Resources()
if seNames is None:
seNames = resources.getEligibleStorageElements()
if not seNames['OK']:
return seNames
seNames = seNames['Value']
for seName in seNames:
result = getSEProtocolOption(seName, 'Host')
if result['OK']:
seHosts.append(result['Value'])
return S_OK(list(set(seHosts)))
def getStorageElementEndpoints(storageElements=None):
resources = Resources.Resources()
if storageElements is None:
storageElements = resources.getEligibleStorageElements()
if not storageElements['OK']:
return storageElements
storageElements = storageElements['Value']
storageElementEndpoints = []
for se in storageElements:
seEndpoint = getStorageElementEndpoint(se)
if not seEndpoint['OK']:
continue
storageElementEndpoints.append(seEndpoint['Value'])
return S_OK(list(set(storageElementEndpoints)))
def beginExecution(self):
self.gridEnv = self.am_getOption("GridEnv", getGridEnv())
# The SiteDirector is for a particular user community
self.vo = self.am_getOption("Community", '')
if not self.vo:
self.vo = CSGlobals.getVO()
# The SiteDirector is for a particular user group
self.group = self.am_getOption("Group", '')
# self.voGroups contain all the eligible user groups for pilots submutted by this SiteDirector
self.voGroups = []
# Choose the group for which pilots will be submitted. This is a hack until
# we will be able to match pilots to VOs.
if not self.group:
if self.vo:
result = Registry.getGroupsForVO(self.vo)
if not result['OK']:
return result
for group in result['Value']:
if 'NormalUser' in Registry.getPropertiesForGroup(group):
self.voGroups.append(group)
else:
self.voGroups = [self.group]
result = findGenericPilotCredentials(vo=self.vo)
if not result['OK']:
return result
self.pilotDN, self.pilotGroup = result['Value']
self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
self.platforms = []
self.sites = []
self.defaultSubmitPools = ''
if self.group:
self.defaultSubmitPools = Registry.getGroupOption(
self.group, 'SubmitPools', '')
elif self.vo:
self.defaultSubmitPools = Registry.getVOOption(
self.vo, 'SubmitPools', '')
self.pilot = self.am_getOption('PilotScript', DIRAC_PILOT)
self.install = DIRAC_INSTALL
self.workingDirectory = self.am_getOption('WorkDirectory')
self.maxQueueLength = self.am_getOption('MaxQueueLength', 86400 * 3)
self.pilotLogLevel = self.am_getOption('PilotLogLevel', 'INFO')
self.maxJobsInFillMode = self.am_getOption('MaxJobsInFillMode',
self.maxJobsInFillMode)
self.maxPilotsToSubmit = self.am_getOption('MaxPilotsToSubmit',
self.maxPilotsToSubmit)
self.pilotWaitingFlag = self.am_getOption('PilotWaitingFlag', True)
self.pilotWaitingTime = self.am_getOption('MaxPilotWaitingTime', 7200)
# Flags
self.updateStatus = self.am_getOption('UpdatePilotStatus', True)
self.getOutput = self.am_getOption('GetPilotOutput', True)
self.sendAccounting = self.am_getOption('SendPilotAccounting', True)
# Get the site description dictionary
siteNames = None
if not self.am_getOption('Site', 'Any').lower() == "any":
siteNames = self.am_getOption('Site', [])
ceTypes = None
if not self.am_getOption('CETypes', 'Any').lower() == "any":
ceTypes = self.am_getOption('CETypes', [])
ces = None
if not self.am_getOption('CEs', 'Any').lower() == "any":
ces = self.am_getOption('CEs', [])
self._resources = Resources.Resources(vo=self.vo)
result = self._resources.getEligibleQueuesInfo(siteList=siteNames,
ceList=ces,
ceTypeList=ceTypes,
mode='Direct')
if not result['OK']:
return result
resourceDict = result['Value']
result = self.getQueues(resourceDict)
if not result['OK']:
return result
#if not siteNames:
# siteName = gConfig.getValue( '/DIRAC/Site', 'Unknown' )
# if siteName == 'Unknown':
# return S_OK( 'No site specified for the SiteDirector' )
# else:
# siteNames = [siteName]
#self.siteNames = siteNames
if self.updateStatus:
self.log.always('Pilot status update requested')
if self.getOutput:
self.log.always('Pilot output retrieval requested')
if self.sendAccounting:
self.log.always('Pilot accounting sending requested')
self.log.always('Sites:', siteNames)
self.log.always('CETypes:', ceTypes)
self.log.always('CEs:', ces)
self.log.always('PilotDN:', self.pilotDN)
self.log.always('PilotGroup:', self.pilotGroup)
self.log.always('MaxPilotsToSubmit:', self.maxPilotsToSubmit)
self.log.always('MaxJobsInFillMode:', self.maxJobsInFillMode)
self.localhost = socket.getfqdn()
self.proxy = ''
if self.queueDict:
self.log.always("Agent will serve queues:")
for queue in self.queueDict:
self.log.always("Site: %s, CE: %s, Queue: %s" %
(self.queueDict[queue]['Site'],
self.queueDict[queue]['CEName'], queue))
return S_OK()
from DIRAC.FrameworkSystem.Client.BundleDeliveryClient import BundleDeliveryClient
bdc = BundleDeliveryClient()
result = bdc.syncCAs()
if result['OK']:
result = bdc.syncCRLs()
except:
DIRAC.gLogger.exception('Could not import BundleDeliveryClient')
pass
if not skipCAChecks:
Script.localCfg.deleteOption('/DIRAC/Security/SkipCAChecks')
if ceName or siteName:
# This is used in the pilot context, we should have a proxy and access to CS
Script.enableCS()
resources = Resources.Resources(vo=vo)
if not siteName:
if ceName:
result = resources.getSiteForResource('Computing', ceName)
if result['OK']:
site = result['Value']
result = resources.getSiteFullName(site)
if result['OK']:
siteName = result['Value']
if siteName:
DIRAC.gLogger.notice('Setting /LocalSite/Site = %s' % siteName)
Script.localCfg.addDefaultEntry('/LocalSite/Site', siteName)
DIRAC.__siteName = False
if ceName:
