def getSite(self, elementType, elementName):
"""
Given an element name, return its site
"""
if elementType == 'StorageElement':
elementType = 'SE'
domainNames = gConfig.getSections('Resources/Sites')
if not domainNames['OK']:
return domainNames
domainNames = domainNames['Value']
for domainName in domainNames:
sites = gConfig.getSections('Resources/Sites/%s' % domainName)
if not sites['OK']:
continue
for site in sites['Value']:
elements = gConfig.getValue('Resources/Sites/%s/%s/%s' % (domainName, site, elementType), '')
if elementName in elements:
return site
return ''
def getComputingElements():
"""
Gets all computing elements from /Resources/Sites/<>/<>/CE
"""
_basePath = 'Resources/Sites'
ces = []
domainNames = gConfig.getSections(_basePath)
if not domainNames['OK']:
return domainNames
domainNames = domainNames['Value']
for domainName in domainNames:
domainSites = gConfig.getSections('%s/%s' % (_basePath, domainName))
if not domainSites['OK']:
return domainSites
domainSites = domainSites['Value']
for site in domainSites:
siteCEs = gConfig.getSections('%s/%s/%s/CEs' % (_basePath, domainName, site))
if not siteCEs['OK']:
# return siteCEs
gLogger.error(siteCEs['Message'])
continue
siteCEs = siteCEs['Value']
ces.extend(siteCEs)
# Remove duplicated ( just in case )
ces = list(set(ces))
return S_OK(ces)
def getOpsSection():
"""
Where is the shifters section?
"""
vo = CSGlobals.getVO()
setup = CSGlobals.getSetup()
if vo:
res = gConfig.getSections( '/Operations/%s/%s/Shifter' % (vo, setup) )
if res['OK']:
return S_OK( '/Operations/%s/%s/Shifter' % ( vo, setup ) )
res = gConfig.getSections( '/Operations/%s/Defaults/Shifter' % vo )
if res['OK']:
return S_OK( '/Operations/%s/Defaults/Shifter' % vo )
else:
res = gConfig.getSections( '/Operations/%s/Shifter' % setup )
if res['OK']:
return S_OK( '/Operations/%s/Shifter' % setup )
res = gConfig.getSections( '/Operations/Defaults/Shifter' )
if res['OK']:
return S_OK( '/Operations/Defaults/Shifter' )
return S_ERROR( "No shifter section" )
def getComponentsStatus( self, conditionDict = {} ):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
"""
result = self.__getComponents( conditionDict )
if not result[ 'OK' ]:
return result
statusSet = result[ 'Value' ]
requiredComponents = {}
result = gConfig.getSections( "/DIRAC/Setups" )
if not result[ 'OK' ]:
return result
for setup in result[ 'Value' ]:
if not self.__checkCondition( conditionDict, "Setup", setup ):
continue
#Iterate through systems
result = gConfig.getOptionsDict( "/DIRAC/Setups/%s" % setup )
if not result[ 'OK' ]:
return result
systems = result[ 'Value' ]
for system in systems:
instance = systems[ system ]
#Check defined agents and serviecs
for type in ( 'agent' , 'service' ):
#Get entries for the instance of a system
result = gConfig.getSections( "/Systems/%s/%s/%s" % ( system, instance, "%ss" % type.capitalize() ) )
if not result[ 'OK' ]:
self.log.warn( "Opps, sytem seems to be defined wrong\n", "System %s at %s: %s" % ( system, instance, result[ 'Message' ] ) )
continue
components = result[ 'Value' ]
for component in components:
componentName = "%s/%s" % ( system, component )
compDict = self.__getComponentDefinitionFromCS( system, setup, instance, type, component )
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#Walk the URLs
result = gConfig.getOptionsDict( "/Systems/%s/%s/URLs" % ( system, instance ) )
if not result[ 'OK' ]:
self.log.warn ( "There doesn't to be defined the URLs section for %s in %s instance" % ( system, instance ) )
else:
serviceURLs = result[ 'Value' ]
for service in serviceURLs:
for url in List.fromChar( serviceURLs[ service ] ):
loc = url[ url.find( "://" ) + 3: ]
iS = loc.find( "/" )
componentName = loc[ iS + 1: ]
loc = loc[ :iS ]
hostname, port = loc.split( ":" )
compDict = { 'ComponentName' : componentName,
'Type' : 'service',
'Setup' : setup,
'Host' : hostname,
'Port' : int( port )
}
if self.__componentMatchesCondition( compDict, requiredComponents, conditionDict ):
statusSet.addUniqueToSet( requiredComponents, compDict )
#WALK THE DICT
statusSet.setComponentsAsRequired( requiredComponents )
return S_OK( ( statusSet.getRequiredComponents(),
self.__mainFields[1:] + self.__versionFields + ( 'Status', 'Message' ) ) )
def _showSoftware():
"""Show available software"""
clip = _Params()
clip.registerSwitches()
Script.parseCommandLine()
from DIRAC import gConfig, gLogger
base = '/Operations/Defaults/AvailableTarBalls'
platforms = gConfig.getSections(base)
for platform in platforms['Value']:
gLogger.notice("For platform %s, here are the available software" % platform)
apps = gConfig.getSections(base + "/" + platform)
for app in apps['Value']:
if clip.software and app.lower() != clip.software.lower():
continue
gLogger.notice(" - %s" % app)
versions = gConfig.getSections(base + "/" + platform + "/" + app)
if clip.appsOnly:
continue
for vers in versions['Value']:
gLogger.notice(" * %s" % vers)
depsb = gConfig.getSections(base + "/" + platform + "/" + app + "/" + vers)
if 'Dependencies' in depsb['Value']:
gLogger.notice(" Depends on")
deps = gConfig.getSections( os.path.join( base, platform, app, vers , "Dependencies") )
for dep in deps['Value']:
depversions = gConfig.getOption(base + "/" + platform + "/" + app + "/" + vers + "/Dependencies/" + dep + "/version")
gLogger.notice(" %s %s" % (dep, depversions['Value']))
if not len(versions['Value']):
gLogger.notice(" No version available")
def getSites():
"""
Gets all sites from /Resources/Sites
"""
_basePath = 'Resources/Sites'
sites = []
domainNames = gConfig.getSections(_basePath)
if not domainNames['OK']:
return domainNames
domainNames = domainNames['Value']
for domainName in domainNames:
domainSites = gConfig.getSections('%s/%s' % (_basePath, domainName))
if not domainSites['OK']:
return domainSites
domainSites = domainSites['Value']
sites.extend(domainSites)
# Remove duplicated ( just in case )
sites = list(set(sites))
return S_OK(sites)
def getCESiteMapping( gridName = '' ):
""" Returns a dictionary of all CEs and their associated site, e.g.
{'ce101.cern.ch':'LCG.CERN.ch', ...]}
Assumes CS structure of: /Resources/Sites/<GRIDNAME>/<SITENAME>
"""
ceSiteMapping = {}
gridTypes = gConfig.getSections( '/Resources/Sites/', [] )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid, [] )
if not sites['OK']: #gConfig returns S_ERROR for empty sections until version
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
if sites:
for candidate in sites['Value']:
siteCEs = gConfig.getValue( '/Resources/Sites/%s/%s/CE' % ( grid, candidate ), [] )
for ce in siteCEs:
if ceSiteMapping.has_key( ce ):
current = ceSiteMapping[ce]
gLogger.warn( 'CE %s already has a defined site %s but it is also defined for %s' % ( ce, current, candidate ) )
else:
ceSiteMapping[ce] = candidate
return S_OK( ceSiteMapping )
def _updateSiteList(self):
sitesInConfig = []
sitesInDB = []
#Calculate Max pledges for Cluster and GRIDs
for directory in gConfig.getSections('/Resources/Sites')['Value']:
for site in gConfig.getSections('/Resources/Sites/' + directory)['Value']:
sitesInConfig.append(site)
result = self.samdb_dao.getSiteList()
if result['OK']:
for site in result['Value']:
sitesInDB.append(site[0])
else:
gLogger.error('Failed to update site list')
return
siteToAdd = []
siteToDelete = []
gLogger.info(siteToAdd)
gLogger.info(sitesInDB)
for site in sitesInConfig:
if site not in sitesInDB:
siteToAdd.append(site)
for site in sitesInDB:
if site not in sitesInConfig:
siteToDelete.append(site)
for site in siteToDelete:
self.samdb_dao.deleteSite(site)
for site in siteToAdd:
self.samdb_dao.addNewSite(site)
return S_OK()
def __instantiateSEs(self):
""" Get storage config for all the SE in the dirac config. """
SEDict = {}
configPath = 'Resources/StorageElements'
res = gConfig.getSections(configPath)
if not res['OK']:
return S_ERROR('Not able to get StorageConfig: %s') % res['Message']
SEList = res['Value']
for SE in SEList:
# se = StorageElement(SE, protocols='GFAL2_HTTP')
# se.getParameters()
seConfigPath = os.path.join(configPath, SE)
res = gConfig.getSections(seConfigPath)
if not res['OK']:
continue
# contains 'AccessProtocol.x'
accessProtocols = res['Value']
for entry in accessProtocols:
protocolConfigPath = os.path.join(seConfigPath, entry)
res = gConfig.getOptionsDict(protocolConfigPath)
if not res['OK']:
continue
res = res['Value']
# there should only be one GFAL2_HTTP plugin defined
if res.get('PluginName', None) == 'GFAL2_HTTP':
if not SEDict.get(res['Host'], None):
SEDict[res['Host']] = {}
SEDict[res['Host']][SE] = res
return S_OK(SEDict)
def getSitesForSE( storageElement, gridName = '' ):
""" Given a DIRAC SE name this method returns a list of corresponding sites.
Optionally restrict to Grid specified by name.
"""
finalSites = []
gridTypes = gConfig.getSections( '/Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if gridName in gridTypes:
gridTypes = [gridName]
else:
return S_ERROR( 'Grid type %s not in list: %s' % ( gridName, ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']: #gConfig returns S_ERROR for empty sections until version
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
if sites:
siteList = sites['Value']
for candidate in siteList:
siteSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
if storageElement in siteSEs:
finalSites.append( candidate )
return S_OK( finalSites )
def export_getSiteSummarySelectors(self):
""" Get all the distinct selector values for the site summary web portal page
"""
resultDict = {}
statusList = ['Good','Fair','Poor','Bad','Idle']
resultDict['Status'] = statusList
maskStatus = ['Active','Banned','NoMask','Reduced']
resultDict['MaskStatus'] = maskStatus
gridTypes = []
result = gConfig.getSections('Resources/Sites/',[])
if result['OK']:
gridTypes = result['Value']
resultDict['GridType'] = gridTypes
siteList = []
for grid in gridTypes:
result = gConfig.getSections('Resources/Sites/%s' % grid,[])
if result['OK']:
siteList += result['Value']
countryList = []
for site in siteList:
if site.find('.') != -1:
grid,sname,country = site.split('.')
country = country.lower()
if country not in countryList:
countryList.append(country)
countryList.sort()
resultDict['Country'] = countryList
siteList.sort()
resultDict['Site'] = siteList
return S_OK(resultDict)
def __getSections( self, path ):
result = []
userData = self.getSessionData()
retVal = gConfig.getSections( '/DIRAC/Setups' )
if retVal['OK']:
setups = [ i.split( '-' )[-1] for i in retVal['Value']]
setup = userData['setup'].split( '-' )[-1]
leaf = True if path.find( 'Agents' ) != -1 or path.find( 'Services' ) != -1 else False
retVal = gConfig.getSections( path )
if retVal['OK']:
records = retVal['Value']
for i in records:
if i in setups and i != setup:
continue
if i == setup:
path = "%s/%s" % ( path, i )
result = self.__getSections( path )
if i not in [setup, 'Databases', 'URLs']:
id = "%s/%s" % ( path, i )
components = path.split( '/' )
if len( components ) > 2:
componentName = "%s/%s" % ( components[2], i )
else:
componentName = i
result += [{'text':i, 'qtip': "Systems", "leaf" : leaf, 'component' : componentName, 'id':id}]
else:
result = []
return result
def _getAccessParams( self, element ):
'''
get the access host and port for the specified ce.
'''
_basePath = 'Resources/Sites'
domains = gConfig.getSections( _basePath )
if not domains[ 'OK' ]:
return domains
domains = domains[ 'Value' ]
for domain in domains:
sites = gConfig.getSections( '%s/%s' % ( _basePath, domain ) )
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
for site in sites:
ces = gConfig.getValue( '%s/%s/%s/CE' % ( _basePath, domain, site ), '' ).split(',')
ces = map(lambda str : str.strip(), ces)
if element in ces:
host = gConfig.getValue('%s/%s/%s/CEs/%s/SSHHost' % ( _basePath, domain, site, element ))
if host:
idx = host.find('/')
if idx != -1: host = host[ 0 : idx ]
return S_OK((host, 22))
else:
return S_OK((element, 8443))
return S_ERROR('%s is not a vaild CE.' % element)
def web_getComponentNames( self ):
result = None
userData = self.getSessionData()
setup = userData['setup'].split( '-' )[-1]
systemList = []
system = None
if "system" in self.request.arguments:
system = self.request.arguments[ 'system' ][-1]
componentTypes = ['Services', 'Agents']
if "ComponentType" in self.request.arguments:
componentTypes = self.request.arguments['ComponentType']
retVal = gConfig.getSections( '/Systems' )
if retVal['OK']:
systems = retVal['Value']
for i in systems:
for compType in componentTypes:
compPath = '/Systems/%s/%s/%s' % ( i, setup, compType )
retVal = gConfig.getSections( compPath )
if retVal['OK']:
components = retVal['Value']
systemList += [ {"Name":j} for j in components ]
result = { "success" : "true" , "result" : systemList }
else:
result = { "success" : "false" , "error" : result['Message'] }
self.finish( result )
def getSiteCEMapping( gridName = '' ):
""" Returns a dictionary of all sites and their CEs as a list, e.g.
{'LCG.CERN.ch':['ce101.cern.ch',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteCEMapping = {}
gridTypes = gConfig.getSections( 'Resources/Sites/', [] )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid, [] )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
for candidate in sites['Value']:
candidateCEs = gConfig.getValue( '/Resources/Sites/%s/%s/CE' % ( grid, candidate ), [] )
if candidateCEs:
siteCEMapping[candidate] = candidateCEs
else:
gLogger.debug( 'No CEs defined for site %s' % candidate )
return S_OK( siteCEMapping )
def getSiteForCE( computingElement ):
""" Given a Grid CE name this method returns the DIRAC site name.
WARNING: if two or more sites happen to have the same ceName/queueName, then only the first found is returned
"""
finalSite = ''
gridTypes = gConfig.getSections( '/Resources/Sites/', [] )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid, [] )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
if sites:
siteList = sites['Value']
for candidate in siteList:
siteCEs = gConfig.getValue( '/Resources/Sites/%s/%s/CE' % ( grid, candidate ), [] )
if computingElement in siteCEs:
finalSite = candidate
break
return S_OK( finalSite )
def getSESiteMapping( gridName = '' ):
""" Returns a dictionary of all SEs and their associated site(s), e.g.
{'CERN-RAW':'LCG.CERN.ch','CERN-RDST':'LCG.CERN.ch',...]}
Although normally one site exists for a given SE, it is possible over all
Grid types to have multiple entries.
If gridName is specified, result is restricted to that Grid type.
Assumes CS structure of: /Resources/Sites/<GRIDNAME>/<SITENAME>
"""
seSiteMapping = {}
gridTypes = gConfig.getSections( '/Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']: #gConfig returns S_ERROR for empty sections until version
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
if sites:
for candidate in sites['Value']:
siteSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
for se in siteSEs:
if se not in seSiteMapping:
seSiteMapping[se] = []
seSiteMapping[se].append( candidate )
return S_OK( seSiteMapping )
def getServicePorts(self, setup="", printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
:return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue("/DIRAC/Setup", "")
setupList = gConfig.getSections("/DIRAC/Setups", [])
if not setupList["OK"]:
return S_ERROR("Could not get /DIRAC/Setups sections")
setupList = setupList["Value"]
if not setup in setupList:
return S_ERROR("Setup %s is not in allowed list: %s" % (setup, ", ".join(setupList)))
serviceSetups = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup)
if not serviceSetups["OK"]:
return S_ERROR("Could not get /DIRAC/Setups/%s options" % setup)
serviceSetups = serviceSetups["Value"] # dict
systemList = gConfig.getSections("/Systems")
if not systemList["OK"]:
return S_ERROR("Could not get Systems sections")
systemList = systemList["Value"]
result = {}
for system in systemList:
if serviceSetups.has_key(system):
path = "/Systems/%s/%s/Services" % (system, serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList["OK"]:
self.log.warn("Could not get sections in %s" % path)
else:
servicesList = servicesList["Value"]
if not servicesList:
servicesList = []
self.log.verbose("System: %s ServicesList: %s" % (system, ", ".join(servicesList)))
for service in servicesList:
spath = "%s/%s/Port" % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose("Found port for %s/%s = %s" % (system, service, servicePort))
result["%s/%s" % (system, service)] = servicePort
else:
self.log.warn("No port found for %s" % spath)
else:
self.log.warn("%s is not defined in /DIRAC/Setups/%s" % (system, setup))
if printOutput:
print self.pPrint.pformat(result)
return S_OK(result)
def getExtraConditions(self, site):
""" Get extra conditions allowing site throttling
"""
# Find Site job limits
grid, siteName, country = site.split(".")
siteSection = "/Resources/Sites/%s/%s" % (grid, site)
result = gConfig.getSections(siteSection)
if not result["OK"]:
return result
if not "JobLimits" in result["Value"]:
return S_OK({})
result = gConfig.getSections("%s/JobLimits" % siteSection)
if not result["OK"]:
return result
sections = result["Value"]
limitDict = {}
resultDict = {}
if sections:
for section in sections:
result = gConfig.getOptionsDict("%s/JobLimits/%s" % (siteSection, section))
if not result["OK"]:
return result
optionDict = result["Value"]
if optionDict:
limitDict[section] = []
for k, v in optionDict.items():
limitDict[section].append((k, int(v)))
if not limitDict:
return S_OK({})
# Check if the site exceeding the given limits
fields = limitDict.keys()
for field in fields:
for key, value in limitDict[field]:
if int(value) > 0:
result = jobDB.getCounters("Jobs", ["Status"], {"Site": site, field: key})
if not result["OK"]:
return result
count = 0
if result["Value"]:
for countDict, number in result["Value"]:
if countDict["Status"] in ["Running", "Matched"]:
count += number
if count > value:
if not resultDict.has_key(field):
resultDict[field] = []
resultDict[field].append(key)
gLogger.verbose(
"Job Limit imposed at %s on %s/%s/%d, %d jobs already deployed"
% (site, field, key, value, count)
)
else:
if not resultDict.has_key(field):
resultDict[field] = []
resultDict[field].append(key)
gLogger.verbose("Jobs prohibited at %s for %s/%s" % (site, field, key))
return S_OK(resultDict)
def getServicePorts( self, setup = '', printOutput = False ):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
@return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue( '/DIRAC/Setup', '' )
setupList = gConfig.getSections( '/DIRAC/Setups', [] )
if not setupList['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups sections' )
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR( 'Setup %s is not in allowed list: %s' % ( setup, ', '.join( setupList ) ) )
serviceSetups = gConfig.getOptionsDict( '/DIRAC/Setups/%s' % setup )
if not serviceSetups['OK']:
return S_ERROR( 'Could not get /DIRAC/Setups/%s options' % setup )
serviceSetups = serviceSetups['Value'] #dict
systemList = gConfig.getSections( '/Systems' )
if not systemList['OK']:
return S_ERROR( 'Could not get Systems sections' )
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key( system ):
path = '/Systems/%s/%s/Services' % ( system, serviceSetups[system] )
servicesList = gConfig.getSections( path )
if not servicesList['OK']:
self.log.warn( 'Could not get sections in %s' % path )
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose( 'System: %s ServicesList: %s' % ( system, ', '.join( servicesList ) ) )
for service in servicesList:
spath = '%s/%s/Port' % ( path, service )
servicePort = gConfig.getValue( spath, 0 )
if servicePort:
self.log.verbose( 'Found port for %s/%s = %s' % ( system, service, servicePort ) )
result['%s/%s' % ( system, service )] = servicePort
else:
self.log.warn( 'No port found for %s' % spath )
else:
self.log.warn( '%s is not defined in /DIRAC/Setups/%s' % ( system, setup ) )
if printOutput:
print self.pPrint.pformat( result )
return S_OK( result )
def getExtraConditions( self, site ):
""" Get extra conditions allowing site throttling
"""
# Find Site job limits
grid = site.split( '.' )[0]
siteSection = '/Resources/Sites/%s/%s' % ( grid, site )
result = gConfig.getSections( siteSection )
if not result['OK']:
return result
if not 'JobLimits' in result['Value']:
return S_OK( {} )
result = gConfig.getSections( '%s/JobLimits' % siteSection )
if not result['OK']:
return result
sections = result['Value']
limitDict = {}
resultDict = {}
if sections:
for section in sections:
result = gConfig.getOptionsDict( '%s/JobLimits/%s' % ( siteSection, section ) )
if not result['OK']:
return result
optionDict = result['Value']
if optionDict:
limitDict[section] = []
for key, value in optionDict.items():
limitDict[section].append( ( key, int( value ) ) )
if not limitDict:
return S_OK( {} )
# Check if the site exceeding the given limits
fields = limitDict.keys()
for field in fields:
for key, value in limitDict[field]:
if int( value ) > 0:
result = gJobDB.getCounters( 'Jobs', ['Status'], {'Site':site, field:key} )
if not result['OK']:
return result
count = 0
if result['Value']:
for countDict, number in result['Value']:
if countDict['Status'] in ["Running", "Matched"]:
count += number
if count > value:
if not resultDict.has_key( field ):
resultDict[field] = []
resultDict[field].append( key )
gLogger.verbose( 'Job Limit imposed at %s on %s/%s/%d,'
' %d jobs already deployed' % ( site, field, key, value, count ) )
else:
if not resultDict.has_key( field ):
resultDict[field] = []
resultDict[field].append( key )
gLogger.verbose( 'Jobs prohibited at %s for %s/%s' % ( site, field, key ) )
return S_OK( resultDict )
def getSiteForCE( ce ):
_basePath = 'Resources/Sites'
domains = gConfig.getSections( _basePath )[ 'Value' ]
for domain in domains:
sites = gConfig.getSections( '%s/%s' % ( _basePath, domain ) )[ 'Value' ]
for site in sites:
ces = gConfig.getValue( '%s/%s/%s/CE' % ( _basePath, domain, site ) , '' ).split( ',' )
ces = map(lambda str : str.strip(), ces);
if ce in ces:
return site
def __getConnectParam(self, element, elementType):
"""
get the access host and port for the specified ce or cloud.
"""
if elementType == "ComputingElement":
_basePath = "Resources/Sites"
domains = gConfig.getSections(_basePath)
if not domains["OK"]:
return domains
domains = domains["Value"]
for domain in domains:
sites = gConfig.getSections("%s/%s" % (_basePath, domain))
if not sites["OK"]:
return sites
sites = sites["Value"]
for site in sites:
ces = gConfig.getValue("%s/%s/%s/CE" % (_basePath, domain, site), "").split(",")
ces = map(lambda str: str.strip(), ces)
if element in ces:
host = gConfig.getValue("%s/%s/%s/CEs/%s/SSHHost" % (_basePath, domain, site, element))
if host:
idx = host.find("/")
if idx != -1:
host = host[0:idx]
return S_OK((host, 22))
else:
return S_OK((element, 8443))
if elementType == "CLOUD":
_basePath = "Resources/VirtualMachines/CloudEndpoints"
_searchKey = ("ex_force_auth_url", "occiURI", "endpointURL")
endpoints = gConfig.getSections(_basePath)
if not endpoints["OK"]:
return endpoints
endpoints = endpoints["Value"]
for endpoint in endpoints:
site = gConfig.getValue("%s/%s/siteName" % (_basePath, endpoint))
if site == element:
url = None
for key in _searchKey:
url = gConfig.getValue("%s/%s/%s" % (_basePath, endpoint, key))
if url:
break
return S_OK(re.match(r"https?://(.+):([0-9]+).*", url).groups())
return S_ERROR("%s is not a vaild %s." % (element, elementType))
def web_getSelectionData( self ):
data = {}
userData = self.getSessionData()
setup = userData['setup'].split( '-' )[-1]
hosts = []
result = Registry.getHosts()
if result['OK']:
hosts = [ [i] for i in result['Value'] ]
data['Hosts'] = hosts
componentTypes = ['Services', 'Agents', 'Executors']
if "ComponentType" in self.request.arguments:
componentTypes = self.request.arguments['ComponentType']
retVal = gConfig.getSections( '/Systems' )
components = []
componentNames = []
data['ComponentModule'] = []
data['ComponentName'] = []
if retVal['OK']:
systems = retVal['Value']
for i in systems:
for compType in componentTypes:
compPath = '/Systems/%s/%s/%s' % ( i, setup, compType )
retVal = gConfig.getSections( compPath )
if retVal['OK']:
records = retVal['Value']
componentNames += [ [cnames] for cnames in records ]
for record in records:
modulepath = "%s/%s/Module" % ( compPath, record )
module = gConfig.getValue( modulepath, '' )
if module != '' and module not in components:
components += [module]
data['ComponentModule'].append( [module] )
elif record not in components and module == '':
data['ComponentModule'].append( [record] )
components += [record]
data['ComponentName'] = componentNames
data['ComponentName'].sort()
data['ComponentModule'].sort()
else:
data = { "success" : "false" , "error" : result['Message'] }
self.finish( data )
def getSitesForSE( se ):
_basePath = 'Resources/Sites'
seSites = []
domains = gConfig.getSections( _basePath )[ 'Value' ]
for domain in domains:
sites = gConfig.getSections( '%s/%s' % ( _basePath, domain ) )[ 'Value' ]
for site in sites:
ses = gConfig.getValue( '%s/%s/%s/SE' % ( _basePath, domain, site ), '' ).split( ',' )
ses = map(lambda str : str.strip(), ses);
if se in ses:
seSites.append(site)
return seSites
def getSiteSEMapping( gridName = '' ):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
siteSEMapping = {}
gridTypes = gConfig.getSections( 'Resources/Sites/' )
if not gridTypes['OK']:
gLogger.warn( 'Problem retrieving sections in /Resources/Sites' )
return gridTypes
gridTypes = gridTypes['Value']
if gridName:
if not gridName in gridTypes:
return S_ERROR( 'Could not get sections for /Resources/Sites/%s' % gridName )
gridTypes = [gridName]
gLogger.debug( 'Grid Types are: %s' % ( ', '.join( gridTypes ) ) )
for grid in gridTypes:
sites = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not sites['OK']:
gLogger.warn( 'Problem retrieving /Resources/Sites/%s section' % grid )
return sites
for candidate in sites['Value']:
candidateSEs = gConfig.getValue( '/Resources/Sites/%s/%s/SE' % ( grid, candidate ), [] )
if candidateSEs:
siteSEMapping[candidate] = candidateSEs
else:
gLogger.debug( 'No SEs defined for site %s' % candidate )
# Add Sites from the SiteLocalSEMapping in the CS
vo = getVO()
setup = getDIRACSetup()
cfgLocalSEPath = cfgPath( 'Operations', vo, setup, 'SiteLocalSEMapping' )
result = gConfig.getOptionsDict( cfgLocalSEPath )
if result['OK']:
mapping = result['Value']
for site in mapping:
ses = gConfig.getValue( cfgPath( cfgLocalSEPath, site ), [] )
if not ses:
continue
if gridName and site not in sites:
continue
if site not in siteSEMapping:
siteSEMapping[site] = []
for se in ses:
if se not in siteSEMapping[site]:
siteSEMapping[site].append( se )
return S_OK( siteSEMapping )
def getSites():
""" Get the list of all the sites defined in the CS
"""
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites' ) )
if not result['OK']:
return result
grids = result['Value']
sites = []
for grid in grids:
result = gConfig.getSections( cfgPath( gBaseResourcesSection, 'Sites', grid ) )
if not result['OK']:
return result
sites += result['Value']
return S_OK( sites )
def getSites():
""" Get the list of all the sites defined in the CS
"""
result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites"))
if not result["OK"]:
return result
grids = result["Value"]
sites = []
for grid in grids:
result = gConfig.getSections(cfgPath(gBaseResourcesSection, "Sites", grid))
if not result["OK"]:
return result
sites += result["Value"]
return S_OK(sites)
def web_getSysInfo(self):
userData = self.getSessionData()
DN = str(userData["user"]["DN"])
group = str(userData["user"]["group"])
# TODO: remove hosts code after v6r7 since it will be built-in
result = gConfig.getSections("/Registry/Hosts")
if not result[ "Value" ]:
self.finish({ "success" : "false" , "error" : result[ "Message" ] })
return
hosts = result[ "Value" ]
callback = []
import pprint
for host in hosts:
client = SystemAdministratorClient(host , None , delegatedDN=DN ,
delegatedGroup=group)
resultHost = yield self.threadTask(client.getHostInfo)
if resultHost[ "OK" ]:
rec = resultHost["Value"]
rec["Host"] = host
callback.append(resultHost["Value"])
else:
callback.append({"Host":host})
total = len(callback)
if not total > 0:
self.finish({ "success" : "false" , "error" : "No system information found" })
return
self.finish({ "success" : "true" , "result" : callback , "total" : total })
def getRawSchema():
"""
Load the schema from the CS
"""
base = "%s/Schema" % ( BASECS )
schema = []
explore = [ ( "", schema ) ]
while len( explore ):
parentName, parentData = explore.pop( 0 )
fullName = "%s/%s" % ( base, parentName )
result = gConfig.getSections( fullName )
if not result[ 'OK' ]:
continue
sectionsList = result[ 'Value' ]
for sName in sectionsList:
sData = []
parentData.append( ( "%s/%s" % ( parentName, sName ), sData ) )
explore.append( ( sName, sData ) )
result = gConfig.getOptions( fullName )
if not result[ 'OK' ]:
continue
optionsList = result[ 'Value' ]
for opName in optionsList:
opVal = gConfig.getValue( "%s/%s" % ( fullName, opName ) )
if opVal.find( "link|" ) == 0:
parentData.append( ( "link", opName, opVal[5:] ) )
else:
parentData.append( ( "app", opName, opVal ) )
return schema
def getVOMSAttributes( self, proxy, switch = "all" ):
"""
Return VOMS proxy attributes as list elements if switch="all" (default) OR
return the string prepared to be stored in DB if switch="db" OR
return the string of elements to be used as the option string in voms-proxy-init
if switch="option".
If a given proxy is a grid proxy, then function will return an empty list.
"""
# Get all possible info from voms proxy
result = self.getVOMSProxyInfo( proxy, "all" )
if not result["OK"]:
return S_ERROR( 'Failed to extract info from proxy: %s' % result[ 'Message' ] )
vomsInfoOutput = List.fromChar( result["Value"], "\n" )
#Get a list of known VOMS attributes
validVOMSAttrs = []
result = gConfig.getOptions( "/Registry/VOMS/Mapping" )
if result[ 'OK' ]:
for group in result[ 'Value' ]:
vA = gConfig.getValue( "/Registry/VOMS/Mapping/%s" % group, "" )
if vA and vA not in validVOMSAttrs:
validVOMSAttrs.append( vA )
result = gConfig.getSections( "/Registry/Groups" )
if result[ 'OK' ]:
for group in result[ 'Value' ]:
vA = gConfig.getValue( "/Registry/Groups/%s/VOMSRole" % group, "" )
if vA and vA not in validVOMSAttrs:
validVOMSAttrs.append( vA )
# Parse output of voms-proxy-info command
attributes = []
voName = ''
nickName = ''
for line in vomsInfoOutput:
fields = List.fromChar( line, ":" )
key = fields[0]
value = " ".join( fields[1:] )
if key == "VO":
voName = value
elif key == "attribute":
# Cut off unsupported Capability selection part
if value.find( "nickname" ) == 0:
nickName = "=".join( List.fromChar( value, "=" )[ 1: ] )
else:
value = value.replace( "/Capability=NULL" , "" )
value = value.replace( "/Role=NULL" , "" )
if value and value not in attributes and value in validVOMSAttrs:
attributes.append( value )
# Sorting and joining attributes
if switch == "db":
returnValue = ":".join( attributes )
elif switch == "option":
if len( attributes ) > 1:
returnValue = voName + " -order " + ' -order '.join( attributes )
elif attributes:
returnValue = voName + ":" + attributes[0]
else:
returnValue = voName
elif switch == 'nickname':
returnValue = nickName
elif switch == 'all':
returnValue = attributes
return S_OK( returnValue )
def _lookForCE(self):
knownces = self.am_getOption('BannedCEs', [])
result = gConfig.getSections('/Resources/Sites')
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict('/Resources/Sites/%s/%s' %
(grid, site))['Value']
ces = List.fromChar(opt.get('CE', ''))
knownces += ces
response = ldapCEState('', vo=self.vo)
if not response['OK']:
self.log.error("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCEState, '', self.vo)
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split(":")[0]
if not cename in knownces:
newces[cename] = None
self.log.debug("newce", cename)
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCluster, ce)
continue
clusters = response['Value']
if len(clusters) != 1:
self.log.warn("Error in cluster leng",
" CE %s Leng %d" % (ce, len(clusters)))
if len(clusters) == 0:
continue
cluster = clusters[0]
fkey = cluster.get('GlueForeignKey', [])
if type(fkey) == type(''):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count('GlueSiteUniqueID'):
nameBDII = entry.split('=')[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % (ce, nameBDII)
self.log.info(cestring)
response = ldapCE(ce)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(ldapCE, ce)
continue
ceinfos = response['Value']
if len(ceinfos):
ceinfo = ceinfos[0]
systemName = ceinfo.get('GlueHostOperatingSystemName',
'Unknown')
systemVersion = ceinfo.get('GlueHostOperatingSystemVersion',
'Unknown')
systemRelease = ceinfo.get('GlueHostOperatingSystemRelease',
'Unknown')
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % (
systemName, systemVersion, systemRelease)
self.log.info(osstring)
response = ldapCEState(ce, vo=self.vo)
if not response['OK']:
self.log.warn("Error during BDII request", response['Message'])
response = self._checkAlternativeBDIISite(
ldapCEState, ce, self.vo)
continue
newcestring = "\n\n%s\n%s" % (cestring, osstring)
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get('GlueCEUniqueID', 'UnknownName')
queuestatus = cestate.get('GlueCEStateStatus', 'UnknownStatus')
queuestring = "%s %s" % (queuename, queuestatus)
self.log.info(queuestring)
newcestring += "\n%s" % queuestring
if queuestatus.count('Production'):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append(
'dirac-admin-add-site DIRACSiteName %s %s' %
(nameBDII, ce))
if body:
body = "We are glade to inform You about new CE(s) possibly suitable for %s:\n" % self.vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % (body, possibleNewSite)
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return S_OK()
def registerUser(self, paramcopy):
# Unfortunately there is no way to get rid of empty text values in JS, so i have to hardcode it on server side. Hate it!
default_values = [
"John Smith", "jsmith", "*****@*****.**",
"+33 9 10 00 10 00", "Select prefered virtual organization(s)"
]
default_values.append("Select your country")
default_values.append(
"Any additional information you want to provide to administrators")
dn = getUserDN()
username = getUsername()
if not username == "anonymous":
return {
"success":
"false",
"error":
"You are already registered in DIRAC with username: %s" %
username
}
else:
if not dn:
return {
"success":
"false",
"error":
"You have to load certificate to your browser before trying to register"
}
body = ""
userMail = False
vo = []
for i in paramcopy:
if not paramcopy[i] in default_values:
if i == "email":
userMail = paramcopy[i]
if i == "vo":
vo = paramcopy[i].split(",")
body = body + str(i) + ' - "' + str(paramcopy[i]) + '"\n'
if not userMail:
return {
"success": "false",
"error": "Can not get your email from the request"
}
gLogger.info("!!! VO: ", vo)
# TODO Check for previous requests
if not len(vo) > 0:
mails = gConfig.getValue("/Website/UserRegistrationEmail", [])
else:
mails = []
for i in vo:
i = i.strip()
voadm = gConfig.getValue("/Registry/VO/%s/VOAdmin" % i, "")
failsafe = False
if voadm:
tmpmail = gConfig.getValue(
"/Registry/Users/%s/Email" % voadm, "")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error(
"Can not find value for option /Registry/Users/%s/Email Trying failsafe option"
% voadm)
failsafe = True
else:
gLogger.error(
"Can not find value for option /Registry/VO/%s/VOAdmin Trying failsafe option"
% i)
failsafe = True
if failsafe:
failsafe = gConfig.getValue(
"/Website/UserRegistrationEmail", [])
if len(failsafe) > 0:
for j in failsafe:
mails.append(j)
else:
gLogger.error(
"Can not find value for failsafe option /Website/UserRegistrationEmail User registration for VO %s is failed"
% i)
mails = uniqueElements(mails)
if not len(mails) > 0:
groupList = list()
allGroups = gConfig.getSections("/Registry/Groups")
if not allGroups["OK"]:
return {
"success": "false",
"error": "No groups found at this DIRAC installation"
}
allGroups = allGroups["Value"]
for j in allGroups:
props = getProperties(j)
if "UserAdministrator" in props: # property which usd for user administration
groupList.append(j)
groupList = uniqueElements(groupList)
if not len(groupList) > 0:
return {
"success": "false",
"error":
"No groups, resposible for user administration, found"
}
userList = list()
for i in groupList:
users = gConfig.getValue("/Registry/Groups/%s/Users" % i, [])
for j in users:
userList.append(j)
userList = uniqueElements(userList)
if not len(userList) > 0:
return {
"success":
"false",
"error":
"Can not find a person resposible for user administration, your request can not be approuved"
}
mails = list()
mail2name = dict()
for i in userList:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i, "")
if tmpmail:
mails.append(tmpmail)
else:
gLogger.error(
"Can not find value for option /Registry/Users/%s/Email"
% i)
mails = uniqueElements(mails)
if not len(mails) > 0:
return {
"success":
"false",
"error":
"Can not find an email of the person resposible for the users administration, your request can not be approuved"
}
gLogger.info("Admins emails: ", mails)
if not len(mails) > 0:
return {
"success": "false",
"error": "Can not find any emails of DIRAC Administrators"
}
allUsers = gConfig.getSections("/Registry/Users")
if not allUsers["OK"]:
return {
"success": "false",
"error": "No users found at this DIRAC installation"
}
allUsers = allUsers["Value"]
mail2name = dict()
for i in allUsers:
tmpmail = gConfig.getValue("/Registry/Users/%s/Email" % i, "")
if tmpmail in mails:
mail2name[tmpmail] = gConfig.getValue(
"/Registry/Users/%s/FullName" % i, i)
sentFailed = list()
sentSuccess = list()
errorMessage = list()
ntc = NotificationClient(getRPCClient)
for i in mails:
i = i.strip()
result = ntc.sendMail(i, "New user has registered", body, userMail,
False)
if not result["OK"]:
sentFailed.append(mail2name[i])
errorMessage.append(result["Message"])
else:
sentSuccess.append(mail2name[i])
gLogger.info("Sent success: ", sentSuccess)
gLogger.info("Sent failure: ", sentFailed)
errorMessage = uniqueElements(errorMessage)
if len(sentSuccess) == 0:
if not len(errorMessage) > 0:
return {
"success":
"false",
"error":
"No messages were sent to administrators due techincal reasons"
}
errorMessage = ", ".join(errorMessage)
return {"success": "false", "error": errorMessage}
sName = ", ".join(sentSuccess)
fName = ", ".join(sentFailed)
if len(sentFailed) > 0:
return {
"success":
"true",
"result":
"Your registration request were sent successfuly to %s. Failed to sent request to %s."
% (sName, fName)
}
return {
"success":
"true",
"result":
"Your registration request were sent successfuly to %s." % sName
}
def __init__(self, *args, **kwargs):
""" c'tor """
# # call base class ctor
AgentModule.__init__(self, *args, **kwargs)
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption("RequestsPerCycle",
self.__requestsPerCycle)
self.log.info("Requests/cycle = %d" % self.__requestsPerCycle)
self.__minProcess = self.am_getOption("MinProcess", self.__minProcess)
self.log.info("ProcessPool min process = %d" % self.__minProcess)
self.__maxProcess = self.am_getOption("MaxProcess", 4)
self.log.info("ProcessPool max process = %d" % self.__maxProcess)
self.__queueSize = self.am_getOption("ProcessPoolQueueSize",
self.__queueSize)
self.log.info("ProcessPool queue size = %d" % self.__queueSize)
self.__poolTimeout = int(
self.am_getOption("ProcessPoolTimeout", self.__poolTimeout))
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout)
self.__poolSleep = int(
self.am_getOption("ProcessPoolSleep", self.__poolSleep))
self.log.info("ProcessPool sleep time = %d seconds" % self.__poolSleep)
self.__taskTimeout = int(
self.am_getOption("ProcessTaskTimeout", self.__taskTimeout))
self.log.info("ProcessTask timeout = %d seconds" % self.__taskTimeout)
# # keep config path and agent name
self.agentName = self.am_getModuleParam("fullName")
self.__configPath = PathFinder.getAgentSection(self.agentName)
# # operation handlers over here
opHandlersPath = "%s/%s" % (self.__configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
self.log.error(opHandlers["Message"])
raise AgentConfigError(
"OperationHandlers section not found in CS under %s" %
self.__configPath)
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
self.operationHandlers = []
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % (opHandlersPath, opHandler)
opLocation = gConfig.getValue(opHandlerPath, "")
if not opLocation:
self.log.error("%s not set for %s operation handler" %
(opHandlerPath, opHandler))
continue
self.timeOuts[opHandler] = {
"PerFile": self.__fileTimeout,
"PerOperation": self.__operationTimeout
}
opTimeout = gConfig.getValue(
"%s/%s/TimeOut" % (opHandlersPath, opHandler), 0)
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue(
"%s/%s/TimeOutPerFile" % (opHandlersPath, opHandler), 0)
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.operationHandlers.append(opLocation)
self.log.info("Operation handlers:")
for itemTuple in enumerate(self.operationHandlers):
self.log.info("[%s] %s" % itemTuple)
# # handlers dict
self.handlersDict = dict()
# # common monitor activity
gMonitor.registerActivity("Iteration", "Agent Loops",
"RequestExecutingAgent", "Loops/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Processed", "Request Processed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
gMonitor.registerActivity("Done", "Request Completed",
"RequestExecutingAgent", "Requests/min",
gMonitor.OP_SUM)
# # create request dict
self.__requestCache = dict()
self.FTSMode = self.am_getOption("FTSMode", False)
def doTheWhizardInstallation():
"""Do the instalation for new whizard version Copy libraries, create tarball,
upload processList file add entry in configuration system
"""
res = checkSLCVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
res = checkGFortranVersion()
if not res['OK']:
gLogger.error(res['Message'])
dexit(1)
cliParams = Params()
cliParams.registerSwitches()
Script.parseCommandLine( ignoreErrors= False)
whizardResultFolder = cliParams.path
platform = cliParams.platform
whizard_version = cliParams.version
appVersion = whizard_version
beam_spectra_version = cliParams.beam_spectra
if not whizardResultFolder or not whizard_version or not beam_spectra_version:
Script.showHelp()
dexit(2)
from ILCDIRAC.Core.Utilities.ProcessList import ProcessList
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from ILCDIRAC.Core.Utilities.FileUtils import upload
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
diracAdmin = DiracAdmin()
modifiedCS = False
softwareSection = "/Operations/Defaults/AvailableTarBalls"
processlistLocation = "ProcessList/Location"
appName = "whizard"
ops = Operations()
path_to_process_list = ops.getValue(processlistLocation, "")
if not path_to_process_list:
gLogger.error("Could not find process list location in CS")
dexit(2)
gLogger.verbose("Getting process list from file catalog")
datMan = DataManager()
res = datMan.getFile(path_to_process_list)
if not res['OK']:
gLogger.error("Error while getting process list from storage")
dexit(2)
gLogger.verbose("done")
##just the name of the local file in current working directory
processlist = os.path.basename(path_to_process_list)
if not os.path.exists(processlist):
gLogger.error("Process list does not exist locally")
dexit(2)
pl = ProcessList(processlist)
startDir = os.getcwd()
inputlist = {}
os.chdir(whizardResultFolder)
folderlist = os.listdir(whizardResultFolder)
whiz_here = folderlist.count("whizard")
if whiz_here == 0:
gLogger.error("whizard executable not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizprc_here = folderlist.count("whizard.prc")
if whizprc_here == 0:
gLogger.error("whizard.prc not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
whizmdl_here = folderlist.count("whizard.mdl")
if whizmdl_here == 0:
gLogger.error("whizard.mdl not found in %s, please check" % whizardResultFolder)
os.chdir(startDir)
dexit(2)
gLogger.verbose("Preparing process list")
## FIXME:: What is this doing exactly? Is this necessary? -- APS, JFS
for f in folderlist:
if f.count(".in"):
infile = open(f, "r")
found_detail = False
for line in infile:
if line.count("decay_description"):
currprocess = f.split(".template.in")[0]
inputlist[currprocess] = {}
inputlist[currprocess]["InFile"] = f.rstrip("~")
inputlist[currprocess]["Detail"] = line.split("\"")[1]
found_detail = True
if line.count("process_id") and found_detail:
process_id = line.split("\"")[1]
inputlist[currprocess]["Model"] = ""
inputlist[currprocess]["Generator"] = ""
inputlist[currprocess]["Restrictions"] = ""
for process in process_id.split():
print "Looking for detail of process %s" % (process)
process_detail = getDetailsFromPRC("whizard.prc", process)
inputlist[currprocess]["Model"] = process_detail["Model"]
inputlist[currprocess]["Generator"] = process_detail["Generator"]
if len(inputlist[currprocess]["Restrictions"]):
inputlist[currprocess]["Restrictions"] = inputlist[currprocess]["Restrictions"] + ", " + process_detail["Restrictions"]
else:
inputlist[currprocess]["Restrictions"] = process_detail["Restrictions"]
#if len(inputlist[currprocess].items()):
# inputlist.append(processdict)
## END FIXEME
##Update inputlist with what was found looking in the prc file
processes = readPRCFile("whizard.prc")
inputlist.update(processes)
##get from cross section files the cross sections for the processes in inputlist
#Need full process list
for f in folderlist:
if f.count("cross_sections_"):
crossfile = open(f, "r")
for line in crossfile:
line = line.rstrip().lstrip()
if not len(line):
continue
if line[0] == "#" or line[0] == "!":
continue
if len(line.split()) < 2:
continue
currprocess = line.split()[0]
if currprocess in inputlist:
inputlist[currprocess]['CrossSection'] = line.split()[1]
gLogger.notice("Preparing Tarball")
##Make a folder in the current directory of the user to store the whizard libraries, executable et al.
localWhizardFolderRel = ("whizard" + whizard_version) # relative path
localWhizardFolder = os.path.join(startDir, localWhizardFolderRel)
if not os.path.exists(localWhizardFolder):
os.makedirs(localWhizardFolder)
localWhizardLibFolder = os.path.join(localWhizardFolder,'lib')
if os.path.exists(localWhizardLibFolder):
shutil.rmtree(localWhizardLibFolder)
os.makedirs(localWhizardLibFolder) ##creates the lib folder
whizardLibraries = getListOfLibraries(os.path.join(whizardResultFolder, "whizard"))
copyLibsCall = ["rsync","-avzL"]
for lib in whizardLibraries:
copyLibsCall.append(lib)
copyLibsCall.append(localWhizardLibFolder)
subprocess.Popen(copyLibsCall, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
for fileName in folderlist:
shutil.copy(fileName, localWhizardFolder)
##Get the list of md5 sums for all the files in the folder to be tarred
os.chdir( localWhizardFolder )
subprocess.call(["find . -type f -exec md5sum {} > ../md5_checksum.md5 \\; && mv ../md5_checksum.md5 ."], shell=True)
os.chdir(startDir)
##Create the Tarball
gLogger.notice("Creating Tarball...")
appTar = localWhizardFolder + ".tgz"
myappTar = tarfile.open(appTar, "w:gz")
myappTar.add(localWhizardFolderRel)
myappTar.close()
md5sum = md5.md5(open( appTar, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
gLogger.notice("Registering new Tarball in CS")
tarballurl = {}
av_platforms = gConfig.getSections(softwareSection, [])
if av_platforms['OK']:
if platform not in av_platforms['Value']:
gLogger.error("Platform %s unknown, available are %s." % (platform, ", ".join(av_platforms['Value'])))
gLogger.error("If yours is missing add it in CS")
dexit(255)
else:
gLogger.error("Could not find all platforms available in CS")
dexit(255)
av_apps = gConfig.getSections("%s/%s" % (softwareSection, platform), [])
if not av_apps['OK']:
gLogger.error("Could not find all applications available in CS")
dexit(255)
if appName.lower() in av_apps['Value']:
versions = gConfig.getSections("%s/%s/%s" % (softwareSection, platform, appName.lower()),
[])
if not versions['OK']:
gLogger.error("Could not find all versions available in CS")
dexit(255)
if appVersion in versions['Value']:
gLogger.error('Application %s %s for %s already in CS, nothing to do' % (appName.lower(), appVersion, platform))
dexit(0)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform, appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()), "")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
if result['OK']:
modifiedCS = True
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
else:
result = diracAdmin.csSetOption("%s/%s/%s/%s/TarBall" % (softwareSection, platform,
appName.lower(), appVersion),
os.path.basename(appTar))
if result['OK']:
modifiedCS = True
tarballurl = gConfig.getOption("%s/%s/%s/TarBallURL" % (softwareSection, platform, appName.lower()),
"")
if len(tarballurl['Value']) > 0:
res = upload(tarballurl['Value'], appTar)
if not res['OK']:
gLogger.error("Upload to %s failed" % tarballurl['Value'])
dexit(255)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Md5Sum" % (softwareSection, platform, appName.lower(), appVersion),
md5sum)
result = diracAdmin.csSetOption("%s/%s/%s/%s/Dependencies/beam_spectra/version" % (softwareSection,
platform,
appName.lower(),
appVersion),
beam_spectra_version)
gLogger.verbose("Done uploading the tar ball")
os.remove(appTar)
#Set for all new processes the TarBallURL
for process in inputlist.keys():
inputlist[process]['TarBallCSPath'] = tarballurl['Value'] + os.path.basename(appTar)
pl.updateProcessList(inputlist)
pl.writeProcessList()
raw_input("Do you want to upload the process list? Press ENTER to proceed or CTRL-C to abort!")
pl.uploadProcessListToFileCatalog(path_to_process_list, appVersion)
#Commit the changes if nothing has failed and the CS has been modified
if modifiedCS:
result = diracAdmin.csCommitChanges(False)
gLogger.verbose(result)
gLogger.notice('All done OK!')
dexit(0)
def getVMTypes(siteList=None, ceList=None, vmTypeList=None, vo=None):
"""Get CE/vmType options filtered by the provided parameters."""
result = gConfig.getSections("/Resources/Sites")
if not result["OK"]:
return result
resultDict = {}
grids = result["Value"]
for grid in grids:
result = gConfig.getSections("/Resources/Sites/%s" % grid)
if not result["OK"]:
continue
sites = result["Value"]
for site in sites:
if siteList is not None and site not in siteList:
continue
if vo:
voList = gConfig.getValue(
"/Resources/Sites/%s/%s/VO" % (grid, site), [])
if voList and vo not in voList:
continue
result = gConfig.getSections("/Resources/Sites/%s/%s/Cloud" %
(grid, site))
if not result["OK"]:
continue
ces = result["Value"]
for ce in ces:
if ceList is not None and ce not in ceList:
continue
if vo:
voList = gConfig.getValue(
"/Resources/Sites/%s/%s/Cloud/%s/VO" %
(grid, site, ce), [])
if voList and vo not in voList:
continue
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/Cloud/%s" % (grid, site, ce))
if not result["OK"]:
continue
ceOptionsDict = result["Value"]
result = gConfig.getSections(
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes" %
(grid, site, ce))
if not result["OK"]:
result = gConfig.getSections(
"/Resources/Sites/%s/%s/Cloud/%s/Images" %
(grid, site, ce))
if not result["OK"]:
return result
vmTypes = result["Value"]
for vmType in vmTypes:
if vmTypeList is not None and vmType not in vmTypeList:
continue
if vo:
voList = gConfig.getValue(
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/VO" %
(grid, site, ce, vmType), [])
if not voList:
voList = gConfig.getValue(
"/Resources/Sites/%s/%s/Cloud/%s/Images/%s/VO"
% (grid, site, ce, vmType), [])
if voList and vo not in voList:
continue
resultDict.setdefault(site, {})
resultDict[site].setdefault(ce, ceOptionsDict)
resultDict[site][ce].setdefault("VMTypes", {})
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s" %
(grid, site, ce, vmType))
if not result["OK"]:
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/Cloud/%s/Images/%s" %
(grid, site, ce, vmType))
if not result["OK"]:
continue
vmTypeOptionsDict = result["Value"]
resultDict[site][ce]["VMTypes"][vmType] = vmTypeOptionsDict
return S_OK(resultDict)
def getComponentsStatus(self, conditionDict={}):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
"""
result = self.__getComponents(conditionDict)
if not result['OK']:
return result
statusSet = result['Value']
requiredComponents = {}
result = gConfig.getSections("/DIRAC/Setups")
if not result['OK']:
return result
for setup in result['Value']:
if not self.__checkCondition(conditionDict, "Setup", setup):
continue
# Iterate through systems
result = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup)
if not result['OK']:
return result
systems = result['Value']
for system in systems:
instance = systems[system]
# Check defined agents and serviecs
for cType in ('agent', 'service'):
# Get entries for the instance of a system
result = gConfig.getSections(
"/Systems/%s/%s/%s" %
(system, instance, "%ss" % cType.capitalize()))
if not result['OK']:
self.log.warn(
"Opps, sytem seems to be defined wrong\n",
"System %s at %s: %s" %
(system, instance, result['Message']))
continue
components = result['Value']
for component in components:
componentName = "%s/%s" % (system, component)
compDict = self.__getComponentDefinitionFromCS(
system, setup, instance, cType, component)
if self.__componentMatchesCondition(
compDict, requiredComponents, conditionDict):
statusSet.addUniqueToSet(requiredComponents,
compDict)
# Walk the URLs
result = gConfig.getOptionsDict("/Systems/%s/%s/URLs" %
(system, instance))
if not result['OK']:
self.log.warn(
"There doesn't to be defined the URLs section for %s in %s instance"
% (system, instance))
else:
serviceURLs = result['Value']
for service in serviceURLs:
for url in List.fromChar(serviceURLs[service]):
loc = url[url.find("://") + 3:]
iS = loc.find("/")
componentName = loc[iS + 1:]
loc = loc[:iS]
hostname, port = loc.split(":")
compDict = {
'ComponentName': componentName,
'Type': 'service',
'Setup': setup,
'Host': hostname,
'Port': int(port)
}
if self.__componentMatchesCondition(
compDict, requiredComponents,
conditionDict):
statusSet.addUniqueToSet(
requiredComponents, compDict)
# WALK THE DICT
statusSet.setComponentsAsRequired(requiredComponents)
return S_OK((statusSet.getRequiredComponents(), self.__mainFields[1:] +
self.__versionFields + ('Status', 'Message')))
def getSiteSEMapping(self):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
"""
if self.siteSEMapping:
return S_OK(self.siteSEMapping)
# Get the list of SEs and keep a mapping of those using an Alias or a
# BaseSE
storageElements = gConfig.getSections('Resources/StorageElements')
if not storageElements['OK']:
gLogger.warn('Problem retrieving storage elements',
storageElements['Message'])
return storageElements
storageElements = storageElements['Value']
equivalentSEs = {}
for se in storageElements:
for option in ('BaseSE', 'Alias'):
originalSE = gConfig.getValue(
'Resources/StorageElements/%s/%s' % (se, option))
if originalSE:
equivalentSEs.setdefault(originalSE, []).append(se)
break
siteSEMapping = {}
gridTypes = gConfig.getSections('Resources/Sites/')
if not gridTypes['OK']:
gLogger.warn(
'Problem retrieving sections in /Resources/Sites', gridTypes['Message'])
return gridTypes
gridTypes = gridTypes['Value']
gLogger.debug('Grid Types are: %s' % (', '.join(gridTypes)))
# Get a list of sites and their local SEs
siteSet = set()
storageElementSet = set()
siteSEMapping[LOCAL] = {}
for grid in gridTypes:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
gLogger.warn('Problem retrieving /Resources/Sites/%s section' % grid)
return result
sites = result['Value']
siteSet.update(sites)
for site in sites:
candidateSEs = gConfig.getValue(
'/Resources/Sites/%s/%s/SE' % (grid, site), [])
if candidateSEs:
candidateSEs += [
eqSE for se in candidateSEs for eqSE in equivalentSEs.get(se, [])]
siteSEMapping[LOCAL].setdefault(site, set()).update(candidateSEs)
storageElementSet.update(candidateSEs)
# Add Sites from the SiteSEMappingByProtocol in the CS
siteSEMapping[PROTOCOL] = {}
cfgLocalSEPath = cfgPath('SiteSEMappingByProtocol')
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if result['OK']:
sites = result['Value']
for site in sites:
candidates = set(self.__opsHelper.getValue(
cfgPath(cfgLocalSEPath, site), []))
ses = set(resolveSEGroup(candidates - siteSet)
) | (candidates & siteSet)
# If a candidate is a site, then all local SEs are eligible
for candidate in ses & siteSet:
ses.remove(candidate)
ses.update(siteSEMapping[LOCAL][candidate])
siteSEMapping[PROTOCOL].setdefault(site, set()).update(ses)
# Add Sites from the SiteSEMappingByDownload in the CS, else
# SiteLocalSEMapping (old convention)
siteSEMapping[DOWNLOAD] = {}
cfgLocalSEPath = cfgPath('SiteSEMappingByDownload')
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if not result['OK']:
cfgLocalSEPath = cfgPath('SiteLocalSEMapping')
result = self.__opsHelper.getOptionsDict(cfgLocalSEPath)
if result['OK']:
sites = result['Value']
for site in sites:
candidates = set(self.__opsHelper.getValue(
cfgPath(cfgLocalSEPath, site), []))
ses = set(resolveSEGroup(candidates - siteSet)
) | (candidates & siteSet)
# If a candidate is a site, then all local SEs are eligible
for candidate in ses & siteSet:
ses.remove(candidate)
ses.update(siteSEMapping[LOCAL][candidate])
siteSEMapping[DOWNLOAD].setdefault(site, set()).update(ses)
self.siteSEMapping = siteSEMapping
# Add storage elements that may not be associated with a site
result = gConfig.getSections('/Resources/StorageElements')
if not result['OK']:
gLogger.warn(
'Problem retrieving /Resources/StorageElements section', result['Message'])
return result
self.storageElementSet = storageElementSet | set(result['Value'])
self.siteSet = siteSet
return S_OK(siteSEMapping)
def setSiteProtocols(self, site, protocolsList, printOutput=False):
"""Allows to set the defined protocols for each SE for a given site.
"""
result = self.__checkSiteIsValid(site)
if not result['OK']:
return result
siteSection = '/Resources/Sites/%s/%s/SE' % (site.split('.')[0], site)
siteSEs = gConfig.getValue(siteSection, [])
if not siteSEs:
return S_ERROR('No SEs found for site %s in section %s' %
(site, siteSection))
defaultProtocols = gConfig.getValue(
'/Resources/StorageElements/DefaultProtocols', [])
self.log.verbose('Default list of protocols are %s' %
(string.join(defaultProtocols, ', ')))
for protocol in protocolsList:
if not protocol in defaultProtocols:
return S_ERROR(
'Requested to set protocol %s in list but %s is not in default list of protocols:\n%s'
%
(protocol, protocol, string.join(defaultProtocols, ', ')))
modifiedCS = False
result = self._promptUser(
'Do you want to add the following default protocols: %s for SE(s):\n%s'
% (string.join(protocolsList, ', '), string.join(siteSEs, ', ')))
if not result['OK']:
return result
for se in siteSEs:
sections = gConfig.getSections('/Resources/StorageElements/%s/' %
(se))
if not sections['OK']:
return sections
for section in sections['Value']:
if gConfig.getValue(
'/Resources/StorageElements/%s/%s/ProtocolName' %
(se, section), '') == 'SRM2':
path = '/Resources/StorageElements/%s/%s/ProtocolsList' % (
se, section)
self.log.verbose('Setting %s to %s' %
(path, string.join(protocolsList, ', ')))
result = self.csSetOption(path,
string.join(protocolsList, ', '))
if not result['OK']:
return result
modifiedCS = True
if modifiedCS:
result = self.csCommitChanges(False)
if not result['OK']:
return S_ERROR('CS Commit failed with message = %s' %
(result['Message']))
else:
if printOutput:
print 'Successfully committed changes to CS'
else:
if printOutput:
print 'No modifications to CS required'
return S_OK()
def loadModules(self, modulesList, hideExceptions=False):
"""
Load all modules required in moduleList
"""
for modName in modulesList:
gLogger.verbose("Checking %s" % modName)
#if it's a executor modName name just load it and be done with it
if modName.find("/") > -1:
gLogger.verbose(
"Module %s seems to be a valid name. Try to load it!" %
modName)
result = self.loadModule(modName,
hideExceptions=hideExceptions)
if not result['OK']:
return result
continue
#Check if it's a system name
#Look in the CS
system = modName
#Can this be generated with sectionFinder?
csPath = "%s/Executors" % PathFinder.getSystemSection(
system, (system, ))
gLogger.verbose("Exploring %s to discover modules" % csPath)
result = gConfig.getSections(csPath)
if result['OK']:
#Add all modules in the CS :P
for modName in result['Value']:
result = self.loadModule("%s/%s" % (system, modName),
hideExceptions=hideExceptions)
if not result['OK']:
return result
#Look what is installed
parentModule = None
for rootModule in getInstalledExtensions():
if system.find("System") != len(system) - 6:
parentImport = "%s.%sSystem.%s" % (rootModule, system,
self.__csSuffix)
else:
parentImport = "%s.%s.%s" % (rootModule, system,
self.__csSuffix)
#HERE!
result = self.__recurseImport(parentImport)
if not result['OK']:
return result
parentModule = result['Value']
if parentModule:
break
if not parentModule:
continue
parentPath = parentModule.__path__[0]
gLogger.notice("Found modules path at %s" % parentImport)
for entry in os.listdir(parentPath):
if entry[-3:] != ".py" or entry == "__init__.py":
continue
if not os.path.isfile(os.path.join(parentPath, entry)):
continue
modName = "%s/%s" % (system, entry[:-3])
gLogger.verbose("Trying to import %s" % modName)
result = self.loadModule(modName,
hideExceptions=hideExceptions,
parentModule=parentModule)
return S_OK()
Get status of the available Storage Elements
Usage:
%s [<options>]
""" % Script.scriptName)
Script.parseCommandLine()
import DIRAC
from DIRAC import gConfig, gLogger
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.Core.Utilities.List import sortList
storageCFGBase = "/Resources/StorageElements"
res = gConfig.getSections(storageCFGBase, True)
if not res['OK']:
gLogger.error('Failed to get storage element info')
gLogger.error(res['Message'])
DIRAC.exit(-1)
gLogger.info("%s %s %s" % ('Storage Element'.ljust(25),
'Read Status'.rjust(15), 'Write Status'.rjust(15)))
seList = sortList(res['Value'])
resourceStatus = ResourceStatus()
res = resourceStatus.getStorageElementStatus(seList)
if not res['OK']:
gLogger.error("Failed to get StorageElement status for %s" % str(seList))
def getImages( siteList = None, ceList = None, imageList = None, vo = None ):
""" Get CE/image options according to the specified selection
"""
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return result
resultDict = {}
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
continue
sites = result['Value']
for site in sites:
if siteList is not None and not site in siteList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/VO' % ( grid, site ), [] )
if voList and not vo in voList:
continue
result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud' % ( grid, site ) )
if not result['OK']:
continue
ces = result['Value']
for ce in ces:
if ceList is not None and ce not in ceList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/VO' % ( grid, site, ce ), [] )
if voList and not vo in voList:
continue
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s' % ( grid, site, ce ) )
if not result['OK']:
continue
ceOptionsDict = result['Value']
result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud/%s/VMTypes' % ( grid, site, ce ) )
if not result['OK']:
result = gConfig.getSections( '/Resources/Sites/%s/%s/Cloud/%s/Images' % ( grid, site, ce ) )
if not result['OK']:
return result
images = result['Value']
for image in images:
if imageList is not None and not image in imageList:
continue
if vo:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s/VO' % ( grid, site, ce, image ), [] )
if not voList:
voList = gConfig.getValue( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s/VO' % ( grid, site, ce, image ), [] )
if voList and not vo in voList:
continue
resultDict.setdefault( site, {} )
resultDict[site].setdefault( ce, ceOptionsDict )
resultDict[site][ce].setdefault( 'Images', {} )
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/VMTypes/%s' % ( grid, site, ce, image ) )
if not result['OK']:
result = gConfig.getOptionsDict( '/Resources/Sites/%s/%s/Cloud/%s/Images/%s' % ( grid, site, ce, image ) )
if not result['OK']:
continue
imageOptionsDict = result['Value']
resultDict[site][ce]['Images'][image] = imageOptionsDict
return S_OK( resultDict )
def _getCSDict(self):
""" Gets minimal info for running a pilot, from the CS
:returns: pilotDict (containing pilots run info)
:rtype: dict
"""
pilotDict = {'Setups': {}, 'CEs': {}}
gLogger.info('-- Getting the content of the CS --')
# These are in fact not only setups: they may be "Defaults" sections, or VOs, in multi-VOs installations
setups = gConfig.getSections('/Operations/')
if not setups['OK']:
gLogger.error(setups['Message'])
return setups
setups = setups['Value']
try:
setups.remove('SoftwareDistribution') # TODO: remove this section
except (AttributeError, ValueError):
pass
# Something inside? (for multi-VO setups)
for vo in setups:
setupsFromVOs = gConfig.getSections('/Operations/%s' % vo)
if not setupsFromVOs['OK']:
continue
else:
setups.append("%s/%s" % (vo, setupsFromVOs))
gLogger.verbose('From Operations/[Setup]/Pilot')
for setup in setups:
self._getPilotOptionsPerSetup(setup, pilotDict)
gLogger.verbose('From Resources/Sites')
sitesSection = gConfig.getSections('/Resources/Sites/')
if not sitesSection['OK']:
gLogger.error(sitesSection['Message'])
return sitesSection
for grid in sitesSection['Value']:
gridSection = gConfig.getSections('/Resources/Sites/' + grid)
if not gridSection['OK']:
gLogger.error(gridSection['Message'])
return gridSection
for site in gridSection['Value']:
ceList = gConfig.getSections('/Resources/Sites/' + grid + '/' +
site + '/CEs/')
if not ceList['OK']:
# Skip but log it
gLogger.error('Site ' + site + ' has no CEs! - skipping')
continue
for ce in ceList['Value']:
ceType = gConfig.getValue('/Resources/Sites/' + grid +
'/' + site + '/CEs/' + ce +
'/CEType')
if ceType is None:
# Skip but log it
gLogger.error('CE ' + ce + ' at ' + site +
' has no option CEType! - skipping')
else:
pilotDict['CEs'][ce] = {
'Site': site,
'GridCEType': ceType
}
defaultSetup = gConfig.getValue('/DIRAC/DefaultSetup')
if defaultSetup:
pilotDict['DefaultSetup'] = defaultSetup
gLogger.verbose('From DIRAC/Configuration')
pilotDict['ConfigurationServers'] = gConfig.getServersList()
gLogger.verbose("Got %s" % str(pilotDict))
return pilotDict
def _getPilotOptionsPerSetup(self, setup, pilotDict):
""" Given a setup, returns its pilot options in a dictionary
"""
options = gConfig.getOptionsDict('/Operations/%s/Pilot' % setup)
if not options['OK']:
gLogger.warn(
"Section /Operations/%s/Pilot does not exist: skipping" %
setup)
return
# We include everything that's in the Pilot section for this setup
if setup == self.pilotSetup:
self.pilotVOVersion = options['Value']['Version']
pilotDict['Setups'][setup] = options['Value']
ceTypesCommands = gConfig.getOptionsDict(
'/Operations/%s/Pilot/Commands' % setup)
if ceTypesCommands['OK']:
# It's ok if the Pilot section doesn't list any Commands too
pilotDict['Setups'][setup]['Commands'] = {}
for ceType in ceTypesCommands['Value']:
# FIXME: inconsistent that we break Commands down into a proper list but other things are comma-list strings
pilotDict['Setups'][setup]['Commands'][
ceType] = ceTypesCommands['Value'][ceType].split(', ')
# pilotDict['Setups'][setup]['Commands'][ceType] = ceTypesCommands['Value'][ceType]
if 'CommandExtensions' in pilotDict['Setups'][setup]:
# FIXME: inconsistent that we break CommandExtensionss down into a proper
# list but other things are comma-list strings
pilotDict['Setups'][setup]['CommandExtensions'] = pilotDict[
'Setups'][setup]['CommandExtensions'].split(', ')
# pilotDict['Setups'][setup]['CommandExtensions'] = pilotDict['Setups'][setup]['CommandExtensions']
# Getting the details aboout the MQ Services to be used for logging, if any
if 'LoggingMQService' in pilotDict['Setups'][setup]:
loggingMQService = gConfig.getOptionsDict(
'/Resources/MQServices/%s' %
pilotDict['Setups'][setup]['LoggingMQService'])
if not loggingMQService['OK']:
gLogger.error(loggingMQService['Message'])
return loggingMQService
pilotDict['Setups'][setup]['Logging'] = {}
pilotDict['Setups'][setup]['Logging']['Host'] = loggingMQService[
'Value']['Host']
pilotDict['Setups'][setup]['Logging']['Port'] = loggingMQService[
'Value']['Port']
loggingMQServiceQueuesSections = gConfig.getSections(
'/Resources/MQServices/%s/Queues' %
pilotDict['Setups'][setup]['LoggingMQService'])
if not loggingMQServiceQueuesSections['OK']:
gLogger.error(loggingMQServiceQueuesSections['Message'])
return loggingMQServiceQueuesSections
pilotDict['Setups'][setup]['Logging']['Queue'] = {}
for queue in loggingMQServiceQueuesSections['Value']:
loggingMQServiceQueue = gConfig.getOptionsDict(
'/Resources/MQServices/%s/Queues/%s' %
(pilotDict['Setups'][setup]['LoggingMQService'], queue))
if not loggingMQServiceQueue['OK']:
gLogger.error(loggingMQServiceQueue['Message'])
return loggingMQServiceQueue
pilotDict['Setups'][setup]['Logging']['Queue'][
queue] = loggingMQServiceQueue['Value']
queuesRes = gConfig.getSections(
'/Resources/MQServices/%s/Queues' %
pilotDict['Setups'][setup]['LoggingMQService'])
if not queuesRes['OK']:
return queuesRes
queues = queuesRes['Value']
queuesDict = {}
for queue in queues:
queueOptionRes = gConfig.getOptionsDict(
'/Resources/MQServices/%s/Queues/%s' %
(pilotDict['Setups'][setup]['LoggingMQService'], queue))
if not queueOptionRes['OK']:
return queueOptionRes
queuesDict[queue] = queueOptionRes['Value']
pilotDict['Setups'][setup]['Logging']['Queues'] = queuesDict
def _resolveCECandidates( self, taskQueueDict ):
"""
Return a list of CEs for this TaskQueue
"""
# assume user knows what they're doing and avoid site mask e.g. sam jobs
if 'GridCEs' in taskQueueDict and taskQueueDict['GridCEs']:
self.log.info( 'CEs requested by TaskQueue %s:' % taskQueueDict['TaskQueueID'],
', '.join( taskQueueDict['GridCEs'] ) )
return taskQueueDict['GridCEs']
# Get the mask
ret = jobDB.getSiteMask()
if not ret['OK']:
self.log.error( 'Can not retrieve site Mask from DB:', ret['Message'] )
return []
siteMask = ret['Value']
if not siteMask:
self.log.error( 'Site mask is empty' )
return []
self.log.verbose( 'Site Mask: %s' % ', '.join( siteMask ) )
# remove banned sites from siteMask
if 'BannedSites' in taskQueueDict:
for site in taskQueueDict['BannedSites']:
if site in siteMask:
siteMask.remove( site )
self.log.verbose( 'Removing banned site %s from site Mask' % site )
# remove from the mask if a Site is given
siteMask = [ site for site in siteMask if 'Sites' not in taskQueueDict or site in taskQueueDict['Sites'] ]
if not siteMask:
# pilot can not be submitted
self.log.info( 'No Valid Site Candidate in Mask for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
return []
self.log.info( 'Site Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
# Get CE's associates to the given site Names
ceMask = []
for grid in self.targetGrids:
section = '/Resources/Sites/%s' % grid
ret = gConfig.getSections( section )
if not ret['OK']:
# this is hack, maintained until LCG is added as TargetGrid for the gLite SubmitPool
section = '/Resources/Sites/LCG'
ret = gConfig.getSections( section )
if not ret['OK']:
self.log.error( 'Could not obtain CEs from CS', ret['Message'] )
continue
gridSites = ret['Value']
for siteName in gridSites:
if siteName in siteMask:
ret = gConfig.getValue( '%s/%s/CE' % ( section, siteName ), [] )
for ce in ret:
submissionMode = gConfig.getValue( '%s/%s/CEs/%s/SubmissionMode' % ( section, siteName, ce ), 'gLite' )
if submissionMode == self.gridMiddleware and ce not in ceMask:
ceMask.append( ce )
if not ceMask:
self.log.info( 'No CE Candidate found for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
self.log.verbose( 'CE Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( ceMask ) )
return ceMask
def getServicePorts(self, setup='', printOutput=False):
"""Checks the service ports for the specified setup. If not given this is
taken from the current installation (/DIRAC/Setup)
Example usage:
>>> print diracAdmin.getServicePorts()
{'OK': True, 'Value':''}
@return: S_OK,S_ERROR
"""
if not setup:
setup = gConfig.getValue('/DIRAC/Setup', '')
setupList = gConfig.getSections('/DIRAC/Setups', [])
if not setupList['OK']:
return S_ERROR('Could not get /DIRAC/Setups sections')
setupList = setupList['Value']
if not setup in setupList:
return S_ERROR('Setup %s is not in allowed list: %s' %
(setup, string.join(setupList, ', ')))
serviceSetups = gConfig.getOptionsDict('/DIRAC/Setups/%s' % setup)
if not serviceSetups['OK']:
return S_ERROR('Could not get /DIRAC/Setups/%s options' % setup)
serviceSetups = serviceSetups['Value'] #dict
systemList = gConfig.getSections('/Systems')
if not systemList['OK']:
return S_ERROR('Could not get Systems sections')
systemList = systemList['Value']
result = {}
for system in systemList:
if serviceSetups.has_key(system):
path = '/Systems/%s/%s/Services' % (system,
serviceSetups[system])
servicesList = gConfig.getSections(path)
if not servicesList['OK']:
self.log.warn('Could not get sections in %s' % path)
else:
servicesList = servicesList['Value']
if not servicesList:
servicesList = []
self.log.verbose('System: %s ServicesList: %s' %
(system, string.join(servicesList, ', ')))
for service in servicesList:
spath = '%s/%s/Port' % (path, service)
servicePort = gConfig.getValue(spath, 0)
if servicePort:
self.log.verbose('Found port for %s/%s = %s' %
(system, service, servicePort))
result['%s/%s' % (system, service)] = servicePort
else:
self.log.warn('No port found for %s' % spath)
else:
self.log.warn('%s is not defined in /DIRAC/Setups/%s' %
(system, setup))
if printOutput:
print self.pPrint.pformat(result)
return S_OK(result)
def getComponentsStatus(self, conditionDict={}):
"""
Get the status of the defined components in the CS compared to the ones that are known in the DB
:type condDict: dictionary
:param condDict: The dictionary containing the conditions.
:return: S_OK with the requires results.
"""
result = self.__getComponents(conditionDict)
if not result["OK"]:
return result
statusSet = result["Value"]
requiredComponents = {}
result = gConfig.getSections("/DIRAC/Setups")
if not result["OK"]:
return result
for setup in result["Value"]:
if not self.__checkCondition(conditionDict, "Setup", setup):
continue
# Iterate through systems
result = gConfig.getOptionsDict("/DIRAC/Setups/%s" % setup)
if not result["OK"]:
return result
systems = result["Value"]
for system in systems:
instance = systems[system]
# Check defined agents and services
for cType in ("agent", "service"):
# Get entries for the instance of a system
result = gConfig.getSections(
"/Systems/%s/%s/%s" %
(system, instance, "%ss" % cType.capitalize()))
if not result["OK"]:
continue
components = result["Value"]
for component in components:
compDict = self.__getComponentDefinitionFromCS(
system, setup, instance, cType, component)
if self.__componentMatchesCondition(
compDict, requiredComponents, conditionDict):
statusSet.addUniqueToSet(requiredComponents,
compDict)
# Walk the URLs
systemURLs = getSystemURLs(
system, setup) # verify URLs in getSystemURLs method
for service in systemURLs:
# systemURLs is a dict that contain a list of URLs for service
if not systemURLs[service]:
self.log.error("Not found URL for %s service." %
service)
continue
url = parse.urlparse(systemURLs[service][0])
if self.__componentMatchesCondition(
dict(
Setup=setup,
Port=url.port,
Host=url.hostname,
Type="service",
ComponentName=system + "/" + service,
),
requiredComponents,
conditionDict,
):
statusSet.addUniqueToSet(requiredComponents, compDict)
# WALK THE DICT
statusSet.setComponentsAsRequired(requiredComponents)
return S_OK((statusSet.getRequiredComponents(), self.__mainFields[1:] +
self.__versionFields + ("Status", "Message")))
def __infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self.__checkAlternativeBDIISite(
ldapSite, name)
if result['OK']:
bdiiSites = result['Value']
if len(bdiiSites) == 0:
self.log.warn(name, "Error in BDII: leng = 0")
else:
if not len(bdiiSites) == 1:
self.log.warn(
name, "Warning in BDII: leng = %d" %
len(bdiiSites))
bdiiSite = bdiiSites[0]
try:
longitude = bdiiSite['GlueSiteLongitude']
latitude = bdiiSite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in BDII coordinates")
newcoor = "Unknown"
try:
newmail = bdiiSite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in BDII mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Mail'), newmail)
changed = True
ceList = List.fromChar(opt.get('CE', ''))
if not ceList:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in ceList:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
ceType = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
ceType = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in BDII for %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiiCE = result['Value'][0]
except:
self.log.warn('Error in BDII for %s' % ce, result)
bdiiCE = None
if bdiiCE:
try:
newWNTmpDir = bdiiCE['GlueSubClusterWNTmpDir']
except:
newWNTmpDir = 'Unknown'
if wnTmpDir != newWNTmpDir and newWNTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newWNTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newWNTmpDir)
else:
self.csAPI.modifyValue(section, newWNTmpDir)
changed = True
try:
newArch = bdiiCE[
'GlueHostArchitecturePlatformType']
except:
newArch = 'Unknown'
if arch != newArch and newArch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newArch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newArch)
else:
self.csAPI.modifyValue(section, newArch)
changed = True
try:
newOS = '_'.join(
(bdiiCE['GlueHostOperatingSystemName'],
bdiiCE['GlueHostOperatingSystemVersion'],
bdiiCE['GlueHostOperatingSystemRelease']))
except:
newOS = 'Unknown'
if os != newOS and newOS != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newOS)))
if os == 'Unknown':
self.csAPI.setOption(section, newOS)
else:
self.csAPI.modifyValue(section, newOS)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newOS, ce, site)
try:
newSI00 = bdiiCE['GlueHostBenchmarkSI00']
except:
newSI00 = 'Unknown'
if si00 != newSI00 and newSI00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
try:
rte = bdiiCE[
'GlueHostApplicationSoftwareRunTimeEnvironment']
for vo in self.voName:
if vo.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newPilot = 'True'
else:
newPilot = 'False'
else:
newPilot = 'Unknown'
except:
newPilot = 'Unknown'
if pilot != newPilot and newPilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newPilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newPilot)
else:
self.csAPI.modifyValue(section, newPilot)
changed = True
newVO = ''
for vo in self.voName:
result = ldapCEState(ce, vo) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in BDII for queue %s' % ce,
result['Message'])
result = self.__checkAlternativeBDIISite(
ldapCEState, ce, vo)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in BDII for queue %s' % ce,
result['Massage'])
continue
newCEType = 'Unknown'
for queue in queues:
try:
queueType = queue['GlueCEImplementationName']
except:
queueType = 'Unknown'
if newCEType == 'Unknown':
newCEType = queueType
else:
if queueType != newCEType:
self.log.warn(
'Error in BDII for CE %s ' % ce,
'different CE types %s %s' %
(newCEType, queueType))
if newCEType == 'ARC-CE':
newCEType = 'ARC'
if ceType != newCEType and newCEType != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(ceType, newCEType)))
if ceType == 'Unknown':
self.csAPI.setOption(section, newCEType)
else:
self.csAPI.modifyValue(section, newCEType)
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split(
'/')[-1]
except:
self.log.warn('Error in queueName ', queue)
continue
try:
newMaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newMaxCPUTime = None
newSI00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newSI00 = cap.split('=')[-1]
except:
newSI00 = None
queueSection = cfgPath(ceSection, 'Queues',
queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues",
result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
allowedVOs = ['']
else:
queueOpt = result['Value']
maxCPUTime = queueOpt.get(
'maxCPUTime', 'Unknown')
si00 = queueOpt.get('SI00', 'Unknown')
if newVO == '': # Remember previous iteration, if none - read from conf
allowedVOs = queueOpt.get('VO',
'').split(",")
else: # Else use newVO, as it can contain changes, which aren't in conf yet
allowedVOs = newVO.split(",")
if newMaxCPUTime and (maxCPUTime != newMaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newMaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(
section, newMaxCPUTime)
else:
self.csAPI.modifyValue(
section, newMaxCPUTime)
changed = True
if newSI00 and (si00 != newSI00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newSI00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newSI00)
else:
self.csAPI.modifyValue(section, newSI00)
changed = True
modifyVO = True # Flag saying if we need VO option to change
newVO = ''
if allowedVOs != ['']:
for allowedVO in allowedVOs:
allowedVO = allowedVO.strip(
) # Get rid of spaces
newVO += allowedVO
if allowedVO == vo: # Current VO has been already in list
newVO = ''
modifyVO = False # Don't change anything
break # Skip next 'if', proceed to next VO
newVO += ', '
if modifyVO:
section = cfgPath(queueSection, 'VO')
newVO += vo
self.log.info(
section, " -> ".join(
('%s' % allowedVOs, newVO)))
if allowedVOs == ['']:
self.csAPI.setOption(section, newVO)
else:
self.csAPI.modifyValue(section, newVO)
changed = True
if changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commit()
else:
self.log.info("No changes found")
return S_OK()
def getQueues(siteList=None,
ceList=None,
ceTypeList=None,
community=None,
mode=None):
"""Get CE/queue options according to the specified selection"""
result = gConfig.getSections("/Resources/Sites")
if not result["OK"]:
return result
resultDict = {}
grids = result["Value"]
for grid in grids:
result = gConfig.getSections("/Resources/Sites/%s" % grid)
if not result["OK"]:
continue
sites = result["Value"]
for site in sites:
if siteList and site not in siteList:
continue
if community:
comList = gConfig.getValue(
"/Resources/Sites/%s/%s/VO" % (grid, site), [])
if comList and community.lower() not in [
cl.lower() for cl in comList
]:
continue
siteCEParameters = {}
result = gConfig.getOptionsDict("/Resources/Sites/%s/%s/CEs" %
(grid, site))
if result["OK"]:
siteCEParameters = result["Value"]
result = gConfig.getSections("/Resources/Sites/%s/%s/CEs" %
(grid, site))
if not result["OK"]:
continue
ces = result["Value"]
for ce in ces:
if mode:
ceMode = gConfig.getValue(
"/Resources/Sites/%s/%s/CEs/%s/SubmissionMode" %
(grid, site, ce), "Direct")
if not ceMode or ceMode.lower() != mode.lower():
continue
if ceTypeList:
ceType = gConfig.getValue(
"/Resources/Sites/%s/%s/CEs/%s/CEType" %
(grid, site, ce), "")
if not ceType or ceType not in ceTypeList:
continue
if ceList and ce not in ceList:
continue
if community:
comList = gConfig.getValue(
"/Resources/Sites/%s/%s/CEs/%s/VO" % (grid, site, ce),
[])
if comList and community.lower() not in [
cl.lower() for cl in comList
]:
continue
ceOptionsDict = dict(siteCEParameters)
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/CEs/%s" % (grid, site, ce))
if not result["OK"]:
continue
ceOptionsDict.update(result["Value"])
result = gConfig.getSections(
"/Resources/Sites/%s/%s/CEs/%s/Queues" % (grid, site, ce))
if not result["OK"]:
continue
queues = result["Value"]
for queue in queues:
if community:
comList = gConfig.getValue(
"/Resources/Sites/%s/%s/CEs/%s/Queues/%s/VO" %
(grid, site, ce, queue), [])
if comList and community.lower() not in [
cl.lower() for cl in comList
]:
continue
resultDict.setdefault(site, {})
resultDict[site].setdefault(ce, ceOptionsDict)
resultDict[site][ce].setdefault("Queues", {})
result = gConfig.getOptionsDict(
"/Resources/Sites/%s/%s/CEs/%s/Queues/%s" %
(grid, site, ce, queue))
if not result["OK"]:
continue
queueOptionsDict = result["Value"]
resultDict[site][ce]["Queues"][queue] = queueOptionsDict
return S_OK(resultDict)
def getQueues(siteList=None,
ceList=None,
ceTypeList=None,
community=None,
mode=None):
""" Get CE/queue options according to the specified selection
"""
result = gConfig.getSections('/Resources/Sites')
if not result['OK']:
return result
resultDict = {}
grids = result['Value']
for grid in grids:
result = gConfig.getSections('/Resources/Sites/%s' % grid)
if not result['OK']:
continue
sites = result['Value']
for site in sites:
if siteList is not None and not site in siteList:
continue
if community:
comList = gConfig.getValue(
'/Resources/Sites/%s/%s/VO' % (grid, site), [])
if comList and not community in comList:
continue
siteCEParameters = {}
result = gConfig.getOptionsDict('/Resources/Sites/%s/%s/CEs' %
(grid, site))
if result['OK']:
siteCEParameters = result['Value']
result = gConfig.getSections('/Resources/Sites/%s/%s/CEs' %
(grid, site))
if not result['OK']:
continue
ces = result['Value']
for ce in ces:
if mode:
ceMode = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/SubmissionMode' %
(grid, site, ce), 'Direct')
if not ceMode or ceMode != mode:
continue
if ceTypeList:
ceType = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/CEType' %
(grid, site, ce), '')
if not ceType or not ceType in ceTypeList:
continue
if ceList is not None and not ce in ceList:
continue
if community:
comList = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/VO' % (grid, site, ce),
[])
if comList and not community in comList:
continue
ceOptionsDict = dict(siteCEParameters)
result = gConfig.getOptionsDict(
'/Resources/Sites/%s/%s/CEs/%s' % (grid, site, ce))
if not result['OK']:
continue
ceOptionsDict.update(result['Value'])
result = gConfig.getSections(
'/Resources/Sites/%s/%s/CEs/%s/Queues' % (grid, site, ce))
if not result['OK']:
continue
queues = result['Value']
for queue in queues:
if community:
comList = gConfig.getValue(
'/Resources/Sites/%s/%s/CEs/%s/Queues/%s/VO' %
(grid, site, ce, queue), [])
if comList and not community in comList:
continue
resultDict.setdefault(site, {})
resultDict[site].setdefault(ce, ceOptionsDict)
resultDict[site][ce].setdefault('Queues', {})
result = gConfig.getOptionsDict(
'/Resources/Sites/%s/%s/CEs/%s/Queues/%s' %
(grid, site, ce, queue))
if not result['OK']:
continue
queueOptionsDict = result['Value']
resultDict[site][ce]['Queues'][queue] = queueOptionsDict
return S_OK(resultDict)
def getSEDefinition(self, seID):
""" Get the Storage Element definition
"""
if isinstance(seID, str):
result = self.getSEID(seID)
if not result['OK']:
return result
seID = result['Value']
if seID in self.db.seDefinitions:
if (time.time() - self.db.seDefinitions[seID]['LastUpdate']) < self.seUpdatePeriod:
if self.db.seDefinitions[seID]['SEDict']:
return S_OK(self.db.seDefinitions[seID])
se = self.db.seDefinitions[seID]['SEName']
else:
result = self.getSEName(seID)
if not result['OK']:
return result
se = result['Value']
self.db.seDefinitions[seID] = {}
self.db.seDefinitions[seID]['SEName'] = se
self.db.seDefinitions[seID]['SEDict'] = {}
self.db.seDefinitions[seID]['LastUpdate'] = 0.
# We have to refresh the SE definition from the CS
result = gConfig.getSections('/Resources/StorageElements/%s' % se)
if not result['OK']:
return result
pluginSection = result['Value'][0]
result = gConfig.getOptionsDict('/Resources/StorageElements/%s/%s' % (se, pluginSection))
if not result['OK']:
return result
seDict = result['Value']
self.db.seDefinitions[seID]['SEDict'] = seDict
# Get VO paths if any
voPathDict = None
result = gConfig.getOptionsDict('/Resources/StorageElements/%s/%s/VOPath' % (se, pluginSection))
if result['OK']:
voPathDict = result['Value']
if seDict:
# A.T. Ports can be multiple, this can be better done using the Storage plugin
# to provide the replica prefix to keep implementations in one place
if 'Port' in seDict:
ports = seDict['Port']
if ',' in ports:
portList = [x.strip() for x in ports.split(',')]
random.shuffle(portList)
seDict['Port'] = portList[0]
tmpDict = dict(seDict)
tmpDict['FileName'] = ''
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict']['PFNPrefix'] = result['Value']
if voPathDict is not None:
for vo in voPathDict:
tmpDict['Path'] = voPathDict[vo]
result = pfnunparse(tmpDict)
if result['OK']:
self.db.seDefinitions[seID]['SEDict'].setdefault("VOPrefix", {})[vo] = result['Value']
self.db.seDefinitions[seID]['LastUpdate'] = time.time()
return S_OK(self.db.seDefinitions[seID])
def initialize(self):
"""initialize agent"""
# # ProcessPool related stuff
self.__requestsPerCycle = self.am_getOption("RequestsPerCycle", self.__requestsPerCycle)
self.log.info("Requests/cycle = %d" % self.__requestsPerCycle)
self.__minProcess = self.am_getOption("MinProcess", self.__minProcess)
self.log.info("ProcessPool min process = %d" % self.__minProcess)
self.__maxProcess = self.am_getOption("MaxProcess", self.__maxProcess)
self.log.info("ProcessPool max process = %d" % self.__maxProcess)
self.__queueSize = self.am_getOption("ProcessPoolQueueSize", self.__queueSize)
self.log.info("ProcessPool queue size = %d" % self.__queueSize)
self.__poolTimeout = int(self.am_getOption("ProcessPoolTimeout", self.__poolTimeout))
self.log.info("ProcessPool timeout = %d seconds" % self.__poolTimeout)
self.__poolSleep = int(self.am_getOption("ProcessPoolSleep", self.__poolSleep))
self.log.info("ProcessPool sleep time = %d seconds" % self.__poolSleep)
self.__bulkRequest = self.am_getOption("BulkRequest", self.__bulkRequest)
self.log.info("Bulk request size = %d" % self.__bulkRequest)
# Check if monitoring is enabled
if "Monitoring" in Operations().getMonitoringBackends(monitoringType="RMSMonitoring"):
# Enable RMS monitoring
self.__rmsMonitoring = True
self.log.info("Enable ES RMS Monitoring = %s" % self.__rmsMonitoring)
# # keep config path and agent name
self.agentName = self.am_getModuleParam("fullName")
self.__configPath = PathFinder.getAgentSection(self.agentName)
# # operation handlers over here
opHandlersPath = "%s/%s" % (self.__configPath, "OperationHandlers")
opHandlers = gConfig.getSections(opHandlersPath)
if not opHandlers["OK"]:
self.log.error(opHandlers["Message"])
raise AgentConfigError("OperationHandlers section not found in CS under %s" % self.__configPath)
opHandlers = opHandlers["Value"]
self.timeOuts = dict()
# # handlers dict
self.handlersDict = dict()
for opHandler in opHandlers:
opHandlerPath = "%s/%s/Location" % (opHandlersPath, opHandler)
opLocation = gConfig.getValue(opHandlerPath, "")
if not opLocation:
self.log.error("%s not set for %s operation handler" % (opHandlerPath, opHandler))
continue
self.timeOuts[opHandler] = {"PerFile": self.__fileTimeout, "PerOperation": self.__operationTimeout}
opTimeout = gConfig.getValue("%s/%s/TimeOut" % (opHandlersPath, opHandler), 0)
if opTimeout:
self.timeOuts[opHandler]["PerOperation"] = opTimeout
fileTimeout = gConfig.getValue("%s/%s/TimeOutPerFile" % (opHandlersPath, opHandler), 0)
if fileTimeout:
self.timeOuts[opHandler]["PerFile"] = fileTimeout
self.handlersDict[opHandler] = opLocation
self.log.info("Operation handlers:")
for item in enumerate(self.handlersDict.items()):
opHandler = item[1][0]
self.log.info(
"[%s] %s: %s (timeout: %d s + %d s per file)"
% (
item[0],
item[1][0],
item[1][1],
self.timeOuts[opHandler]["PerOperation"],
self.timeOuts[opHandler]["PerFile"],
)
)
if self.__rmsMonitoring:
self.rmsMonitoringReporter = MonitoringReporter(monitoringType="RMSMonitoring")
gThreadScheduler.addPeriodicTask(100, self.__rmsMonitoringReporting)
# # create request dict
self.__requestCache = dict()
return S_OK()
def getCPUTime(cpuNormalizationFactor):
""" Trying to get CPUTime left for execution (in seconds).
It will first look to get the work left looking for batch system information useing the TimeLeft utility.
If it succeeds, it will convert it in real second, and return it.
If it fails, it tries to get it from the static info found in CS.
If it fails, it returns the default, which is a large 9999999, that we may consider as "Infinite".
This is a generic method, independent from the middleware of the resource if TimeLeft doesn't return a value
args:
cpuNormalizationFactor (float): the CPU power of the current Worker Node.
If not passed in, it's get from the local configuration
returns:
cpuTimeLeft (int): the CPU time left, in seconds
"""
cpuTimeLeft = 0.
cpuWorkLeft = gConfig.getValue('/LocalSite/CPUTimeLeft', 0)
if not cpuWorkLeft:
# Try and get the information from the CPU left utility
result = TimeLeft().getTimeLeft()
if result['OK']:
cpuWorkLeft = result['Value']
if cpuWorkLeft > 0:
# This is in HS06sseconds
# We need to convert in real seconds
if not cpuNormalizationFactor: # if cpuNormalizationFactor passed in is 0, try get it from the local cfg
cpuNormalizationFactor = gConfig.getValue(
'/LocalSite/CPUNormalizationFactor', 0.0)
if cpuNormalizationFactor:
cpuTimeLeft = cpuWorkLeft / cpuNormalizationFactor # this is a float
if not cpuTimeLeft:
# now we know that we have to find the CPUTimeLeft by looking in the CS
# this is not granted to be correct as the CS units may not be real seconds
gridCE = gConfig.getValue('/LocalSite/GridCE')
ceQueue = gConfig.getValue('/LocalSite/CEQueue')
if not ceQueue:
# we have to look for a ceQueue in the CS
# A bit hacky. We should better profit from something generic
gLogger.warn(
"No CEQueue in local configuration, looking to find one in CS")
siteName = gConfig.getValue('/LocalSite/Site')
queueSection = '/Resources/Sites/%s/%s/CEs/%s/Queues' % (
siteName.split('.')[0], siteName, gridCE)
res = gConfig.getSections(queueSection)
if not res['OK']:
raise RuntimeError(res['Message'])
queues = res['Value']
cpuTimes = [
gConfig.getValue(queueSection + '/' + queue + '/maxCPUTime',
9999999.) for queue in queues
]
# These are (real, wall clock) minutes - damn BDII!
cpuTimeLeft = min(cpuTimes) * 60
else:
queueInfo = getQueueInfo('%s/%s' % (gridCE, ceQueue))
cpuTimeLeft = 9999999.
if not queueInfo['OK'] or not queueInfo['Value']:
gLogger.warn(
"Can't find a CE/queue, defaulting CPUTime to %d" %
cpuTimeLeft)
else:
queueCSSection = queueInfo['Value']['QueueCSSection']
# These are (real, wall clock) minutes - damn BDII!
cpuTimeInMinutes = gConfig.getValue(
'%s/maxCPUTime' % queueCSSection, 0.)
if cpuTimeInMinutes:
cpuTimeLeft = cpuTimeInMinutes * 60.
gLogger.info("CPUTime for %s: %f" %
(queueCSSection, cpuTimeLeft))
else:
gLogger.warn(
"Can't find maxCPUTime for %s, defaulting CPUTime to %f"
% (queueCSSection, cpuTimeLeft))
return int(cpuTimeLeft)
def _getCSDict(self):
""" Gets minimal info for running a pilot, from the CS
:returns: pilotDict (containing pilots run info)
:rtype: S_OK, S_ERROR, value is pilotDict
"""
pilotDict = {'Setups': {}, 'CEs': {}, 'GenericPilotDNs': []}
self.log.info('-- Getting the content of the CS --')
# These are in fact not only setups: they may be "Defaults" sections, or VOs, in multi-VOs installations
setupsRes = gConfig.getSections('/Operations/')
if not setupsRes['OK']:
self.log.error("Can't get sections from Operations",
setupsRes['Message'])
return setupsRes
setupsInOperations = setupsRes['Value']
# getting the setup(s) in this CS, and comparing with what we found in Operations
setupsInDIRACRes = gConfig.getSections('DIRAC/Setups')
if not setupsInDIRACRes['OK']:
self.log.error("Can't get sections from DIRAC/Setups",
setupsInDIRACRes['Message'])
return setupsInDIRACRes
setupsInDIRAC = setupsInDIRACRes['Value']
# Handling the case of multi-VO CS
if not set(setupsInDIRAC).intersection(set(setupsInOperations)):
vos = list(setupsInOperations)
for vo in vos:
setupsFromVOs = gConfig.getSections('/Operations/%s' % vo)
if not setupsFromVOs['OK']:
continue
else:
setupsInOperations = setupsFromVOs['Value']
self.log.verbose('From Operations/[Setup]/Pilot')
for setup in setupsInOperations:
self._getPilotOptionsPerSetup(setup, pilotDict)
self.log.verbose('From Resources/Sites')
sitesSection = gConfig.getSections('/Resources/Sites/')
if not sitesSection['OK']:
self.log.error("Can't get sections from Resources",
sitesSection['Message'])
return sitesSection
for grid in sitesSection['Value']:
gridSection = gConfig.getSections('/Resources/Sites/' + grid)
if not gridSection['OK']:
self.log.error("Can't get sections from Resources",
gridSection['Message'])
return gridSection
for site in gridSection['Value']:
ceList = gConfig.getSections('/Resources/Sites/' + grid + '/' +
site + '/CEs/')
if not ceList['OK']:
# Skip but log it
self.log.error('Site has no CEs! - skipping', site)
continue
for ce in ceList['Value']:
ceType = gConfig.getValue('/Resources/Sites/' + grid +
'/' + site + '/CEs/' + ce +
'/CEType')
localCEType = gConfig.getValue('/Resources/Sites/' + grid +
'/' + site + '/CEs/' + ce +
'/LocalCEType')
if ceType is None:
# Skip but log it
self.log.error('CE has no option CEType!',
ce + ' at ' + site)
pilotDict['CEs'][ce] = {'Site': site}
else:
pilotDict['CEs'][ce] = {
'Site': site,
'GridCEType': ceType
}
if localCEType is not None:
pilotDict['CEs'][ce].setdefault(
'LocalCEType', localCEType)
defaultSetup = gConfig.getValue('/DIRAC/DefaultSetup')
if defaultSetup:
pilotDict['DefaultSetup'] = defaultSetup
self.log.debug('From DIRAC/Configuration')
pilotDict['ConfigurationServers'] = gConfig.getServersList()
self.log.debug("Got pilotDict", str(pilotDict))
return S_OK(pilotDict)
def execute(self):
""" Main execution method
"""
result = gConfig.getSections("/DIRAC/Setups")
if not result['OK']:
return result
validSetups = result['Value']
self.log.info("Valid setups for this cycle are %s" %
", ".join(validSetups))
#Get the WMS Snapshot!
result = self.jobDB.getSummarySnapshot(self.__jobDBFields)
now = Time.dateTime()
if not result['OK']:
self.log.error(
"Can't get the JobDB summary",
"%s: won't commit at this cycle" % result['Message'])
else:
values = result['Value'][1]
if self.retryOnce:
self.log.verbose(
"Adding to records to commit those not committed within the previous cycle"
)
acWMSListAdded = []
for record in values:
recordSetup = record[0]
if recordSetup not in validSetups:
self.log.error("Setup %s is not valid" % recordSetup)
continue
if recordSetup not in self.dsClients:
self.log.info("Creating DataStore client for %s" %
recordSetup)
self.dsClients[recordSetup] = DataStoreClient(
setup=recordSetup, retryGraceTime=900)
record = record[1:]
rD = {}
for fV in self.__summaryDefinedFields:
rD[fV[0]] = fV[1]
for iP in range(len(self.__summaryKeyFieldsMapping)):
fieldName = self.__summaryKeyFieldsMapping[iP]
rD[self.__renameFieldsMapping.get(fieldName,
fieldName)] = record[iP]
record = record[len(self.__summaryKeyFieldsMapping):]
for iP in range(len(self.__summaryValueFieldsMapping)):
rD[self.__summaryValueFieldsMapping[iP]] = int(record[iP])
acWMS = WMSHistory()
acWMS.setStartTime(now)
acWMS.setEndTime(now)
acWMS.setValuesFromDict(rD)
retVal = acWMS.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s -> %s" % (retVal['Message'], rD))
else:
self.dsClients[recordSetup].addRegister(acWMS)
acWMSListAdded.append(acWMS)
if self.retryOnce and self.retryValues:
for acWMSCumulated in self.retryValues:
retVal = acWMSCumulated.checkValues()
if not retVal['OK']:
self.log.error("Invalid accounting record ",
"%s" % (retVal['Message']))
else:
self.dsClients[recordSetup].addRegister(acWMSCumulated)
for setup in self.dsClients:
self.log.info("Sending records for setup %s" % setup)
result = self.dsClients[setup].commit()
if not result['OK']:
self.log.error(
"Couldn't commit wms history for setup %s" % setup,
result['Message'])
# Re-creating the client: for new connection, and for avoiding accumulating too large of a backlog
self.dsClients[setup] = DataStoreClient(setup=setup,
retryGraceTime=900)
if not self.retryOnce:
self.log.info("Will try again at next cycle")
self.retryOnce = True
self.retryValues = acWMSListAdded
else:
self.log.warn("Won't retry one more time")
self.retryOnce = False
self.retryValues = []
else:
self.log.info("Sent %s records for setup %s" %
(result['Value'], setup))
self.retryOnce = False
return S_OK()
def configHelper(voList):
"""
A helper function to gather necessary Rucio client options from the CS.
:param volist: list of VO names, or a VO name (str)
:return: a dictionary of a form {vo: params, vo: params,}
:rtype: dict
"""
log = gLogger.getLocalSubLogger("RucioSynchronizerHelper")
if isinstance(voList, str):
voList = [voList]
clientConfig = {}
log.debug("VO list to consider for synchronization: ", voList)
# locate RucioFileCatalog type in resources first
result = gConfig.getSections("/Resources/FileCatalogs")
catNames = []
if result["OK"]:
catalogs = result["Value"]
log.debug("File catalogs defined in Resources", catalogs)
for catalog in catalogs:
result = gConfig.getOptionsDict(getCatalogPath(catalog))
if result["OK"]:
options = result["Value"]
log.debug("Rucio Catalog candidate options", options)
if options.get("Status", None) == "Active" and options.get(
"CatalogType", None) == "RucioFileCatalog":
catNames.append(catalog)
else:
log.error("No catalogs defined in Resources.")
return S_ERROR("No catalogs defined in Resources.")
log.info(
"Active FileCatalogs candidates of type RucioFileCatalog found in Resources:",
catNames)
# we found (possibly more that one) candidate, now we look for it in Operations
# to find out which one is used by which VO. There can be only one
# Rucio catalog per VO.
for vo in voList:
opHelper = Operations(vo=vo)
result = opHelper.getSections("/Services/Catalogs")
if result["OK"]:
catSections = set(result["Value"])
else:
log.warn("No Services/Catalogs section in Operations, for ",
"VO=%s (skipped)" % vo)
continue
selectedCatalog = list(catSections.intersection(catNames))
if len(selectedCatalog) > 1:
log.error(
"VO %s: Services/Catalogs section mis-configured."
" More that one Rucio file catalog",
"[VO: %s, Catalogs: %s]" % (vo, selectedCatalog),
)
continue
if not selectedCatalog:
log.warn("VO is not using RucioFileCatalog (VO skipped)",
"[VO: %s]" % vo)
continue
# check if the section name is in the catalog list to use.
# if the list is not empty it has to contain the selected catalog.
fileCatalogs = opHelper.getValue("/Services/Catalogs/CatalogList", [])
if fileCatalogs and selectedCatalog[0] not in fileCatalogs:
log.warn(
"VO is not using RucioFileCatalog - it is not in the catalog list",
"[VO: %s]" % vo)
continue
# now collect Rucio specific parameters for the VO
params = {}
result = gConfig.getOptionsDict(getCatalogPath(selectedCatalog[0]))
if result["OK"]:
optDict = result["Value"]
params["rucioHost"] = optDict.get("RucioHost", None)
params["authHost"] = optDict.get("AuthHost", None)
params["privilegedAccount"] = optDict.get("PrivilegedAccount",
"root")
clientConfig[vo] = params
log.info("RSEs and users will be configured in Rucio for the VO:",
vo)
else:
log.error(result["Message"])
return clientConfig
Script.setUsageMessage("""
Show storage quotas for specified users or for all registered users if nobody is specified
Usage:
%s [user1 ...]
""" % Script.scriptName)
Script.parseCommandLine()
users = Script.getPositionalArgs()
from DIRAC import gLogger, gConfig
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
if not users:
res = gConfig.getSections('/Registry/Users')
if not res['OK']:
gLogger.error("Failed to retrieve user list from CS", res['Message'])
DIRAC.exit(2)
users = res['Value']
gLogger.notice("-" * 30)
gLogger.notice("%s|%s" % ('Username'.ljust(15), 'Quota (GB)'.rjust(15)))
gLogger.notice("-" * 30)
for user in sorted(users):
quota = gConfig.getValue('/Registry/Users/%s/Quota' % user, 0)
if not quota:
quota = gConfig.getValue('/Registry/DefaultStorageQuota')
gLogger.notice("%s|%s" % (user.ljust(15), str(quota).rjust(15)))
gLogger.notice("-" * 30)
DIRAC.exit(0)
def execute(self):
""" execution in one agent's cycle
:param self: self reference
"""
# Check if CopyToSE is valid
if not self.CopyToSE.lower() == 'none':
# Get allowed SEs
res = gConfig.getSections(self.DIRACCfgSEPath)
if not res['OK']:
gLogger.warn(
'Could not retrieve SE info from DIRAC cfg path %s' %
self.DIRACCfgSEPath)
if res['OK'] and res['Value']:
if self.CopyToSE not in res['Value']:
gLogger.error(
'Could not find CopyToSE - %s - in DIRAC cfg' %
self.CopyToSE)
return S_ERROR('CopyToSE %s is not valid' % self.CopyToSE)
dest_se = self.CopyToSE
se_data_dir = self.SEDataDirPath
local_data_dir = self.LocalDataDirPath
for rootdir, subdirs, filenames in os.walk(local_data_dir):
for filename in filenames:
if filename.endswith('.mat') or filename.endswith(
'.MAT') or filename.endswith(
'.egg') or filename.endswith(
'_meta.json') or filename.endswith(
'.msk') or filename.endswith('.Setup'):
gLogger.info('Matched local file: ' + filename)
pfn = os.path.join(rootdir, filename)
lfn = os.path.join(
se_data_dir,
os.path.join(rootdir,
filename).split(local_data_dir)[-1])
### Check if file already exists ###
cmd = 'dirac-dms-lfn-accessURL ' + lfn + ' ' + dest_se
gLogger.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if "No such file" not in output:
gLogger.info('File already exists ... removing.')
cmd = '/bin/rm ' + pfn + ' <<< y'
gLogger.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
gLogger.info('File successfully removed.')
else:
gLogger.error(
'Problem removing file! rm returned {}'.
format(status))
gLogger.info('rm cmd was: "{}"'.format(cmd))
continue
else:
### Upload file ###
cmd = 'dirac-dms-add-file -ddd -o /Resources/Sites/Test=true '
gLogger.info("local file is {}".format(lfn))
cmd += lfn + ' '
gLogger.info("will move file to {}".format(pfn))
cmd += pfn + ' '
cmd += dest_se
gLogger.info("full upload command is:\n{}".format(cmd))
initialTime = time.time()
status, output = commands.getstatusoutput(cmd)
elapsedTime = time.time() - initialTime
if status == 0:
gLogger.info(
'Upload successful in {} s. Removing local file...'
.format(elapsedTime))
cmd = '/bin/rm ' + pfn + ' <<< y'
gLogger.debug(cmd)
status, output = commands.getstatusoutput(cmd)
if status == 0:
gLogger.info('File successfully removed.')
else:
gLogger.error(
'Problem removing file! rm returned {}'.
format(status))
gLogger.info('rm cmd was: "{}"'.format(cmd))
else:
gLogger.error('Failed to upload file ' + lfn)
### Do metadata for the dir
if filename.endswith('_meta.json'):
meta_json_string = open(filename).read()
meta_python_unicode_dict = json.loads(
meta_json_string) # converts to unicode
meta_python_dict = {} # this one is utf encoded
for item in meta_python_unicode_dict.items():
key = item[0].encode('utf-8')
if item[1] is None:
value = 'null'
elif isinstance(item[1], (int, float)):
value = item[1]
elif item[1].isdigit():
value = int(item[1].encode(
'utf-8')) # encode '0' and '1' as int
else:
value = item[1].encode('utf-8')
meta_python_dict[key] = value
gLogger.info('Meta Data from file (%s) is: %s' %
(filename, meta_python_dict))
return S_OK()
def _infoFromCE(self):
sitesSection = cfgPath('Resources', 'Sites')
result = gConfig.getSections(sitesSection)
if not result['OK']:
return
grids = result['Value']
for grid in grids:
gridSection = cfgPath(sitesSection, grid)
result = gConfig.getSections(gridSection)
if not result['OK']:
return
sites = result['Value']
changed = False
body = ""
for site in sites:
# if site[-2:]!='ru':
# continue
siteSection = cfgPath(gridSection, site)
opt = gConfig.getOptionsDict(siteSection)['Value']
name = opt.get('Name', '')
if name:
coor = opt.get('Coordinates', 'Unknown')
mail = opt.get('Mail', 'Unknown')
result = ldapSite(name)
if not result['OK']:
self.log.warn("BDII site %s: %s" %
(name, result['Message']))
result = self._checkAlternativeBDIISite(ldapSite, name)
if result['OK']:
bdiisites = result['Value']
if len(bdiisites) == 0:
self.log.warn(name, "Error in bdii: leng = 0")
else:
if not len(bdiisites) == 1:
self.log.warn(
name, "Warning in bdii: leng = %d" %
len(bdiisites))
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % (longitude, latitude)
except:
self.log.warn("Error in bdii coor")
newcoor = "Unknown"
try:
newmail = bdiisite[
'GlueSiteSysAdminContact'].split(
":")[-1].strip()
except:
self.log.warn("Error in bdii mail")
newmail = "Unknown"
self.log.debug("%s %s %s" %
(name, newcoor, newmail))
if newcoor != coor:
self.log.info("%s" % (name),
"%s -> %s" % (coor, newcoor))
if coor == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Coordinates'),
newcoor)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newcoor)
changed = True
if newmail != mail:
self.log.info("%s" % (name),
"%s -> %s" % (mail, newmail))
if mail == 'Unknown':
self.csAPI.setOption(
cfgPath(siteSection, 'Mail'), newmail)
else:
self.csAPI.modifyValue(
cfgPath(siteSection, 'Coordinates'),
newmail)
changed = True
celist = List.fromChar(opt.get('CE', ''))
if not celist:
self.log.warn(site, 'Empty site list')
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath(siteSection, 'CEs', ce)
result = gConfig.getOptionsDict(ceSection)
if not result['OK']:
self.log.debug("Section CE", result['Message'])
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get('wnTmpDir', 'Unknown')
arch = ceopt.get('architecture', 'Unknown')
os = ceopt.get('OS', 'Unknown')
si00 = ceopt.get('SI00', 'Unknown')
pilot = ceopt.get('Pilot', 'Unknown')
cetype = ceopt.get('CEType', 'Unknown')
result = ldapCE(ce)
if not result['OK']:
self.log.warn('Error in bdii for %s' % ce,
result['Message'])
result = self._checkAlternativeBDIISite(ldapCE, ce)
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn('Error in bdii for %s' % ce, result)
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath(ceSection, 'wnTmpDir')
self.log.info(section, " -> ".join(
(wnTmpDir, newwnTmpDir)))
if wnTmpDir == 'Unknown':
self.csAPI.setOption(section, newwnTmpDir)
else:
self.csAPI.modifyValue(section, newwnTmpDir)
changed = True
try:
newarch = bdiice[
'GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath(ceSection, 'architecture')
self.log.info(section, " -> ".join(
(arch, newarch)))
if arch == 'Unknown':
self.csAPI.setOption(section, newarch)
else:
self.csAPI.modifyValue(section, newarch)
changed = True
try:
newos = '_'.join(
(bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease']))
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath(ceSection, 'OS')
self.log.info(section, " -> ".join((os, newos)))
if os == 'Unknown':
self.csAPI.setOption(section, newos)
else:
self.csAPI.modifyValue(section, newos)
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % (
os, newos, ce, site)
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath(ceSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
try:
rte = bdiice[
'GlueHostApplicationSoftwareRunTimeEnvironment']
if self.vo == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath(ceSection, 'Pilot')
self.log.info(section, " -> ".join(
(pilot, newpilot)))
if pilot == 'Unknown':
self.csAPI.setOption(section, newpilot)
else:
self.csAPI.modifyValue(section, newpilot)
changed = True
result = ldapService(ce)
if not result['OK']:
result = self._checkAlternativeBDIISite(
ldapService, ce)
if result['OK']:
services = result['Value']
newcetype = 'LCG'
for service in services:
if service['GlueServiceType'].count('CREAM'):
newcetype = "CREAM"
else:
newcetype = 'Unknown'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath(ceSection, 'CEType')
self.log.info(section, " -> ".join(
(cetype, newcetype)))
if cetype == 'Unknown':
self.csAPI.setOption(section, newcetype)
else:
self.csAPI.modifyValue(section, newcetype)
changed = True
result = ldapCEState(ce, vo=self.vo) #getBDIICEVOView
if not result['OK']:
self.log.warn('Error in bdii for queue %s' % ce,
result['Message'])
result = self._checkAlternativeBDIISite(
ldapCEState, ce, self.vo)
continue
try:
queues = result['Value']
except:
self.log.warn('Error in bdii for queue %s' % ce,
result['Massage'])
continue
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split('/')[-1]
except:
self.log.warn('error in queuename ', queue)
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type(caps) == type(''):
caps = [caps]
for cap in caps:
if cap.count('CPUScalingReferenceSI00'):
newsi00 = cap.split('=')[-1]
except:
newsi00 = None
queueSection = cfgPath(ceSection, 'Queues', queueName)
result = gConfig.getOptionsDict(queueSection)
if not result['OK']:
self.log.warn("Section Queues", result['Message'])
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get('maxCPUTime', 'Unknown')
si00 = queueopt.get('SI00', 'Unknown')
if newmaxCPUTime and (maxCPUTime != newmaxCPUTime):
section = cfgPath(queueSection, 'maxCPUTime')
self.log.info(
section, " -> ".join(
(maxCPUTime, newmaxCPUTime)))
if maxCPUTime == 'Unknown':
self.csAPI.setOption(section, newmaxCPUTime)
else:
self.csAPI.modifyValue(section, newmaxCPUTime)
changed = True
if newsi00 and (si00 != newsi00):
section = cfgPath(queueSection, 'SI00')
self.log.info(section, " -> ".join(
(si00, newsi00)))
if si00 == 'Unknown':
self.csAPI.setOption(section, newsi00)
else:
self.csAPI.modifyValue(section, newsi00)
changed = True
if False and changed:
self.log.info(body)
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False)
return self.csAPI.commitChanges(sortUsers=False)
else:
self.log.info("No changes found")
return S_OK()
gLogger.notice(
printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
if __name__ == '__main__':
if not voName:
# Get the current VO
result = getVOfromProxyGroup()
if not result['OK']:
gLogger.error('No proxy found, please login')
DIRACExit(-1)
voName = result['Value']
else:
result = gConfig.getSections('/Registry/VO')
if not result['OK']:
gLogger.error('Failed to contact the CS')
DIRACExit(-1)
if voName not in result['Value']:
gLogger.error('Invalid VO name')
DIRACExit(-1)
if not (ceFlag or seFlag):
gLogger.error('Resource type is not specified')
DIRACExit(-1)
if ceFlag:
result = printCEInfo(voName)
if not result['OK']:
gLogger.error(result['Message'])