def getVOMSServerInfo( requestedVO = '' ):
""" Get information on VOMS servers for the given VO or for all of them
"""
vomsDict = {}
# For backward compatibility check the VOMS section first
result = gConfig.getSections( '%s/VOMS/Servers' % gBaseRegistrySection )
if result['OK']:
voNames = result['Value']
for vo in voNames:
if requestedVO and vo != requestedVO:
continue
vomsDict.setdefault( vo, {} )
vomsDict[vo]['VOMSName'] = vo
result = gConfig.getSections( '%s/VOMS/Servers/%s' % (gBaseRegistrySection, vo) )
if result['OK']:
serverList = result['Value']
vomsDict[vo].setdefault( "Servers", {} )
for server in serverList:
DN = gConfig.getValue( '%s/VOMS/Servers/%s/%s/DN' % (gBaseRegistrySection, vo, server), '' )
CA = gConfig.getValue( '%s/VOMS/Servers/%s/%s/CA' % (gBaseRegistrySection, vo, server), '' )
port = gConfig.getValue( '%s/VOMS/Servers/%s/%s/Port' % (gBaseRegistrySection, vo, server), 0 )
vomsDict[vo]['Servers'].setdefault( server, {} )
vomsDict[vo]['Servers'][server]['DN'] = DN
vomsDict[vo]['Servers'][server]['CA'] = CA
vomsDict[vo]['Servers'][server]['Port'] = port
result = getVOs()
if result['OK']:
voNames = result['Value']
for vo in voNames:
if requestedVO and vo != requestedVO:
continue
vomsName = getVOOption( vo, 'VOMSName', '' )
if not vomsName:
continue
vomsDict.setdefault( vo, {} )
vomsDict[vo]['VOMSName'] = getVOOption( vo, 'VOMSName', '' )
result = gConfig.getSections( '%s/VO/%s/VOMSServers' % (gBaseRegistrySection, vo) )
if result['OK']:
serverList = result['Value']
vomsDict[vo].setdefault( "Servers", {} )
for server in serverList:
vomsDict[vo]['Servers'].setdefault( server, {} )
DN = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/DN' % (gBaseRegistrySection, vo, server), '' )
CA = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/CA' % (gBaseRegistrySection, vo, server), '' )
port = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/Port' % (gBaseRegistrySection, vo, server), 0 )
vomsDict[vo]['Servers'][server]['DN'] = DN
vomsDict[vo]['Servers'][server]['CA'] = CA
vomsDict[vo]['Servers'][server]['Port'] = port
return S_OK( vomsDict )
def getVOMSServerInfo( requestedVO = '' ):
""" Get information on VOMS servers for the given VO or for all of them
"""
vomsDict = {}
# For backward compatibility check the VOMS section first
result = gConfig.getSections( '%s/VOMS/Servers' % gBaseRegistrySection )
if result['OK']:
voNames = result['Value']
for vo in voNames:
if requestedVO and vo != requestedVO:
continue
vomsDict.setdefault( vo, {} )
vomsDict[vo]['VOMSName'] = vo
result = gConfig.getSections( '%s/VOMS/Servers/%s' % (gBaseRegistrySection, vo) )
if result['OK']:
serverList = result['Value']
vomsDict[vo].setdefault( "Servers", {} )
for server in serverList:
DN = gConfig.getValue( '%s/VOMS/Servers/%s/%s/DN' % (gBaseRegistrySection, vo, server), '' )
CA = gConfig.getValue( '%s/VOMS/Servers/%s/%s/CA' % (gBaseRegistrySection, vo, server), '' )
port = gConfig.getValue( '%s/VOMS/Servers/%s/%s/Port' % (gBaseRegistrySection, vo, server), 0 )
vomsDict[vo]['Servers'].setdefault( server, {} )
vomsDict[vo]['Servers'][server]['DN'] = DN
vomsDict[vo]['Servers'][server]['CA'] = CA
vomsDict[vo]['Servers'][server]['Port'] = port
result = getVOs()
if result['OK']:
voNames = result['Value']
for vo in voNames:
if requestedVO and vo != requestedVO:
continue
vomsName = getVOOption( vo, 'VOMSName', '' )
if not vomsName:
continue
vomsDict.setdefault( vo, {} )
vomsDict[vo]['VOMSName'] = getVOOption( vo, 'VOMSName', '' )
result = gConfig.getSections( '%s/VO/%s/VOMSServers' % (gBaseRegistrySection, vo) )
if result['OK']:
serverList = result['Value']
vomsDict[vo].setdefault( "Servers", {} )
for server in serverList:
vomsDict[vo]['Servers'].setdefault( server, {} )
DN = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/DN' % (gBaseRegistrySection, vo, server), '' )
CA = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/CA' % (gBaseRegistrySection, vo, server), '' )
port = gConfig.getValue( '%s/VO/%s/VOMSServers/%s/Port' % (gBaseRegistrySection, vo, server), 0 )
vomsDict[vo]['Servers'][server]['DN'] = DN
vomsDict[vo]['Servers'][server]['CA'] = CA
vomsDict[vo]['Servers'][server]['Port'] = port
return S_OK( vomsDict )
def getSchemaSections(self, path=""):
retDict = gConfig.getSections(
"%s/Schema/%s" % (self.webSection, path), listOrdered=True)
if retDict['OK']:
return retDict['Value']
else:
return []
def getAllGroups():
""" Get all groups
:return: list
"""
result = gConfig.getSections("%s/Groups" % gBaseRegistrySection)
return result['Value'] if result['OK'] else []
def getVOs():
""" Get all the configured VOs
"""
voName = getVO()
if voName:
return S_OK([voName])
return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def getVOs():
""" Get all the configured VOs
"""
voName = getVO()
if voName:
return S_OK([voName])
return gConfig.getSections('%s/VO' % gBaseRegistrySection)
def initializeParameters( self ):
""" Initialize the CE parameters after they are collected from various sources
"""
# Collect global defaults first
result = gConfig.getSections( '/Resources/Computing' )
if result['OK'] and result['Value']:
ceSections = result['Value']
if 'CEDefaults' in ceSections:
result = self.__getCEParameters( '/Resources/Computing/CEDefaults' ) #can be overwritten by other sections
if not result['OK']:
self.log.warn( result['Message'] )
if self.ceName in ceSections:
result = self.__getCEParameters( '/Resources/Computing/%s' % self.ceName )
if not result['OK']:
self.log.warn( result['Message'] )
# Get local CE configuration
localConfigDict = getCEConfigDict( self.ceName )
self.ceParameters.update( localConfigDict )
result = self.__getSiteParameters()
if not result['OK']:
self.log.warn( result['Message'] )
result = self.__getResourceRequirements()
if not result['OK']:
self.log.warn( result['Message'] )
self._addCEConfigDefaults()
def initializeParameters(self):
""" Initialize the CE parameters after they are collected from various sources
"""
# Collect global defaults first
result = gConfig.getSections("/Resources/Computing")
if result["OK"] and result["Value"]:
ceSections = result["Value"]
if "CEDefaults" in ceSections:
result = self.__getCEParameters(
"/Resources/Computing/CEDefaults"
) # can be overwritten by other sections
if not result["OK"]:
self.log.warn(result["Message"])
if self.ceName in ceSections:
result = self.__getCEParameters("/Resources/Computing/%s" % self.ceName)
if not result["OK"]:
self.log.warn(result["Message"])
# Get local CE configuration
localConfigDict = getCEConfigDict(self.ceName)
self.ceParameters.update(localConfigDict)
result = self.__getSiteParameters()
if not result["OK"]:
self.log.warn(result["Message"])
self._addCEConfigDefaults()
def initializeParameters( self ):
""" Initialize the CE parameters after they are collected from various sources
"""
# Collect global defaults first
result = gConfig.getSections( '/Resources/Computing' )
if result['OK'] and result['Value']:
ceSections = result['Value']
if 'CEDefaults' in ceSections:
result = self.__getCEParameters( '/Resources/Computing/CEDefaults' ) #can be overwritten by other sections
if not result['OK']:
self.log.warn( result['Message'] )
if self.ceName in ceSections:
result = self.__getCEParameters( '/Resources/Computing/%s' % self.ceName )
if not result['OK']:
self.log.warn( result['Message'] )
# Get local CE configuration
localConfigDict = getCEConfigDict( self.ceName )
self.ceParameters.update( localConfigDict )
result = self.__getSiteParameters()
if not result['OK']:
self.log.warn( result['Message'] )
self._addCEConfigDefaults()
def getAllUsers():
""" Get all users
:return: list
"""
result = gConfig.getSections("%s/Users" % gBaseRegistrySection)
return result['Value'] if result['OK'] else []
def getVOs():
""" Get all the configured VOs
:return: S_OK(list)/S_ERROR()
"""
voName = getVO()
return S_OK([voName]) if voName else gConfig.getSections(
'%s/VO' % gBaseRegistrySection)
def getGroupsWithVOMSAttribute( vomsAttr ):
retVal = gConfig.getSections( "%s/Groups" % ( gBaseSecuritySection ) )
if not retVal[ 'OK' ]:
return []
groups = []
for group in retVal[ 'Value' ]:
if vomsAttr == gConfig.getValue( "%s/Groups/%s/VOMSRole" % ( gBaseSecuritySection, group ), "" ):
groups.append( group )
return groups
def getHostnameForDN( dn ):
retVal = gConfig.getSections( "%s/Hosts" % gBaseSecuritySection )
if not retVal[ 'OK' ]:
return retVal
hostList = retVal[ 'Value' ]
for hostname in hostList:
if dn in gConfig.getValue( "%s/Hosts/%s/DN" % ( gBaseSecuritySection, hostname ), [] ):
return S_OK( hostname )
return S_ERROR( "No hostname found for dn %s" % dn )
def getGroupsForVO(vo):
""" Get groups for VO
:param str vo: DIRAC VO name
:return: S_OK(list)/S_ERROR()
"""
if getVO():
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
return __getGroupsWithAttr('VO', vo)
def getUsernameForID(ID, usersList=False):
if not usersList:
retVal = gConfig.getSections("%s/Users" % gBaseRegistrySection)
if not retVal['OK']:
return retVal
usersList = retVal['Value']
for username in usersList:
if ID in gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []):
return S_OK(username)
return S_ERROR("No username found for ID %s" % ID)
def getUsernameForID(ID, usersList=False):
if not usersList:
retVal = gConfig.getSections("%s/Users" % gBaseRegistrySection)
if not retVal['OK']:
return retVal
usersList = retVal['Value']
for username in usersList:
if ID in gConfig.getValue("%s/Users/%s/ID" % (gBaseRegistrySection, username), []):
return S_OK(username)
return S_ERROR("No username found for ID %s" % ID)
def getUsernameForDN( dn, usersList = False ):
if not usersList:
retVal = gConfig.getSections( "%s/Users" % gBaseSecuritySection )
if not retVal[ 'OK' ]:
return retVal
usersList = retVal[ 'Value' ]
for username in usersList:
if dn in gConfig.getValue( "%s/Users/%s/DN" % ( gBaseSecuritySection, username ), [] ):
return S_OK( username )
return S_ERROR( "No username found for dn %s" % dn )
def getSites( grids = None ):
if grids == None:
grids = ['LCG']
if isinstance(grids, basestring):
grids = [grids]
sites = []
for grid in grids:
s = gConfig.getSections('%s/Sites/%s' %( g_BaseResourcesSection, grid ), True)
if not s['OK']:
return s
sites = sites + s['Value']
return S_OK(sites)
def getGroupsForVO(vo):
""" Get groups for VO
:param str vo: DIRAC VO name
:return: S_OK(list)/S_ERROR()
"""
if getVO(): # tries to get default VO in /DIRAC/VirtualOrganization
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
if not vo:
return S_ERROR("No VO requested")
return __getGroupsWithAttr('VO', vo)
def getGroupsForUser( username ):
retVal = gConfig.getSections( "%s/Groups" % gBaseSecuritySection )
if not retVal[ 'OK' ]:
return retVal
groupsList = retVal[ 'Value' ]
userGroups = []
for group in groupsList:
if username in gConfig.getValue( "%s/Groups/%s/Users" % ( gBaseSecuritySection, group ), [] ):
userGroups.append( group )
if not userGroups:
return S_ERROR( "No groups found for user %s" % username )
userGroups.sort()
return S_OK( userGroups )
def getGroupsForUser(username):
retVal = gConfig.getSections("%s/Groups" % gBaseSecuritySection)
if not retVal["OK"]:
return retVal
groupsList = retVal["Value"]
userGroups = []
for group in groupsList:
if username in gConfig.getValue("%s/Groups/%s/Users" % (gBaseSecuritySection, group), []):
userGroups.append(group)
if not userGroups:
return S_ERROR("No groups found for user %s" % username)
userGroups.sort()
return S_OK(userGroups)
def __getGroupsWithAttr( attrName, value ):
retVal = gConfig.getSections( "%s/Groups" % gBaseSecuritySection )
if not retVal[ 'OK' ]:
return retVal
groupsList = retVal[ 'Value' ]
groups = []
for group in groupsList:
if value in gConfig.getValue( "%s/Groups/%s/%s" % ( gBaseSecuritySection, group, attrName ), [] ):
groups.append( group )
if not groups:
return S_ERROR( "No groups found for %s=%s" % ( attrName,value ) )
groups.sort()
return S_OK( groups )
def getSites(grids=None):
if grids == None:
grids = ['LCG']
if isinstance(grids, basestring):
grids = [grids]
sites = []
for grid in grids:
s = gConfig.getSections('%s/Sites/%s' % (g_BaseResourcesSection, grid),
True)
if not s['OK']:
return s
sites = sites + s['Value']
return S_OK(sites)
def getGroupsWithVOMSAttribute(vomsAttr):
""" Search groups with VOMS attribute
:param str vomsAttr: VOMS attribute
:return: list
"""
groups = []
for group in gConfig.getSections("%s/Groups" % (gBaseRegistrySection)).get(
'Value', []):
if vomsAttr == gConfig.getValue(
"%s/Groups/%s/VOMSRole" % (gBaseRegistrySection, group), ""):
groups.append(group)
return groups
def getStorageElements( hostName = None ):
SEs = gConfig.getSections('%s/StorageElements' %g_BaseResourcesSection)
if not SEs['OK']:
return SEs
SEs = SEs['Value']
if hostName != None:
removeSEs = []
if isinstance(hostName, basestring):
hostName = [hostName]
for SE in SEs:
host = gConfig.getValue('%s/StorageElements/%s/AccessProtocol.1/Host' %(g_BaseResourcesSection, SE) )
if host not in hostName:
removeSEs.append(SE)
for SE in removeSEs:
SEs.remove(SE)
return S_OK(SEs)
def getHostnameForDN(dn):
""" Search host name for host DN
:param str dn: host DN
:return: S_OK()/S_ERROR()
"""
result = gConfig.getSections("%s/Hosts" % gBaseRegistrySection)
if not result['OK']:
return result
hostList = result['Value']
for hostname in hostList:
if dn in gConfig.getValue(
"%s/Hosts/%s/DN" % (gBaseRegistrySection, hostname), []):
return S_OK(hostname)
return S_ERROR("No hostname found for dn %s" % dn)
def getUsernameForDN(dn, usersList=None):
""" Find DIRAC user for DN
:param str dn: user DN
:param list usersList: list of possible users
:return: S_OK(str)/S_ERROR()
"""
if not usersList:
result = gConfig.getSections("%s/Users" % gBaseRegistrySection)
if not result['OK']:
return result
usersList = result['Value']
for username in usersList:
if dn in gConfig.getValue(
"%s/Users/%s/DN" % (gBaseRegistrySection, username), []):
return S_OK(username)
return S_ERROR("No username found for dn %s" % dn)
def getStorageElements(hostName=None):
SEs = gConfig.getSections('%s/StorageElements' % g_BaseResourcesSection)
if not SEs['OK']:
return SEs
SEs = SEs['Value']
if hostName != None:
removeSEs = []
if isinstance(hostName, basestring):
hostName = [hostName]
for SE in SEs:
host = gConfig.getValue(
'%s/StorageElements/%s/AccessProtocol.1/Host' %
(g_BaseResourcesSection, SE))
if host not in hostName:
removeSEs.append(SE)
for SE in removeSEs:
SEs.remove(SE)
return S_OK(SEs)
def getUsernameForID(ID, usersList=None):
""" Get DIRAC user name by ID
:param str ID: user ID
:param list usersList: list of DIRAC user names
:return: S_OK(str)/S_ERROR()
"""
if not usersList:
result = gConfig.getSections("%s/Users" % gBaseRegistrySection)
if not result['OK']:
return result
usersList = result['Value']
for username in usersList:
if ID in gConfig.getValue(
"%s/Users/%s/ID" % (gBaseRegistrySection, username), []):
return S_OK(username)
return S_ERROR("No username found for ID %s" % ID)
def __getGroupsWithAttr(attrName, value):
""" Get all possible groups with some attribute
:param str attrName: attribute name
:param str value: attribute value
:return: S_OK(list)/S_ERROR() -- contain list of groups
"""
result = gConfig.getSections("%s/Groups" % gBaseRegistrySection)
if not result['OK']:
return result
groupsList = result['Value']
groups = []
for group in groupsList:
if value in gConfig.getValue(
"%s/Groups/%s/%s" % (gBaseRegistrySection, group, attrName),
[]):
groups.append(group)
groups.sort()
return S_OK(groups) if groups else S_ERROR("No groups found for %s=%s" %
(attrName, value))
def getDNProperty(userDN, value, defaultValue=None):
""" Get property from DNProperties section by user DN
:param str userDN: user DN
:param str value: option that need to get
:param defaultValue: default value
:return: S_OK()/S_ERROR() -- str or list that contain option value
"""
result = getUsernameForDN(userDN)
if not result['OK']:
return result
pathDNProperties = "%s/Users/%s/DNProperties" % (gBaseRegistrySection,
result['Value'])
result = gConfig.getSections(pathDNProperties)
if result['OK']:
for section in result['Value']:
if userDN == gConfig.getValue("%s/%s/DN" %
(pathDNProperties, section)):
return S_OK(
gConfig.getValue(
"%s/%s/%s" % (pathDNProperties, section, value),
defaultValue))
return S_OK(defaultValue)
def getLFCSites():
lfcL = gConfig.getSections('%s/FileCatalogs/LcgFileCatalogCombined' %g_BaseResourcesSection,
True)
return lfcL
def getHosts():
return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getAllGroups():
retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection)
if not retVal['OK']:
return []
return retVal['Value']
def getAllUsers():
retVal = gConfig.getSections( "%s/Users" % gBaseSecuritySection )
if not retVal[ 'OK' ]:
return []
return retVal[ 'Value' ]
def getHosts(): return gConfig.getSections( '%s/Hosts' % gBaseSecuritySection )
def getSetups( self ):
setupsList = gConfig.getSections( "/DIRAC/Setups" )
if not setupsList[ 'OK' ]:
return []
return setupsList[ 'Value' ]
def getHosts():
""" Get all hosts
:return: S_OK()/S_ERROR()
"""
return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getHosts():
return gConfig.getSections('%s/Hosts' % gBaseRegistrySection)
def getAllGroups():
retVal = gConfig.getSections("%s/Groups" % gBaseRegistrySection)
if not retVal['OK']:
return []
return retVal['Value']
def getHosts():
return gConfig.getSections('%s/Hosts' % gBaseSecuritySection)
def getGroupsForVO(vo):
if getVO():
return gConfig.getSections("%s/Groups" % gBaseRegistrySection)
return __getGroupsWithAttr('VO', vo)
def getVOMSEndpoints():
voms_S = gConfig.getSections("%s/VOMS/Servers/lhcb/" %(g_BaseRegistrySection) ,'')
return voms_S
def web_getData(self):
siteMaxJobs = {}
#Calculate Max pledges for Cluster and GRIDs
for dir in gConfig.getSections('/Resources/Sites')['Value']:
for site in gConfig.getSections('/Resources/Sites/'+dir)['Value']:
if 'CEs' in gConfig.getSections('/Resources/Sites/'+dir+'/'+site)['Value']:
for ce in gConfig.getSections('/Resources/Sites/'+dir+'/'+site+'/CEs')['Value']:
for queue in gConfig.getSections('/Resources/Sites/'+dir+'/'+site+'/CEs/'+ce+'/Queues')['Value']:
siteMaxJobs[site] = siteMaxJobs.get(site, 0) + int(gConfig.getValue('/Resources/Sites/'+dir+'/'+site+'/CEs/'+ce+'/Queues/'+queue+'/MaxTotalJobs'))
#Calculate Max pledges for clouds
for vm in gConfig.getSections('/Resources/VirtualMachines/CloudEndpoints')['Value']:
site = gConfig.getValue('/Resources/VirtualMachines/CloudEndpoints/'+vm+'/siteName')
limit = gConfig.getValue('/Resources/VirtualMachines/CloudEndpoints/'+vm+'/maxEndpointInstances')
if site is not None and limit is not None:
siteMaxJobs[site] = int(limit)
pprint(siteMaxJobs)
for site in siteMaxJobs:
siteMaxJobs[site] = {'max': siteMaxJobs[site], 'current':0}
#Get current Running jobs
import MySQLdb #@UnresolvedImport
db = MySQLdb.connect(host="diracdb2.ihep.ac.cn", # your host, usually localhost
user="******", # your username
passwd="###DIRAC_DB_PASS###", # your password
db="JobDB")
cur = db.cursor()
cur.execute("select Site, count(*) from Jobs where Status='Running' group by Site")
for row in cur.fetchall() :
if row[0] not in siteMaxJobs:
siteMaxJobs[row[0]] = {'max':0}
siteMaxJobs[row[0]]['current'] = row[1]
#Get Weekly Done and Failed jobs
for site in siteMaxJobs:
siteMaxJobs[site]['week_done'] = 0
siteMaxJobs[site]['week_failed'] = 0
cur.execute("select Site, count(*), min(SubmissionTime), max(SubmissionTime) from Jobs where (Status='Done') and SubmissionTime>DATE_SUB(NOW(), INTERVAL 168 HOUR) group by Site")
for row in cur.fetchall():
if row[0] not in siteMaxJobs:
siteMaxJobs[row[0]] = {'max':0, 'current':0, 'week_done':0, 'week_failed':0}
siteMaxJobs[row[0]]['week_done'] = row[1]
cur.execute("select Site, count(*), min(SubmissionTime), max(SubmissionTime) from Jobs where (Status='Failed') and SubmissionTime>DATE_SUB(NOW(), INTERVAL 168 HOUR) group by Site")
for row in cur.fetchall():
if row[0] not in siteMaxJobs:
siteMaxJobs[row[0]] = {'max':0, 'current':0, 'week_done':0, 'week_failed':0}
siteMaxJobs[row[0]]['week_failed'] = row[1]
cur.close()
res = []
for site in siteMaxJobs:
res.append({'site':site, 'max':siteMaxJobs[site]['max'], 'current':siteMaxJobs[site]['current'], 'week_done':siteMaxJobs[site]['week_done'], 'week_failed':siteMaxJobs[site]['week_failed']})
#from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
#db = JobDB()
#cmd = 'SELECT count(*) FROM Jobs'
#res = db._query( cmd )
#print res
print siteMaxJobs
self.write({"result":res})
def getSchemaSections( self, path = "" ):
retDict = gConfig.getSections( "%s/Schema/%s" % ( self.webSection, path ), listOrdered = True )
if retDict[ 'OK' ]:
return retDict[ 'Value' ]
else:
return []
def getGroupsForVO( vo ):
if getVO():
return gConfig.getSections( "%s/Groups" % gBaseSecuritySection )
return __getGroupsWithAttr( 'VO', vo )
