def ldapTag(ce,vo):
# returns the list of tags that are associated with this CE
res = ldapsearchBDII( filt = "(&(objectClass=GlueSubCluster)(GlueChunkKey=GlueClusterUniqueID=%s))"%ce, attr = "GlueHostApplicationSoftwareRuntimeEnvironment")
tags = []
if res['OK'] and len(res['Value'])!=0:
tagdict = res['Value'][-1]
if tagdict['attr'].has_key('GlueHostApplicationSoftwareRunTimeEnvironment'):
tags = [tag for tag in tagdict['attr']['GlueHostApplicationSoftwareRunTimeEnvironment'] if vo in tag]
return tags
def ldapCEs(vo):
# returns the list of CEs that are associated with the correct VO
res = ldapsearchBDII(
filt="(&(objectClass=GlueCE)(GlueCEAccessControlBaseRule=VO:%s))" % vo,
attr="GlueCEUniqueID",
host=None,
base=None)
celist = [
res['Value'][i]['attr']['GlueCEUniqueID'].split(":")[0]
for i, value in enumerate(res['Value']) if res['OK']
]
return celist
def getCEStatus( self ):
""" Method to return information on running and pending jobs.
"""
vo = ''
result = getVOfromProxyGroup()
if result['OK']:
vo = result['Value']
else: # A backup solution which may work
vo = self.ceParameters['VO']
voFilters = '(GlueCEAccessControlBaseRule=VOMS:/%s/*)' % vo
voFilters += '(GlueCEAccessControlBaseRule=VOMS:/%s)' % vo
voFilters += '(GlueCEAccessControlBaseRule=VO:%s)' % vo
filt = '(&(GlueCEUniqueID=%s*)(|%s))' % ( self.ceHost, voFilters )
result = ldapsearchBDII( filt, attr=None, host=None, base=None )
ces = result['Value']
filt = '(&(objectClass=GlueVOView)(|%s))' % ( voFilters )
dn = ces[0]['dn']
result = ldapsearchBDII( filt, attr=None, host=None, base = dn )
stats = result['Value'][0]['attr']
result['RunningJobs'] = int(stats["GlueCEStateRunningJobs"])
result['WaitingJobs'] = int(stats["GlueCEStateTotalJobs"])
result['SubmittedJobs'] = 0
return result
def ldapTag(ce, vo):
# returns the list of tags that are associated with this CE
res = ldapsearchBDII(
filt=
"(&(objectClass=GlueSubCluster)(GlueChunkKey=GlueClusterUniqueID=%s))"
% ce,
attr="GlueHostApplicationSoftwareRuntimeEnvironment")
tags = []
if res['OK'] and len(res['Value']) != 0:
tagdict = res['Value'][-1]
if tagdict['attr'].has_key(
'GlueHostApplicationSoftwareRunTimeEnvironment'):
tags = [
tag for tag in tagdict['attr']
['GlueHostApplicationSoftwareRunTimeEnvironment'] if vo in tag
]
return tags
def getOccupancy(self, **kwargs):
"""Returns the space information given by BDII
Total and Free space are taken from GlueSATotalOnlineSize and GlueSAFreeOnlineSize, respectively.
:returns: S_OK with dict (keys: Total, Free)
"""
sTokenDict = {"Total": 0, "Free": 0}
BDIIAttr = ["GlueSATotalOnlineSize", "GlueSAFreeOnlineSize"]
filt = "(&(GlueSAAccessControlBaseRule=VO:%s)(GlueChunkKey=GlueSEUniqueID=%s))" % (self.vo, self.host)
ret = ldapsearchBDII(filt, BDIIAttr, host=self.bdii)
if not ret["OK"]:
return ret
for value in ret["Value"]:
if "attr" in value:
attr = value["attr"]
sTokenDict["Total"] = float(attr.get(BDIIAttr[0], 0)) * 1024 * 1024 * 1024
sTokenDict["Free"] = float(attr.get(BDIIAttr[1], 0)) * 1024 * 1024 * 1024
return S_OK(sTokenDict)
def __getGlue2ExecutionEnvironmentInfo(host, executionEnvironments):
"""Find all the executionEnvironments.
:param str host: BDII host to query
:param list executionEnvironments: list of the execution environments to get some information from
:returns: result of the ldapsearch for all executionEnvironments, Glue2 schema
"""
listOfValues = []
# break up to avoid argument list too long, it started failing at about 1900 entries
for exeEnvs in breakListIntoChunks(executionEnvironments, 1000):
exeFilter = ""
for execEnv in exeEnvs:
exeFilter += "(GLUE2ResourceID=%s)" % execEnv
filt = "(&(objectClass=GLUE2ExecutionEnvironment)(|%s))" % exeFilter
response = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2")
if not response["OK"]:
return response
if not response["Value"]:
sLog.error("No information found for %s" % executionEnvironments)
continue
listOfValues += response["Value"]
if not listOfValues:
return S_ERROR("No information found for executionEnvironments")
return S_OK(listOfValues)
def __ldapsearchBDII(*args, **kwargs): """ wrap `DIRAC.Core.Utilities.Grid.ldapsearchBDII` to avoid circular import """ from DIRAC.Core.Utilities.Grid import ldapsearchBDII return ldapsearchBDII(*args, **kwargs)
def __ldapsearchBDII(*args, **kwargs):
""" wrap `DIRAC.Core.Utilities.Grid.ldapsearchBDII` to avoid circular import """
from DIRAC.Core.Utilities.Grid import ldapsearchBDII
return ldapsearchBDII(*args, **kwargs)
def getGlue2CEInfo(vo, host=None):
"""call ldap for GLUE2 and get information
:param str vo: Virtual Organisation
:param str host: host to query for information
:returns: result structure with result['Value'][siteID]['CEs'][ceID]['Queues'][queueName]. For
each siteID, ceID, queueName all the GLUE2 parameters are retrieved
"""
# get all Policies allowing given VO
filt = "(&(objectClass=GLUE2Policy)(|(GLUE2PolicyRule=VO:%s)(GLUE2PolicyRule=vo:%s)))" % (vo, vo)
polRes = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2")
if not polRes["OK"]:
return S_ERROR("Failed to get policies for this VO")
polRes = polRes["Value"]
sLog.notice("Found %s policies for this VO %s" % (len(polRes), vo))
# get all shares for this policy
# create an or'ed list of all the shares and then call the search
listOfSitesWithPolicies = set()
shareFilter = ""
for policyValues in polRes:
# skip entries without GLUE2DomainID in the DN because we cannot associate them to a site
if "GLUE2DomainID" not in policyValues["attr"]["dn"]:
continue
shareID = policyValues["attr"].get("GLUE2MappingPolicyShareForeignKey", None)
policyID = policyValues["attr"]["GLUE2PolicyID"]
siteName = policyValues["attr"]["dn"].split("GLUE2DomainID=")[1].split(",", 1)[0]
listOfSitesWithPolicies.add(siteName)
if shareID is None: # policy not pointing to ComputingInformation
sLog.debug("Policy %s does not point to computing information" % (policyID,))
continue
sLog.verbose("%s policy %s pointing to %s " % (siteName, policyID, shareID))
sLog.debug("Policy values:\n%s" % pformat(policyValues))
shareFilter += "(GLUE2ShareID=%s)" % shareID
filt = "(&(objectClass=GLUE2Share)(|%s))" % shareFilter
shareRes = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2")
if not shareRes["OK"]:
sLog.error("Could not get share information", shareRes["Message"])
return shareRes
shareInfoLists = {}
for shareInfo in shareRes["Value"]:
if "GLUE2DomainID" not in shareInfo["attr"]["dn"]:
continue
if "GLUE2ComputingShare" not in shareInfo["objectClass"]:
sLog.debug("Share %r is not a ComputingShare: \n%s" % (shareID, pformat(shareInfo)))
continue
sLog.debug("Found computing share:\n%s" % pformat(shareInfo))
siteName = shareInfo["attr"]["dn"].split("GLUE2DomainID=")[1].split(",", 1)[0]
shareInfoLists.setdefault(siteName, []).append(shareInfo["attr"])
siteInfo = __getGlue2ShareInfo(host, shareInfoLists)
if not siteInfo["OK"]:
sLog.error("Could not get CE info for", "%s: %s" % (shareID, siteInfo["Message"]))
return siteInfo
siteDict = siteInfo["Value"]
sLog.debug("Found Sites:\n%s" % pformat(siteDict))
sitesWithoutShares = set(siteDict) - listOfSitesWithPolicies
if sitesWithoutShares:
sLog.error("Found some sites without any shares", pformat(sitesWithoutShares))
else:
sLog.notice("Found information for all known sites")
# remap siteDict to assign CEs to known sites,
# in case their names differ from the "gocdb name" in the CS.
newSiteDict = {}
ceSiteMapping = getCESiteMapping().get("Value", {})
# pylint thinks siteDict is a tuple, so we cast
for siteName, infoDict in dict(siteDict).items():
for ce, ceInfo in infoDict.get("CEs", {}).items():
ceSiteName = ceSiteMapping.get(ce, siteName)
gocSiteName = getGOCSiteName(ceSiteName).get("Value", siteName)
newSiteDict.setdefault(gocSiteName, {}).setdefault("CEs", {})[ce] = ceInfo
return S_OK(newSiteDict)
def ldapCEs(vo):
# returns the list of CEs that are associated with the correct VO
res = ldapsearchBDII( filt = "(&(objectClass=GlueCE)(GlueCEAccessControlBaseRule=VO:%s))"%vo, attr = "GlueCEUniqueID", host = None, base = None)
celist = [res['Value'][i]['attr']['GlueCEUniqueID'].split(":")[0] for i,value in enumerate(res['Value']) if res['OK']]
return celist