def ldapTag(ce,vo): # returns the list of tags that are associated with this CE res = ldapsearchBDII( filt = "(&(objectClass=GlueSubCluster)(GlueChunkKey=GlueClusterUniqueID=%s))"%ce, attr = "GlueHostApplicationSoftwareRuntimeEnvironment") tags = [] if res['OK'] and len(res['Value'])!=0: tagdict = res['Value'][-1] if tagdict['attr'].has_key('GlueHostApplicationSoftwareRunTimeEnvironment'): tags = [tag for tag in tagdict['attr']['GlueHostApplicationSoftwareRunTimeEnvironment'] if vo in tag] return tags
def ldapCEs(vo): # returns the list of CEs that are associated with the correct VO res = ldapsearchBDII( filt="(&(objectClass=GlueCE)(GlueCEAccessControlBaseRule=VO:%s))" % vo, attr="GlueCEUniqueID", host=None, base=None) celist = [ res['Value'][i]['attr']['GlueCEUniqueID'].split(":")[0] for i, value in enumerate(res['Value']) if res['OK'] ] return celist
def getCEStatus( self ): """ Method to return information on running and pending jobs. """ vo = '' result = getVOfromProxyGroup() if result['OK']: vo = result['Value'] else: # A backup solution which may work vo = self.ceParameters['VO'] voFilters = '(GlueCEAccessControlBaseRule=VOMS:/%s/*)' % vo voFilters += '(GlueCEAccessControlBaseRule=VOMS:/%s)' % vo voFilters += '(GlueCEAccessControlBaseRule=VO:%s)' % vo filt = '(&(GlueCEUniqueID=%s*)(|%s))' % ( self.ceHost, voFilters ) result = ldapsearchBDII( filt, attr=None, host=None, base=None ) ces = result['Value'] filt = '(&(objectClass=GlueVOView)(|%s))' % ( voFilters ) dn = ces[0]['dn'] result = ldapsearchBDII( filt, attr=None, host=None, base = dn ) stats = result['Value'][0]['attr'] result['RunningJobs'] = int(stats["GlueCEStateRunningJobs"]) result['WaitingJobs'] = int(stats["GlueCEStateTotalJobs"]) result['SubmittedJobs'] = 0 return result
def ldapTag(ce, vo): # returns the list of tags that are associated with this CE res = ldapsearchBDII( filt= "(&(objectClass=GlueSubCluster)(GlueChunkKey=GlueClusterUniqueID=%s))" % ce, attr="GlueHostApplicationSoftwareRuntimeEnvironment") tags = [] if res['OK'] and len(res['Value']) != 0: tagdict = res['Value'][-1] if tagdict['attr'].has_key( 'GlueHostApplicationSoftwareRunTimeEnvironment'): tags = [ tag for tag in tagdict['attr'] ['GlueHostApplicationSoftwareRunTimeEnvironment'] if vo in tag ] return tags
def getOccupancy(self, **kwargs): """Returns the space information given by BDII Total and Free space are taken from GlueSATotalOnlineSize and GlueSAFreeOnlineSize, respectively. :returns: S_OK with dict (keys: Total, Free) """ sTokenDict = {"Total": 0, "Free": 0} BDIIAttr = ["GlueSATotalOnlineSize", "GlueSAFreeOnlineSize"] filt = "(&(GlueSAAccessControlBaseRule=VO:%s)(GlueChunkKey=GlueSEUniqueID=%s))" % (self.vo, self.host) ret = ldapsearchBDII(filt, BDIIAttr, host=self.bdii) if not ret["OK"]: return ret for value in ret["Value"]: if "attr" in value: attr = value["attr"] sTokenDict["Total"] = float(attr.get(BDIIAttr[0], 0)) * 1024 * 1024 * 1024 sTokenDict["Free"] = float(attr.get(BDIIAttr[1], 0)) * 1024 * 1024 * 1024 return S_OK(sTokenDict)
def __getGlue2ExecutionEnvironmentInfo(host, executionEnvironments): """Find all the executionEnvironments. :param str host: BDII host to query :param list executionEnvironments: list of the execution environments to get some information from :returns: result of the ldapsearch for all executionEnvironments, Glue2 schema """ listOfValues = [] # break up to avoid argument list too long, it started failing at about 1900 entries for exeEnvs in breakListIntoChunks(executionEnvironments, 1000): exeFilter = "" for execEnv in exeEnvs: exeFilter += "(GLUE2ResourceID=%s)" % execEnv filt = "(&(objectClass=GLUE2ExecutionEnvironment)(|%s))" % exeFilter response = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2") if not response["OK"]: return response if not response["Value"]: sLog.error("No information found for %s" % executionEnvironments) continue listOfValues += response["Value"] if not listOfValues: return S_ERROR("No information found for executionEnvironments") return S_OK(listOfValues)
def __ldapsearchBDII(*args, **kwargs): """ wrap `DIRAC.Core.Utilities.Grid.ldapsearchBDII` to avoid circular import """ from DIRAC.Core.Utilities.Grid import ldapsearchBDII return ldapsearchBDII(*args, **kwargs)
def getGlue2CEInfo(vo, host=None): """call ldap for GLUE2 and get information :param str vo: Virtual Organisation :param str host: host to query for information :returns: result structure with result['Value'][siteID]['CEs'][ceID]['Queues'][queueName]. For each siteID, ceID, queueName all the GLUE2 parameters are retrieved """ # get all Policies allowing given VO filt = "(&(objectClass=GLUE2Policy)(|(GLUE2PolicyRule=VO:%s)(GLUE2PolicyRule=vo:%s)))" % (vo, vo) polRes = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2") if not polRes["OK"]: return S_ERROR("Failed to get policies for this VO") polRes = polRes["Value"] sLog.notice("Found %s policies for this VO %s" % (len(polRes), vo)) # get all shares for this policy # create an or'ed list of all the shares and then call the search listOfSitesWithPolicies = set() shareFilter = "" for policyValues in polRes: # skip entries without GLUE2DomainID in the DN because we cannot associate them to a site if "GLUE2DomainID" not in policyValues["attr"]["dn"]: continue shareID = policyValues["attr"].get("GLUE2MappingPolicyShareForeignKey", None) policyID = policyValues["attr"]["GLUE2PolicyID"] siteName = policyValues["attr"]["dn"].split("GLUE2DomainID=")[1].split(",", 1)[0] listOfSitesWithPolicies.add(siteName) if shareID is None: # policy not pointing to ComputingInformation sLog.debug("Policy %s does not point to computing information" % (policyID,)) continue sLog.verbose("%s policy %s pointing to %s " % (siteName, policyID, shareID)) sLog.debug("Policy values:\n%s" % pformat(policyValues)) shareFilter += "(GLUE2ShareID=%s)" % shareID filt = "(&(objectClass=GLUE2Share)(|%s))" % shareFilter shareRes = ldapsearchBDII(filt=filt, attr=None, host=host, base="o=glue", selectionString="GLUE2") if not shareRes["OK"]: sLog.error("Could not get share information", shareRes["Message"]) return shareRes shareInfoLists = {} for shareInfo in shareRes["Value"]: if "GLUE2DomainID" not in shareInfo["attr"]["dn"]: continue if "GLUE2ComputingShare" not in shareInfo["objectClass"]: sLog.debug("Share %r is not a ComputingShare: \n%s" % (shareID, pformat(shareInfo))) continue sLog.debug("Found computing share:\n%s" % pformat(shareInfo)) siteName = shareInfo["attr"]["dn"].split("GLUE2DomainID=")[1].split(",", 1)[0] shareInfoLists.setdefault(siteName, []).append(shareInfo["attr"]) siteInfo = __getGlue2ShareInfo(host, shareInfoLists) if not siteInfo["OK"]: sLog.error("Could not get CE info for", "%s: %s" % (shareID, siteInfo["Message"])) return siteInfo siteDict = siteInfo["Value"] sLog.debug("Found Sites:\n%s" % pformat(siteDict)) sitesWithoutShares = set(siteDict) - listOfSitesWithPolicies if sitesWithoutShares: sLog.error("Found some sites without any shares", pformat(sitesWithoutShares)) else: sLog.notice("Found information for all known sites") # remap siteDict to assign CEs to known sites, # in case their names differ from the "gocdb name" in the CS. newSiteDict = {} ceSiteMapping = getCESiteMapping().get("Value", {}) # pylint thinks siteDict is a tuple, so we cast for siteName, infoDict in dict(siteDict).items(): for ce, ceInfo in infoDict.get("CEs", {}).items(): ceSiteName = ceSiteMapping.get(ce, siteName) gocSiteName = getGOCSiteName(ceSiteName).get("Value", siteName) newSiteDict.setdefault(gocSiteName, {}).setdefault("CEs", {})[ce] = ceInfo return S_OK(newSiteDict)
def ldapCEs(vo): # returns the list of CEs that are associated with the correct VO res = ldapsearchBDII( filt = "(&(objectClass=GlueCE)(GlueCEAccessControlBaseRule=VO:%s))"%vo, attr = "GlueCEUniqueID", host = None, base = None) celist = [res['Value'][i]['attr']['GlueCEUniqueID'].split(":")[0] for i,value in enumerate(res['Value']) if res['OK']] return celist