def getCEsFromCS():
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSiteCEMapping
res = getSiteCEMapping()
if not res['OK']:
return res
knownCEs = []
for site in res['Value']:
knownCEs = knownCEs + res['Value'][site]
return S_OK(knownCEs)
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
site = self.getSite(elementType, elementName)
if not site:
return S_ERROR('No site')
siteStatus = rsClient.selectStatusElement('Site', 'Status', name=site,
meta={'columns': ['StatusType', 'Status']})
if not siteStatus['OK']:
return siteStatus
tree = {site: {'statusTypes': dict(siteStatus['Value'])}}
result = getSiteCEMapping()
if not result['OK']:
return result
ces = result['Value'][site]
cesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ces,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not cesStatus['OK']:
return cesStatus
res = DMSHelpers().getSiteSEMapping()
if not res['OK']:
self.log.error('Could not get site to SE mapping', res['Message'])
return S_OK()
ses = res['Value'][1].get(site, [])
sesStatus = rsClient.selectStatusElement('Resource', 'Status', name=list(ses),
meta={'columns': ['Name', 'StatusType', 'Status']})
if not sesStatus['OK']:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList['Value']:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]['ces'] = feedTree(cesStatus)
tree[site]['ses'] = feedTree(sesStatus)
return S_OK(tree)
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
if siteNames is None:
res = getSites()
if not res["OK"]:
self.log.error("Error getting sites", res["Message"])
return res
siteNames = res["Value"]
if isinstance(siteNames, six.string_types):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
result = getSiteCEMapping()
if not result["OK"]:
self.log.error("Error getting sites/CEs mapping",
result["Message"])
return result
res = {}
res["ces"] = result["Value"][siteName]
# Convert StorageElements to host names
result = DMSHelpers().getSiteSEMapping()
if not result["OK"]:
self.log.error("Error getting sites/SEs mapping",
result["Message"])
sitesRes[siteName] = res
continue
ses = result["Value"][1].get(siteName, [])
result = getStorageElementsHosts(ses)
if not result["OK"]:
self.log.error("Error getting storage element hosts",
result["Message"])
return result
# Remove duplicates
res["ses"] = list(set(result["Value"]))
sitesRes[siteName] = res
return S_OK(sitesRes)
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
result = self.getSite(elementType, elementName)
if not result["OK"]:
return result
site = result["Value"]
siteStatus = self.rsClient.selectStatusElement(
"Site",
"Status",
name=site,
meta={"columns": ["StatusType", "Status"]})
if not siteStatus["OK"]:
return siteStatus
tree = {site: {"statusTypes": dict(siteStatus["Value"])}}
result = getSiteCEMapping()
if not result["OK"]:
return result
ces = result["Value"][site]
cesStatus = self.rsClient.selectStatusElement(
"Resource",
"Status",
name=ces,
meta={"columns": ["Name", "StatusType", "Status"]})
if not cesStatus["OK"]:
return cesStatus
res = DMSHelpers().getSiteSEMapping()
if not res["OK"]:
self.log.error("Could not get site to SE mapping", res["Message"])
return S_OK()
ses = res["Value"][1].get(site, [])
sesStatus = self.rsClient.selectStatusElement(
"Resource",
"Status",
name=list(ses),
meta={"columns": ["Name", "StatusType", "Status"]})
if not sesStatus["OK"]:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList["Value"]:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]["ces"] = feedTree(cesStatus)
tree[site]["ses"] = feedTree(sesStatus)
return S_OK(tree)
def getSiteComputingElements(siteName):
from DIRAC.ConfigurationSystem.Client.Helpers.Resources import getSiteCEMapping
res = getSiteCEMapping()
if not res['OK']:
return res
return res['Value'][siteName]
def __lookForNewCEs(self):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption('BannedCEs', [])
res = getSiteCEMapping()
if not res['OK']:
return res
knownCEs = set()
for site in res['Value']:
knownCEs = knownCEs.union(set(res['Value'][site]))
knownCEs = knownCEs.union(set(bannedCEs))
for vo in self.voName:
result = self.__getBdiiCEInfo(vo)
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs(vo, bdiiInfo=bdiiInfo, ceBlackList=knownCEs)
if not result['OK']:
self.log.error('Failed to get unused CEs', result['Message'])
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set(siteDict[site].keys()) # pylint: disable=no-member
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
newCEString = "CE: %s, GOCDB Site Name: %s" % (ce, site)
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % (systemTuple)
newCEString = "\n%s\n%s\n" % (newCEString, osString)
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get(
'GlueCEStateStatus', 'UnknownStatus')
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get(
'GlueCEImplementationName', '')
queueString += " %s %s %s\n" % (
queue, queueStatus, ceType)
if queueString:
ceString += newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info(body)
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail(self.addressTo,
self.subject,
body,
self.addressFrom,
localAttempt=False,
avoidSpam=True)
if not result['OK']:
self.log.error(
'Can not send new site notification mail',
result['Message'])
return S_OK()