def doMaster(self):
""" Master method, which looks little bit spaghetti code, sorry !
- It gets all sites and transforms them into gocSites.
- It gets all the storage elements and transforms them into their hosts
- It gets the the CEs (FTS and file catalogs will come).
"""
gocSites = getGOCSites()
if not gocSites['OK']:
return gocSites
gocSites = gocSites['Value']
sesHosts = getStorageElementsHosts()
if not sesHosts['OK']:
return sesHosts
sesHosts = sesHosts['Value']
resources = sesHosts if sesHosts else []
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer['OK'] and ftsServer['Value']:
resources.extend(ftsServer['Value'])
# TODO: file catalogs need also to use their hosts
# fc = CSHelpers.getFileCatalogs()
# if fc[ 'OK' ]:
# resources = resources + fc[ 'Value' ]
ce = getComputingElements()
if ce['OK'] and ce['Value']:
resources.extend(ce['Value'])
self.log.verbose('Processing Sites',
', '.join(gocSites if gocSites else ['NONE']))
siteRes = self.doNew(('Site', gocSites))
if not siteRes['OK']:
self.metrics['failed'].append(siteRes['Message'])
self.log.verbose('Processing Resources',
', '.join(resources if resources else ['NONE']))
resourceRes = self.doNew(('Resource', resources))
if not resourceRes['OK']:
self.metrics['failed'].append(resourceRes['Message'])
return S_OK(self.metrics)
def doMaster(self):
"""Master method, which looks little bit spaghetti code, sorry !
- It gets all sites and transforms them into gocSites.
- It gets all the storage elements and transforms them into their hosts
- It gets the the CEs (FTS and file catalogs will come).
"""
gocSites = getGOCSites()
if not gocSites["OK"]:
return gocSites
gocSites = gocSites["Value"]
sesHosts = getStorageElementsHosts()
if not sesHosts["OK"]:
return sesHosts
sesHosts = sesHosts["Value"]
resources = sesHosts if sesHosts else []
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer["OK"] and ftsServer["Value"]:
resources.extend(ftsServer["Value"])
# TODO: file catalogs need also to use their hosts
# fc = CSHelpers.getFileCatalogs()
# if fc[ 'OK' ]:
# resources = resources + fc[ 'Value' ]
res = getCESiteMapping()
if res["OK"] and res["Value"]:
resources.extend(list(res["Value"]))
self.log.verbose("Processing Sites",
", ".join(gocSites if gocSites else ["NONE"]))
siteRes = self.doNew(("Site", gocSites))
if not siteRes["OK"]:
self.metrics["failed"].append(siteRes["Message"])
self.log.verbose("Processing Resources",
", ".join(resources if resources else ["NONE"]))
resourceRes = self.doNew(("Resource", resources))
if not resourceRes["OK"]:
self.metrics["failed"].append(resourceRes["Message"])
return S_OK(self.metrics)
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
if siteNames is None:
res = getSites()
if not res["OK"]:
self.log.error("Error getting sites", res["Message"])
return res
siteNames = res["Value"]
if isinstance(siteNames, six.string_types):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
result = getSiteCEMapping()
if not result["OK"]:
self.log.error("Error getting sites/CEs mapping",
result["Message"])
return result
res = {}
res["ces"] = result["Value"][siteName]
# Convert StorageElements to host names
result = DMSHelpers().getSiteSEMapping()
if not result["OK"]:
self.log.error("Error getting sites/SEs mapping",
result["Message"])
sitesRes[siteName] = res
continue
ses = result["Value"][1].get(siteName, [])
result = getStorageElementsHosts(ses)
if not result["OK"]:
self.log.error("Error getting storage element hosts",
result["Message"])
return result
# Remove duplicates
res["ses"] = list(set(result["Value"]))
sitesRes[siteName] = res
return S_OK(sitesRes)
def __removeNonExistingResourcesFromRM(self):
'''
Remove resources from DowntimeCache table that no longer exist in the CS.
'''
if not getServiceURL("ResourceStatus/ResourceManagement"):
gLogger.verbose(
'ResourceManagement is not installed, skipping removal of non existing resources...'
)
return S_OK()
sesHosts = getStorageElementsHosts()
if not sesHosts['OK']:
return sesHosts
sesHosts = sesHosts['Value']
resources = sesHosts
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer['OK']:
resources.extend(ftsServer['Value'])
ce = CSHelpers.getComputingElements()
if ce['OK']:
resources.extend(ce['Value'])
downtimes = self.rManagement.selectDowntimeCache()
if not downtimes['OK']:
return downtimes
# Remove hosts that no longer exist in the CS
for host in downtimes['Value']:
gLogger.verbose('Checking if %s is still in the CS' % host[0])
if host[0] not in resources:
gLogger.verbose('%s is no longer in CS, removing entry...' %
host[0])
result = self.rManagement.deleteDowntimeCache(name=host[0])
if not result['OK']:
return result
return S_OK()
def __removeNonExistingResourcesFromRM(self):
"""
Remove resources from DowntimeCache table that no longer exist in the CS.
"""
if not getServiceURL("ResourceStatus/ResourceManagement"):
gLogger.verbose(
"ResourceManagement is not installed, skipping removal of non existing resources..."
)
return S_OK()
sesHosts = getStorageElementsHosts()
if not sesHosts["OK"]:
return sesHosts
sesHosts = sesHosts["Value"]
resources = sesHosts
ftsServer = getFTS3Servers(hostOnly=True)
if ftsServer["OK"]:
resources.extend(ftsServer["Value"])
res = getCESiteMapping()
if res["OK"]:
resources.extend(list(res["Value"]))
downtimes = self.rManagement.selectDowntimeCache()
if not downtimes["OK"]:
return downtimes
# Remove hosts that no longer exist in the CS
for host in downtimes["Value"]:
gLogger.verbose("Checking if %s is still in the CS" % host[0])
if host[0] not in resources:
gLogger.verbose("%s is no longer in CS, removing entry..." %
host[0])
result = self.rManagement.deleteDowntimeCache(name=host[0])
if not result["OK"]:
return result
return S_OK()
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
gLogger.info('getSitesResources')
if siteNames is None:
siteNames = getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
if isinstance(siteNames, basestring):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
res = {}
res['ces'] = CSHelpers.getSiteComputingElements(siteName)
# Convert StorageElements to host names
res = DMSHelpers().getSiteSEMapping()
if not res['OK']:
return res
ses = res['Value'][1].get(siteName, [])
sesHosts = getStorageElementsHosts(ses)
if not sesHosts['OK']:
return sesHosts
# Remove duplicates
res['ses'] = list(set(sesHosts['Value']))
sitesRes[siteName] = res
return S_OK(sitesRes)