def export_getTestHistory(self, elementType, element, fromDate, toDate):
gLogger.info('getTestHistory')
if fromDate > toDate:
return S_ERROR('from date can not be after the to date.')
selectElements = []
if elementType == 'Site':
if element.split('.')[ 0 ] == 'CLOUD':
selectElements.append( element )
else:
selectElements += CSHelpers.getSiteComputingElements(element)
selectElements += CSHelpers.getSiteStorageElements(element)
else:
selectElements = [ element ]
queryRes = rmClient.selectSAMResultLog(
elementName = selectElements,
meta = { 'newer' : ['LastCheckTime', fromDate ],
'older' : [ 'LastCheckTime', toDate ],
'columns' : [ 'ElementName', 'TestType', 'Status', 'LastCheckTime' ] }
)
if not queryRes[ 'OK' ]:
return queryRes
records = queryRes[ 'Value' ]
testHistory = {}
for record in records:
key = record[ 0 ] + '-' + record[ 1 ]
if key not in testHistory:
testHistory[ key ] = []
testHistory[ key ].append(( record[ 3 ], record[ 2 ] ))
return S_OK(testHistory)
def export_getSAMSummary(self, siteName, vo):
"""
Returns SAM tests status for the elements of the given site.
:return: S_OK( { element : { 'ElementType' :
'WMSTest' :
'CVMFSTest' :
'BOSSTest' :
'SETest' : } } ) / S_ERROR
"""
gLogger.info('getSAMSummary')
siteType = siteName.split('.')[ 0 ]
if 'CLOUD' == siteType:
ces = [ siteName ]
else:
ces = CSHelpers.getSiteComputingElements(siteName)
ses = CSHelpers.getSiteStorageElements(siteName)
samSummary = {}
for ce in ces:
samSummary[ ce ] = { 'ElementType' : 'ComputingElement' }
for se in ses:
samSummary[ se ] = { 'ElementType' : 'StorageElement' }
lastCheckTime = datetime.utcnow().replace(microsecond = 0) - timedelta(hours = 24)
queryRes = rmClient.selectResourceSAMStatus(elementName = ces, vO = vo,
meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
if not queryRes[ 'OK' ]:
return queryRes
records = queryRes[ 'Value' ]
columns = queryRes[ 'Columns' ]
if ses != []:
queryRes = rmClient.selectResourceSAMStatus(elementName = ses,
meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
if not queryRes[ 'OK' ]:
return queryRes
records += queryRes[ 'Value' ]
for record in records:
samDict = dict(zip(columns, record))
elementName = samDict[ 'ElementName' ]
samSummary[ elementName ][ 'Status' ] = samDict[ 'Status' ]
tests = [ test.strip() for test in samDict[ 'Tests' ].split(',') ]
queryRes = rmClient.selectSAMResult(elementName = elementName, testType = tests,
meta = { 'newer' : [ 'LastCheckTime', lastCheckTime ] })
if not queryRes[ 'OK' ]:
return queryRes
testRecords = queryRes[ 'Value' ]
testColumns = queryRes[ 'Columns' ]
for testRecord in testRecords:
testDict = dict(zip(testColumns, testRecord))
samSummary[ elementName ][ testDict[ 'TestType' ] ] = testDict[ 'Status' ]
return S_OK(samSummary)
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
gLogger.info('getTree')
site = self.getSite(elementType, elementName)
if not site:
return S_ERROR('No site')
siteStatus = rsClient.selectStatusElement(
'Site',
'Status',
name=site,
meta={'columns': ['StatusType', 'Status']})
if not siteStatus['OK']:
return siteStatus
tree = {site: {'statusTypes': dict(siteStatus['Value'])}}
ces = CSHelpers.getSiteComputingElements(site)
cesStatus = rsClient.selectStatusElement(
'Resource',
'Status',
name=ces,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not cesStatus['OK']:
return cesStatus
ses = CSHelpers.getSiteStorageElements(site)
sesStatus = rsClient.selectStatusElement(
'Resource',
'Status',
name=ses,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not sesStatus['OK']:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList['Value']:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]['ces'] = feedTree(cesStatus)
tree[site]['ses'] = feedTree(sesStatus)
return S_OK(tree)
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
gLogger.info('getTree')
site = self.getSite(elementType, elementName)
if not site:
return S_ERROR('No site')
siteStatus = rsClient.selectStatusElement('Site', 'Status', name=site,
meta={'columns': ['StatusType', 'Status']})
if not siteStatus['OK']:
return siteStatus
tree = {site: {'statusTypes': dict(siteStatus['Value'])}}
ces = CSHelpers.getSiteComputingElements(site)
cesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ces,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not cesStatus['OK']:
return cesStatus
ses = CSHelpers.getSiteStorageElements(site)
sesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ses,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not sesStatus['OK']:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList['Value']:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]['ces'] = feedTree(cesStatus)
tree[site]['ses'] = feedTree(sesStatus)
return S_OK(tree)
def export_getSitesStorageSummary(self, sitesName):
"""
Return the dictionary with storage summary information for the given sites.
:return: S_OK( { site : { 'MaxStorage' : 1000.00
'FreeStorage' : 200.00
'StorageUsage' : 80.0
} } ) | S_ERROR
"""
gLogger.info('getSitesStorageSummary')
sitesStorageSummary = {}
ses = set()
for siteName in sitesName:
se = CSHelpers.getSiteStorageElements(siteName)
if se:
sitesStorageSummary[ siteName ] = se[ 0 ]
ses.add(se[ 0 ])
if len(ses) == 0:
return sitesStorageSummary
queryRes = rmClient.selectStorageCache(sE = list(ses))
if not queryRes[ 'OK' ]:
return queryRes
records = queryRes[ 'Value' ]
columns = queryRes[ 'Columns' ]
seInfo = {}
for record in records:
recordDict = dict(zip(columns, record))
seName = recordDict.pop('SE')
seDict = {}
seDict[ 'MaxStorage' ] = math.floor(float(recordDict[ 'Occupied' ] + recordDict[ 'Free' ]) / 1024 / 1024 / 1024 * 100) / 100
seDict[ 'FreeStorage' ] = math.floor(float(recordDict[ 'Free' ]) / 1024 / 1024 / 1024 * 100) / 100
seDict[ 'StorageUsage' ] = recordDict[ 'Usage' ]
seInfo[ seName ] = seDict
for siteName, seName in sitesStorageSummary.items():
sitesStorageSummary[ siteName ] = seInfo[ seName ]
return S_OK(sitesStorageSummary)
def export_getSiteResource(self, siteName):
"""
Returns the dictionary with CEs and SEs for the given site.
:return: S_OK( { 'ComputingElement' : celist, 'StorageElement' : selist } ) | S_ERROR
"""
gLogger.info('getSiteResource')
siteType = siteName.split('.')[ 0 ]
if siteType == 'CLOUD':
ces = []
else:
ces = CSHelpers.getSiteComputingElements(siteName)
ses = CSHelpers.getSiteStorageElements(siteName)
return S_OK({ 'ComputingElement' : ces, 'StorageElement' : ses })
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
gLogger.info('getSitesResources')
if siteNames is None:
siteNames = CSHelpers.getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
if isinstance(siteNames, basestring):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
res = {}
res['ces'] = CSHelpers.getSiteComputingElements(siteName)
# Convert StorageElements to host names
ses = CSHelpers.getSiteStorageElements(siteName)
sesHosts = CSHelpers.getStorageElementsHosts(ses)
if not sesHosts['OK']:
return sesHosts
# Remove duplicates
res['ses'] = list(set(sesHosts['Value']))
sitesRes[siteName] = res
return S_OK(sitesRes)
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
gLogger.info('getSitesResources')
if siteNames is None:
siteNames = CSHelpers.getSites()
if not siteNames['OK']:
return siteNames
siteNames = siteNames['Value']
if isinstance(siteNames, basestring):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
res = {}
res['ces'] = CSHelpers.getSiteComputingElements(siteName)
# Convert StorageElements to host names
ses = CSHelpers.getSiteStorageElements(siteName)
sesHosts = CSHelpers.getStorageElementsHosts(ses)
if not sesHosts['OK']:
return sesHosts
# Remove duplicates
res['ses'] = list(set(sesHosts['Value']))
sitesRes[siteName] = res
return S_OK(sitesRes)
def execute(self):
"""
The main method of the agent. It get elements which need to be tested and
evaluated from CS. Then it instantiates TestExecutor and StatusEvaluate and
calls their main method to finish all the work.
"""
from BESDIRAC.ResourceStatusSystem.SAM.SAMTest import TestConfiguration
self.tests = TestConfiguration.TESTS
self.__loadTestObj()
self.testExecutor = TestExecutor( self.tests, self.apis )
self.statusEvaluator = StatusEvaluator( self.apis )
elements = []
sitesCEs = {}
# CE tests
noTestSites = [ site.strip() for site in self.am_getOption( 'noTestSite', '' ).split( ',' ) if site != '' ]
diracAdmin = DiracAdmin()
activeSites = diracAdmin.getSiteMask()
# wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
# activeSites = wmsAdmin.getSiteMask()
if not activeSites[ 'OK' ]:
return activeSites
activeSites = [ site for site in activeSites[ 'Value' ] if site not in noTestSites ]
gLogger.info('Active sites: %s', activeSites)
for siteName in activeSites:
domain = siteName.split('.')[ 0 ]
vos = BESUtils.getSiteVO( siteName )
if 'CLOUD' != domain:
siteCEs = CSHelpers.getSiteComputingElements( siteName )
sitesCEs[ siteName ] = siteCEs
for ce in siteCEs:
elements.append( { 'ElementName' : ce,
'ElementType' : 'ComputingElement',
'VO' : vos } )
gLogger.debug("List of elements: %s" % ce)
else:
sitesCEs[ siteName ] = [ siteName ]
elements.append( { 'ElementName' : siteName,
'ElementType' : 'CLOUD',
'VO' : vos } )
# SE tests
ses = gConfig.getValue( 'Resources/StorageElementGroups/SE-USER' )
for se in ses.split( ', ' ):
seSites = BESUtils.getSitesForSE( se )
for seSite in seSites:
gLogger.debug( 'Site for SE %s: %s' % (se, seSite) )
if seSite not in activeSites:
continue
vos = BESUtils.getSiteVO( seSite )
gLogger.debug( 'vos for SE %s under site %s: %s' % (se, seSite, vos) )
if len(vos) == 0:
continue
vo = vos[0]
elements.append( { 'ElementName' : se,
'ElementType' : 'StorageElement',
'VO' : vo } )
gLogger.info( 'VO for SE %s: %s' % ( se, vo ) )
break
lastCheckTime = datetime.utcnow().replace(microsecond = 0)
self.elementsStatus = {}
threads = []
for elementDict in elements:
t = threading.Thread( target = self._execute, args = ( elementDict, ) )
threads.append( t )
t.start()
for thread in threads:
thread.join()
for siteName in activeSites:
seList = CSHelpers.getSiteStorageElements( siteName )
se = ''
if [] != seList:
se = seList[ 0 ]
try:
seStatus = self.elementsStatus[ se ][ 'all' ]
except KeyError:
seStatus = None
voStatus = { 'all' : [] }
for ce in sitesCEs[ siteName ]:
if not self.elementsStatus.has_key( ce ):
continue
for vo, status in self.elementsStatus[ ce ].items():
if vo not in voStatus:
voStatus[ vo ] = []
voStatus[ vo ].append( status )
for vo, ceStatusList in voStatus.items():
if ceStatusList == [] and seStatus == None:
continue
res = self.statusEvaluator.evaluateSiteStatus( siteName, ceStatusList, seStatus, vo = vo, lastCheckTime = lastCheckTime)
if not res[ 'OK' ]:
gLogger.error( 'StatusEvaluator.evaluateSiteStatus: %s' % res[ 'Message' ] )
break
return S_OK()
def execute(self):
"""
The main method of the agent. It get elements which need to be tested and
evaluated from CS. Then it instantiates TestExecutor and StatusEvaluate and
calls their main method to finish all the work.
"""
elements = []
sitesCEs = {}
ses = gConfig.getValue( 'Resources/StorageElementGroups/SE-USER' )
for se in ses.split( ', ' ):
elements.append( { 'ElementName' : se,
'ElementType' : 'StorageElement' } )
wmsAdmin = RPCClient('WorkloadManagement/WMSAdministrator')
activeSites = wmsAdmin.getSiteMask()
if not activeSites[ 'OK' ]:
return activeSites
activeSites = activeSites[ 'Value' ]
for siteName in activeSites:
domain = siteName.split('.')[ 0 ]
vos = BESUtils.getSiteVO( siteName )
if 'CLOUD' != domain:
siteCEs = CSHelpers.getSiteComputingElements( siteName )
sitesCEs[ siteName ] = siteCEs
for ce in siteCEs:
elements.append( { 'ElementName' : ce,
'ElementType' : 'ComputingElement',
'VO' : vos } )
else:
sitesCEs[ siteName ] = [ siteName ]
elements.append( { 'ElementName' : siteName,
'ElementType' : 'CLOUD',
'VO' : vos } )
lastCheckTime = datetime.utcnow().replace(microsecond = 0)
self.elementsStatus = {}
threads = []
for elementDict in elements:
t = threading.Thread( target = self._execute, args = ( elementDict, ) )
threads.append( t )
t.start()
for thread in threads:
thread.join()
for siteName in activeSites:
seList = CSHelpers.getSiteStorageElements( siteName )
se = ''
if [] != seList:
se = seList[ 0 ]
try:
seStatus = self.elementsStatus[ se ][ 'all' ]
except KeyError:
seStatus = None
voStatus = { 'all' : [] }
for ce in sitesCEs[ siteName ]:
if not self.elementsStatus.has_key( ce ):
continue
for vo, status in self.elementsStatus[ ce ].items():
if vo not in voStatus:
voStatus[ vo ] = []
voStatus[ vo ].append( status )
for vo, ceStatusList in voStatus.items():
if ceStatusList == [] and seStatus == None:
continue
res = self.statusEvaluator.evaluateSiteStatus( siteName, ceStatusList, seStatus, vo = vo, lastCheckTime = lastCheckTime)
if not res[ 'OK' ]:
gLogger.error( 'StatusEvaluator.evaluateSiteStatus: %s' % res[ 'Message' ] )
break
return S_OK()
