def doCommand( self ):
"""
Returns running and runned jobs, querying the WMSHistory
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every sites)
:returns:
"""
if not 'hours' in self.args:
return S_ERROR( 'Number of hours not specified' )
hours = self.args[ 'hours' ]
sites = None
if 'sites' in self.args:
sites = self.args[ 'sites' ]
if sites is None:
#FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = Resources.getSites()
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
if not sites:
return S_ERROR( 'Sites is empty' )
fromD = datetime.utcnow() - timedelta( hours = hours )
toD = datetime.utcnow()
runJobs = self.rClient.getReport( 'WMSHistory', 'NumberOfJobs', fromD, toD,
{}, 'Site')
if not runJobs[ 'OK' ]:
return runJobs
runJobs = runJobs[ 'Value' ]
if not 'data' in runJobs:
return S_ERROR( 'Missing data key' )
if not 'granularity' in runJobs:
return S_ERROR( 'Missing granularity key' )
singlePlots = {}
for site, value in runJobs[ 'data' ].items():
if site in sites:
plot = {}
plot[ 'data' ] = { site: value }
plot[ 'granularity' ] = runJobs[ 'granularity' ]
singlePlots[ site ] = plot
return S_OK( singlePlots )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def test_ResourcesGetters():
res = Resources.getSites()
assert res['OK'] is True, res['Message']
assert res['Value'] == ['DIRAC.Jenkins.ch'], res['Value']
res = Resources.getSiteCEMapping()
assert res['OK'] is True, res['Message']
assert res['Value'] == {
'DIRAC.Jenkins.ch': ['jenkins.cern.ch']
}, res['Value']
res = Resources.getCESiteMapping()
assert res['OK'] is True, res['Message']
assert res['Value'] == {
'jenkins.cern.ch': 'DIRAC.Jenkins.ch'
}, res['Value']
res = Resources.getCESiteMapping('jenkins.cern.ch')
assert res['OK'] is True, res['Message']
assert res['Value'] == {
'jenkins.cern.ch': 'DIRAC.Jenkins.ch'
}, res['Value']
res = Resources.getCESiteMapping('not-here')
assert res['OK'] is True, res['Message']
assert res['Value'] == {}, res['Value']
def test_ResourcesGetters():
res = Resources.getSites()
assert res["OK"] is True, res["Message"]
assert res["Value"] == ["DIRAC.Jenkins.ch"], res["Value"]
res = Resources.getSiteCEMapping()
assert res["OK"] is True, res["Message"]
assert res["Value"] == {
"DIRAC.Jenkins.ch": ["jenkins.cern.ch"]
}, res["Value"]
res = Resources.getCESiteMapping()
assert res["OK"] is True, res["Message"]
assert res["Value"] == {
"jenkins.cern.ch": "DIRAC.Jenkins.ch"
}, res["Value"]
res = Resources.getCESiteMapping("jenkins.cern.ch")
assert res["OK"] is True, res["Message"]
assert res["Value"] == {
"jenkins.cern.ch": "DIRAC.Jenkins.ch"
}, res["Value"]
res = Resources.getCESiteMapping("not-here")
assert res["OK"] is True, res["Message"]
assert res["Value"] == {}, res["Value"]
def export_getSitesResources( self, siteNames ):
resources = Resources.Resources()
if siteNames is None:
siteNames = Resources.getSites()
if not siteNames[ 'OK' ]:
return siteNames
siteNames = siteNames[ 'Value' ]
if isinstance( siteNames, str ):
siteNames = [ siteNames ]
sitesRes = {}
for siteName in siteNames:
res = {}
res[ 'ces' ] = resources.getEligibleResources( 'Computing', { 'Site': siteName } )
ses = resources.getEligibleStorageElements( { 'Site': siteName } )
sesHosts = CSHelpers.getStorageElementsHosts( ses )
if not sesHosts[ 'OK' ]:
return sesHosts
res[ 'ses' ] = list( set( sesHosts[ 'Value' ] ) )
sitesRes[ siteName ] = res
return S_OK( sitesRes )
def doCommand(self):
"""
Returns failed jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if not 'hours' in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
sites = None
if 'sites' in self.args:
sites = self.args['sites']
if sites is None:
#FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = Resources.getSites()
if not sites['OK']:
return sites
sites = sites['Value']
if not sites:
return S_ERROR('Sites is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
failedPilots = self.rClient.getReport('Pilot', 'NumberOfPilots', fromD,
toD, {
'GridStatus': ['Aborted'],
'Site': sites
}, 'Site')
if not failedPilots['OK']:
return failedPilots
failedPilots = failedPilots['Value']
if not 'data' in failedPilots:
return S_ERROR('Missing data key')
if not 'granularity' in failedPilots:
return S_ERROR('Missing granularity key')
singlePlots = {}
for site, value in failedPilots['data'].items():
if site in sites:
plot = {}
plot['data'] = {site: value}
plot['granularity'] = failedPilots['granularity']
singlePlots[site] = plot
return S_OK(singlePlots)
def doCommand( self ):
"""
Returns failed jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if not 'hours' in self.args:
return S_ERROR( 'Number of hours not specified' )
hours = self.args[ 'hours' ]
sites = None
if 'sites' in self.args:
sites = self.args[ 'sites' ]
if sites is None:
#FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = Resources.getSites()
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
if not sites:
return S_ERROR( 'Sites is empty' )
fromD = datetime.utcnow() - timedelta( hours = hours )
toD = datetime.utcnow()
failedPilots = self.rClient.getReport( 'Pilot', 'NumberOfPilots', fromD, toD,
{ 'GridStatus' : [ 'Aborted' ],
'Site' : sites
}, 'Site' )
if not failedPilots[ 'OK' ]:
return failedPilots
failedPilots = failedPilots[ 'Value' ]
if not 'data' in failedPilots:
return S_ERROR( 'Missing data key' )
if not 'granularity' in failedPilots:
return S_ERROR( 'Missing granularity key' )
singlePlots = {}
for site, value in failedPilots[ 'data' ].items():
if site in sites:
plot = {}
plot[ 'data' ] = { site: value }
plot[ 'granularity' ] = failedPilots[ 'granularity' ]
singlePlots[ site ] = plot
return S_OK( singlePlots )
def getGOCSites(diracSites=None):
# FIXME: THIS SHOULD GO INTO Resources HELPER
if diracSites is None:
diracSites = Resources.getSites()
if not diracSites["OK"]:
return diracSites
diracSites = diracSites["Value"]
gocSites = []
for diracSite in diracSites:
gocSite = getGOCSiteName(diracSite)
if not gocSite["OK"]:
continue
gocSites.append(gocSite["Value"])
return S_OK(list(set(gocSites)))
def getGOCSites(diracSites=None):
#FIXME: THIS SHOULD GO INTO Resources HELPER
if diracSites is None:
diracSites = Resources.getSites()
if not diracSites['OK']:
return diracSites
diracSites = diracSites['Value']
gocSites = []
for diracSite in diracSites:
gocSite = getGOCSiteName(diracSite)
if not gocSite['OK']:
continue
gocSites.append(gocSite['Value'])
return S_OK(list(set(gocSites)))
def doMaster( self ):
'''
Master method, which looks little bit spaguetti code, sorry !
- It gets all Sites.
- It gets all StorageElements
As there is no bulk query, it compares with what we have on the database.
It queries a portion of them.
'''
sites = Resources.getSites()
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
ses = self.resources.getEligibleStorageElements()
if not ses[ 'OK' ]:
return ses
ses = ses[ 'Value' ]
elementNames = sites + ses
# sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } )
# if not sourceQuery[ 'OK' ]:
# return sourceQuery
# sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ]
#
# sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) )
gLogger.info( 'Processing %s' % ', '.join( elementNames ) )
for metric in [ 'Quality', 'FailedTransfers' ]:
for direction in [ 'Source', 'Destination' ]:
# 2 hours of window
result = self.doNew( ( 2, elementNames, direction, metric ) )
if not result[ 'OK' ]:
self.metrics[ 'failed' ].append( result )
return S_OK( self.metrics )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def doCommand(self):
"""
# Returns simple pilots efficiency
#
# :attr:`args`:
# - args[0]: string - should be a ValidElement
#
# - args[1]: string - should be the name of the ValidElement
#
# returns:
# {
# 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad'
# }
"""
if not 'element' in self.args:
return self.returnERROR(S_ERROR('element is missing'))
element = self.args['element']
if not 'siteName' in self.args:
return self.returnERROR(S_ERROR('siteName is missing'))
siteName = self.args['siteName']
# If siteName is None, we take all sites
if siteName is None:
siteName = Resources.getSites()
if not siteName['OK']:
return self.returnERROR(siteName)
siteName = siteName['Value']
if element == 'Site':
results = self.wmsAdmin.getPilotSummaryWeb({'GridSite': siteName},
[], 0, 300)
elif element == 'Resource':
results = self.wmsAdmin.getPilotSummaryWeb(
{'ExpandSite': siteName}, [], 0, 300)
else:
return self.returnERROR(S_ERROR('%s is a wrong element' % element))
if not results['OK']:
return self.returnERROR(results)
results = results['Value']
if not 'ParameterNames' in results:
return self.returnERROR(S_ERROR('Malformed result dictionary'))
params = results['ParameterNames']
if not 'Records' in results:
return self.returnERROR(S_ERROR('Malformed result dictionary'))
records = results['Records']
pilotResults = []
for record in records:
pilotDict = dict(zip(params, record))
try:
pilotDict['PilotsPerJob'] = float(pilotDict['PilotsPerJob'])
pilotDict['PilotsJobEff'] = float(pilotDict['PilotsJobEff'])
except KeyError, e:
return self.returnERROR(S_ERROR(e))
except ValueError, e:
return self.returnERROR(S_ERROR(e))
def doCommand(self):
"""
Returns running and runned jobs, querying the WMSHistory
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every sites)
:returns:
"""
if not 'hours' in self.args:
return S_ERROR('Number of hours not specified')
hours = self.args['hours']
sites = None
if 'sites' in self.args:
sites = self.args['sites']
if sites is None:
#FIXME: pointing to the CSHelper instead
# sources = self.rsClient.getSite( meta = {'columns': 'SiteName'} )
# if not sources[ 'OK' ]:
# return sources
# sources = [ si[0] for si in sources[ 'Value' ] ]
sites = Resources.getSites()
if not sites['OK']:
return sites
sites = sites['Value']
if not sites:
return S_ERROR('Sites is empty')
fromD = datetime.utcnow() - timedelta(hours=hours)
toD = datetime.utcnow()
runJobs = self.rClient.getReport('WMSHistory', 'NumberOfJobs', fromD,
toD, {}, 'Site')
if not runJobs['OK']:
return runJobs
runJobs = runJobs['Value']
if not 'data' in runJobs:
return S_ERROR('Missing data key')
if not 'granularity' in runJobs:
return S_ERROR('Missing granularity key')
singlePlots = {}
for site, value in runJobs['data'].items():
if site in sites:
plot = {}
plot['data'] = {site: value}
plot['granularity'] = runJobs['granularity']
singlePlots[site] = plot
return S_OK(singlePlots)
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
def export_getSites( self ):
'''
Returns list of all sites considered by RSS
'''
gLogger.info( 'getSites' )
return Resources.getSites()
def doCommand( self ):
"""
# Returns simple pilots efficiency
#
# :attr:`args`:
# - args[0]: string - should be a ValidElement
#
# - args[1]: string - should be the name of the ValidElement
#
# returns:
# {
# 'Result': 'Good'|'Fair'|'Poor'|'Idle'|'Bad'
# }
"""
if not 'element' in self.args:
return self.returnERROR( S_ERROR( 'element is missing' ) )
element = self.args[ 'element' ]
if not 'siteName' in self.args:
return self.returnERROR( S_ERROR( 'siteName is missing' ) )
siteName = self.args[ 'siteName' ]
# If siteName is None, we take all sites
if siteName is None:
siteName = Resources.getSites()
if not siteName[ 'OK' ]:
return self.returnERROR( siteName )
siteName = siteName[ 'Value' ]
if element == 'Site':
results = self.wmsAdmin.getPilotSummaryWeb( { 'GridSite' : siteName }, [], 0, 300 )
elif element == 'Resource':
results = self.wmsAdmin.getPilotSummaryWeb( { 'ExpandSite' : siteName }, [], 0, 300 )
else:
return self.returnERROR( S_ERROR( '%s is a wrong element' % element ) )
if not results[ 'OK' ]:
return self.returnERROR( results )
results = results[ 'Value' ]
if not 'ParameterNames' in results:
return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) )
params = results[ 'ParameterNames' ]
if not 'Records' in results:
return self.returnERROR( S_ERROR( 'Malformed result dictionary' ) )
records = results[ 'Records' ]
pilotResults = []
for record in records:
pilotDict = dict( zip( params , record ))
try:
pilotDict[ 'PilotsPerJob' ] = float( pilotDict[ 'PilotsPerJob' ] )
pilotDict[ 'PilotsJobEff' ] = float( pilotDict[ 'PilotsJobEff' ] )
except KeyError, e:
return self.returnERROR( S_ERROR( e ) )
except ValueError, e:
return self.returnERROR( S_ERROR( e ) )
