def test_addAndRemoveStorageOccupancy():
# just inserting one record
record = createStorageOccupancyAccountingRecord()
record.setStartTime()
record.setEndTime()
res = gDataStoreClient.addRegister(record)
assert res['OK']
res = gDataStoreClient.commit()
assert res['OK']
rc = ReportsClient()
res = rc.listReports('StorageOccupancy')
assert res['OK']
res = rc.listUniqueKeyValues('StorageOccupancy')
assert res['OK']
res = rc.getReport('StorageOccupancy', 'Free and Used Space',
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
{}, 'StorageElement')
assert res['OK']
# now removing that record
res = gDataStoreClient.remove(record)
assert res['OK']
def test_addAndRemove():
# just inserting one record
record = createAccountingRecord()
record.setStartTime()
record.setEndTime()
res = gDataStoreClient.addRegister(record)
assert res['OK']
res = gDataStoreClient.commit()
assert res['OK']
rc = ReportsClient()
res = rc.listReports('DataOperation')
assert res['OK']
res = rc.listUniqueKeyValues('DataOperation')
assert res['OK']
res = rc.getReport('DataOperation', 'Successful transfers',
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
{}, 'Destination')
assert res['OK']
# now removing that record
res = gDataStoreClient.remove(record)
assert res['OK']
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
#Get unique key values
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
callback["selectionValues"] = json.dumps(retVal['Value'])
#Cache for plotsList?
data = AccountingPlotOldHandler.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
data = json.dumps(retVal['Value'])
AccountingPlotOldHandler.__keysCache.add(
"reportsList:%s" % typeName, 300, data)
callback["plotsList"] = data
self.write(json.dumps({"success": "true", "result": callback}))
def getPlotListAndSelectionValues(self):
result = {}
try:
typeName = str(request.params['typeName'])
except:
return S_ERROR("Missing or invalid type name!")
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK'] and 'rpcStub' in retVal:
del (retVal['rpcStub'])
return retVal
selectionValues = retVal['Value']
data = AccountingplotsController.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
return retVal
data = simplejson.dumps(retVal['Value'])
AccountingplotsController.__keysCache.add(
"reportsList:%s" % typeName, 300, data)
try:
plotsList = eval(data)
except:
return S_ERROR('Failed to convert a string to a list!')
return S_OK({'SelectionData': selectionValues, 'PlotList': plotsList})
def getPlotData( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.getReport( *params )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
rawData = retVal[ 'Value' ]
groupKeys = rawData[ 'data' ].keys()
groupKeys.sort()
if 'granularity' in rawData:
granularity = rawData[ 'granularity' ]
data = rawData['data']
tS = int( Time.toEpoch( params[2] ) )
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join( groupKeys )
for timeSlot in range( timeStart, int( Time.toEpoch( params[3] ) ), granularity ):
lineData = [ str( timeSlot ) ]
for key in groupKeys:
if timeSlot in data[ key ]:
lineData.append( str( data[ key ][ timeSlot ] ) )
else:
lineData.append( "" )
strData += "%s\n" % ",".join( lineData )
else:
strData = "%s\n" % ",".join( groupKeys )
strData += ",".join( [ str( rawData[ 'data' ][ k ] ) for k in groupKeys ] )
response.headers['Content-type'] = 'text/csv'
response.headers['Content-Disposition'] = 'attachment; filename="%s.csv"' % md5( str( params ) ).hexdigest()
response.headers['Content-Length'] = len( strData )
return strData
def _getHistoryData(self, timeSpan, groupToUse):
"""Get history data from Accounting WMSHistory database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
reportsClient = ReportsClient()
reportCondition = {"Status": ["Running"]}
if not groupToUse:
reportGrouping = "UserGroup"
else:
reportGrouping = "User"
reportCondition = {"UserGroup": groupToUse}
now = datetime.datetime.utcnow()
result = reportsClient.getReport(
"WMSHistory",
"AverageNumberOfJobs",
now - datetime.timedelta(seconds=timeSpan),
now,
reportCondition,
reportGrouping,
{"lastSeconds": timeSpan},
)
return result
def test_addAndRemoveDataOperation():
# just inserting one record
record = createDataOperationAccountingRecord()
record.setStartTime()
record.setEndTime()
res = gDataStoreClient.addRegister(record)
assert res['OK']
res = gDataStoreClient.commit()
assert res['OK']
rc = ReportsClient()
res = rc.listReports('DataOperation')
assert res['OK']
res = rc.listUniqueKeyValues('DataOperation')
assert res['OK']
res = rc.getReport('DataOperation', 'Successful transfers',
datetime.datetime.utcnow(), datetime.datetime.utcnow(),
{}, 'Destination')
assert res['OK']
# now removing that record
res = gDataStoreClient.remove(record)
assert res['OK']
def getPlotData( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.getReport( *params )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render( "/error.mako" )
rawData = retVal[ 'Value' ]
groupKeys = rawData[ 'data' ].keys()
groupKeys.sort()
if 'granularity' in rawData:
granularity = rawData[ 'granularity' ]
data = rawData['data']
tS = int( Time.toEpoch( params[2] ) )
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join( groupKeys )
for timeSlot in range( timeStart, int( Time.toEpoch( params[3] ) ), granularity ):
lineData = [ str( timeSlot ) ]
for key in groupKeys:
if timeSlot in data[ key ]:
lineData.append( str( data[ key ][ timeSlot ] ) )
else:
lineData.append( "" )
strData += "%s\n" % ",".join( lineData )
else:
strData = "%s\n" % ",".join( groupKeys )
strData += ",".join( [ str( rawData[ 'data' ][ k ] ) for k in groupKeys ] )
response.headers['Content-type'] = 'text/csv'
response.headers['Content-Disposition'] = 'attachment; filename="%s.csv"' % md5( str( params ) ).hexdigest()
response.headers['Content-Length'] = len( strData )
return strData
def __init__(self, args=None, clients=None):
super(FailedJobsBySiteSplittedCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(TransferQualityCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(SuccessfullPilotsByCESplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(FailedPilotsBySiteSplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __queryForPlot(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
return retVal
params = retVal['Value']
repClient = ReportsClient(rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.generateDelayedPlot(*params)
return retVal
def __queryForPlot( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
return retVal
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.generateDelayedPlot( *params )
return retVal
def __init__(self, args=None, clients=None):
super(DIRACAccountingCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def initialize(self):
self.__log = gLogger.getSubLogger("WMSHistoryCorrector")
self.__reportsClient = ReportsClient()
self.__usageHistory = {}
self.__slices = {}
self.__lastHistoryUpdate = 0
self.__globalCorrectionFactor = 5
self._fillSlices()
return S_OK()
def __init__( self, baseCSPath, group ):
self.__log = gLogger.getSubLogger( "WMSHistoryCorrector" )
self.__baseCSPath = baseCSPath
self.__group = group
self.__reportsClient = ReportsClient()
self.__usageHistory = {}
self.__slices = {}
self.__lastHistoryUpdate = 0
self.__globalCorrectionFactor = 5
self._fillSlices()
def __init__(self, args=None, clients=None):
super(TransferCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(RunningJobsBySiteSplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
def __init__(self, args=None, clients=None):
super(RunningJobsBySiteSplittedCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
if "ReportGenerator" in self.apis:
self.rgClient = self.apis["ReportGenerator"]
else:
self.rgClient = Client(url="Accounting/ReportGenerator")
self.rClient.rpcClient = self.rgClient
def __init__( self, args = None, clients = None ):
super( TransferQualityCommand, self ).__init__( args, clients )
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
self.rClient.rpcClient = self.rgClient
def doCommand(self, CEs = None):
"""
Returns failed pilots using the DIRAC accounting system for every CE
for the last self.args[0] hours
:params:
:attr:`CEs`: list of CEs (when not given, take every CE)
:returns:
"""
if CEs is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
CEs = RPC_RSS.getCEsList()
if not CEs['OK']:
raise RSSException, where(self, self.doCommand) + " " + CEs['Message']
else:
CEs = CEs['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
failed_pilots = self.client.getReport('Pilot', 'NumberOfPilots', fromD, toD,
{'GridStatus':['Aborted'], 'GridCE':CEs}, 'GridCE')
if not failed_pilots['OK']:
raise RSSException, where(self, self.doCommand) + " " + failed_pilots['Message']
else:
failed_pilots = failed_pilots['Value']
except:
gLogger.exception("Exception when calling FailedPilotsByCESplitted_Command")
return {}
listOfCEs = failed_pilots['data'].keys()
plotGran = failed_pilots['granularity']
singlePlots = {}
for CE in listOfCEs:
if CE in CEs:
plot = {}
plot['data'] = {CE: failed_pilots['data'][CE]}
plot['granularity'] = plotGran
singlePlots[CE] = plot
resToReturn = {'Pilot': singlePlots}
return resToReturn
def setCommandClient(self,
comm,
cObj,
RPCWMSAdmin=None,
RPCAccounting=None):
client = None
if comm == 'JobsEffSimpleEveryOne_Command':
from DIRAC.ResourceStatusSystem.Client.JobsClient import JobsClient
client = JobsClient()
cObj.setRPC(RPCWMSAdmin)
elif comm == 'PilotsEffSimpleEverySites_Command':
from DIRAC.ResourceStatusSystem.Client.PilotsClient import PilotsClient
client = PilotsClient()
cObj.setRPC(RPCWMSAdmin)
elif comm in ('TransferQualityEverySEs_Command',
'TransferQualityEverySEsSplitted_Command'):
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
client = ReportsClient(rpcClient=RPCAccounting)
cObj.setRPC(RPCAccounting)
cObj.setClient(client)
def doCommand(self, sites = None):
"""
Returns running and runned jobs, querying the WMSHistory
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every sites)
:returns:
"""
if sites is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
sites = RPC_RSS.getSitesList()
if not sites['OK']:
raise RSSException, where(self, self.doCommand) + " " + sites['Message']
else:
sites = sites['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
run_jobs = self.client.getReport('WMSHistory', 'NumberOfJobs', fromD, toD,
{}, 'Site')
if not run_jobs['OK']:
raise RSSException, where(self, self.doCommand) + " " + run_jobs['Message']
else:
run_jobs = run_jobs['Value']
except:
gLogger.exception("Exception when calling RunningJobsBySiteSplitted_Command")
return {}
listOfSites = run_jobs['data'].keys()
plotGran = run_jobs['granularity']
singlePlots = {}
for site in listOfSites:
if site in sites:
plot = {}
plot['data'] = {site: run_jobs['data'][site]}
plot['granularity'] = plotGran
singlePlots[site] = plot
resToReturn = {'WMSHistory': singlePlots}
return resToReturn
def do_getSummary( self, args ):
"""
Gets a summary
Usage : getSummary <Summary name> <startdate YYYYMMDDHHMM> <enddate YYYYMMDDHHMM> (<field name> <field value>)*
"""
try:
argList = List.fromChar( args, " " )
if len( argList ) < 3:
gLogger.error( "Missing arguments!" )
return
startDT = self.__getDatetimeFromArg( argList[1] )
if not startDT:
gLogger.error( "Start time has invalid format" )
endDT = self.__getDatetimeFromArg( argList[2] )
if not endDT:
gLogger.error( "End time has invalid format" )
gLogger.info( "Start time is %s" % startDT )
gLogger.info( "End time is %s" % endDT )
sumArgs = {}
for iP in range( 3, len( argList ), 2 ):
key = argList[ iP ]
if key in sumArgs:
sumArgs[ key ].append( argList[ iP + 1 ] )
else:
sumArgs[ key ] = [ argList[ iP + 1 ] ]
retVal = ReportsClient().generateSummary( argList[ 0 ], startDT, endDT, sumArgs )
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
printer = pprint.PrettyPrinter(indent=4)
printer.pprint( retVal[ 'Value' ][0] )
for data in retVal[ 'Value' ][1]:
printer.pprint( data )
except:
self.showTraceback()
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message'])
return res
rsClass = res['Value']
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message'])
return res
rmClass = res['Value']
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = rsClass()
self.clients['ResourceManagementClient'] = rmClass()
self.clients['WMSAdministrator'] = WMSAdministratorClient()
self.clients['Pilots'] = PilotManagerClient()
self.cCaller = CommandCaller
return S_OK()
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
callback = {"success":"false", "error":retVal[ 'Message' ]}
self.finish( callback )
params = retVal[ 'Value' ]
'''self.finish({'success' : 'true', 'result' : params})'''
repClient = ReportsClient( rpcClient = RPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.getReport(*params)
if not retVal[ 'OK' ]:
callback = {"success":"false", "error":retVal[ 'Message' ]}
self.finish( callback )
rawData = retVal[ 'Value' ]
groupKeys = rawData[ 'data' ].keys()
self.finish({'success' : 'true', 'result' : groupKeys})
def generateAccountingPlot(self):
try:
site = str(request.params['site'])
plotName = str(request.params['plotName'])
plotTime = str(request.params['plotTime'])
height = int(request.params['height'])
width = int(request.params['width'])
except:
S_ERROR("Oops, Invalid parameters!")
extraParams = {'height': height, 'width': width}
if plotName == 'CPU Used':
typeName = "Job"
reportName = "CPUUsed"
grouping = "FinalMajorStatus"
condDict = {'Site': [site]}
extraParams['plotTitle'] = "CPU used for site %s" % site
elif plotName == "Running jobs":
typeName = "WMSHistory"
reportName = "NumberOfJobs"
grouping = "JobGroup"
condDict = {'Site': [site], 'Status': ['Running']}
extraParams['plotTitle'] = "Jobs running for site %s" % site
else:
return S_ERROR("Oops, invalid plot name!")
if plotTime == "Last day":
extraParams['lastSeconds'] = 86400
elif plotTime == "Last week":
extraParams['lastSeconds'] = 604800
elif plotTime == "Last month":
extraParams['lastSeconds'] = 2592000
else:
return S_ERROR("Oops, invalid time!")
end = datetime.datetime.utcnow()
start = end - datetime.timedelta(seconds=extraParams['lastSeconds'])
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
result = repClient.generateDelayedPlot(typeName, reportName, start,
end, condDict, grouping,
extraParams)
if not result['OK']:
return S_ERROR(result['Message'])
return S_OK(result['Value']['plot'])
def initialize( self ):
self.__log = gLogger.getSubLogger( "WMSHistoryCorrector" )
self.__reportsClient = ReportsClient()
self.__usageHistory = {}
self.__slices = {}
self.__lastHistoryUpdate = 0
self.__globalCorrectionFactor = 5
self._fillSlices()
return S_OK()
def __init__(self, args=None, clients=None):
super(SuccessfullPilotsByCESplittedCommand,
self).__init__(args, clients)
self.resources = Resources.Resources()
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
def web_generatePlot(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
callback = {"success":"false", "error":retVal[ 'Message' ]}
self.finish( callback )
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = RPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.generateDelayedPlot( *params )
if not retVal[ 'OK' ]:
callback = {"success":"false", "error":retVal[ 'Message' ]}
self.finish( callback )
rawData = retVal[ 'Value' ]
'''groupKeys = rawData[ 'data' ].keys()'''
self.log.always("11111111111111111111111111111111111111111111111111111111111")
self.log.always(retVal[ 'Value' ][ 'plot' ])
self.log.always("22222222222222222222222222222222222222222222222222222222222")
self.finish({'success' : 'true', 'result' : rawData['plot']})
def __showPlotPage( self, typeName, templateFile ):
#Get unique key values
retVal = self.__getUniqueKeyValues( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
c.selectionValues = simplejson.dumps( retVal[ 'Value' ] )
#Cache for plotsList?
data = AcctController.__keysCache.get( "reportsList:%s" % typeName )
if not data:
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.listReports( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
data = simplejson.dumps( retVal[ 'Value' ] )
AcctController.__keysCache.add( "reportsList:%s" % typeName, 300, data )
c.plotsList = data
return render ( templateFile )
def __showPlotPage( self, typeName, templateFile ):
#Get unique key values
retVal = self.__getUniqueKeyValues( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
c.selectionValues = simplejson.dumps( retVal[ 'Value' ] )
#Cache for plotsList?
data = AccountingplotsController.__keysCache.get( "reportsList:%s" % typeName )
if not data:
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.listReports( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
data = simplejson.dumps( retVal[ 'Value' ] )
AccountingplotsController.__keysCache.add( "reportsList:%s" % typeName, 300, data )
c.plotsList = data
return render ( templateFile )
def generateAccountingPlot( self ):
try:
site = str( request.params[ 'site' ] )
plotName = str( request.params[ 'plotName' ] )
plotTime = str( request.params[ 'plotTime' ] )
height = int( request.params[ 'height' ] )
width = int( request.params[ 'width' ] )
except:
S_ERROR( "Oops, Invalid parameters!" )
extraParams = { 'height' : height, 'width' : width }
if plotName == 'CPU Used':
typeName = "Job"
reportName = "CPUUsed"
grouping = "FinalMajorStatus"
condDict = { 'Site' : [ site ] }
extraParams[ 'plotTitle' ] = "CPU used for site %s" % site
elif plotName == "Running jobs":
typeName = "WMSHistory"
reportName = "NumberOfJobs"
grouping = "JobGroup"
condDict = { 'Site' : [ site ], 'Status' : [ 'Running' ] }
extraParams[ 'plotTitle' ] = "Jobs running for site %s" % site
else:
return S_ERROR( "Oops, invalid plot name!" )
if plotTime == "Last day":
extraParams[ 'lastSeconds' ] = 86400
elif plotTime == "Last week":
extraParams[ 'lastSeconds' ] = 604800
elif plotTime == "Last month":
extraParams[ 'lastSeconds' ] = 2592000
else:
return S_ERROR( "Oops, invalid time!" )
end = datetime.datetime.utcnow()
start = end - datetime.timedelta( seconds = extraParams[ 'lastSeconds' ] )
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
result = repClient.generateDelayedPlot( typeName, reportName, start, end, condDict, grouping, extraParams )
if not result[ 'OK' ]:
return S_ERROR( result[ 'Message' ] )
return S_OK( result[ 'Value' ][ 'plot' ] )
def _getHistoryData(self, timeSpan, groupToUse):
""" Get history data from Accounting WMSHistory database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
reportsClient = ReportsClient()
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
else:
reportGrouping = 'User'
reportCondition = {'UserGroup': groupToUse}
now = Time.dateTime()
result = reportsClient.getReport(
'WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now, reportCondition,
reportGrouping, {'lastSeconds': timeSpan})
return result
def _getHistoryData(self, timeSpan, groupToUse):
""" Get history data from Accounting WMSHistory database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
reportsClient = ReportsClient()
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
else:
reportGrouping = 'User'
reportCondition = {'UserGroup': groupToUse}
now = Time.dateTime()
result = reportsClient.getReport('WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now,
reportCondition, reportGrouping,
{'lastSeconds': timeSpan})
return result
def do_connect(self, args):
"""
Tries to connect to the server
Usage: connect
"""
gLogger.info("Trying to connect to server")
self.connected = False
self.prompt = "(%s)> " % colorize("Not connected", "red")
retVal = ReportsClient().ping()
if retVal['OK']:
self.prompt = "(%s)> " % colorize("Connected", "green")
self.connected = True
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
#Get unique key values
retVal = self.__getUniqueKeyValues( typeName )
if not retVal[ 'OK' ]:
self.write(json.dumps({"success":"false", "result":"", "error":retVal[ 'Message' ]}))
return
callback["selectionValues"] = simplejson.dumps( retVal[ 'Value' ] )
#Cache for plotsList?
data = AccountingPlotHandler.__keysCache.get( "reportsList:%s" % typeName )
if not data:
repClient = ReportsClient( rpcClient = RPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.listReports( typeName )
if not retVal[ 'OK' ]:
self.write(json.dumps({"success":"false", "result":"", "error":retVal[ 'Message' ]}))
return
data = simplejson.dumps( retVal[ 'Value' ] )
AccountingPlotHandler.__keysCache.add( "reportsList:%s" % typeName, 300, data )
callback["plotsList"] = data
self.write(json.dumps({"success":"true", "result":callback}))
def getJobsHistory():
result = gOAManager.authorize()
if not result[ 'OK' ]:
bottle.abort( 401, result[ 'Message' ] )
condDict = {}
if 'allOwners' not in bottle.request.params:
condDict[ 'User' ] = gOAData.userName
timeSpan = 86400
if 'timeSpan' in bottle.request.params:
try:
timeSpan = max( 86400, int( bottle.request.params[ 'timeSpan' ] ) )
except ValueError:
bottle.abort( 400, "timeSpan has to be an integer!" )
print "[DEBUG] condDict is %s" % condDict
rpg = ReportsClient( rpcClient = getRPCClient("Accounting/ReportGenerator"), transferClient = getTransferClient("Accounting/ReportGenerator") )
end = datetime.datetime.utcnow()
start = end - datetime.timedelta( seconds = timeSpan )
result = rpg.getReport( "WMSHistory", "NumberOfJobs", start, end, condDict, 'Status' )
if not result[ 'OK' ]:
bottle.abort( 500, "Server Error: %s" % result[ 'Message' ] )
return result[ 'Value' ]
def do_plotView( self, args ):
"""
Gets a summary
Usage : getSummary <Summary name> <startdate YYYYMMDDHHMM> <enddate YYYYMMDDHHMM> <destLocation> (<field name> <field value>)*
"""
try:
argList = List.fromChar( args, " " )
if len( argList ) < 4:
gLogger.error( "Missing arguments!" )
return
startDT = self.__getDatetimeFromArg( argList[1] )
if not startDT:
gLogger.error( "Start time has invalid format" )
endDT = self.__getDatetimeFromArg( argList[2] )
if not endDT:
gLogger.error( "End time has invalid format" )
gLogger.info( "Start time is %s" % startDT )
gLogger.info( "End time is %s" % endDT )
sumArgs = {}
for iP in range( 4, len( argList ), 2 ):
key = argList[ iP ]
if key in sumArgs:
sumArgs[ key ].append( argList[ iP + 1 ] )
else:
sumArgs[ key ] = [ argList[ iP + 1 ] ]
repClient = ReportsClient()
retVal = repClient.plotView( argList[ 0 ], startDT, endDT, sumArgs )
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
destDir = argList[3]
plotImg = retVal[ 'Value' ]
print "Downloading %s plot to %s.." % ( plotImg, destDir )
retVal = repClient.getPlotToDirectory( plotImg, destDir )
if not retVal[ 'OK' ]:
print " Error: %s" % retVal[ 'Message' ]
else:
print " done (%s/%s)!" % ( destDir, plotImg )
except:
self.showTraceback()
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
self.am_setOption('shifterProxy', 'DataManager')
self.rmClient = ResourceManagementClient()
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['SpaceTokenOccupancy'] = [{'SpaceTokenOccupancy': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportGenerator'] = RPCClient(
'Accounting/ReportGenerator')
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = ResourceStatusClient()
self.clients['ResourceManagementClient'] = ResourceManagementClient()
self.clients['WMSAdministrator'] = RPCClient(
'WorkloadManagement/WMSAdministrator')
self.cCaller = CommandCaller
return S_OK()
def __init__(self, args=None, clients=None):
super(DIRACAccountingCommand, self).__init__(args, clients)
if "ReportGenerator" in self.apis:
self.rgClient = self.apis["ReportGenerator"]
else:
self.rgClient = RPCClient("Accounting/ReportGenerator")
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
self.rClient.rpcClient = self.rgClient
def getPlotListAndSelectionValues(self):
result = {}
try:
typeName = str( request.params[ 'typeName' ] )
except:
return S_ERROR( "Missing or invalid type name!" )
retVal = self.__getUniqueKeyValues( typeName )
if not retVal[ 'OK' ] and 'rpcStub' in retVal:
del( retVal[ 'rpcStub' ] )
return retVal
selectionValues = retVal['Value']
data = AcctController.__keysCache.get( "reportsList:%s" % typeName )
if not data:
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.listReports( typeName )
if not retVal[ 'OK' ]:
return retVal
data = simplejson.dumps( retVal[ 'Value' ] )
AcctController.__keysCache.add( "reportsList:%s" % typeName, 300, data )
try:
plotsList = eval(data)
except:
return S_ERROR('Failed to convert a string to a list!')
return S_OK({'SelectionData':selectionValues, 'PlotList':plotsList})
def do_listViews( self, args ):
"""
Get a list of available views
Usage : listViews
"""
try:
retVal = ReportsClient().listViews()
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
print "Available summaries:"
for summary in retVal[ 'Value' ]:
print " %s" % summary
except:
self.showTraceback()
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
self.finish(rawData['data'])
def __init__( self, args = None, clients = None ):
super( SuccessfullPilotsByCESplittedCommand, self ).__init__( args, clients )
self.resources = Resources.Resources()
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
self.rClient.rpcClient = self.rgClient
def __init__( self, args = None, clients = None ):
super( TransferCommand, self ).__init__( args, clients )
self.resources = Resources.Resources()
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def __getPlotSrc(self,name,time,type):
rc = ReportsClient()
time = str(time)
now = datetime.datetime.utcnow()
if time == 'day':
timeSpan = now - datetime.timedelta( seconds = 86400 )
elif time == 'week':
timeSpan = now - datetime.timedelta( seconds = 86400 * 7 )
elif time == 'month':
timeSpan = now - datetime.timedelta( seconds = 86400 * 30 )
elif time == 'year':
timeSpan = now - datetime.timedelta( seconds = 86400 * 360 )
else:
timeSpan = now - datetime.timedelta( seconds = 86400 * 7 )
if len(name) < 1:
c.result = {"success":"false","error":"Recived empty value"}
else:
result = rc.listReports("Job")
if result["OK"]:
plots = result["Value"]
if type == 'jobsBySite':
result = rc.generatePlot("Job",plots[8],timeSpan,now,{"Site":name},"Site")
elif type == 'jobCPUbySite':
result = rc.generatePlot("Job",plots[0],timeSpan,now,{"Site":name},"Site")
elif type == 'CPUUsedBySite':
result = rc.generatePlot("Job",plots[2],timeSpan,now,{"Site":name},"Site")
else:
result = rc.generatePlot("Job",plots[8],timeSpan,now,{"Site":name},"Site")
if result["OK"]:
result = result["Value"]
result = result["plot"]
gLogger.info("result:",result)
c.result = {"success":"true","result":result}
else:
c.result = {"success":"false","error":result["Message"]}
else:
c.result = {"success":"false","error":result["Message"]}
gLogger.info("getPlotSrc:",c.result)
return c.result
def doCommand(self):
"""
Return getQuality from DIRAC's accounting ReportsClient
`args`: a tuple
- args[0]: string: should be a ValidRes
- args[1]: string should be the name of the ValidRes
- args[2]: optional dateTime object: a "from" date
- args[3]: optional dateTime object: a "to" date
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
super(TransferQuality_Command, self).doCommand()
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
try:
if self.args[2] is None:
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = 2)
else:
fromD = self.args[2]
except:
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = 2)
try:
if self.args[3] is None:
toD = datetime.datetime.utcnow()
else:
toD = self.args[3]
except:
toD = datetime.datetime.utcnow()
try:
pr_quality = self.client.getReport('DataOperation', 'Quality', fromD, toD,
{'OperationType':'putAndRegister',
'Destination':[self.args[1]]}, 'Channel')
if not pr_quality['OK']:
raise RSSException, where(self, self.doCommand) + " " + pr_quality['Message']
except:
gLogger.exception("Exception when calling ReportsClient for %s %s" %(self.args[0], self.args[1]))
return {'Result':'Unknown'}
pr_q_d = pr_quality['Value']['data']
if pr_q_d == {}:
return {'Result':None}
else:
if len(pr_q_d) == 1:
values = []
for k in pr_q_d.keys():
for n in pr_q_d[k].values():
values.append(n)
return {'Result':sum(values)/len(values)}
else:
values = []
for n in pr_q_d['Total'].values():
values.append(n)
return {'Result':sum(values)/len(values)}
def doCommand(self):
"""
Returns jobs accounting info for sites in the last 24h
`args`:
- args[0]: string - should be a ValidRes
- args[1]: string - should be the name of the ValidRes
- args[2]: string - should be 'Job' or 'Pilot' or 'DataOperation'
or 'WMSHistory' (??) or 'SRM' (??)
- args[3]: string - should be the plot to generate (e.g. CPUEfficiency)
- args[4]: dictionary - e.g. {'Format': 'LastHours', 'hours': 24}
- args[5]: string - should be the grouping
- args[6]: dictionary - optional conditions
"""
super(DIRACAccounting_Command, self).doCommand()
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
granularity = self.args[0]
name = self.args[1]
accounting = self.args[2]
plot = self.args[3]
period = self.args[4]
if period['Format'] == 'LastHours':
fromT = datetime.datetime.utcnow()-datetime.timedelta(hours = period['hours'])
toT = datetime.datetime.utcnow()
elif period['Format'] == 'Periods':
#TODO
pass
grouping = self.args[5]
try:
if self.args[6] is not None:
conditions = self.args[6]
else:
raise Exception
except:
conditions = {}
if accounting == 'Job' or accounting == 'Pilot':
if granularity == 'Resource':
conditions['GridCE'] = [name]
elif granularity == 'Service':
conditions['Site'] = [name.split('@').pop()]
elif granularity == 'Site':
conditions['Site'] = [name]
else:
raise InvalidRes, where(self, self.doCommand)
elif accounting == 'DataOperation':
conditions['Destination'] = [name]
try:
res = self.client.getReport(accounting, plot, fromT, toT, conditions, grouping)
if res['OK']:
return {'Result':res['Value']}
else:
raise RSSException, where(self, self.doCommand) + ' ' + res['Message']
except:
gLogger.exception("Exception when calling ReportsClient for " + granularity + " " + name )
return {'Result':'Unknown'}
def doCommand(self, sources = None, SEs = None):
"""
Returns transfer quality using the DIRAC accounting system for every SE
of a single site for the last self.args[0] hours
:params:
:attr:`sources`: list of source sites (when not given, take every site)
:attr:`SEs`: list of storage elements (when not given, take every SE)
:returns:
"""
if SEs is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
SEs = RPC_RSS.getStorageElementsList()
if not SEs['OK']:
raise RSSException, where(self, self.doCommand) + " " + SEs['Message']
else:
SEs = SEs['Value']
if sources is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
sources = RPC_RSS.getSitesList()
if not sources['OK']:
raise RSSException, where(self, self.doCommand) + " " + sources['Message']
else:
sources = sources['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
qualityAll = self.client.getReport('DataOperation', 'Quality', fromD, toD,
{'OperationType':'putAndRegister',
'Source':sources + SEs, 'Destination':sources + SEs},
'Destination')
if not qualityAll['OK']:
raise RSSException, where(self, self.doCommand) + " " + qualityAll['Message']
else:
qualityAll = qualityAll['Value']
except:
gLogger.exception("Exception when calling TransferQualityByDestSplittedSite_Command")
return {}
listOfDest = qualityAll['data'].keys()
try:
storSitesWeb = RPC_RSS.getStorageElementsStatusWeb({'StorageElementName':listOfDest}, [], 0, 300)
except NameError:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
storSitesWeb = RPC_RSS.getStorageElementsStatusWeb({'StorageElementName':listOfDest}, [], 0, 300)
if not storSitesWeb['OK']:
raise RSSException, where(self, self.doCommand) + " " + storSitesWeb['Message']
else:
storSitesWeb = storSitesWeb['Value']['Records']
SESiteMapping = {}
siteSEMapping = {}
for r in storSitesWeb:
sites = r[2].split(' ')[:-1]
SESiteMapping[r[0]] = sites
for SE in SESiteMapping.keys():
for site in SESiteMapping[SE]:
try:
l = siteSEMapping[site]
l.append(SE)
siteSEMapping[site] = l
except KeyError:
siteSEMapping[site] = [SE]
plotGran = qualityAll['granularity']
singlePlots = {}
for site in siteSEMapping.keys():
plot = {}
plot['data'] = {}
for SE in siteSEMapping[site]:
plot['data'][SE] = qualityAll['data'][SE]
plot['granularity'] = plotGran
singlePlots[site] = plot
resToReturn = {'DataOperation': singlePlots}
return resToReturn
class FailedPilotsBySiteSplitted_Command(Command):
def doCommand(self, sites = None):
"""
Returns failed jobs using the DIRAC accounting system for every site
for the last self.args[0] hours
:params:
:attr:`sites`: list of sites (when not given, take every site)
:returns:
"""
if sites is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
sites = RPC_RSS.getSitesList()
if not sites['OK']:
raise RSSException, where(self, self.doCommand) + " " + sites['Message']
else:
sites = sites['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
failed_pilots = self.client.getReport('Pilot', 'NumberOfPilots', fromD, toD,
{'GridStatus':['Aborted'], 'Site':sites}, 'Site')
if not failed_pilots['OK']:
raise RSSException, where(self, self.doCommand) + " " + failed_pilots['Message']
else:
failed_pilots = failed_pilots['Value']
except:
gLogger.exception("Exception when calling FailedPilotsBySiteSplitted_Command")
return {}
listOfSites = failed_pilots['data'].keys()
plotGran = failed_pilots['granularity']
singlePlots = {}
for site in listOfSites:
if site in sites:
plot = {}
plot['data'] = {site: failed_pilots['data'][site]}
plot['granularity'] = plotGran
singlePlots[site] = plot
resToReturn = {'Pilot': singlePlots}
return resToReturn
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
def doCommand(self, sources = None, SEs = None):
"""
Returns failed transfer using the DIRAC accounting system for every SE
for the last self.args[0] hours
:params:
:attr:`sources`: list of source sites (when not given, take every site)
:attr:`SEs`: list of storage elements (when not given, take every SE)
:returns:
"""
if SEs is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
SEs = RPC_RSS.getStorageElementsList()
if not SEs['OK']:
raise RSSException, where(self, self.doCommand) + " " + SEs['Message']
else:
SEs = SEs['Value']
if sources is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
sources = RPC_RSS.getSitesList()
if not sources['OK']:
raise RSSException, where(self, self.doCommand) + " " + sources['Message']
else:
sources = sources['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
ft_source = self.client.getReport('DataOperation', 'FailedTransfers',
fromD, toD,
{'OperationType':'putAndRegister',
'Source':sources + SEs, 'Destination':sources + SEs,
'FinalStatus':['Failed']},
'Source')
if not ft_source['OK']:
raise RSSException, where(self, self.doCommand) + " " + ft_source['Message']
else:
ft_source = ft_source['Value']
except:
gLogger.exception("Exception when calling FailedTransfersBySourceSplitted_Command")
return {}
listOfSources = ft_source['data'].keys()
plotGran = ft_source['granularity']
singlePlots = {}
for source in listOfSources:
if source in sources:
plot = {}
plot['data'] = {source: ft_source['data'][source]}
plot['granularity'] = plotGran
singlePlots[source] = plot
resToReturn = {'DataOperation': singlePlots}
return resToReturn
class TransferQualityByDestSplitted_Command(Command):
def doCommand(self, sources = None, SEs = None):
"""
Returns transfer quality using the DIRAC accounting system for every SE
for the last self.args[0] hours
:params:
:attr:`sources`: list of source sites (when not given, take every site)
:attr:`SEs`: list of storage elements (when not given, take every SE)
:returns:
"""
if SEs is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
SEs = RPC_RSS.getStorageElementsList()
if not SEs['OK']:
raise RSSException, where(self, self.doCommand) + " " + SEs['Message']
else:
SEs = SEs['Value']
if sources is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
RPC_RSS = RPCClient("ResourceStatus/ResourceStatus")
sources = RPC_RSS.getSitesList()
if not sources['OK']:
raise RSSException, where(self, self.doCommand) + " " + sources['Message']
else:
sources = sources['Value']
if self.RPC is None:
from DIRAC.Core.DISET.RPCClient import RPCClient
self.RPC = RPCClient("Accounting/ReportGenerator", timeout = self.timeout)
if self.client is None:
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
self.client = ReportsClient(rpcClient = self.RPC)
fromD = datetime.datetime.utcnow()-datetime.timedelta(hours = self.args[0])
toD = datetime.datetime.utcnow()
try:
qualityAll = self.client.getReport('DataOperation', 'Quality', fromD, toD,
{'OperationType':'putAndRegister',
'Source':sources + SEs, 'Destination':sources + SEs},
'Destination')
if not qualityAll['OK']:
raise RSSException, where(self, self.doCommand) + " " + qualityAll['Message']
else:
qualityAll = qualityAll['Value']
except:
gLogger.exception("Exception when calling TransferQualityByDestSplitted_Command")
return {}
listOfDestSEs = qualityAll['data'].keys()
plotGran = qualityAll['granularity']
singlePlots = {}
for SE in listOfDestSEs:
plot = {}
plot['data'] = {SE: qualityAll['data'][SE]}
plot['granularity'] = plotGran
singlePlots[SE] = plot
resToReturn = {'DataOperation': singlePlots}
return resToReturn
doCommand.__doc__ = Command.doCommand.__doc__ + doCommand.__doc__
class TransferCommand( Command ):
'''
Transfer "master" Command
'''
def __init__( self, args = None, clients = None ):
super( TransferCommand, self ).__init__( args, clients )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis[ 'ResourceManagementClient' ]
else:
self.rmClient = ResourceManagementClient()
def _storeCommand( self, results ):
'''
Stores the results of doNew method on the database.
'''
for result in results:
resQuery = self.rmClient.addOrModifyTransferCache( result[ 'SourceName' ],
result[ 'DestinationName' ],
result[ 'Metric' ],
result[ 'Value' ] )
if not resQuery[ 'OK' ]:
return resQuery
return S_OK()
def _prepareCommand( self ):
'''
TransferChannelCommand requires four arguments:
- hours : <int>
- direction : Source | Destination
- elementName : <str>
- metric : Quality | FailedTransfers
GGUSTickets are associated with gocDB names, so we have to transform the
diracSiteName into a gocSiteName.
'''
if not 'hours' in self.args:
return S_ERROR( 'Number of hours not specified' )
hours = self.args[ 'hours' ]
if not 'direction' in self.args:
return S_ERROR( 'direction is missing' )
direction = self.args[ 'direction' ]
if direction not in [ 'Source', 'Destination' ]:
return S_ERROR( 'direction is not Source nor Destination' )
if not 'name' in self.args:
return S_ERROR( '"name" is missing' )
name = self.args[ 'name' ]
if not 'metric' in self.args:
return S_ERROR( 'metric is missing' )
metric = self.args[ 'metric' ]
if metric not in [ 'Quality', 'FailedTransfers' ]:
return S_ERROR( 'metric is not Quality nor FailedTransfers' )
return S_OK( ( hours, name, direction, metric ) )
def doNew( self, masterParams = None ):
'''
Gets the parameters to run, either from the master method or from its
own arguments.
For every elementName ( cannot process bulk queries.. ) contacts the
accounting client. It reurns dictionaries like { 'X -> Y' : { id: 100%.. } }
If there are ggus tickets, are recorded and then returned.
'''
if masterParams is not None:
hours, name, direction, metric = masterParams
else:
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
hours, name, direction, metric = params[ 'Value' ]
toD = datetime.utcnow()
fromD = toD - timedelta( hours = hours )
# dictionary with conditions for the accounting
transferDict = {
'OperationType' : 'putAndRegister',
direction : name
}
if metric == 'FailedTransfers':
transferDict[ 'FinalStatus' ] = [ 'Failed' ]
transferResults = self.rClient.getReport( 'DataOperation', metric, fromD,
toD, transferDict, 'Channel' )
if not transferResults[ 'OK' ]:
return transferResults
transferResults = transferResults[ 'Value' ]
if not 'data' in transferResults:
return S_ERROR( 'Missing data key' )
transferResults = transferResults[ 'data' ]
uniformResult = []
for channel, elementDict in transferResults.items():
try:
source, destination = channel.split( ' -> ' )
except ValueError:
continue
channelDict = {}
channelDict[ 'SourceName' ] = source
channelDict[ 'DestinationName' ] = destination
channelDict[ 'Metric' ] = metric
channelDict[ 'Value' ] = sum( elementDict.values() ) / len( elementDict.values() )
uniformResult.append( channelDict )
storeRes = self._storeCommand( uniformResult )
if not storeRes[ 'OK' ]:
return storeRes
# Compute mean of all transfer channels
value = 0
for channelDict in uniformResult:
value += channelDict[ 'Value' ]
if uniformResult:
value = float( value ) / len( uniformResult )
else:
value = None
return S_OK( { 'Mean' : value, 'Name' : name } )
def doCache( self ):
'''
Method that reads the cache table and tries to read from it. It will
return a list of dictionaries if there are results.
'''
params = self._prepareCommand()
if not params[ 'OK' ]:
return params
_hours, name, direction, metric = params[ 'Value' ]
sourceName, destinationName = None, None
if direction == 'Source':
sourceName = name
if direction == 'Destination':
destinationName = name
result = self.rmClient.selectTransferCache( sourceName, destinationName, metric )
if not result[ 'OK' ]:
return result
result = [ dict( zip( result[ 'Columns' ], res ) ) for res in result[ 'Value' ] ]
# Compute mean of all transfer channels
value = 0
for channelDict in result:
value += channelDict[ 'Value' ]
if result:
value = float( value ) / len( result )
else:
value = None
return S_OK( { 'Mean' : value, 'Name' : name } )
def doMaster( self ):
'''
Master method, which looks little bit spaguetti code, sorry !
- It gets all Sites.
- It gets all StorageElements
As there is no bulk query, it compares with what we have on the database.
It queries a portion of them.
'''
sites = CSHelpers.getSites()
if not sites[ 'OK' ]:
return sites
sites = sites[ 'Value' ]
ses = CSHelpers.getStorageElements()
if not ses[ 'OK' ]:
return ses
ses = ses[ 'Value' ]
elementNames = sites + ses
# sourceQuery = self.rmClient.selectTransferCache( meta = { 'columns' : [ 'SourceName' ] } )
# if not sourceQuery[ 'OK' ]:
# return sourceQuery
# sourceQuery = [ element[0] for element in sourceQuery[ 'Value' ] ]
#
# sourceElementsToQuery = list( set( elementNames ).difference( set( sourceQuery ) ) )
gLogger.info( 'Processing %s' % ', '.join( elementNames ) )
for metric in [ 'Quality', 'FailedTransfers' ]:
for direction in [ 'Source', 'Destination' ]:
# 2 hours of window
result = self.doNew( ( 2, elementNames, direction, metric ) )
if not result[ 'OK' ]:
self.metrics[ 'failed' ].append( result )
return S_OK( self.metrics )
################################################################################
#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF#EOF
class TransferQualityCommand( Command ):
def __init__( self, args = None, clients = None ):
super( TransferQualityCommand, self ).__init__( args, clients )
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
self.rClient.rpcClient = self.rgClient
def doCommand( self ):
"""
Return getQuality from DIRAC's accounting ReportsClient
`args`: a tuple
- args[0]: string: should be a ValidElement
- args[1]: string should be the name of the ValidElement
- args[2]: optional dateTime object: a "from" date
- args[3]: optional dateTime object: a "to" date
:returns:
{'Result': None | a float between 0.0 and 100.0}
"""
if not 'fromDate' in self.args:
fromDate = datetime.utcnow() - timedelta( hours = 2 )
else:
fromDate = self.args[ 'fromDate' ]
if not 'toDate' in self.args:
toDate = datetime.utcnow()
else:
toDate = self.args[ 'toDate' ]
if not 'name' in self.args:
return S_ERROR( 'name not specified' )
name = self.args[ 'name' ]
results = self.rClient.getReport( 'DataOperation', 'Quality', fromDate, toDate,
{ 'OperationType' : 'putAndRegister',
'Destination' : [ name ]
}, 'Channel' )
if not results[ 'OK' ]:
return results
pr_q_d = results[ 'Value' ][ 'data' ]
#FIXME: WHAT the hell is this doing ?
values = []
if len( pr_q_d ) == 1:
for k in pr_q_d.keys():
for n in pr_q_d[ k ].values():
values.append( n )
res = sum( values ) / len( values )
else:
for n in pr_q_d[ 'Total' ].values():
values.append(n)
res = sum( values ) / len( values )
return S_OK( res )
