def test_addAndRemoveStorageOccupancy():
# just inserting one record
record = createStorageOccupancyAccountingRecord()
record.setStartTime()
record.setEndTime()
res = gDataStoreClient.addRegister(record)
assert res["OK"]
res = gDataStoreClient.commit()
assert res["OK"]
rc = ReportsClient()
res = rc.listReports("StorageOccupancy")
assert res["OK"]
res = rc.listUniqueKeyValues("StorageOccupancy")
assert res["OK"]
res = rc.getReport(
"StorageOccupancy",
"Free and Used Space",
datetime.datetime.utcnow(),
datetime.datetime.utcnow(),
{},
"StorageElement",
)
assert res["OK"]
# now removing that record
res = gDataStoreClient.remove(record)
assert res["OK"]
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
#Get unique key values
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
callback["selectionValues"] = simplejson.dumps(retVal['Value'])
#Cache for plotsList?
data = AccountingPlotHandler.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
data = simplejson.dumps(retVal['Value'])
AccountingPlotHandler.__keysCache.add("reportsList:%s" % typeName,
300, data)
callback["plotsList"] = data
self.write(json.dumps({"success": "true", "result": callback}))
def test_addAndRemoveDataOperation():
# just inserting one record
record = createDataOperationAccountingRecord()
record.setStartTime()
record.setEndTime()
res = gDataStoreClient.addRegister(record)
assert res["OK"]
res = gDataStoreClient.commit()
assert res["OK"]
rc = ReportsClient()
res = rc.listReports("DataOperation")
assert res["OK"]
res = rc.listUniqueKeyValues("DataOperation")
assert res["OK"]
res = rc.getReport(
"DataOperation",
"Successful transfers",
datetime.datetime.utcnow(),
datetime.datetime.utcnow(),
{},
"Destination",
)
assert res["OK"]
# now removing that record
res = gDataStoreClient.remove(record)
assert res["OK"]
def _getHistoryData(self, timeSpan, groupToUse):
"""Get history data from Accounting WMSHistory database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
reportsClient = ReportsClient()
reportCondition = {"Status": ["Running"]}
if not groupToUse:
reportGrouping = "UserGroup"
else:
reportGrouping = "User"
reportCondition = {"UserGroup": groupToUse}
now = Time.dateTime()
result = reportsClient.getReport(
"WMSHistory",
"AverageNumberOfJobs",
now - datetime.timedelta(seconds=timeSpan),
now,
reportCondition,
reportGrouping,
{"lastSeconds": timeSpan},
)
return result
def setCommandClient(self,
comm,
cObj,
RPCWMSAdmin=None,
RPCAccounting=None):
client = None
if comm == 'JobsEffSimpleEveryOne_Command':
from DIRAC.ResourceStatusSystem.Client.JobsClient import JobsClient
client = JobsClient()
cObj.setRPC(RPCWMSAdmin)
elif comm == 'PilotsEffSimpleEverySites_Command':
from DIRAC.ResourceStatusSystem.Client.PilotsClient import PilotsClient
client = PilotsClient()
cObj.setRPC(RPCWMSAdmin)
elif comm in ('TransferQualityEverySEs_Command',
'TransferQualityEverySEsSplitted_Command'):
from DIRAC.AccountingSystem.Client.ReportsClient import ReportsClient
client = ReportsClient(rpcClient=RPCAccounting)
cObj.setRPC(RPCAccounting)
cObj.setClient(client)
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message'])
return res
rsClass = res['Value']
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message'])
return res
rmClass = res['Value']
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = rsClass()
self.clients['ResourceManagementClient'] = rmClass()
self.clients['WMSAdministrator'] = WMSAdministratorClient()
self.clients['Pilots'] = PilotManagerClient()
self.cCaller = CommandCaller
return S_OK()
def __init__(self, args=None, clients=None):
super(FailedJobsBySiteSplittedCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
def __queryForPlot( self ):
retVal = self.__parseFormParams()
if not retVal[ 'OK' ]:
return retVal
params = retVal[ 'Value' ]
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.generateDelayedPlot( *params )
return retVal
def __init__(self, args=None, clients=None):
super(DIRACAccountingCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(SuccessfullPilotsByCESplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(FailedPilotsBySiteSplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def __init__(self, args=None, clients=None):
super(TransferQualityCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
def initialize(self):
self.__log = gLogger.getSubLogger("WMSHistoryCorrector")
self.__reportsClient = ReportsClient()
self.__usageHistory = {}
self.__slices = {}
self.__lastHistoryUpdate = 0
self.__globalCorrectionFactor = 5
self._fillSlices()
return S_OK()
def __init__( self, baseCSPath, group ):
self.__log = gLogger.getSubLogger( "WMSHistoryCorrector" )
self.__baseCSPath = baseCSPath
self.__group = group
self.__reportsClient = ReportsClient()
self.__usageHistory = {}
self.__slices = {}
self.__lastHistoryUpdate = 0
self.__globalCorrectionFactor = 5
self._fillSlices()
def do_connect(self, args):
"""
Tries to connect to the server
Usage: connect
"""
gLogger.info("Trying to connect to server")
self.connected = False
self.prompt = "(%s)> " % colorize("Not connected", "red")
retVal = ReportsClient().ping()
if retVal['OK']:
self.prompt = "(%s)> " % colorize("Connected", "green")
self.connected = True
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
self.am_setOption('shifterProxy', 'DataManager')
self.rmClient = ResourceManagementClient()
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['SpaceTokenOccupancy'] = [{'SpaceTokenOccupancy': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportGenerator'] = RPCClient(
'Accounting/ReportGenerator')
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = ResourceStatusClient()
self.clients['ResourceManagementClient'] = ResourceManagementClient()
self.clients['WMSAdministrator'] = RPCClient(
'WorkloadManagement/WMSAdministrator')
self.cCaller = CommandCaller
return S_OK()
def __init__(self, args=None, clients=None):
super(TransferCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
if "ResourceManagementClient" in self.apis:
self.rmClient = self.apis["ResourceManagementClient"]
else:
self.rmClient = ResourceManagementClient()
def __init__(self, args=None, clients=None):
super(RunningJobsBySiteSplittedCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
def __init__(self, args=None, clients=None):
super(RunningJobsBySiteSplittedCommand, self).__init__(args, clients)
if "ReportsClient" in self.apis:
self.rClient = self.apis["ReportsClient"]
else:
self.rClient = ReportsClient()
if "ReportGenerator" in self.apis:
self.rgClient = self.apis["ReportGenerator"]
else:
self.rgClient = Client(url="Accounting/ReportGenerator")
self.rClient.rpcClient = self.rgClient
def do_listViews( self, args ):
"""
Get a list of available views
Usage : listViews
"""
try:
retVal = ReportsClient().listViews()
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
print "Available summaries:"
for summary in retVal[ 'Value' ]:
print " %s" % summary
except:
self.showTraceback()
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
self.finish(rawData['data'])
def __init__( self, args = None, clients = None ):
super( TransferQualityCommand, self ).__init__( args, clients )
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis[ 'ReportGenerator' ]
else:
self.rgClient = RPCClient( 'Accounting/ReportGenerator' )
if 'ReportsClient' in self.apis:
self.rClient = self.apis[ 'ReportsClient' ]
else:
self.rClient = ReportsClient()
self.rClient.rpcClient = self.rgClient
def generateAccountingPlot(self):
try:
site = str(request.params['site'])
plotName = str(request.params['plotName'])
plotTime = str(request.params['plotTime'])
height = int(request.params['height'])
width = int(request.params['width'])
except:
S_ERROR("Oops, Invalid parameters!")
extraParams = {'height': height, 'width': width}
if plotName == 'CPU Used':
typeName = "Job"
reportName = "CPUUsed"
grouping = "FinalMajorStatus"
condDict = {'Site': [site]}
extraParams['plotTitle'] = "CPU used for site %s" % site
elif plotName == "Running jobs":
typeName = "WMSHistory"
reportName = "NumberOfJobs"
grouping = "JobGroup"
condDict = {'Site': [site], 'Status': ['Running']}
extraParams['plotTitle'] = "Jobs running for site %s" % site
else:
return S_ERROR("Oops, invalid plot name!")
if plotTime == "Last day":
extraParams['lastSeconds'] = 86400
elif plotTime == "Last week":
extraParams['lastSeconds'] = 604800
elif plotTime == "Last month":
extraParams['lastSeconds'] = 2592000
else:
return S_ERROR("Oops, invalid time!")
end = datetime.datetime.utcnow()
start = end - datetime.timedelta(seconds=extraParams['lastSeconds'])
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
result = repClient.generateDelayedPlot(typeName, reportName, start,
end, condDict, grouping,
extraParams)
if not result['OK']:
return S_ERROR(result['Message'])
return S_OK(result['Value']['plot'])
def __init__(self, args=None, clients=None):
super(SuccessfullPilotsByCESplittedCommand,
self).__init__(args, clients)
self.resources = Resources.Resources()
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
# Get unique key values
retVal = yield self.threadTask(self.__getUniqueKeyValues, typeName)
if not retVal['OK']:
self.finish({
"success": "false",
"result": "",
"error": retVal['Message']
})
return
records = {}
for record in retVal['Value']: # may have more than 1000 of records.
# do not show all of them in the web portal
length = len(retVal['Value'][record])
if length > 10000:
records[record] = retVal['Value'][record][length - 5000:]
message = "The %s accounting type contains to many rows: %s - > %d. Note: Only 1000 rows are returned!" % (
typeName, record, length)
gLogger.warn(message)
else:
records[record] = retVal['Value'][record]
callback["selectionValues"] = records
# Cache for plotsList?
data = AccountingPlotHandler.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient()
retVal = yield self.threadTask(repClient.listReports, typeName)
if not retVal['OK']:
self.finish({
"success": "false",
"result": "",
"error": retVal['Message']
})
return
data = retVal['Value']
AccountingPlotHandler.__keysCache.add("reportsList:%s" % typeName,
300, data)
callback["plotsList"] = data
self.finish({"success": "true", "result": callback})
def __showPlotPage( self, typeName, templateFile ):
#Get unique key values
retVal = self.__getUniqueKeyValues( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
c.selectionValues = simplejson.dumps( retVal[ 'Value' ] )
#Cache for plotsList?
data = AccountingplotsController.__keysCache.get( "reportsList:%s" % typeName )
if not data:
repClient = ReportsClient( rpcClient = getRPCClient( "Accounting/ReportGenerator" ) )
retVal = repClient.listReports( typeName )
if not retVal[ 'OK' ]:
c.error = retVal[ 'Message' ]
return render ( "/error.mako" )
data = simplejson.dumps( retVal[ 'Value' ] )
AccountingplotsController.__keysCache.add( "reportsList:%s" % typeName, 300, data )
c.plotsList = data
return render ( templateFile )
def __init__(self, args=None, clients=None):
super(AccountingCommand, self).__init__(args, clients)
if 'ReportsClient' in self.apis:
self.rClient = self.apis['ReportsClient']
else:
self.rClient = ReportsClient()
if 'ReportGenerator' in self.apis:
self.rgClient = self.apis['ReportGenerator']
else:
self.rgClient = RPCClient('Accounting/ReportGenerator')
self.rClient.rpcClient = self.rgClient
if 'ResourceManagementClient' in self.apis:
self.rmClient = self.apis['ResourceManagementClient']
else:
self.rmClient = ResourceManagementClient()
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])),
granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
def _getHistoryData(self, timeSpan, groupToUse):
""" Get history data from Accounting WMSHistory database
:param int timeSpan: time span
:param str groupToUse: requested user group
:return: dictionary with history data
"""
reportsClient = ReportsClient()
reportCondition = {'Status': ['Running']}
if not groupToUse:
reportGrouping = 'UserGroup'
else:
reportGrouping = 'User'
reportCondition = {'UserGroup': groupToUse}
now = Time.dateTime()
result = reportsClient.getReport(
'WMSHistory', 'AverageNumberOfJobs',
now - datetime.timedelta(seconds=timeSpan), now, reportCondition,
reportGrouping, {'lastSeconds': timeSpan})
return result
def do_plotView( self, args ):
"""
Gets a summary
Usage : getSummary <Summary name> <startdate YYYYMMDDHHMM> <enddate YYYYMMDDHHMM> <destLocation> (<field name> <field value>)*
"""
try:
argList = List.fromChar( args, " " )
if len( argList ) < 4:
gLogger.error( "Missing arguments!" )
return
startDT = self.__getDatetimeFromArg( argList[1] )
if not startDT:
gLogger.error( "Start time has invalid format" )
endDT = self.__getDatetimeFromArg( argList[2] )
if not endDT:
gLogger.error( "End time has invalid format" )
gLogger.info( "Start time is %s" % startDT )
gLogger.info( "End time is %s" % endDT )
sumArgs = {}
for iP in range( 4, len( argList ), 2 ):
key = argList[ iP ]
if key in sumArgs:
sumArgs[ key ].append( argList[ iP + 1 ] )
else:
sumArgs[ key ] = [ argList[ iP + 1 ] ]
repClient = ReportsClient()
retVal = repClient.plotView( argList[ 0 ], startDT, endDT, sumArgs )
if not retVal[ 'OK' ]:
gLogger.error( "Error: %s" % retVal[ 'Message' ] )
return
destDir = argList[3]
plotImg = retVal[ 'Value' ]
print "Downloading %s plot to %s.." % ( plotImg, destDir )
retVal = repClient.getPlotToDirectory( plotImg, destDir )
if not retVal[ 'OK' ]:
print " Error: %s" % retVal[ 'Message' ]
else:
print " done (%s/%s)!" % ( destDir, plotImg )
except:
self.showTraceback()
