def initializeOptimizer( cls ):
""" Initialization of the Agent.
"""
random.seed()
cls.__SEStatus = DictCache.DictCache()
cls.__sitesForSE = DictCache.DictCache()
try:
from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB
except ImportError, excp :
return S_ERROR( "Could not import JobDB: %s" % str( excp ) )
def __init__(self):
self.graphsLocation = os.path.join(
gConfig.getValue('/LocalSite/InstancePath', rootPath), 'data',
'accountingPlots')
self.cachedGraphs = {}
self.alive = True
self.purgeThread = threading.Thread(target=self.purgeExpired)
self.purgeThread.start()
self.__dataCache = DictCache()
self.__graphCache = DictCache(deleteFunction=_deleteGraph)
self.__dataLifeTime = 600
self.__graphLifeTime = 3600
def __init__( self, plotsLocation = False ): self.plotsLocation = plotsLocation self.alive = True self.__graphCache = DictCache( deleteFunction = _deleteGraph ) self.__graphLifeTime = 600 self.purgeThread = threading.Thread( target = self.purgeExpired ) self.purgeThread.start()
def initializeOptimizer(cls):
"""Initialize specific parameters
"""
cls.ex_setProperty('shifterProxy', 'DataManager')
cls.__SEStatus = DictCache.DictCache()
try:
cls.__replicaMan = ReplicaManager()
except Exception, e:
msg = 'Failed to create ReplicaManager'
cls.log.exception(msg)
return S_ERROR(msg + str(e))
def __init__( self, maxQueueSize = 10 ):
random.seed()
DB.__init__( self, 'TaskQueueDB', 'WorkloadManagement/TaskQueueDB', maxQueueSize )
self.__multiValueDefFields = ( 'Sites', 'GridCEs', 'GridMiddlewares', 'BannedSites',
'LHCbPlatforms', 'PilotTypes', 'SubmitPools', 'JobTypes' )
self.__multiValueMatchFields = ( 'GridCE', 'Site', 'GridMiddleware', 'LHCbPlatform',
'PilotType', 'SubmitPool', 'JobType' )
self.__bannedJobMatchFields = ( 'Site', )
self.__strictRequireMatchFields = ( 'SubmitPool', 'LHCbPlatform', 'PilotType' )
self.__singleValueDefFields = ( 'OwnerDN', 'OwnerGroup', 'Setup', 'CPUTime' )
self.__mandatoryMatchFields = ( 'Setup', 'CPUTime' )
self.__defaultCPUSegments = maxCPUSegments
self.__maxMatchRetry = 3
self.__jobPriorityBoundaries = ( 0.001, 10 )
self.__groupShares = {}
self.__deleteTQWithDelay = DictCache( self.__deleteTQIfEmpty )
self.__opsHelper = Operations()
self.__ensureInsertionIsSingle = False
self.__sharesCorrector = SharesCorrector( self.__opsHelper )
result = self.__initializeDB()
if not result[ 'OK' ]:
raise Exception( "Can't create tables: %s" % result[ 'Message' ] )
class AccountingPlotOldHandler(WebHandler):
AUTH_PROPS = "authenticated"
__keysCache = DictCache.DictCache()
def __getUniqueKeyValues(self, typeName):
sessionData = SessionData().getData()
userGroup = sessionData["user"]["group"]
if 'NormalUser' in CS.getPropertiesForGroup(userGroup):
cacheKey = (sessionData["user"]["username"], userGroup,
sessionData["setup"], typeName)
else:
cacheKey = (userGroup, sessionData["setup"], typeName)
data = AccountingPlotOldHandler.__keysCache.get(cacheKey)
if not data:
rpcClient = RPCClient("Accounting/ReportGenerator")
retVal = rpcClient.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del (retVal['rpcStub'])
if not retVal['OK']:
return retVal
#Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
AccountingPlotOldHandler.__keysCache.add(cacheKey, 300, data)
return data
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
#Get unique key values
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
callback["selectionValues"] = json.dumps(retVal['Value'])
#Cache for plotsList?
data = AccountingPlotOldHandler.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
data = json.dumps(retVal['Value'])
AccountingPlotOldHandler.__keysCache.add(
"reportsList:%s" % typeName, 300, data)
callback["plotsList"] = data
self.write(json.dumps({"success": "true", "result": callback}))
def __parseFormParams(self):
params = self.request.arguments
pD = {}
extraParams = {}
pinDates = False
for name in params:
if name.find("_") != 0:
continue
value = params[name][0]
name = name[1:]
pD[name] = str(value)
print pD
#Personalized title?
if 'plotTitle' in pD:
extraParams['plotTitle'] = pD['plotTitle']
del (pD['plotTitle'])
#Pin dates?
if 'pinDates' in pD:
pinDates = pD['pinDates']
del (pD['pinDates'])
pinDates = pinDates.lower() in ("yes", "y", "true", "1")
#Get plotname
if not 'grouping' in pD:
return S_ERROR("Missing grouping!")
grouping = pD['grouping']
#Get plotname
if not 'typeName' in pD:
return S_ERROR("Missing type name!")
typeName = pD['typeName']
del (pD['typeName'])
#Get plotname
if not 'plotName' in pD:
return S_ERROR("Missing plot name!")
reportName = pD['plotName']
del (pD['plotName'])
#Get times
if not 'timeSelector' in pD:
return S_ERROR("Missing time span!")
#Find the proper time!
pD['timeSelector'] = int(pD['timeSelector'])
if pD['timeSelector'] > 0:
end = Time.dateTime()
start = end - datetime.timedelta(seconds=pD['timeSelector'])
if not pinDates:
extraParams['lastSeconds'] = pD['timeSelector']
else:
if 'endTime' not in pD:
end = False
else:
end = Time.fromString(pD['endTime'])
del (pD['endTime'])
if 'startTime' not in pD:
return S_ERROR("Missing starTime!")
else:
start = Time.fromString(pD['startTime'])
del (pD['startTime'])
del (pD['timeSelector'])
for k in pD:
if k.find("ex_") == 0:
extraParams[k[3:]] = pD[k]
#Listify the rest
for selName in pD:
pD[selName] = List.fromChar(pD[selName], ",")
return S_OK(
(typeName, reportName, start, end, pD, grouping, extraParams))
def web_generatePlot(self):
callback = {}
retVal = self.__queryForPlot()
if retVal['OK']:
callback = {'success': True, 'data': retVal['Value']['plot']}
else:
callback = {'success': False, 'errors': retVal['Message']}
self.write(json.dumps(callback))
def __queryForPlot(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
return retVal
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.generateDelayedPlot(*params)
return retVal
@asyncGen
def web_getPlotImg(self):
"""
Get plot image
"""
callback = {}
if 'file' not in self.request.arguments:
callback = {
"success": "false",
"error": "Maybe you forgot the file?"
}
self.finish(json.dumps(callback))
return
plotImageFile = str(self.request.arguments['file'][0])
if plotImageFile.find(".png") < -1:
callback = {"success": "false", "error": "Not a valid image!"}
self.finish(json.dumps(callback))
return
transferClient = TransferClient("Accounting/ReportGenerator")
tempFile = tempfile.TemporaryFile()
retVal = yield self.threadTask(transferClient.receiveFile, tempFile,
plotImageFile)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(json.dumps(callback))
return
tempFile.seek(0)
data = tempFile.read()
self.set_header('Content-type', 'image/png')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest())
self.set_header('Content-Length', len(data))
self.set_header('Content-Transfer-Encoding', 'Binary')
self.set_header('Cache-Control',
"no-cache, no-store, must-revalidate, max-age=0")
self.set_header('Pragma', "no-cache")
self.set_header('Expires', (
datetime.datetime.utcnow() -
datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT"))
self.finish(data)
@asyncGen
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])),
granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header(
'Content-Disposition',
'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
@asyncGen
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
self.finish(json.dumps(rawData['data']))
class MonitoringHandler(WebHandler):
AUTH_PROPS = "authenticated"
__keysCache = DictCache.DictCache()
def __getUniqueKeyValues(self, typeName):
sessionData = self.getSessionData()
cacheKey = (sessionData["user"].get("username", ""),
sessionData["user"].get("group", ""),
sessionData["setup"],
typeName)
data = MonitoringHandler.__keysCache.get(cacheKey)
if not data:
client = MonitoringClient()
retVal = client.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del(retVal['rpcStub'])
if not retVal['OK']:
return retVal
# Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue("/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
MonitoringHandler.__keysCache.add(cacheKey, 300, data)
return data
@asyncGen
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
# Get unique key values
retVal = yield self.threadTask(self.__getUniqueKeyValues, typeName)
if not retVal['OK']:
self.finish({"success": "false", "result": "", "error": retVal['Message']})
return
records = {}
for record in retVal['Value']: # may have more than 1000 of records.
# do not show all of them in the web portal
length = len(retVal['Value'][record])
if length > 10000:
records[record] = retVal['Value'][record][length - 5000:]
message = "The %s accounting type contains to many rows: %s - > %d. Note: Only 1000 rows are returned!" % (
typeName, record, length)
gLogger.warn(message)
else:
records[record] = retVal['Value'][record]
callback["selectionValues"] = records
# Cache for plotsList?
data = MonitoringHandler.__keysCache.get("reportsList:%s" % typeName)
if not data:
repClient = MonitoringClient()
retVal = yield self.threadTask(repClient.listReports, typeName)
if not retVal['OK']:
self.finish({"success": "false", "result": "", "error": retVal['Message']})
return
data = retVal['Value']
MonitoringHandler.__keysCache.add("reportsList:%s" % typeName, 300, data)
callback["plotsList"] = data
self.finish({"success": "true", "result": callback})
def __parseFormParams(self):
params = self.request.arguments
pD = {}
extraParams = {}
pinDates = False
for name in params:
if name.find("_") != 0:
continue
value = params[name][0]
name = name[1:]
pD[name] = str(value)
# Personalized title?
if 'plotTitle' in pD:
extraParams['plotTitle'] = pD['plotTitle']
del(pD['plotTitle'])
# Pin dates?
if 'pinDates' in pD:
pinDates = pD['pinDates']
del(pD['pinDates'])
pinDates = pinDates.lower() in ("yes", "y", "true", "1")
# Get plotname
if 'grouping' not in pD:
return S_ERROR("Missing grouping!")
grouping = pD['grouping']
# Get plotname
if 'typeName' not in pD:
return S_ERROR("Missing type name!")
typeName = pD['typeName']
del(pD['typeName'])
# Get plotname
if 'plotName' not in pD:
return S_ERROR("Missing plot name!")
reportName = pD['plotName']
del(pD['plotName'])
# Get times
if 'timeSelector' not in pD:
return S_ERROR("Missing time span!")
# Find the proper time!
pD['timeSelector'] = int(pD['timeSelector'])
if pD['timeSelector'] > 0:
end = Time.dateTime()
start = end - datetime.timedelta(seconds=pD['timeSelector'])
if not pinDates:
extraParams['lastSeconds'] = pD['timeSelector']
else:
if 'endTime' not in pD:
end = False
else:
end = Time.fromString(pD['endTime'])
del(pD['endTime'])
if 'startTime' not in pD:
return S_ERROR("Missing starTime!")
else:
start = Time.fromString(pD['startTime'])
del(pD['startTime'])
del(pD['timeSelector'])
for k in pD:
if k.find("ex_") == 0:
extraParams[k[3:]] = pD[k]
# Listify the rest
for selName in pD:
if selName == 'grouping':
pD[selName] = [pD[selName]]
else:
try:
pD[selName] = json.loads(pD[selName])
except ValueError:
pD[selName] = List.fromChar(pD[selName], ",")
return S_OK((typeName, reportName, start, end, pD, grouping, extraParams))
@asyncGen
def web_generatePlot(self):
callback = {}
retVal = yield self.threadTask(self.__queryForPlot)
if retVal['OK']:
callback = {'success': True, 'data': retVal['Value']['plot']}
else:
callback = {'success': False, 'errors': retVal['Message']}
self.finish(callback)
def __queryForPlot(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
return retVal
params = retVal['Value']
repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = repClient.generateDelayedPlot(*params)
return retVal
@asyncGen
def web_getPlotImg(self):
"""
Get plot image
"""
callback = {}
if 'file' not in self.request.arguments:
callback = {"success": "false", "error": "Maybe you forgot the file?"}
self.finish(callback)
return
plotImageFile = str(self.request.arguments['file'][0])
if plotImageFile.find(".png") < -1:
callback = {"success": "false", "error": "Not a valid image!"}
self.finish(callback)
return
transferClient = TransferClient("Monitoring/Monitoring")
tempFile = tempfile.TemporaryFile()
retVal = yield self.threadTask(transferClient.receiveFile, tempFile, plotImageFile)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
return
tempFile.seek(0)
data = tempFile.read()
self.set_header('Content-type', 'image/png')
self.set_header('Content-Disposition', 'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest())
self.set_header('Content-Length', len(data))
self.set_header('Content-Transfer-Encoding', 'Binary')
#self.set_header( 'Cache-Control', "no-cache, no-store, must-revalidate, max-age=0" )
#self.set_header( 'Pragma', "no-cache" )
#self.set_header( 'Expires', ( datetime.datetime.utcnow() - datetime.timedelta( minutes = -10 ) ).strftime( "%d %b %Y %H:%M:%S GMT" ) )
self.finish(data)
@asyncGen
def web_getPlotImgFromCache(self):
"""
Get plot image from cache.
"""
callback = {}
if 'file' not in self.request.arguments:
callback = {"success": "false", "error": "Maybe you forgot the file?"}
self.finish(callback)
return
plotImageFile = str(self.request.arguments['file'][0])
retVal = extractRequestFromFileId(plotImageFile)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Value']}
self.finish(callback)
return
fields = retVal['Value']
if "extraArgs" in fields: # in order to get the plot from the cache we have to clean the extraArgs...
plotTitle = ""
if 'plotTitle' in fields["extraArgs"]:
plotTitle = fields["extraArgs"]["plotTitle"]
fields["extraArgs"] = {}
fields["extraArgs"]["plotTitle"] = plotTitle
else:
fields["extraArgs"] = {}
retVal = codeRequestInFileId(fields)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Value']}
self.finish(callback)
return
plotImageFile = retVal['Value']['plot']
transferClient = TransferClient("Monitoring/Monitoring")
tempFile = tempfile.TemporaryFile()
retVal = yield self.threadTask(transferClient.receiveFile, tempFile, plotImageFile)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
return
tempFile.seek(0)
data = tempFile.read()
self.set_header('Content-type', 'image/png')
self.set_header('Content-Disposition', 'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest())
self.set_header('Content-Length', len(data))
self.set_header('Content-Transfer-Encoding', 'Binary')
self.set_header('Cache-Control', "no-cache, no-store, must-revalidate, max-age=0")
self.set_header('Pragma', "no-cache")
self.set_header(
'Expires', (datetime.datetime.utcnow() - datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT"))
self.finish(data)
@asyncGen
def web_getCsvPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
# print rawData['data']
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
self.set_header('Content-type', 'text/csv')
self.set_header('Content-Disposition', 'attachment; filename="%s.csv"' % md5(str(params)).hexdigest())
self.set_header('Content-Length', len(strData))
self.finish(strData)
@asyncGen
def web_getPlotData(self):
callback = {}
retVal = self.__parseFormParams()
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
params = retVal['Value']
repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring"))
retVal = yield self.threadTask(repClient.getReport, *params)
if not retVal['OK']:
callback = {"success": "false", "error": retVal['Message']}
self.finish(callback)
rawData = retVal['Value']
self.finish(rawData['data'])
class AccountingplotsController(BaseController):
__keysCache = DictCache()
def __getUniqueKeyValues(self, typeName):
userGroup = getSelectedGroup()
if 'NormalUser' in CS.getPropertiesForGroup(userGroup):
cacheKey = (getUserName(), userGroup, getSelectedSetup(), typeName)
else:
cacheKey = (userGroup, getSelectedSetup(), typeName)
data = AccountingplotsController.__keysCache.get(cacheKey)
if not data:
rpcClient = getRPCClient("Accounting/ReportGenerator")
retVal = rpcClient.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del (retVal['rpcStub'])
if not retVal['OK']:
return retVal
#Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
AccountingplotsController.__keysCache.add(cacheKey, 300, data)
return data
def index(self):
# Return a rendered template
# return render('/some/template.mako')
# or, Return a response
return defaultRedirect()
def dataOperation(self):
return self.__showPlotPage("DataOperation",
"/systems/accounting/dataOperation.mako")
def job(self):
return self.__showPlotPage("Job", "/systems/accounting/job.mako")
def WMSHistory(self):
return self.__showPlotPage("WMSHistory",
"/systems/accounting/WMSHistory.mako")
def pilot(self):
return self.__showPlotPage("Pilot", "/systems/accounting/Pilot.mako")
def SRMSpaceTokenDeployment(self):
return self.__showPlotPage(
"SRMSpaceTokenDeployment",
"/systems/accounting/SRMSpaceTokenDeployment.mako")
def plotPage(self):
try:
typeName = str(request.params['typeName'])
except:
c.errorMessage = "Oops. missing type"
return render("/error.mako")
return self.__showPlotPage(typeName,
"/systems/accounting/%s.mako" % typeName)
def __showPlotPage(self, typeName, templateFile):
#Get unique key values
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK']:
c.error = retVal['Message']
return render("/error.mako")
c.selectionValues = simplejson.dumps(retVal['Value'])
#Cache for plotsList?
data = AccountingplotsController.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
c.error = retVal['Message']
return render("/error.mako")
data = simplejson.dumps(retVal['Value'])
AccountingplotsController.__keysCache.add(
"reportsList:%s" % typeName, 300, data)
c.plotsList = data
return render(templateFile)
@jsonify
def getKeyValuesForType(self):
try:
typeName = str(request.params['typeName'])
except:
return S_ERROR("Missing or invalid type name!")
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK'] and 'rpcStub' in retVal:
del (retVal['rpcStub'])
return retVal
def __parseFormParams(self):
pD = {}
extraParams = {}
pinDates = False
for name in request.params:
if name.find("_") != 0:
continue
value = request.params[name]
name = name[1:]
pD[name] = str(value)
#Personalized title?
if 'plotTitle' in pD:
extraParams['plotTitle'] = pD['plotTitle']
del (pD['plotTitle'])
#Pin dates?
if 'pinDates' in pD:
pinDates = pD['pinDates']
del (pD['pinDates'])
pinDates = pinDates.lower() in ("yes", "y", "true", "1")
#Get plotname
if not 'grouping' in pD:
return S_ERROR("Missing grouping!")
grouping = pD['grouping']
#Get plotname
if not 'typeName' in pD:
return S_ERROR("Missing type name!")
typeName = pD['typeName']
del (pD['typeName'])
#Get plotname
if not 'plotName' in pD:
return S_ERROR("Missing plot name!")
reportName = pD['plotName']
del (pD['plotName'])
#Get times
if not 'timeSelector' in pD:
return S_ERROR("Missing time span!")
#Find the proper time!
pD['timeSelector'] = int(pD['timeSelector'])
if pD['timeSelector'] > 0:
end = Time.dateTime()
start = end - datetime.timedelta(seconds=pD['timeSelector'])
if not pinDates:
extraParams['lastSeconds'] = pD['timeSelector']
else:
if 'endTime' not in pD:
end = False
else:
end = Time.fromString(pD['endTime'])
del (pD['endTime'])
if 'startTime' not in pD:
return S_ERROR("Missing starTime!")
else:
start = Time.fromString(pD['startTime'])
del (pD['startTime'])
del (pD['timeSelector'])
for k in pD:
if k.find("ex_") == 0:
extraParams[k[3:]] = pD[k]
#Listify the rest
for selName in pD:
pD[selName] = List.fromChar(pD[selName], ",")
return S_OK(
(typeName, reportName, start, end, pD, grouping, extraParams))
def __translateToExpectedExtResult(self, retVal):
if retVal['OK']:
return {'success': True, 'data': retVal['Value']['plot']}
else:
return {'success': False, 'errors': retVal['Message']}
def __queryForPlot(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
return retVal
params = retVal['Value']
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
retVal = repClient.generateDelayedPlot(*params)
return retVal
def getPlotData(self):
retVal = self.__parseFormParams()
if not retVal['OK']:
c.error = retVal['Message']
return render("/error.mako")
params = retVal['Value']
repClient = ReportsClient(
rpcClient=getRPCClient("Accounting/ReportGenerator"))
retVal = repClient.getReport(*params)
if not retVal['OK']:
c.error = retVal['Message']
return render("/error.mako")
rawData = retVal['Value']
groupKeys = rawData['data'].keys()
groupKeys.sort()
if 'granularity' in rawData:
granularity = rawData['granularity']
data = rawData['data']
tS = int(Time.toEpoch(params[2]))
timeStart = tS - tS % granularity
strData = "epoch,%s\n" % ",".join(groupKeys)
for timeSlot in range(timeStart, int(Time.toEpoch(params[3])),
granularity):
lineData = [str(timeSlot)]
for key in groupKeys:
if timeSlot in data[key]:
lineData.append(str(data[key][timeSlot]))
else:
lineData.append("")
strData += "%s\n" % ",".join(lineData)
else:
strData = "%s\n" % ",".join(groupKeys)
strData += ",".join([str(rawData['data'][k]) for k in groupKeys])
response.headers['Content-type'] = 'text/csv'
response.headers[
'Content-Disposition'] = 'attachment; filename="%s.csv"' % md5(
str(params)).hexdigest()
response.headers['Content-Length'] = len(strData)
return strData
@jsonify
def generatePlot(self):
return self.__translateToExpectedExtResult(self.__queryForPlot())
def generatePlotAndGetHTML(self):
retVal = self.__queryForPlot()
if not retVal['OK']:
return "<h2>Can't regenerate plot: %s</h2>" % retVal['Message']
return "<img src='getPlotImg?file=%s'/>" % retVal['Value']['plot']
def getPlotImg(self):
"""
Get plot image
"""
if 'file' not in request.params:
c.error = "Maybe you forgot the file?"
return render("/error.mako")
plotImageFile = str(request.params['file'])
if plotImageFile.find(".png") < -1:
c.error = "Not a valid image!"
return render("/error.mako")
transferClient = getTransferClient("Accounting/ReportGenerator")
tempFile = tempfile.TemporaryFile()
retVal = transferClient.receiveFile(tempFile, plotImageFile)
if not retVal['OK']:
c.error = retVal['Message']
return render("/error.mako")
tempFile.seek(0)
data = tempFile.read()
response.headers['Content-type'] = 'image/png'
response.headers[
'Content-Disposition'] = 'attachment; filename="%s.png"' % md5(
plotImageFile).hexdigest()
response.headers['Content-Length'] = len(data)
response.headers['Content-Transfer-Encoding'] = 'Binary'
response.headers[
'Cache-Control'] = "no-cache, no-store, must-revalidate, max-age=0"
response.headers['Pragma'] = "no-cache"
response.headers['Expires'] = (
datetime.datetime.utcnow() -
datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT")
return data
def __init__( self ): self.__usersCache = DictCache() self.__proxiesCache = DictCache() self.__vomsProxiesCache = DictCache() self.__pilotProxiesCache = DictCache() self.__filesCache = DictCache( self.__deleteTemporalFile )
class JobMonitorHandler(WebHandler):
AUTH_PROPS = "authenticated"
__dataCache = DictCache.DictCache()
@asyncGen
def web_getJobData(self):
RPC = RPCClient("WorkloadManagement/JobMonitoring", timeout=600)
req = self._request()
result = yield self.threadTask(RPC.getJobPageSummaryWeb, req,
self.globalSort, self.pageNumber,
self.numberOfJobs)
if not result["OK"]:
self.finish({
"success": "false",
"result": [],
"total": 0,
"error": result["Message"]
})
return
result = result["Value"]
if not result.has_key("TotalRecords"):
self.finish({
"success": "false",
"result": [],
"total": -1,
"error": "Data structure is corrupted"
})
return
if not (result["TotalRecords"] > 0):
self.finish({
"success": "false",
"result": [],
"total": 0,
"error": "There were no data matching your selection"
})
return
if not (result.has_key("ParameterNames")
and result.has_key("Records")):
self.finish({
"success": "false",
"result": [],
"total": -1,
"error": "Data structure is corrupted"
})
return
if not (len(result["ParameterNames"]) > 0):
self.finish({
"success": "false",
"result": [],
"total": -1,
"error": "ParameterNames field is missing"
})
return
if not (len(result["Records"]) > 0):
self.finish({
"success": "false",
"result": [],
"total": 0,
"Message": "There are no data to display"
})
return
callback = []
jobs = result["Records"]
head = result["ParameterNames"]
headLength = len(head)
for i in jobs:
tmp = {}
for j in range(0, headLength):
tmp[head[j]] = i[j]
callback.append(tmp)
total = result["TotalRecords"]
if result.has_key("Extras"):
st = self.__dict2string({})
extra = result["Extras"]
timestamp = Time.dateTime().strftime("%Y-%m-%d %H:%M [UTC]")
callback = {
"success": "true",
"result": callback,
"total": total,
"extra": extra,
"request": st,
"date": timestamp
}
else:
callback = {
"success": "true",
"result": callback,
"total": total,
"date": None
}
self.finish(callback)
def __dict2string(self, req):
result = ""
try:
for key, value in req.iteritems():
result = result + str(key) + ": " + ", ".join(value) + "; "
except Exception, x:
pass
gLogger.info("\033[0;31m Exception: \033[0m %s" % x)
result = result.strip()
result = result[:-1]
return result
class AccountingPlotHandler(WebHandler):
AUTH_PROPS = "authenticated"
__keysCache = DictCache()
def __getUniqueKeyValues(self, typeName):
sessionData = SessionData().getData()
userGroup = sessionData["user"]["group"]
if 'NormalUser' in CS.getPropertiesForGroup(userGroup):
cacheKey = (sessionData["user"]["username"], userGroup,
sessionData["setup"], typeName)
else:
cacheKey = (userGroup, sessionData["setup"], typeName)
data = AccountingPlotHandler.__keysCache.get(cacheKey)
if not data:
rpcClient = RPCClient("Accounting/ReportGenerator")
retVal = rpcClient.listUniqueKeyValues(typeName)
if 'rpcStub' in retVal:
del (retVal['rpcStub'])
if not retVal['OK']:
return retVal
#Site ordering based on TierLevel / alpha
if 'Site' in retVal['Value']:
siteLevel = {}
for siteName in retVal['Value']['Site']:
sitePrefix = siteName.split(".")[0].strip()
level = gConfig.getValue(
"/Resources/Sites/%s/%s/MoUTierLevel" %
(sitePrefix, siteName), 10)
if level not in siteLevel:
siteLevel[level] = []
siteLevel[level].append(siteName)
orderedSites = []
for level in sorted(siteLevel):
orderedSites.extend(sorted(siteLevel[level]))
retVal['Value']['Site'] = orderedSites
data = retVal
AccountingPlotHandler.__keysCache.add(cacheKey, 300, data)
return data
def web_getSelectionData(self):
callback = {}
typeName = self.request.arguments["type"][0]
#Get unique key values
retVal = self.__getUniqueKeyValues(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
callback["selectionValues"] = simplejson.dumps(retVal['Value'])
#Cache for plotsList?
data = AccountingPlotHandler.__keysCache.get("reportsList:%s" %
typeName)
if not data:
repClient = ReportsClient(
rpcClient=RPCClient("Accounting/ReportGenerator"))
retVal = repClient.listReports(typeName)
if not retVal['OK']:
self.write(
json.dumps({
"success": "false",
"result": "",
"error": retVal['Message']
}))
return
data = simplejson.dumps(retVal['Value'])
AccountingPlotHandler.__keysCache.add("reportsList:%s" % typeName,
300, data)
callback["plotsList"] = data
self.write(json.dumps({"success": "true", "result": callback}))
