def initializeOptimizer( cls ): """ Initialization of the Agent. """ random.seed() cls.__SEStatus = DictCache.DictCache() cls.__sitesForSE = DictCache.DictCache() try: from DIRAC.WorkloadManagementSystem.DB.JobDB import JobDB except ImportError, excp : return S_ERROR( "Could not import JobDB: %s" % str( excp ) )
def __init__(self): self.graphsLocation = os.path.join( gConfig.getValue('/LocalSite/InstancePath', rootPath), 'data', 'accountingPlots') self.cachedGraphs = {} self.alive = True self.purgeThread = threading.Thread(target=self.purgeExpired) self.purgeThread.start() self.__dataCache = DictCache() self.__graphCache = DictCache(deleteFunction=_deleteGraph) self.__dataLifeTime = 600 self.__graphLifeTime = 3600
def __init__( self, plotsLocation = False ): self.plotsLocation = plotsLocation self.alive = True self.__graphCache = DictCache( deleteFunction = _deleteGraph ) self.__graphLifeTime = 600 self.purgeThread = threading.Thread( target = self.purgeExpired ) self.purgeThread.start()
def initializeOptimizer(cls): """Initialize specific parameters """ cls.ex_setProperty('shifterProxy', 'DataManager') cls.__SEStatus = DictCache.DictCache() try: cls.__replicaMan = ReplicaManager() except Exception, e: msg = 'Failed to create ReplicaManager' cls.log.exception(msg) return S_ERROR(msg + str(e))
def __init__( self, maxQueueSize = 10 ): random.seed() DB.__init__( self, 'TaskQueueDB', 'WorkloadManagement/TaskQueueDB', maxQueueSize ) self.__multiValueDefFields = ( 'Sites', 'GridCEs', 'GridMiddlewares', 'BannedSites', 'LHCbPlatforms', 'PilotTypes', 'SubmitPools', 'JobTypes' ) self.__multiValueMatchFields = ( 'GridCE', 'Site', 'GridMiddleware', 'LHCbPlatform', 'PilotType', 'SubmitPool', 'JobType' ) self.__bannedJobMatchFields = ( 'Site', ) self.__strictRequireMatchFields = ( 'SubmitPool', 'LHCbPlatform', 'PilotType' ) self.__singleValueDefFields = ( 'OwnerDN', 'OwnerGroup', 'Setup', 'CPUTime' ) self.__mandatoryMatchFields = ( 'Setup', 'CPUTime' ) self.__defaultCPUSegments = maxCPUSegments self.__maxMatchRetry = 3 self.__jobPriorityBoundaries = ( 0.001, 10 ) self.__groupShares = {} self.__deleteTQWithDelay = DictCache( self.__deleteTQIfEmpty ) self.__opsHelper = Operations() self.__ensureInsertionIsSingle = False self.__sharesCorrector = SharesCorrector( self.__opsHelper ) result = self.__initializeDB() if not result[ 'OK' ]: raise Exception( "Can't create tables: %s" % result[ 'Message' ] )
class AccountingPlotOldHandler(WebHandler): AUTH_PROPS = "authenticated" __keysCache = DictCache.DictCache() def __getUniqueKeyValues(self, typeName): sessionData = SessionData().getData() userGroup = sessionData["user"]["group"] if 'NormalUser' in CS.getPropertiesForGroup(userGroup): cacheKey = (sessionData["user"]["username"], userGroup, sessionData["setup"], typeName) else: cacheKey = (userGroup, sessionData["setup"], typeName) data = AccountingPlotOldHandler.__keysCache.get(cacheKey) if not data: rpcClient = RPCClient("Accounting/ReportGenerator") retVal = rpcClient.listUniqueKeyValues(typeName) if 'rpcStub' in retVal: del (retVal['rpcStub']) if not retVal['OK']: return retVal #Site ordering based on TierLevel / alpha if 'Site' in retVal['Value']: siteLevel = {} for siteName in retVal['Value']['Site']: sitePrefix = siteName.split(".")[0].strip() level = gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) if level not in siteLevel: siteLevel[level] = [] siteLevel[level].append(siteName) orderedSites = [] for level in sorted(siteLevel): orderedSites.extend(sorted(siteLevel[level])) retVal['Value']['Site'] = orderedSites data = retVal AccountingPlotOldHandler.__keysCache.add(cacheKey, 300, data) return data def web_getSelectionData(self): callback = {} typeName = self.request.arguments["type"][0] #Get unique key values retVal = self.__getUniqueKeyValues(typeName) if not retVal['OK']: self.write( json.dumps({ "success": "false", "result": "", "error": retVal['Message'] })) return callback["selectionValues"] = json.dumps(retVal['Value']) #Cache for plotsList? data = AccountingPlotOldHandler.__keysCache.get("reportsList:%s" % typeName) if not data: repClient = ReportsClient( rpcClient=RPCClient("Accounting/ReportGenerator")) retVal = repClient.listReports(typeName) if not retVal['OK']: self.write( json.dumps({ "success": "false", "result": "", "error": retVal['Message'] })) return data = json.dumps(retVal['Value']) AccountingPlotOldHandler.__keysCache.add( "reportsList:%s" % typeName, 300, data) callback["plotsList"] = data self.write(json.dumps({"success": "true", "result": callback})) def __parseFormParams(self): params = self.request.arguments pD = {} extraParams = {} pinDates = False for name in params: if name.find("_") != 0: continue value = params[name][0] name = name[1:] pD[name] = str(value) print pD #Personalized title? if 'plotTitle' in pD: extraParams['plotTitle'] = pD['plotTitle'] del (pD['plotTitle']) #Pin dates? if 'pinDates' in pD: pinDates = pD['pinDates'] del (pD['pinDates']) pinDates = pinDates.lower() in ("yes", "y", "true", "1") #Get plotname if not 'grouping' in pD: return S_ERROR("Missing grouping!") grouping = pD['grouping'] #Get plotname if not 'typeName' in pD: return S_ERROR("Missing type name!") typeName = pD['typeName'] del (pD['typeName']) #Get plotname if not 'plotName' in pD: return S_ERROR("Missing plot name!") reportName = pD['plotName'] del (pD['plotName']) #Get times if not 'timeSelector' in pD: return S_ERROR("Missing time span!") #Find the proper time! pD['timeSelector'] = int(pD['timeSelector']) if pD['timeSelector'] > 0: end = Time.dateTime() start = end - datetime.timedelta(seconds=pD['timeSelector']) if not pinDates: extraParams['lastSeconds'] = pD['timeSelector'] else: if 'endTime' not in pD: end = False else: end = Time.fromString(pD['endTime']) del (pD['endTime']) if 'startTime' not in pD: return S_ERROR("Missing starTime!") else: start = Time.fromString(pD['startTime']) del (pD['startTime']) del (pD['timeSelector']) for k in pD: if k.find("ex_") == 0: extraParams[k[3:]] = pD[k] #Listify the rest for selName in pD: pD[selName] = List.fromChar(pD[selName], ",") return S_OK( (typeName, reportName, start, end, pD, grouping, extraParams)) def web_generatePlot(self): callback = {} retVal = self.__queryForPlot() if retVal['OK']: callback = {'success': True, 'data': retVal['Value']['plot']} else: callback = {'success': False, 'errors': retVal['Message']} self.write(json.dumps(callback)) def __queryForPlot(self): retVal = self.__parseFormParams() if not retVal['OK']: return retVal params = retVal['Value'] repClient = ReportsClient( rpcClient=RPCClient("Accounting/ReportGenerator")) retVal = repClient.generateDelayedPlot(*params) return retVal @asyncGen def web_getPlotImg(self): """ Get plot image """ callback = {} if 'file' not in self.request.arguments: callback = { "success": "false", "error": "Maybe you forgot the file?" } self.finish(json.dumps(callback)) return plotImageFile = str(self.request.arguments['file'][0]) if plotImageFile.find(".png") < -1: callback = {"success": "false", "error": "Not a valid image!"} self.finish(json.dumps(callback)) return transferClient = TransferClient("Accounting/ReportGenerator") tempFile = tempfile.TemporaryFile() retVal = yield self.threadTask(transferClient.receiveFile, tempFile, plotImageFile) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(json.dumps(callback)) return tempFile.seek(0) data = tempFile.read() self.set_header('Content-type', 'image/png') self.set_header( 'Content-Disposition', 'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest()) self.set_header('Content-Length', len(data)) self.set_header('Content-Transfer-Encoding', 'Binary') self.set_header('Cache-Control', "no-cache, no-store, must-revalidate, max-age=0") self.set_header('Pragma', "no-cache") self.set_header('Expires', ( datetime.datetime.utcnow() - datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT")) self.finish(data) @asyncGen def web_getCsvPlotData(self): callback = {} retVal = self.__parseFormParams() if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) params = retVal['Value'] repClient = ReportsClient( rpcClient=RPCClient("Accounting/ReportGenerator")) retVal = yield self.threadTask(repClient.getReport, *params) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) rawData = retVal['Value'] groupKeys = rawData['data'].keys() groupKeys.sort() # print rawData['data'] if 'granularity' in rawData: granularity = rawData['granularity'] data = rawData['data'] tS = int(Time.toEpoch(params[2])) timeStart = tS - tS % granularity strData = "epoch,%s\n" % ",".join(groupKeys) for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity): lineData = [str(timeSlot)] for key in groupKeys: if timeSlot in data[key]: lineData.append(str(data[key][timeSlot])) else: lineData.append("") strData += "%s\n" % ",".join(lineData) else: strData = "%s\n" % ",".join(groupKeys) strData += ",".join([str(rawData['data'][k]) for k in groupKeys]) self.set_header('Content-type', 'text/csv') self.set_header( 'Content-Disposition', 'attachment; filename="%s.csv"' % md5(str(params)).hexdigest()) self.set_header('Content-Length', len(strData)) self.finish(strData) @asyncGen def web_getPlotData(self): callback = {} retVal = self.__parseFormParams() if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) params = retVal['Value'] repClient = ReportsClient( rpcClient=RPCClient("Accounting/ReportGenerator")) retVal = yield self.threadTask(repClient.getReport, *params) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) rawData = retVal['Value'] groupKeys = rawData['data'].keys() groupKeys.sort() self.finish(json.dumps(rawData['data']))
class MonitoringHandler(WebHandler): AUTH_PROPS = "authenticated" __keysCache = DictCache.DictCache() def __getUniqueKeyValues(self, typeName): sessionData = self.getSessionData() cacheKey = (sessionData["user"].get("username", ""), sessionData["user"].get("group", ""), sessionData["setup"], typeName) data = MonitoringHandler.__keysCache.get(cacheKey) if not data: client = MonitoringClient() retVal = client.listUniqueKeyValues(typeName) if 'rpcStub' in retVal: del(retVal['rpcStub']) if not retVal['OK']: return retVal # Site ordering based on TierLevel / alpha if 'Site' in retVal['Value']: siteLevel = {} for siteName in retVal['Value']['Site']: sitePrefix = siteName.split(".")[0].strip() level = gConfig.getValue("/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) if level not in siteLevel: siteLevel[level] = [] siteLevel[level].append(siteName) orderedSites = [] for level in sorted(siteLevel): orderedSites.extend(sorted(siteLevel[level])) retVal['Value']['Site'] = orderedSites data = retVal MonitoringHandler.__keysCache.add(cacheKey, 300, data) return data @asyncGen def web_getSelectionData(self): callback = {} typeName = self.request.arguments["type"][0] # Get unique key values retVal = yield self.threadTask(self.__getUniqueKeyValues, typeName) if not retVal['OK']: self.finish({"success": "false", "result": "", "error": retVal['Message']}) return records = {} for record in retVal['Value']: # may have more than 1000 of records. # do not show all of them in the web portal length = len(retVal['Value'][record]) if length > 10000: records[record] = retVal['Value'][record][length - 5000:] message = "The %s accounting type contains to many rows: %s - > %d. Note: Only 1000 rows are returned!" % ( typeName, record, length) gLogger.warn(message) else: records[record] = retVal['Value'][record] callback["selectionValues"] = records # Cache for plotsList? data = MonitoringHandler.__keysCache.get("reportsList:%s" % typeName) if not data: repClient = MonitoringClient() retVal = yield self.threadTask(repClient.listReports, typeName) if not retVal['OK']: self.finish({"success": "false", "result": "", "error": retVal['Message']}) return data = retVal['Value'] MonitoringHandler.__keysCache.add("reportsList:%s" % typeName, 300, data) callback["plotsList"] = data self.finish({"success": "true", "result": callback}) def __parseFormParams(self): params = self.request.arguments pD = {} extraParams = {} pinDates = False for name in params: if name.find("_") != 0: continue value = params[name][0] name = name[1:] pD[name] = str(value) # Personalized title? if 'plotTitle' in pD: extraParams['plotTitle'] = pD['plotTitle'] del(pD['plotTitle']) # Pin dates? if 'pinDates' in pD: pinDates = pD['pinDates'] del(pD['pinDates']) pinDates = pinDates.lower() in ("yes", "y", "true", "1") # Get plotname if 'grouping' not in pD: return S_ERROR("Missing grouping!") grouping = pD['grouping'] # Get plotname if 'typeName' not in pD: return S_ERROR("Missing type name!") typeName = pD['typeName'] del(pD['typeName']) # Get plotname if 'plotName' not in pD: return S_ERROR("Missing plot name!") reportName = pD['plotName'] del(pD['plotName']) # Get times if 'timeSelector' not in pD: return S_ERROR("Missing time span!") # Find the proper time! pD['timeSelector'] = int(pD['timeSelector']) if pD['timeSelector'] > 0: end = Time.dateTime() start = end - datetime.timedelta(seconds=pD['timeSelector']) if not pinDates: extraParams['lastSeconds'] = pD['timeSelector'] else: if 'endTime' not in pD: end = False else: end = Time.fromString(pD['endTime']) del(pD['endTime']) if 'startTime' not in pD: return S_ERROR("Missing starTime!") else: start = Time.fromString(pD['startTime']) del(pD['startTime']) del(pD['timeSelector']) for k in pD: if k.find("ex_") == 0: extraParams[k[3:]] = pD[k] # Listify the rest for selName in pD: if selName == 'grouping': pD[selName] = [pD[selName]] else: try: pD[selName] = json.loads(pD[selName]) except ValueError: pD[selName] = List.fromChar(pD[selName], ",") return S_OK((typeName, reportName, start, end, pD, grouping, extraParams)) @asyncGen def web_generatePlot(self): callback = {} retVal = yield self.threadTask(self.__queryForPlot) if retVal['OK']: callback = {'success': True, 'data': retVal['Value']['plot']} else: callback = {'success': False, 'errors': retVal['Message']} self.finish(callback) def __queryForPlot(self): retVal = self.__parseFormParams() if not retVal['OK']: return retVal params = retVal['Value'] repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring")) retVal = repClient.generateDelayedPlot(*params) return retVal @asyncGen def web_getPlotImg(self): """ Get plot image """ callback = {} if 'file' not in self.request.arguments: callback = {"success": "false", "error": "Maybe you forgot the file?"} self.finish(callback) return plotImageFile = str(self.request.arguments['file'][0]) if plotImageFile.find(".png") < -1: callback = {"success": "false", "error": "Not a valid image!"} self.finish(callback) return transferClient = TransferClient("Monitoring/Monitoring") tempFile = tempfile.TemporaryFile() retVal = yield self.threadTask(transferClient.receiveFile, tempFile, plotImageFile) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) return tempFile.seek(0) data = tempFile.read() self.set_header('Content-type', 'image/png') self.set_header('Content-Disposition', 'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest()) self.set_header('Content-Length', len(data)) self.set_header('Content-Transfer-Encoding', 'Binary') #self.set_header( 'Cache-Control', "no-cache, no-store, must-revalidate, max-age=0" ) #self.set_header( 'Pragma', "no-cache" ) #self.set_header( 'Expires', ( datetime.datetime.utcnow() - datetime.timedelta( minutes = -10 ) ).strftime( "%d %b %Y %H:%M:%S GMT" ) ) self.finish(data) @asyncGen def web_getPlotImgFromCache(self): """ Get plot image from cache. """ callback = {} if 'file' not in self.request.arguments: callback = {"success": "false", "error": "Maybe you forgot the file?"} self.finish(callback) return plotImageFile = str(self.request.arguments['file'][0]) retVal = extractRequestFromFileId(plotImageFile) if not retVal['OK']: callback = {"success": "false", "error": retVal['Value']} self.finish(callback) return fields = retVal['Value'] if "extraArgs" in fields: # in order to get the plot from the cache we have to clean the extraArgs... plotTitle = "" if 'plotTitle' in fields["extraArgs"]: plotTitle = fields["extraArgs"]["plotTitle"] fields["extraArgs"] = {} fields["extraArgs"]["plotTitle"] = plotTitle else: fields["extraArgs"] = {} retVal = codeRequestInFileId(fields) if not retVal['OK']: callback = {"success": "false", "error": retVal['Value']} self.finish(callback) return plotImageFile = retVal['Value']['plot'] transferClient = TransferClient("Monitoring/Monitoring") tempFile = tempfile.TemporaryFile() retVal = yield self.threadTask(transferClient.receiveFile, tempFile, plotImageFile) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) return tempFile.seek(0) data = tempFile.read() self.set_header('Content-type', 'image/png') self.set_header('Content-Disposition', 'attachment; filename="%s.png"' % md5(plotImageFile).hexdigest()) self.set_header('Content-Length', len(data)) self.set_header('Content-Transfer-Encoding', 'Binary') self.set_header('Cache-Control', "no-cache, no-store, must-revalidate, max-age=0") self.set_header('Pragma', "no-cache") self.set_header( 'Expires', (datetime.datetime.utcnow() - datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT")) self.finish(data) @asyncGen def web_getCsvPlotData(self): callback = {} retVal = self.__parseFormParams() if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) params = retVal['Value'] repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring")) retVal = yield self.threadTask(repClient.getReport, *params) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) rawData = retVal['Value'] groupKeys = rawData['data'].keys() groupKeys.sort() # print rawData['data'] if 'granularity' in rawData: granularity = rawData['granularity'] data = rawData['data'] tS = int(Time.toEpoch(params[2])) timeStart = tS - tS % granularity strData = "epoch,%s\n" % ",".join(groupKeys) for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity): lineData = [str(timeSlot)] for key in groupKeys: if timeSlot in data[key]: lineData.append(str(data[key][timeSlot])) else: lineData.append("") strData += "%s\n" % ",".join(lineData) else: strData = "%s\n" % ",".join(groupKeys) strData += ",".join([str(rawData['data'][k]) for k in groupKeys]) self.set_header('Content-type', 'text/csv') self.set_header('Content-Disposition', 'attachment; filename="%s.csv"' % md5(str(params)).hexdigest()) self.set_header('Content-Length', len(strData)) self.finish(strData) @asyncGen def web_getPlotData(self): callback = {} retVal = self.__parseFormParams() if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) params = retVal['Value'] repClient = MonitoringClient(rpcClient=RPCClient("Monitoring/Monitoring")) retVal = yield self.threadTask(repClient.getReport, *params) if not retVal['OK']: callback = {"success": "false", "error": retVal['Message']} self.finish(callback) rawData = retVal['Value'] self.finish(rawData['data'])
class AccountingplotsController(BaseController): __keysCache = DictCache() def __getUniqueKeyValues(self, typeName): userGroup = getSelectedGroup() if 'NormalUser' in CS.getPropertiesForGroup(userGroup): cacheKey = (getUserName(), userGroup, getSelectedSetup(), typeName) else: cacheKey = (userGroup, getSelectedSetup(), typeName) data = AccountingplotsController.__keysCache.get(cacheKey) if not data: rpcClient = getRPCClient("Accounting/ReportGenerator") retVal = rpcClient.listUniqueKeyValues(typeName) if 'rpcStub' in retVal: del (retVal['rpcStub']) if not retVal['OK']: return retVal #Site ordering based on TierLevel / alpha if 'Site' in retVal['Value']: siteLevel = {} for siteName in retVal['Value']['Site']: sitePrefix = siteName.split(".")[0].strip() level = gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) if level not in siteLevel: siteLevel[level] = [] siteLevel[level].append(siteName) orderedSites = [] for level in sorted(siteLevel): orderedSites.extend(sorted(siteLevel[level])) retVal['Value']['Site'] = orderedSites data = retVal AccountingplotsController.__keysCache.add(cacheKey, 300, data) return data def index(self): # Return a rendered template # return render('/some/template.mako') # or, Return a response return defaultRedirect() def dataOperation(self): return self.__showPlotPage("DataOperation", "/systems/accounting/dataOperation.mako") def job(self): return self.__showPlotPage("Job", "/systems/accounting/job.mako") def WMSHistory(self): return self.__showPlotPage("WMSHistory", "/systems/accounting/WMSHistory.mako") def pilot(self): return self.__showPlotPage("Pilot", "/systems/accounting/Pilot.mako") def SRMSpaceTokenDeployment(self): return self.__showPlotPage( "SRMSpaceTokenDeployment", "/systems/accounting/SRMSpaceTokenDeployment.mako") def plotPage(self): try: typeName = str(request.params['typeName']) except: c.errorMessage = "Oops. missing type" return render("/error.mako") return self.__showPlotPage(typeName, "/systems/accounting/%s.mako" % typeName) def __showPlotPage(self, typeName, templateFile): #Get unique key values retVal = self.__getUniqueKeyValues(typeName) if not retVal['OK']: c.error = retVal['Message'] return render("/error.mako") c.selectionValues = simplejson.dumps(retVal['Value']) #Cache for plotsList? data = AccountingplotsController.__keysCache.get("reportsList:%s" % typeName) if not data: repClient = ReportsClient( rpcClient=getRPCClient("Accounting/ReportGenerator")) retVal = repClient.listReports(typeName) if not retVal['OK']: c.error = retVal['Message'] return render("/error.mako") data = simplejson.dumps(retVal['Value']) AccountingplotsController.__keysCache.add( "reportsList:%s" % typeName, 300, data) c.plotsList = data return render(templateFile) @jsonify def getKeyValuesForType(self): try: typeName = str(request.params['typeName']) except: return S_ERROR("Missing or invalid type name!") retVal = self.__getUniqueKeyValues(typeName) if not retVal['OK'] and 'rpcStub' in retVal: del (retVal['rpcStub']) return retVal def __parseFormParams(self): pD = {} extraParams = {} pinDates = False for name in request.params: if name.find("_") != 0: continue value = request.params[name] name = name[1:] pD[name] = str(value) #Personalized title? if 'plotTitle' in pD: extraParams['plotTitle'] = pD['plotTitle'] del (pD['plotTitle']) #Pin dates? if 'pinDates' in pD: pinDates = pD['pinDates'] del (pD['pinDates']) pinDates = pinDates.lower() in ("yes", "y", "true", "1") #Get plotname if not 'grouping' in pD: return S_ERROR("Missing grouping!") grouping = pD['grouping'] #Get plotname if not 'typeName' in pD: return S_ERROR("Missing type name!") typeName = pD['typeName'] del (pD['typeName']) #Get plotname if not 'plotName' in pD: return S_ERROR("Missing plot name!") reportName = pD['plotName'] del (pD['plotName']) #Get times if not 'timeSelector' in pD: return S_ERROR("Missing time span!") #Find the proper time! pD['timeSelector'] = int(pD['timeSelector']) if pD['timeSelector'] > 0: end = Time.dateTime() start = end - datetime.timedelta(seconds=pD['timeSelector']) if not pinDates: extraParams['lastSeconds'] = pD['timeSelector'] else: if 'endTime' not in pD: end = False else: end = Time.fromString(pD['endTime']) del (pD['endTime']) if 'startTime' not in pD: return S_ERROR("Missing starTime!") else: start = Time.fromString(pD['startTime']) del (pD['startTime']) del (pD['timeSelector']) for k in pD: if k.find("ex_") == 0: extraParams[k[3:]] = pD[k] #Listify the rest for selName in pD: pD[selName] = List.fromChar(pD[selName], ",") return S_OK( (typeName, reportName, start, end, pD, grouping, extraParams)) def __translateToExpectedExtResult(self, retVal): if retVal['OK']: return {'success': True, 'data': retVal['Value']['plot']} else: return {'success': False, 'errors': retVal['Message']} def __queryForPlot(self): retVal = self.__parseFormParams() if not retVal['OK']: return retVal params = retVal['Value'] repClient = ReportsClient( rpcClient=getRPCClient("Accounting/ReportGenerator")) retVal = repClient.generateDelayedPlot(*params) return retVal def getPlotData(self): retVal = self.__parseFormParams() if not retVal['OK']: c.error = retVal['Message'] return render("/error.mako") params = retVal['Value'] repClient = ReportsClient( rpcClient=getRPCClient("Accounting/ReportGenerator")) retVal = repClient.getReport(*params) if not retVal['OK']: c.error = retVal['Message'] return render("/error.mako") rawData = retVal['Value'] groupKeys = rawData['data'].keys() groupKeys.sort() if 'granularity' in rawData: granularity = rawData['granularity'] data = rawData['data'] tS = int(Time.toEpoch(params[2])) timeStart = tS - tS % granularity strData = "epoch,%s\n" % ",".join(groupKeys) for timeSlot in range(timeStart, int(Time.toEpoch(params[3])), granularity): lineData = [str(timeSlot)] for key in groupKeys: if timeSlot in data[key]: lineData.append(str(data[key][timeSlot])) else: lineData.append("") strData += "%s\n" % ",".join(lineData) else: strData = "%s\n" % ",".join(groupKeys) strData += ",".join([str(rawData['data'][k]) for k in groupKeys]) response.headers['Content-type'] = 'text/csv' response.headers[ 'Content-Disposition'] = 'attachment; filename="%s.csv"' % md5( str(params)).hexdigest() response.headers['Content-Length'] = len(strData) return strData @jsonify def generatePlot(self): return self.__translateToExpectedExtResult(self.__queryForPlot()) def generatePlotAndGetHTML(self): retVal = self.__queryForPlot() if not retVal['OK']: return "<h2>Can't regenerate plot: %s</h2>" % retVal['Message'] return "<img src='getPlotImg?file=%s'/>" % retVal['Value']['plot'] def getPlotImg(self): """ Get plot image """ if 'file' not in request.params: c.error = "Maybe you forgot the file?" return render("/error.mako") plotImageFile = str(request.params['file']) if plotImageFile.find(".png") < -1: c.error = "Not a valid image!" return render("/error.mako") transferClient = getTransferClient("Accounting/ReportGenerator") tempFile = tempfile.TemporaryFile() retVal = transferClient.receiveFile(tempFile, plotImageFile) if not retVal['OK']: c.error = retVal['Message'] return render("/error.mako") tempFile.seek(0) data = tempFile.read() response.headers['Content-type'] = 'image/png' response.headers[ 'Content-Disposition'] = 'attachment; filename="%s.png"' % md5( plotImageFile).hexdigest() response.headers['Content-Length'] = len(data) response.headers['Content-Transfer-Encoding'] = 'Binary' response.headers[ 'Cache-Control'] = "no-cache, no-store, must-revalidate, max-age=0" response.headers['Pragma'] = "no-cache" response.headers['Expires'] = ( datetime.datetime.utcnow() - datetime.timedelta(minutes=-10)).strftime("%d %b %Y %H:%M:%S GMT") return data
def __init__( self ): self.__usersCache = DictCache() self.__proxiesCache = DictCache() self.__vomsProxiesCache = DictCache() self.__pilotProxiesCache = DictCache() self.__filesCache = DictCache( self.__deleteTemporalFile )
class JobMonitorHandler(WebHandler): AUTH_PROPS = "authenticated" __dataCache = DictCache.DictCache() @asyncGen def web_getJobData(self): RPC = RPCClient("WorkloadManagement/JobMonitoring", timeout=600) req = self._request() result = yield self.threadTask(RPC.getJobPageSummaryWeb, req, self.globalSort, self.pageNumber, self.numberOfJobs) if not result["OK"]: self.finish({ "success": "false", "result": [], "total": 0, "error": result["Message"] }) return result = result["Value"] if not result.has_key("TotalRecords"): self.finish({ "success": "false", "result": [], "total": -1, "error": "Data structure is corrupted" }) return if not (result["TotalRecords"] > 0): self.finish({ "success": "false", "result": [], "total": 0, "error": "There were no data matching your selection" }) return if not (result.has_key("ParameterNames") and result.has_key("Records")): self.finish({ "success": "false", "result": [], "total": -1, "error": "Data structure is corrupted" }) return if not (len(result["ParameterNames"]) > 0): self.finish({ "success": "false", "result": [], "total": -1, "error": "ParameterNames field is missing" }) return if not (len(result["Records"]) > 0): self.finish({ "success": "false", "result": [], "total": 0, "Message": "There are no data to display" }) return callback = [] jobs = result["Records"] head = result["ParameterNames"] headLength = len(head) for i in jobs: tmp = {} for j in range(0, headLength): tmp[head[j]] = i[j] callback.append(tmp) total = result["TotalRecords"] if result.has_key("Extras"): st = self.__dict2string({}) extra = result["Extras"] timestamp = Time.dateTime().strftime("%Y-%m-%d %H:%M [UTC]") callback = { "success": "true", "result": callback, "total": total, "extra": extra, "request": st, "date": timestamp } else: callback = { "success": "true", "result": callback, "total": total, "date": None } self.finish(callback) def __dict2string(self, req): result = "" try: for key, value in req.iteritems(): result = result + str(key) + ": " + ", ".join(value) + "; " except Exception, x: pass gLogger.info("\033[0;31m Exception: \033[0m %s" % x) result = result.strip() result = result[:-1] return result
class AccountingPlotHandler(WebHandler): AUTH_PROPS = "authenticated" __keysCache = DictCache() def __getUniqueKeyValues(self, typeName): sessionData = SessionData().getData() userGroup = sessionData["user"]["group"] if 'NormalUser' in CS.getPropertiesForGroup(userGroup): cacheKey = (sessionData["user"]["username"], userGroup, sessionData["setup"], typeName) else: cacheKey = (userGroup, sessionData["setup"], typeName) data = AccountingPlotHandler.__keysCache.get(cacheKey) if not data: rpcClient = RPCClient("Accounting/ReportGenerator") retVal = rpcClient.listUniqueKeyValues(typeName) if 'rpcStub' in retVal: del (retVal['rpcStub']) if not retVal['OK']: return retVal #Site ordering based on TierLevel / alpha if 'Site' in retVal['Value']: siteLevel = {} for siteName in retVal['Value']['Site']: sitePrefix = siteName.split(".")[0].strip() level = gConfig.getValue( "/Resources/Sites/%s/%s/MoUTierLevel" % (sitePrefix, siteName), 10) if level not in siteLevel: siteLevel[level] = [] siteLevel[level].append(siteName) orderedSites = [] for level in sorted(siteLevel): orderedSites.extend(sorted(siteLevel[level])) retVal['Value']['Site'] = orderedSites data = retVal AccountingPlotHandler.__keysCache.add(cacheKey, 300, data) return data def web_getSelectionData(self): callback = {} typeName = self.request.arguments["type"][0] #Get unique key values retVal = self.__getUniqueKeyValues(typeName) if not retVal['OK']: self.write( json.dumps({ "success": "false", "result": "", "error": retVal['Message'] })) return callback["selectionValues"] = simplejson.dumps(retVal['Value']) #Cache for plotsList? data = AccountingPlotHandler.__keysCache.get("reportsList:%s" % typeName) if not data: repClient = ReportsClient( rpcClient=RPCClient("Accounting/ReportGenerator")) retVal = repClient.listReports(typeName) if not retVal['OK']: self.write( json.dumps({ "success": "false", "result": "", "error": retVal['Message'] })) return data = simplejson.dumps(retVal['Value']) AccountingPlotHandler.__keysCache.add("reportsList:%s" % typeName, 300, data) callback["plotsList"] = data self.write(json.dumps({"success": "true", "result": callback}))