class ActiveRequestJobInfo(RESTEntity): """ This class need to move under WMStats server when wmstats server created """ def __init__(self, app, api, config, mount): # main CouchDB database where requests/workloads are stored RESTEntity.__init__(self, app, api, config, mount) wmstats_url = "%s/%s" % (self.config.couch_host, self.config.couch_wmstats_db) reqdb_url = "%s/%s" % (self.config.couch_host, self.config.couch_reqmgr_db) self.wmstats = WMStatsReader(wmstats_url, reqdb_url, reqdbCouchApp="ReqMgr") def validate(self, apiobj, method, api, param, safe): args_length = len(param.args) return @restcall(formats=[('application/json', JSONFormat())]) @tools.expires(secs=-1) def get(self): results = DataCache.getlatestJobData() if results == None or DataCache.islatestJobDataExpired(): results = self.wmstats.getActiveData(jobInfoFlag=True) return rows([results])
def gatherActiveDataStats(self, config): """ gather active data statistics """ self.logger.info("Starting gatherActiveDataStats with jobInfo set to: %s", self.getJobInfo) try: tStart = time.time() if DataCache.islatestJobDataExpired(): wmstatsDB = WMStatsReader(config.wmstats_url, reqdbURL=config.reqmgrdb_url, reqdbCouchApp="ReqMgr", logger=self.logger) self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) jobData = wmstatsDB.getActiveData(WMSTATS_JOB_INFO, jobInfoFlag=self.getJobInfo) self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO) tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False) jobData.update(tempData) self.logger.info("Running setlatestJobData...") DataCache.setlatestJobData(jobData) self.logger.info("DataCache is up-to-date with %d requests data", len(jobData)) except Exception as ex: self.logger.exception("Exception updating DataCache. Error: %s", str(ex)) self.logger.info("Total time loading data from ReqMgr2 and WMStats: %s", time.time() - tStart) return
def gatherActiveDataStats(self, config): """ gather active data statistics """ try: if DataCache.islatestJobDataExpired(): wmstatsDB = WMStatsReader(config.wmstats_url, config.reqmgrdb_url, reqdbCouchApp = "ReqMgr") jobData = wmstatsDB.getActiveData(jobInfoFlag = True) DataCache.setlatestJobData(jobData) self.logger.info("DataCache is updated: %s" % len(jobData)) except Exception as ex: self.logger.error(str(ex)) return
def gatherActiveDataStats(self, config): """ gather active data statistics """ try: if DataCache.islatestJobDataExpired(): wmstatsDB = WMStatsReader(config.wmstats_url, config.reqmgrdb_url, reqdbCouchApp = "ReqMgr") jobData = wmstatsDB.getActiveData(jobInfoFlag = True) DataCache.setlatestJobData(jobData) except Exception as ex: cherrypy.log.error(str(ex)) return
def gatherActiveDataStats(): wmstats_url = "https://cmsweb.cern.ch/couchdb/wmstats" reqmgrdb_url = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache" jobInfoFlag = False tStart = time.time() try: if DataCache.islatestJobDataExpired(): wmstatsDB = WMStatsReader(wmstats_url, reqdbURL=reqmgrdb_url, reqdbCouchApp="ReqMgr") jobData = wmstatsDB.getActiveData(jobInfoFlag=jobInfoFlag) DataCache.setlatestJobData(jobData) print("DataCache is updated: {}".format(len(jobData))) else: print("DataCache is up-to-date") except Exception as ex: print("Exception updating cache. Details: {}\nTraceback: {}".format( str(ex), str(traceback.format_exc()))) print("Total time executing this cycle: {}".format(time.time() - tStart))
class ActiveRequestJobInfo(RESTEntity): """ This class need to move under WMStats server when wmstats server created """ def __init__(self, app, api, config, mount): # main CouchDB database where requests/workloads are stored RESTEntity.__init__(self, app, api, config, mount) wmstats_url = "%s/%s" % (self.config.couch_host, self.config.couch_wmstats_db) reqdb_url = "%s/%s" % (self.config.couch_host, self.config.couch_reqmgr_db) self.wmstats = WMStatsReader(wmstats_url, reqdb_url, reqdbCouchApp = "ReqMgr") def validate(self, apiobj, method, api, param, safe): args_length = len(param.args) return @restcall(formats = [('application/json', JSONFormat())]) @tools.expires(secs=-1) def get(self): results = DataCache.getlatestJobData() if results == None or DataCache.islatestJobDataExpired(): results = self.wmstats.getActiveData(jobInfoFlag = True) return rows([results])
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = 'wmstats_t' self.testInit.setupCouch(dbName, "WMStats") reqDBName = "reqmgrdb_t" self.testInit.setupCouch(reqDBName, "ReqMgr") wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName) reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName) self.reqDBWriter = RequestDBWriter(reqDBURL) self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL) self.wmstatsReader.defaultStale = {} self.wmstatsReader.reqDB.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.reqDBWriter.insertGenericRequest(schema[0]) self.assertEquals(result[0]['ok'], True, 'insert fail') result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed") self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned") self.assertEquals(result, 'Error: document not found') result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], {"Teams": ['teamA']}) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( "not_exist_schema", {"Teams": ['teamA']}) self.assertEquals(result, 'Error: document not found') totalStats = { 'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5 } result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty( "not_exist_schema", totalStats) self.assertEquals(result, 'Error: document not found') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) properties = { "RequestPriority": spec1.priority(), 'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec1.listOutputDatasets() } result = self.reqDBWriter.updateRequestProperty( spec1.name(), properties) self.assertEquals(result, 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") properties = { "RequestPriority": spec2.priority(), 'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec2.listOutputDatasets() } result = self.reqDBWriter.updateRequestProperty( spec2.name(), properties) self.assertEquals(result, 'Error: document not found') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag=False, legacyFormat=True) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestByStatus(["failed"]) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestSummaryWithJobInfo( schema[0]['RequestName']) self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) dbName = 'wmstats_t' self.testInit.setupCouch(dbName, "WMStats") reqDBName = "reqmgrdb_t" self.testInit.setupCouch(reqDBName, "ReqMgr") wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName) reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName) self.reqDBWriter = RequestDBWriter(reqDBURL) self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL) self.wmstatsReader.defaultStale = {} self.wmstatsReader.reqDB.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() result = self.reqDBWriter.insertGenericRequest(schema[0]) self.assertEquals(result[0]['ok'], True, 'insert fail') result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed") self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned") self.assertEquals(result,'Error: document not found') result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']}) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']}) self.assertEquals(result, 'Error: document not found') totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5} result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats) self.assertEquals(result, 'OK', 'update fail') result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats) self.assertEquals(result, 'Error: document not found') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) properties = {"RequestPriority": spec1.priority(), 'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec1.listOutputDatasets()} result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties) self.assertEquals(result, 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") properties = {"RequestPriority": spec2.priority(), 'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(), 'OutputDatasets': spec2.listOutputDatasets()} result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties) self.assertEquals(result, 'Error: document not found') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestByStatus(["failed"]) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName']) self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=self.schema, useDefault=False) dbName = 'wmstats_t' self.testInit.setupCouch(dbName, *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, dbName) self.wmstatsReader = WMStatsReader(self.testInit.couchUrl, dbName) self.wmstatsReader.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals( self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals( self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = { 'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5 } self.assertEquals( self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals( self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals( self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag=False) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.workflowsByStatus(["failed"]) self.assertEquals(requests, [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase): """ """ def setUp(self): """ _setUp_ """ self.schema = [] self.couchApps = ["WMStats"] self.testInit = TestInitCouchApp('WorkQueueServiceTest') self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = self.schema, useDefault = False) dbName = 'wmstats_t' self.testInit.setupCouch(dbName, *self.couchApps) self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, dbName) self.wmstatsReader = WMStatsReader(self.testInit.couchUrl, dbName) self.wmstatsReader.defaultStale = {} return def tearDown(self): """ _tearDown_ Drop all the WMBS tables. """ self.testInit.tearDownCouch() def testWMStatsWriter(self): # test getWork schema = generate_reqmgr_schema() self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail'); self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"), 'ERROR: request not found - not_exist_schema') self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail') self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'), 'ERROR: request not found - not_exist_schema') totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5} self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail') self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats), 'ERROR: request not found - not_exist_schema') spec1 = newWorkload(schema[0]['RequestName']) production = spec1.newTask("Production") production.setTaskType("Merge") production.setSiteWhitelist(['TEST_SITE']) self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail') spec2 = newWorkload("not_exist_schema") production = spec2.newTask("Production") production.setTaskType("Merge") self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2), 'ERROR: request not found - not_exist_schema') requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False) self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requestCollection = RequestInfoCollection(requests) result = requestCollection.getJSONData() self.assertEquals(result.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.getActiveData() self.assertEquals(requests.keys(), [schema[0]['RequestName']]) requests = self.wmstatsReader.workflowsByStatus(["failed"]) self.assertEquals(requests, [schema[0]['RequestName']])