class ActiveRequestJobInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host,
self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host,
self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url,
reqdb_url,
reqdbCouchApp="ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
return
@restcall(formats=[('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
results = DataCache.getlatestJobData()
if results == None or DataCache.islatestJobDataExpired():
results = self.wmstats.getActiveData(jobInfoFlag=True)
return rows([results])
def gatherActiveDataStats(self, config):
"""
gather active data statistics
"""
self.logger.info("Starting gatherActiveDataStats with jobInfo set to: %s", self.getJobInfo)
try:
tStart = time.time()
if DataCache.islatestJobDataExpired():
wmstatsDB = WMStatsReader(config.wmstats_url, reqdbURL=config.reqmgrdb_url,
reqdbCouchApp="ReqMgr", logger=self.logger)
self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO)
jobData = wmstatsDB.getActiveData(WMSTATS_JOB_INFO, jobInfoFlag=self.getJobInfo)
self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO)
tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False)
jobData.update(tempData)
self.logger.info("Running setlatestJobData...")
DataCache.setlatestJobData(jobData)
self.logger.info("DataCache is up-to-date with %d requests data", len(jobData))
except Exception as ex:
self.logger.exception("Exception updating DataCache. Error: %s", str(ex))
self.logger.info("Total time loading data from ReqMgr2 and WMStats: %s", time.time() - tStart)
return
def gatherActiveDataStats(self, config):
"""
gather active data statistics
"""
try:
if DataCache.islatestJobDataExpired():
wmstatsDB = WMStatsReader(config.wmstats_url, config.reqmgrdb_url,
reqdbCouchApp = "ReqMgr")
jobData = wmstatsDB.getActiveData(jobInfoFlag = True)
DataCache.setlatestJobData(jobData)
self.logger.info("DataCache is updated: %s" % len(jobData))
except Exception as ex:
self.logger.error(str(ex))
return
def gatherActiveDataStats(self, config):
"""
gather active data statistics
"""
try:
if DataCache.islatestJobDataExpired():
wmstatsDB = WMStatsReader(config.wmstats_url, config.reqmgrdb_url,
reqdbCouchApp = "ReqMgr")
jobData = wmstatsDB.getActiveData(jobInfoFlag = True)
DataCache.setlatestJobData(jobData)
except Exception as ex:
cherrypy.log.error(str(ex))
return
def gatherActiveDataStats():
wmstats_url = "https://cmsweb.cern.ch/couchdb/wmstats"
reqmgrdb_url = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"
jobInfoFlag = False
tStart = time.time()
try:
if DataCache.islatestJobDataExpired():
wmstatsDB = WMStatsReader(wmstats_url,
reqdbURL=reqmgrdb_url,
reqdbCouchApp="ReqMgr")
jobData = wmstatsDB.getActiveData(jobInfoFlag=jobInfoFlag)
DataCache.setlatestJobData(jobData)
print("DataCache is updated: {}".format(len(jobData)))
else:
print("DataCache is up-to-date")
except Exception as ex:
print("Exception updating cache. Details: {}\nTraceback: {}".format(
str(ex), str(traceback.format_exc())))
print("Total time executing this cycle: {}".format(time.time() - tStart))
class ActiveRequestJobInfo(RESTEntity):
"""
This class need to move under WMStats server when wmstats server created
"""
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
wmstats_url = "%s/%s" % (self.config.couch_host, self.config.couch_wmstats_db)
reqdb_url = "%s/%s" % (self.config.couch_host, self.config.couch_reqmgr_db)
self.wmstats = WMStatsReader(wmstats_url, reqdb_url, reqdbCouchApp = "ReqMgr")
def validate(self, apiobj, method, api, param, safe):
args_length = len(param.args)
return
@restcall(formats = [('application/json', JSONFormat())])
@tools.expires(secs=-1)
def get(self):
results = DataCache.getlatestJobData()
if results == None or DataCache.islatestJobDataExpired():
results = self.wmstats.getActiveData(jobInfoFlag = True)
return rows([results])
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'],
"failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema",
"assigned")
self.assertEquals(result, 'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {
'TotalEstimatedJobs': 100,
'TotalInputEvents': 1000,
'TotalInputLumis': 1234,
'TotalInputFiles': 5
}
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(
"not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {
"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {
"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()
}
result = self.reqDBWriter.updateRequestProperty(
spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"],
jobInfoFlag=False,
legacyFormat=True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(
schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
result = self.reqDBWriter.insertGenericRequest(schema[0])
self.assertEquals(result[0]['ok'], True, 'insert fail')
result = self.reqDBWriter.updateRequestStatus(schema[0]['RequestName'], "failed")
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestStatus("not_exist_schema", "assigned")
self.assertEquals(result,'Error: document not found')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], {"Teams": ['teamA']})
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", {"Teams": ['teamA']})
self.assertEquals(result, 'Error: document not found')
totalStats = {'TotalEstimatedJobs': 100, 'TotalInputEvents': 1000, 'TotalInputLumis': 1234, 'TotalInputFiles': 5}
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty(schema[0]['RequestName'], totalStats)
self.assertEquals(result, 'OK', 'update fail')
result = self.reqDBWriter.updateRequestProperty("not_exist_schema", totalStats)
self.assertEquals(result, 'Error: document not found')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
properties = {"RequestPriority": spec1.priority(),
'SiteWhitelist': spec1.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec1.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec1.name(), properties)
self.assertEquals(result, 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
properties = {"RequestPriority": spec2.priority(),
'SiteWhitelist': spec2.getTopLevelTask()[0].siteWhitelist(),
'OutputDatasets': spec2.listOutputDatasets()}
result = self.reqDBWriter.updateRequestProperty(spec2.name(), properties)
self.assertEquals(result, 'Error: document not found')
requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False, legacyFormat = True)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestByStatus(["failed"])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getRequestSummaryWithJobInfo(schema[0]['RequestName'])
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, dbName)
self.wmstatsReader = WMStatsReader(self.testInit.couchUrl, dbName)
self.wmstatsReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK',
'insert fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'],
"failed"), 'OK',
'update fail')
self.assertEquals(
self.wmstatsWriter.updateRequestStatus("not_exist_schema",
"assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(
self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'),
'OK', 'update fail')
self.assertEquals(
self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {
'total_jobs': 100,
'input_events': 1000,
'input_lumis': 1234,
'input_num_files': 5
}
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'INSERTED',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'],
totalStats), 'UPDATED',
'update fail')
self.assertEquals(
self.wmstatsWriter.insertTotalStats("not_exist_schema",
totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK',
'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
requests = self.wmstatsReader.getRequestByStatus(["failed"],
jobInfoFlag=False)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.workflowsByStatus(["failed"])
self.assertEquals(requests, [schema[0]['RequestName']])
class WMStatsTest(unittest.TestCase):
"""
"""
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = self.schema,
useDefault = False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, *self.couchApps)
self.wmstatsWriter = WMStatsWriter(self.testInit.couchUrl, dbName)
self.wmstatsReader = WMStatsReader(self.testInit.couchUrl, dbName)
self.wmstatsReader.defaultStale = {}
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
self.testInit.tearDownCouch()
def testWMStatsWriter(self):
# test getWork
schema = generate_reqmgr_schema()
self.assertEquals(self.wmstatsWriter.insertRequest(schema[0]), 'OK', 'insert fail');
self.assertEquals(self.wmstatsWriter.updateRequestStatus(schema[0]['RequestName'], "failed"), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateRequestStatus("not_exist_schema", "assigned"),
'ERROR: request not found - not_exist_schema')
self.assertEquals(self.wmstatsWriter.updateTeam(schema[0]['RequestName'], 'teamA'), 'OK', 'update fail')
self.assertEquals(self.wmstatsWriter.updateTeam("not_exist_schema", 'teamA'),
'ERROR: request not found - not_exist_schema')
totalStats = {'total_jobs': 100, 'input_events': 1000, 'input_lumis': 1234, 'input_num_files': 5}
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'INSERTED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats(schema[0]['RequestName'], totalStats), 'UPDATED', 'update fail')
self.assertEquals(self.wmstatsWriter.insertTotalStats("not_exist_schema", totalStats),
'ERROR: request not found - not_exist_schema')
spec1 = newWorkload(schema[0]['RequestName'])
production = spec1.newTask("Production")
production.setTaskType("Merge")
production.setSiteWhitelist(['TEST_SITE'])
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec1), 'OK', 'update fail')
spec2 = newWorkload("not_exist_schema")
production = spec2.newTask("Production")
production.setTaskType("Merge")
self.assertEquals(self.wmstatsWriter.updateFromWMSpec(spec2),
'ERROR: request not found - not_exist_schema')
requests = self.wmstatsReader.getRequestByStatus(["failed"], jobInfoFlag = False)
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requestCollection = RequestInfoCollection(requests)
result = requestCollection.getJSONData()
self.assertEquals(result.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.getActiveData()
self.assertEquals(requests.keys(), [schema[0]['RequestName']])
requests = self.wmstatsReader.workflowsByStatus(["failed"])
self.assertEquals(requests, [schema[0]['RequestName']])
