def getSplitting(requestName): reqmgrUrl='https://cmsweb.cern.ch/reqmgr/reqMgr/' reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) params = workload.getTopLevelTask()[0].jobSplittingParameters() algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm() return params['events_per_job']
def main(): demPolicy = Block() reqmgr = RequestManager(dict = {'endpoint' : 'https://cmsweb.cern.ch/reqmgr/reqMgr'}) result = reqmgr.getRequest('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207') workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment('pdmvserv_HIG-Summer12DR53X-01392_T1_ES_PIC_MSS_1_v0__130724_063344_7207', 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) x,y = demPolicy(wmspec = workload, task = workload.getTopLevelTask()[0]) print x print y
def getSplitting(requestName): reqmgrUrl = 'https://cmsweb.cern.ch/reqmgr/reqMgr/' reqmgr = RequestManager(dict={'endpoint': reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) params = workload.getTopLevelTask()[0].jobSplittingParameters() algo = workload.getTopLevelTask()[0].jobSplittingAlgorithm() return params['events_per_job']
def getRequestInformationAndWorkload(requestName, reqmgrUrl): """ _getRequestInformationAndWorkload_ Retrieve the request information for assignment and the full pickled workload. """ reqmgr = RequestManager(dict = {'endpoint' : reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) return workload, result
def getRequestInformationAndWorkload(requestName, reqmgrUrl): """ _getRequestInformationAndWorkload_ Retrieve the request information for assignment and the full pickled workload. """ reqmgr = RequestManager(dict={'endpoint': reqmgrUrl}) result = reqmgr.getRequest(requestName) workloadDB = Database(result['CouchWorkloadDBName'], result['CouchURL']) workloadPickle = workloadDB.getAttachment(requestName, 'spec') spec = pickle.loads(workloadPickle) workload = WMWorkloadHelper(spec) return workload, result
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8899) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName = userName, groupName = groupName, teamName = teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print("Exception during set up, reason: %s" % msg) raise ex def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad = False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = {'process': {'outputModules_': ['ThisIsAName'], 'ThisIsAName': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} BadTweak = {'process': {'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}, 'ThisIsAName2': {'dataset': {'dataTier': 'RECO', 'filterName': 'Filter'}}}} configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = self.couchDBName) configCache.createUserGroup(groupname = "testGroup", username = '******') if bad: configCache.setPSetTweaks(PSetTweak = BadTweak) else: configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.save() return configCache.getCouchID() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName = "team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request = requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName,"error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName, percent_complete = 100, percent_success = 90) self.reqService.updateRequestStatus(requestName, "running-open")
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName=self.couchDBName) self.config.setPort(8899) self.schemaModules = ["WMCore.RequestManager.RequestDB"] def setUp(self): RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache", "ReqMgr") self.testInit.setupCouch("%s_wmstats" % self.couchDBName, "WMStats") # logging stuff from TestInit is broken, setting myself l = logging.getLogger() l.setLevel(logging.DEBUG) self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) userName = '******' groupName = 'Li' teamName = 'Tang' schema = utils.getAndSetupSchema(self, userName=userName, groupName=groupName, teamName=teamName) schema['ConfigCacheID'] = self.createConfig() schema['CouchDBName'] = self.couchDBName schema['CouchWorkloadDBName'] = self.couchDBName try: r = self.jsonSender.put('request', schema) try: self.requestName = r[0]['RequestName'] except: self.requestName = r[0].values()[0]['RequestName'] except Exception as ex: msg = traceback.format_exc() print "Exception during set up, reason: %s" % msg raise ex def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() def createConfig(self, bad=False): """ _createConfig_ Create a config of some sort that we can load out of ConfigCache """ PSetTweak = { 'process': { 'outputModules_': ['ThisIsAName'], 'ThisIsAName': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } BadTweak = { 'process': { 'outputModules_': ['ThisIsAName1', 'ThisIsAName2'], 'ThisIsAName1': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } }, 'ThisIsAName2': { 'dataset': { 'dataTier': 'RECO', 'filterName': 'Filter' } } } } configCache = ConfigCache(os.environ["COUCHURL"], couchDBName=self.couchDBName) configCache.createUserGroup(groupname="testGroup", username='******') if bad: configCache.setPSetTweaks(PSetTweak=BadTweak) else: configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.save() return configCache.getCouchID() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName="team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request=requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName, "error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName, percent_complete=100, percent_success=90) self.reqService.updateRequestStatus(requestName, "running-open")
class JobUpdaterPoller(BaseWorkerThread): """ _JobUpdaterPoller_ Poller class for the JobUpdater """ def __init__(self, config): """ __init__ """ BaseWorkerThread.__init__(self) self.config = config self.bossAir = BossAirAPI(config = self.config) self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl}) self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl, self.config.WorkQueueManager.dbname) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater") self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority") self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus") def setup(self, parameters = None): """ _setup_ """ pass def terminate(self, parameters = None): """ _terminate_ Terminate gracefully. """ pass def algorithm(self, parameters = None): """ _algorithm_ """ logging.info("Synchronizing priorities with ReqMgr...") self.synchronizeJobPriority() def synchronizeJobPriority(self): """ _synchronizeJobPriority_ Check WMBS and WorkQueue for active workflows and compare with the ReqMgr for priority changes. If a priority change occurs then update the job priority in the batch system and the elements in the local queue that have not been injected yet. """ # Update the priority of workflows that are not in WMBS and just in local queue priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()] for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority'] except Exception, ex: logging.error("Couldn't retrieve the priority of request %s" % workflow) logging.error("Error: %s" % ex) continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow] for workflow in workflowsToUpdate: self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow]) # Check the workflows in WMBS priorityCache = {} workflowsToUpdateWMBS = {} workflowsToCheck = self.listWorkflowsDAO.execute() for workflowEntry in workflowsToCheck: workflow = workflowEntry['name'] if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority'] except Exception, ex: logging.error("Couldn't retrieve the priority of request %s" % workflow) logging.error("Error: %s" % ex) continue requestPriority = priorityCache[workflow] if requestPriority != workflowEntry['workflow_priority']: # Update the workqueue priority for the Available elements self.workqueue.updatePriority(workflow, priorityCache[workflow]) # Check if there are executing jobs for this particular task if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0: self.bossAir.updateJobInformation(workflow, workflowEntry['task'], requestPriority = priorityCache[workflow], taskPriority = workflowEntry['task_priority']) workflowsToUpdateWMBS[workflow] = priorityCache[workflow]
class RequestManagerTest(RESTBaseUnitTest): """ Test RequestMgr Service client It will start RequestMgr RESTService Server DB is whatever env is set This checks whether DS call makes without error and return the results. This test only test service call returns without error. The correctness of each function is tested in test/python/RequestManager_t/RequestMgr_t.py """ def initialize(self): self.couchDBName = "reqmgr_t_0" self.config = RequestManagerConfig( 'WMCore.HTTPFrontEnd.RequestManager.ReqMgrRESTModel') dbUrl = os.environ.get("DATABASE", None) self.config.setDBUrl(dbUrl) self.config.setFormatter('WMCore.WebTools.RESTFormatter') self.config.setupRequestConfig() self.config.setupCouchDatabase(dbName = self.couchDBName) self.config.setPort(8888) self.schemaModules = ["WMCore.RequestManager.RequestDB"] return def setUp(self): """ setUP global values """ RESTBaseUnitTest.setUp(self) self.testInit.setupCouch("%s" % self.couchDBName, "GroupUser", "ConfigCache") self.params = {} self.params['endpoint'] = self.config.getServerUrl() self.reqService = RequestManagerDS(self.params) self.jsonSender = JSONRequests(self.config.getServerUrl()) self.requestSchema = getRequestSchema() self.jsonSender.put('group/PeopleLikeMe') self.jsonSender.put('user/[email protected]') self.jsonSender.put('group/PeopleLikeMe/me') self.jsonSender.put('version/CMSSW_3_5_8') r = self.jsonSender.put('request/' + self.requestSchema['RequestName'], self.requestSchema) self.requestName = r[0]['RequestName'] def tearDown(self): self.config.deleteWorkloadCache() RESTBaseUnitTest.tearDown(self) self.testInit.tearDownCouch() @attr("integration") def testA_RequestManagerService(self): requestName = self.requestName request = self.reqService.getRequest(requestName) # minimal test : it's return type and the some value inside self.assertEqual(type(request), dict) self.assertTrue(len(request) > 0) # Test putTeam self.reqService.putTeam("team_usa") self.assertTrue('team_usa' in self.jsonSender.get('team')[0]) self.jsonSender.put('assignment/%s/%s' % ("team_usa", requestName)) request = self.reqService.getAssignment(teamName = "team_usa") self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) request = self.reqService.getAssignment(request = requestName) self.assertEqual(type(request), list) self.assertTrue(len(request) > 0) self.reqService.sendMessage(requestName,"error") self.reqService.putWorkQueue(requestName, "http://test_url") self.reqService.reportRequestProgress(requestName) self.reqService.reportRequestProgress(requestName, percent_complete = 100, percent_success = 90) self.reqService.reportRequestStatus(requestName, "running") return