def main(): """ _main_ """ if 'WMAGENT_CONFIG' not in os.environ: os.environ['WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) # Instantiating central reqmgr and local workqueue print "ReqMgr2 URL : %s" % sanitizeURL(config.JobUpdater.reqMgr2Url)['url'] print "WorkQueue URL: %s and dbname %s" % (sanitizeURL(config.WorkQueueManager.couchurl)['url'], config.WorkQueueManager.dbname) reqmgr2 = ReqMgr(config.JobUpdater.reqMgr2Url) workqueue = WorkQueue(config.WorkQueueManager.couchurl, config.WorkQueueManager.dbname) print "\nFirst attempt to update prio of wfs that are not in WMBS and only in local queue" priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in workqueue.getAvailableWorkflows()] print "Retrieved %d workflows from workqueue" % len(workflowsToCheck) for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: priorityCache[workflow] = reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority'] except Exception, ex: print "Couldn't retrieve the priority of request %s" % workflow print "Error: %s" % ex continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow]
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec( specName, "file", assignKwargs={'SiteWhitelist': ["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True, **self.queueParams) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox', **self.queueParams) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file", assignKwargs={'SiteWhitelist':["T2_XX_SiteA"]}) globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl, UnittestFlag=True) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') # overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) numWorks = localQ.pullWork({'T2_XX_SiteA': 10}) self.assertTrue(numWorks > 0) # replicate from GQ to LQ manually localQ.backend.pullFromParent(continuous=False) # wait until replication is done time.sleep(2) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') # overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) availableWF = localApi.getAvailableWorkflows() self.assertEqual(availableWF, set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except Exception as ex: self.fail('No exception should be raised.: %s' % str(ex))
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName='workqueue_t', QueueURL=self.testInit.couchUrl) localQ = localQueue(DbName='local_workqueue_t', QueueURL=self.testInit.couchUrl, CacheDir=self.testInit.testDir, ParentQueueCouchUrl='%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName='workqueue_t_inbox') # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce': True, 'group': True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue( localQ.pullWork({'T2_XX_SiteA': 10}, continuousReplication=False) > 0) localQ.processInboundWork(continuous=False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce': True, 'group': True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def testUpdatePriorityService(self): """ _testUpdatePriorityService_ Check that we can update the priority correctly also check the available workflows feature """ specName = "RerecoSpec" specUrl = self.specGenerator.createReRecoSpec(specName, "file") globalQ = globalQueue(DbName = 'workqueue_t', QueueURL = self.testInit.couchUrl) localQ = localQueue(DbName = 'local_workqueue_t', QueueURL = self.testInit.couchUrl, CacheDir = self.testInit.testDir, ParentQueueCouchUrl = '%s/workqueue_t' % self.testInit.couchUrl, ParentQueueInboxCouchDBName = 'workqueue_t_inbox' ) # Try a full chain of priority update and propagation self.assertTrue(globalQ.queueWork(specUrl, "RerecoSpec", "teamA") > 0) globalApi = WorkQueueDS(self.testInit.couchUrl, 'workqueue_t') #overwrite default - can't test with stale view globalApi.defaultOptions = {'reduce' : True, 'group' : True} globalApi.updatePriority(specName, 100) self.assertEqual(globalQ.backend.getWMSpec(specName).priority(), 100) storedElements = globalQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) self.assertTrue(localQ.pullWork({'T2_XX_SiteA' : 10}) > 0) localQ.processInboundWork(continuous = False) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 100) localApi = WorkQueueDS(self.testInit.couchUrl, 'local_workqueue_t') #overwrite default - can't test with stale view localApi.defaultOptions = {'reduce' : True, 'group' : True} localApi.updatePriority(specName, 500) self.assertEqual(localQ.backend.getWMSpec(specName).priority(), 500) storedElements = localQ.backend.getElementsForWorkflow(specName) for element in storedElements: self.assertEqual(element['Priority'], 500) self.assertEqual(localApi.getAvailableWorkflows(), set([(specName, 500)])) # Attempt to update an inexistent workflow in the queue try: globalApi.updatePriority('NotExistent', 2) except: self.fail('No exception should be raised.')
def main(): """ _main_ """ if 'WMAGENT_CONFIG' not in os.environ: os.environ[ 'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py' config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"]) # Instantiating central reqmgr and local workqueue print "ReqMgr2 URL : %s" % sanitizeURL( config.JobUpdater.reqMgr2Url)['url'] print "WorkQueue URL: %s and dbname %s" % (sanitizeURL( config.WorkQueueManager.couchurl)['url'], config.WorkQueueManager.dbname) reqmgr2 = ReqMgr(config.JobUpdater.reqMgr2Url) workqueue = WorkQueue(config.WorkQueueManager.couchurl, config.WorkQueueManager.dbname) print "\nFirst attempt to update prio of wfs that are not in WMBS and only in local queue" priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in workqueue.getAvailableWorkflows()] print "Retrieved %d workflows from workqueue" % len(workflowsToCheck) for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: priorityCache[workflow] = reqmgr2.getRequestByNames( workflow)[workflow]['RequestPriority'] except Exception, ex: print "Couldn't retrieve the priority of request %s" % workflow print "Error: %s" % ex continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow]
class JobUpdaterPoller(BaseWorkerThread): """ _JobUpdaterPoller_ Poller class for the JobUpdater """ def __init__(self, config): """ __init__ """ BaseWorkerThread.__init__(self) self.config = config self.bossAir = BossAirAPI(config=self.config) self.reqmgr2 = ReqMgr(self.config.JobUpdater.reqMgr2Url) self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl, self.config.WorkQueueManager.dbname) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listWorkflowsDAO = self.daoFactory(classname="Workflow.ListForJobUpdater") self.updateWorkflowPrioDAO = self.daoFactory(classname="Workflow.UpdatePriority") self.executingJobsDAO = self.daoFactory(classname="Jobs.GetNumberOfJobsForWorkflowTaskStatus") def setup(self, parameters=None): """ _setup_ """ pass def terminate(self, parameters=None): """ _terminate_ Terminate gracefully. """ pass def algorithm(self, parameters=None): """ _algorithm_ """ try: logging.info("Synchronizing priorities with ReqMgr...") self.synchronizeJobPriority() logging.info("Priorities were synchronized, wait until the next cycle") except CouchConnectionError as ex: msg = "Caught CouchConnectionError exception in JobUpdater\n" msg += "transactions postponed until the next polling cycle\n" msg += str(ex) logging.exception(msg) except CouchConflictError as ex: msg = "Caught CouchConflictError exception in JobUpdater\n" msg += "transactions postponed until the next polling cycle\n" msg += str(ex) logging.exception(msg) except Exception as ex: if 'Connection refused' in str(ex): logging.warn("Failed to sync priorities. Trying in the next cycle") else: msg = "Caught unexpected exception in JobUpdater: %s\n" % str(ex) logging.exception(msg) raise JobUpdaterException(msg) def synchronizeJobPriority(self): """ _synchronizeJobPriority_ Check WMBS and WorkQueue for active workflows and compare with the ReqMgr for priority changes. If a priority change occurs then update the job priority in the batch system and the elements in the local queue that have not been injected yet. """ # Update the priority of workflows that are not in WMBS and just in local queue priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()] for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority'] except Exception as ex: logging.error("Couldn't retrieve the priority of request %s", workflow) logging.error("Error: %s", str(ex)) continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow] logging.info("Found %d workflows to update in workqueue", len(workflowsToUpdate)) for workflow in workflowsToUpdate: self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow]) # Check the workflows in WMBS priorityCache = {} workflowsToUpdateWMBS = {} workflowsToCheck = self.listWorkflowsDAO.execute() for workflowEntry in workflowsToCheck: workflow = workflowEntry['name'] if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr2.getRequestByNames(workflow)[workflow]['RequestPriority'] except Exception as ex: logging.error("Couldn't retrieve the priority of request %s", workflow) logging.error("Error: %s", str(ex)) continue requestPriority = int(priorityCache[workflow]) if requestPriority != int(workflowEntry['workflow_priority']): # Update the workqueue priority for the Available elements self.workqueue.updatePriority(workflow, requestPriority) # Check if there are executing jobs for this particular task if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0: self.bossAir.updateJobInformation(workflow, workflowEntry['task'], requestPriority=priorityCache[workflow], taskPriority=workflowEntry['task_priority']) workflowsToUpdateWMBS[workflow] = priorityCache[workflow] if workflowsToUpdateWMBS: logging.info("Updating %d workflows in WMBS.", len(workflowsToUpdateWMBS)) self.updateWorkflowPrioDAO.execute(workflowsToUpdateWMBS)
class JobUpdaterPoller(BaseWorkerThread): """ _JobUpdaterPoller_ Poller class for the JobUpdater """ def __init__(self, config): """ __init__ """ BaseWorkerThread.__init__(self) self.config = config self.bossAir = BossAirAPI(config=self.config) self.reqmgr2 = ReqMgr(self.config.General.ReqMgr2ServiceURL) self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl, self.config.WorkQueueManager.dbname) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.listWorkflowsDAO = self.daoFactory( classname="Workflow.ListForJobUpdater") self.updateWorkflowPrioDAO = self.daoFactory( classname="Workflow.UpdatePriority") self.executingJobsDAO = self.daoFactory( classname="Jobs.GetNumberOfJobsForWorkflowTaskStatus") def setup(self, parameters=None): """ _setup_ """ pass def terminate(self, parameters=None): """ _terminate_ Terminate gracefully. """ pass @timeFunction def algorithm(self, parameters=None): """ _algorithm_ """ try: logging.info("Synchronizing priorities with ReqMgr...") self.synchronizeJobPriority() logging.info( "Priorities were synchronized, wait until the next cycle") except CouchConnectionError as ex: msg = "Caught CouchConnectionError exception in JobUpdater\n" msg += "transactions postponed until the next polling cycle\n" msg += str(ex) logging.exception(msg) except CouchConflictError as ex: msg = "Caught CouchConflictError exception in JobUpdater\n" msg += "transactions postponed until the next polling cycle\n" msg += str(ex) logging.exception(msg) except Exception as ex: if 'Connection refused' in str(ex): logging.warn( "Failed to sync priorities. Trying in the next cycle") else: msg = "Caught unexpected exception in JobUpdater: %s\n" % str( ex) logging.exception(msg) raise JobUpdaterException(msg) def synchronizeJobPriority(self): """ _synchronizeJobPriority_ Check WMBS and WorkQueue for active workflows and compare with the ReqMgr for priority changes. If a priority change occurs then update the job priority in the batch system and the elements in the local queue that have not been injected yet. """ # Update the priority of workflows that are not in WMBS and just in local queue priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()] for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: result = self.reqmgr2.getRequestByNames(workflow)[0] priorityCache[workflow] = result[workflow][ 'RequestPriority'] except Exception as ex: logging.error( "Couldn't retrieve the priority of request %s", workflow) logging.error("Error: %s", str(ex)) continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow] logging.info("Found %d workflows to update in workqueue", len(workflowsToUpdate)) for workflow in workflowsToUpdate: self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow]) # Check the workflows in WMBS priorityCache = {} workflowsToUpdateWMBS = {} workflowsToCheck = self.listWorkflowsDAO.execute() for workflowEntry in workflowsToCheck: workflow = workflowEntry['name'] if workflow not in priorityCache: try: result = self.reqmgr2.getRequestByNames(workflow)[0] priorityCache[workflow] = result[workflow][ 'RequestPriority'] except Exception as ex: logging.error( "Couldn't retrieve the priority of request %s", workflow) logging.error("Error: %s", str(ex)) continue requestPriority = int(priorityCache[workflow]) if requestPriority != int(workflowEntry['workflow_priority']): # Update the workqueue priority for the Available elements self.workqueue.updatePriority(workflow, requestPriority) # Check if there are executing jobs for this particular task if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0: self.bossAir.updateJobInformation( workflow, workflowEntry['task'], requestPriority=priorityCache[workflow], taskPriority=workflowEntry['task_priority']) workflowsToUpdateWMBS[workflow] = priorityCache[workflow] if workflowsToUpdateWMBS: logging.info("Updating %d workflows in WMBS.", len(workflowsToUpdateWMBS)) self.updateWorkflowPrioDAO.execute(workflowsToUpdateWMBS)
class JobUpdaterPoller(BaseWorkerThread): """ _JobUpdaterPoller_ Poller class for the JobUpdater """ def __init__(self, config): """ __init__ """ BaseWorkerThread.__init__(self) self.config = config self.bossAir = BossAirAPI(config = self.config) self.reqmgr = RequestManager({'endpoint' : self.config.JobUpdater.reqMgrUrl}) self.workqueue = WorkQueue(self.config.WorkQueueManager.couchurl, self.config.WorkQueueManager.dbname) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.listWorkflowsDAO = self.daoFactory(classname = "Workflow.ListForJobUpdater") self.updateWorkflowPrioDAO = self.daoFactory(classname = "Workflow.UpdatePriority") self.executingJobsDAO = self.daoFactory(classname = "Jobs.GetNumberOfJobsForWorkflowTaskStatus") def setup(self, parameters = None): """ _setup_ """ pass def terminate(self, parameters = None): """ _terminate_ Terminate gracefully. """ pass def algorithm(self, parameters = None): """ _algorithm_ """ logging.info("Synchronizing priorities with ReqMgr...") self.synchronizeJobPriority() def synchronizeJobPriority(self): """ _synchronizeJobPriority_ Check WMBS and WorkQueue for active workflows and compare with the ReqMgr for priority changes. If a priority change occurs then update the job priority in the batch system and the elements in the local queue that have not been injected yet. """ # Update the priority of workflows that are not in WMBS and just in local queue priorityCache = {} workflowsToUpdate = {} workflowsToCheck = [x for x in self.workqueue.getAvailableWorkflows()] for workflow, priority in workflowsToCheck: if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority'] except Exception, ex: logging.error("Couldn't retrieve the priority of request %s" % workflow) logging.error("Error: %s" % ex) continue if priority != priorityCache[workflow]: workflowsToUpdate[workflow] = priorityCache[workflow] for workflow in workflowsToUpdate: self.workqueue.updatePriority(workflow, workflowsToUpdate[workflow]) # Check the workflows in WMBS priorityCache = {} workflowsToUpdateWMBS = {} workflowsToCheck = self.listWorkflowsDAO.execute() for workflowEntry in workflowsToCheck: workflow = workflowEntry['name'] if workflow not in priorityCache: try: priorityCache[workflow] = self.reqmgr.getRequest(workflow)['RequestPriority'] except Exception, ex: logging.error("Couldn't retrieve the priority of request %s" % workflow) logging.error("Error: %s" % ex) continue requestPriority = priorityCache[workflow] if requestPriority != workflowEntry['workflow_priority']: # Update the workqueue priority for the Available elements self.workqueue.updatePriority(workflow, priorityCache[workflow]) # Check if there are executing jobs for this particular task if self.executingJobsDAO.execute(workflow, workflowEntry['task']) > 0: self.bossAir.updateJobInformation(workflow, workflowEntry['task'], requestPriority = priorityCache[workflow], taskPriority = workflowEntry['task_priority']) workflowsToUpdateWMBS[workflow] = priorityCache[workflow]