def testCaching(self):
"""
_testCaching_
Verify that JobSubmitter caching works.
"""
config = self.createConfig()
mySubmitterPoller = JobSubmitterPoller(config)
mySubmitterPoller.getThresholds()
mySubmitterPoller.refreshCache()
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 0,
"Error: The job cache should be empty.")
self.injectJobs()
mySubmitterPoller.refreshCache()
# Verify the cache is full
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 20,
"Error: The job cache should contain 20 jobs. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
killWorkflow("wf001", jobCouchConfig = config)
mySubmitterPoller.refreshCache()
# Verify that the workflow is gone from the cache
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 10,
"Error: The job cache should contain 10 jobs. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
killWorkflow("wf002", jobCouchConfig = config)
mySubmitterPoller.refreshCache()
# Verify that the workflow is gone from the cache
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 0,
"Error: The job cache should be empty. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
return
def testCaching(self):
"""
_testCaching_
Verify that JobSubmitter caching works.
"""
config = self.createConfig()
mySubmitterPoller = JobSubmitterPoller(config)
mySubmitterPoller.getThresholds()
mySubmitterPoller.refreshCache()
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 0,
"Error: The job cache should be empty.")
self.injectJobs()
mySubmitterPoller.refreshCache()
# Verify the cache is full
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 20,
"Error: The job cache should contain 20 jobs. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
killWorkflow("wf001", jobCouchConfig = config)
mySubmitterPoller.refreshCache()
# Verify that the workflow is gone from the cache
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 10,
"Error: The job cache should contain 10 jobs. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
killWorkflow("wf002", jobCouchConfig = config)
mySubmitterPoller.refreshCache()
# Verify that the workflow is gone from the cache
self.assertEqual(len(mySubmitterPoller.cachedJobIDs), 0,
"Error: The job cache should be empty. Contains: %i" % len(mySubmitterPoller.cachedJobIDs))
return
def cancelWork(self,
elementIDs=None,
SubscriptionId=None,
WorkflowName=None,
elements=None):
"""Cancel work - delete in wmbs, delete from workqueue db, set canceled in inbox
Elements may be directly provided or determined from series of filter arguments
"""
if not elements:
args = {}
if SubscriptionId:
args['SubscriptionId'] = SubscriptionId
if WorkflowName:
args['RequestName'] = WorkflowName
elements = self.backend.getElements(elementIDs=elementIDs, **args)
# take wf from args in case no elements exist for workflow (i.e. work was negotiating)
requestNames = set([x['RequestName'] for x in elements]) | set(
[wf for wf in [WorkflowName] if wf])
if not requestNames:
return []
inbox_elements = []
for wf in requestNames:
inbox_elements.extend(
self.backend.getInboxElements(WorkflowName=wf))
# if local queue, kill jobs, update parent to Canceled and delete elements
if self.params['LocalQueueFlag']:
# if we can talk to wmbs kill the jobs
if self.params['PopulateFilesets']:
self.logger.info("""Canceling work for workflow(s): %s""" %
(requestNames))
from WMCore.WorkQueue.WMBSHelper import killWorkflow
for workflow in requestNames:
try:
myThread = threading.currentThread()
myThread.dbi = self.conn.dbi
myThread.logger = self.logger
killWorkflow(workflow, self.params["JobDumpConfig"],
self.params["BossAirConfig"])
except Exception, ex:
self.logger.error(
'Aborting %s wmbs subscription failed: %s' %
(workflow, str(ex)))
# Don't update as fails sometimes due to conflicts (#3856)
[
x.load().__setitem__('Status', 'Canceled')
for x in inbox_elements if x['Status'] != 'Canceled'
]
updated_inbox_ids = [
x.id for x in self.backend.saveElements(*inbox_elements)
]
# delete elements - no longer need them
self.backend.deleteElements(*[
x for x in elements if x['ParentQueueId'] in updated_inbox_ids
])
def cancelWork(self, elementIDs = None, SubscriptionId = None, WorkflowName = None, elements = None):
"""Cancel work - delete in wmbs, delete from workqueue db, set canceled in inbox
Elements may be directly provided or determined from series of filter arguments
"""
if not elements:
args = {}
if SubscriptionId:
args['SubscriptionId'] = SubscriptionId
if WorkflowName:
args['RequestName'] = WorkflowName
elements = self.backend.getElements(elementIDs = elementIDs, **args)
# only cancel in global if work has not been passed to a child queue
if not self.params['LocalQueueFlag']:
elements = [x for x in elements if not x['ChildQueueUrl']]
requestNames = set([x['RequestName'] for x in elements])
if not requestNames:
return []
# if we can talk to wmbs kill the jobs
if self.params['PopulateFilesets']:
from WMCore.WorkQueue.WMBSHelper import killWorkflow
self.logger.debug("""Canceling work in wmbs, workflows: %s""" % (requestNames))
for workflow in requestNames:
try:
myThread = threading.currentThread()
myThread.dbi = self.conn.dbi
myThread.logger = self.logger
killWorkflow(workflow, self.params["JobDumpConfig"],
self.params["BossAirConfig"])
except RuntimeError:
#TODO: Check this logic and improve if possible
if SubscriptionId:
self.logger.info("""Cancel update: Only some subscription's canceled.
This might be due to a child subscriptions: %s"""
% elementIDs)
# update parent elements to canceled
for wf in requestNames:
inbox_elements = self.backend.getInboxElements(WorkflowName = wf, returnIdOnly = True)
if not inbox_elements:
raise RuntimeError, "Cant find parent for %s" % wf
self.backend.updateInboxElements(*inbox_elements, Status = 'Canceled')
# delete elements - no longer need them
self.backend.deleteElements(*elements)
return [x.id for x in elements]
def killWorkflowAgent(WorkflowName):
"""
Cancel work for a given workflow - delete in wmbs, delete from workqueue db, set canceled in inbox
"""
# get configuration file path
if not os.environ.has_key("WMAGENT_CONFIG"):
os.environ[
'WMAGENT_CONFIG'] = '/data/srv/wmagent/current/config/wmagent/config.py'
# load config
wmConfig = loadConfigurationFile(os.environ['WMAGENT_CONFIG'])
wqManager = wmConfig.section_('WorkQueueManager')
couchUrl = wqManager.couchurl
dbname = wqManager.dbname
inboxDatabase = wqManager.inboxDatabase
parentQueueCouchUrl = wqManager.queueParams['ParentQueueCouchUrl']
# Creates backend
backend = WorkQueueBackend(couchUrl, dbname, inboxDatabase,
parentQueueCouchUrl)
args = {}
args['RequestName'] = WorkflowName
elements = backend.getElements(**args)
# take wf from args in case no elements exist for workflow (i.e. work was negotiating)
requestNames = set([x['RequestName'] for x in elements]) | set(
[wf for wf in [WorkflowName]])
if not requestNames:
print 'Workflow is not at the backend'
inbox_elements = []
for wf in requestNames:
inbox_elements.extend(backend.getInboxElements(WorkflowName=wf))
print "Canceling work for workflow: %s" % (requestNames)
for workflow in requestNames:
try:
connectToDB()
jobDumpConfig = wmConfig
bossAirConfig = wmConfig
killWorkflow(workflow, jobDumpConfig, bossAirConfig)
except Exception, ex:
print 'Aborting %s wmbs subscription failed: %s' % (workflow,
str(ex))
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
baAPI = BossAirAPI(config=self.config, insertStates=True)
# Create nine jobs
self.setupForKillTest(baAPI=baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", self.config, self.config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
return
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
baAPI = BossAirAPI(config=self.config, insertStates=True)
# Create nine jobs
self.setupForKillTest(baAPI=baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", self.config, self.config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
return
def killWorkflowAgent(WorkflowName):
"""
Cancel work for a given workflow - delete in wmbs, delete from workqueue db, set canceled in inbox
"""
# get configuration file path
if not os.environ.has_key("WMAGENT_CONFIG"):
os.environ["WMAGENT_CONFIG"] = "/data/srv/wmagent/current/config/wmagent/config.py"
# load config
wmConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
wqManager = wmConfig.section_("WorkQueueManager")
couchUrl = wqManager.couchurl
dbname = wqManager.dbname
inboxDatabase = wqManager.inboxDatabase
parentQueueCouchUrl = wqManager.queueParams["ParentQueueCouchUrl"]
# Creates backend
backend = WorkQueueBackend(couchUrl, dbname, inboxDatabase, parentQueueCouchUrl)
args = {}
args["RequestName"] = WorkflowName
elements = backend.getElements(**args)
# take wf from args in case no elements exist for workflow (i.e. work was negotiating)
requestNames = set([x["RequestName"] for x in elements]) | set([wf for wf in [WorkflowName]])
if not requestNames:
print "Workflow is not at the backend"
inbox_elements = []
for wf in requestNames:
inbox_elements.extend(backend.getInboxElements(WorkflowName=wf))
print "Canceling work for workflow: %s" % (requestNames)
for workflow in requestNames:
try:
connectToDB()
jobDumpConfig = wmConfig
bossAirConfig = wmConfig
killWorkflow(workflow, jobDumpConfig, bossAirConfig)
except Exception, ex:
print "Aborting %s wmbs subscription failed: %s" % (workflow, str(ex))
def cancelWork(self, elementIDs = None, SubscriptionId = None, WorkflowName = None, elements = None):
"""Cancel work - delete in wmbs, delete from workqueue db, set canceled in inbox
Elements may be directly provided or determined from series of filter arguments
"""
if not elements:
args = {}
if SubscriptionId:
args['SubscriptionId'] = SubscriptionId
if WorkflowName:
args['RequestName'] = WorkflowName
elements = self.backend.getElements(elementIDs = elementIDs, **args)
# take wf from args in case no elements exist for workflow (i.e. work was negotiating)
requestNames = set([x['RequestName'] for x in elements]) | set([wf for wf in [WorkflowName] if wf])
if not requestNames:
return []
inbox_elements = []
for wf in requestNames:
inbox_elements.extend(self.backend.getInboxElements(WorkflowName = wf))
# if local queue, kill jobs, update parent to Canceled and delete elements
if self.params['LocalQueueFlag']:
# if we can talk to wmbs kill the jobs
if self.params['PopulateFilesets']:
self.logger.info("""Canceling work for workflow(s): %s""" % (requestNames))
from WMCore.WorkQueue.WMBSHelper import killWorkflow
for workflow in requestNames:
try:
myThread = threading.currentThread()
myThread.dbi = self.conn.dbi
myThread.logger = self.logger
killWorkflow(workflow, self.params["JobDumpConfig"],
self.params["BossAirConfig"])
except Exception, ex:
self.logger.error('Aborting %s wmbs subscription failed: %s' % (workflow, str(ex)))
# Don't update as fails sometimes due to conflicts (#3856)
[x.load().__setitem__('Status', 'Canceled') for x in inbox_elements if x['Status'] != 'Canceled']
updated_inbox_ids = [x.id for x in self.backend.saveElements(*inbox_elements)]
# delete elements - no longer need them
self.backend.deleteElements(*[x for x in elements if x['ParentQueueId'] in updated_inbox_ids])
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
configFile = EmulatorSetup.setupWMAgentConfig()
config = loadConfigurationFile(configFile)
baAPI = BossAirAPI(config=config)
# Create nine jobs
self.setupForKillTest(baAPI=baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", config, config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
EmulatorSetup.deleteConfig(configFile)
return
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
configFile = EmulatorSetup.setupWMAgentConfig()
config = loadConfigurationFile(configFile)
baAPI = BossAirAPI(config = config)
# Create nine jobs
self.setupForKillTest(baAPI = baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", config, config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
EmulatorSetup.deleteConfig(configFile)
return