def testB_Submit(self):
"""
WMComponent_t.ErrorHandler_t.ErrorHandler_t:testSubmit()
Mimics creation of component and test jobs failed in submit stage.
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'submitfailed', 'created')
idList = self.getJobs.execute(state='SubmitFailed')
self.assertEqual(len(idList), self.nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='SubmitFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='SubmitCooloff')
self.assertEqual(len(idList), self.nJobs)
return
def testB_Submit(self):
"""
WMComponent_t.ErrorHandler_t.ErrorHandler_t:testSubmit()
Mimics creation of component and test jobs failed in submit stage.
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'submitfailed', 'created')
idList = self.getJobs.execute(state='SubmitFailed')
self.assertEqual(len(idList), self.nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='SubmitFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='SubmitCooloff')
self.assertEqual(len(idList), self.nJobs)
return
def testA_Create(self):
"""
WMComponent_t.ErrorHandler_t.ErrorHandler_t:testCreate()
Mimics creation of component and test jobs failed in create stage.
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
workloadName=workloadName)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'createfailed', 'created')
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), self.nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), 0)
# These should go directly to exhausted
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
# Check that it showed up in ACDC
collection = self.dataCS.getDataCollection(workloadName)
# Now look at what's inside
self.assertTrue(len(collection['filesets']) > 0)
for fileset in collection["filesets"]:
counter = 0
for f in fileset.listFiles():
counter += 1
self.assertTrue(
f['lfn'] in ["/this/is/a/lfnA", "/this/is/a/lfnB"])
self.assertEqual(f['events'], 10)
self.assertEqual(f['size'], 1024)
self.assertEqual(f['parents'], [u'/this/is/a/parent'])
self.assertTrue(
f['runs'][0]['lumis'] in [[12312], [12314, 12315, 12316]],
"Unknown lumi %s" % f['runs'][0]['lumis'])
self.assertTrue(f['merged'], 1)
self.assertTrue(f['first_event'], 88)
self.assertEqual(counter, 20)
return
def testA_Create(self):
"""
WMComponent_t.ErrorHandler_t.ErrorHandler_t:testCreate()
Mimics creation of component and test jobs failed in create stage.
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
workloadName=workloadName)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'createfailed', 'created')
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), self.nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), 0)
# These should go directly to exhausted
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
# Check that it showed up in ACDC
collection = self.dataCS.getDataCollection(workloadName)
# Now look at what's inside
self.assertTrue(len(collection['filesets']) > 0)
for fileset in collection["filesets"]:
counter = 0
for f in fileset.listFiles():
counter += 1
self.assertTrue(f['lfn'] in ["/this/is/a/lfnA", "/this/is/a/lfnB"])
self.assertEqual(f['events'], 10)
self.assertEqual(f['size'], 1024)
self.assertEqual(f['parents'], [u'/this/is/a/parent'])
self.assertTrue(f['runs'][0]['lumis'] in [[12312], [12314, 12315, 12316]],
"Unknown lumi %s" % f['runs'][0]['lumis'])
self.assertEqual(f['merged'], 0)
self.assertEqual(f['first_event'], 88)
self.assertEqual(counter, 20)
return
def testZ_Profile(self):
"""
_testProfile_
Do a full profile of the poller
"""
nJobs = 100
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=nJobs,
workloadPath=workloadPath)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
startTime = time.time()
cProfile.runctx("testErrorHandler.algorithm()",
globals(),
locals(),
filename="profStats.stat")
stopTime = time.time()
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), nJobs)
print("Took %f seconds to run polling algo" % (stopTime - startTime))
p = pstats.Stats('profStats.stat')
p.sort_stats('cumulative')
p.print_stats(0.2)
return
def testD_Exhausted(self):
"""
_testExhausted_
Test that the system can exhaust jobs correctly
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs, retry_count=5,
workloadPath=workloadPath)
config = self.getConfig()
config.ErrorHandler.maxRetries = 1
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testSubscription = Subscription(id=1) # You should only have one
testSubscription.load()
testSubscription.loadData()
# Do we have files to start with?
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 2)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
# Did we fail the files?
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Failed")), 2)
def testD_Exhausted(self):
"""
_testExhausted_
Test that the system can exhaust jobs correctly
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs, retry_count=5,
workloadPath=workloadPath)
config = self.getConfig()
config.ErrorHandler.maxRetries = 1
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testSubscription = Subscription(id=1) # You should only have one
testSubscription.load()
testSubscription.loadData()
# Do we have files to start with?
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 2)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.algorithm(None)
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
# Did we fail the files?
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Failed")), 2)
def testZ_Profile(self):
"""
_testProfile_
Do a full profile of the poller
"""
nJobs = 100
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
testJobGroup = self.createTestJobGroup(nJobs=nJobs, workloadPath=workloadPath)
config = self.getConfig()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), nJobs)
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
startTime = time.time()
cProfile.runctx("testErrorHandler.algorithm()", globals(), locals(), filename="profStats.stat")
stopTime = time.time()
idList = self.getJobs.execute(state='CreateFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), nJobs)
print("Took %f seconds to run polling algo" % (stopTime - startTime))
p = pstats.Stats('profStats.stat')
p.sort_stats('cumulative')
p.print_stats(0.2)
return
def testE_FailJobs(self):
"""
_FailJobs_
Test our ability to fail jobs based on the information in the FWJR
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
fwjrPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t",
"fwjrs/badBackfillJobReport.pkl")
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
fwjrPath=fwjrPath)
badJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
fwjrPath=None,
fileModifier='bad')
config = self.getConfig()
config.ErrorHandler.readFWJR = True
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(badJobGroup.jobs, 'created', 'new')
changer.propagate(badJobGroup.jobs, 'executing', 'created')
changer.propagate(badJobGroup.jobs, 'complete', 'executing')
changer.propagate(badJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.exitCodesNoRetry = [8020]
testErrorHandler.algorithm(None)
# This should exhaust all jobs due to exit code
# Except those with no fwjr
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), self.nJobs)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
config.ErrorHandler.maxFailTime = -10
testErrorHandler2 = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler2.reqAuxDB = None
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler2.algorithm(None)
# This should exhaust all jobs due to timeout
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), self.nJobs)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
config.ErrorHandler.maxFailTime = 24 * 3600
config.ErrorHandler.passExitCodes = [8020]
testErrorHandler3 = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler3.reqAuxDB = None
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler3.algorithm(None)
# This should pass all jobs due to exit code
idList = self.getJobs.execute(state='Created')
self.assertEqual(len(idList), self.nJobs)
return
def testE_FailJobs(self):
"""
_FailJobs_
Test our ability to fail jobs based on the information in the FWJR
"""
workloadName = 'TestWorkload'
self.createWorkload(workloadName=workloadName)
workloadPath = os.path.join(self.testDir, 'workloadTest', workloadName,
'WMSandbox', 'WMWorkload.pkl')
fwjrPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t",
"fwjrs/badBackfillJobReport.pkl")
testJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
fwjrPath=fwjrPath)
badJobGroup = self.createTestJobGroup(nJobs=self.nJobs,
workloadPath=workloadPath,
fwjrPath=None,
fileModifier='bad')
config = self.getConfig()
config.ErrorHandler.readFWJR = True
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(badJobGroup.jobs, 'created', 'new')
changer.propagate(badJobGroup.jobs, 'executing', 'created')
changer.propagate(badJobGroup.jobs, 'complete', 'executing')
changer.propagate(badJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler.reqAuxDB = None
testErrorHandler.setup(None)
testErrorHandler.exitCodesNoRetry = [8020]
testErrorHandler.algorithm(None)
# This should exhaust all jobs due to exit code
# Except those with no fwjr
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), self.nJobs)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
config.ErrorHandler.maxFailTime = -10
testErrorHandler2 = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler2.reqAuxDB = None
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler2.algorithm(None)
# This should exhaust all jobs due to timeout
idList = self.getJobs.execute(state='JobFailed')
self.assertEqual(len(idList), 0)
idList = self.getJobs.execute(state='JobCooloff')
self.assertEqual(len(idList), self.nJobs)
idList = self.getJobs.execute(state='Exhausted')
self.assertEqual(len(idList), self.nJobs)
config.ErrorHandler.maxFailTime = 24 * 3600
config.ErrorHandler.passExitCodes = [8020]
testErrorHandler3 = ErrorHandlerPoller(config)
# set reqAuxDB None for the test,
testErrorHandler3.reqAuxDB = None
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
testErrorHandler3.algorithm(None)
# This should pass all jobs due to exit code
idList = self.getJobs.execute(state='Created')
self.assertEqual(len(idList), self.nJobs)
return
