def testSplitting(self):
""" Test that the PreDAG works when probe jobs finishes
"""
predag = PreDAG()
#Mock the createSubdagSubmission function
predag.createSubdagSubmission = lambda *args: True
self.assertEqual(predag.execute("processing", 5, 0), 0)
def testTailSplitting(self):
""" Test that the PreDAG works when there are tail jobs to create
"""
predag = PreDAG()
self.createStatusCache()
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 1)
def testTailSplittingWithAllProcessed(self):
""" Test that the PreDAG works when there are no tail jobs to create
"""
shutil.rmtree(self.missing_lumidir)
predag = PreDAG()
self.createStatusCache()
self.assertEqual(predag.execute("tail", 6, 1), 0)
self.assertEqual(os.path.exists('RunJobs1.subdag'), False)
def testTailSplittingWithAllProcessed(self):
""" Test that the PreDAG works when there are no tail jobs to create
"""
self.createFakePostjobData(missingLumis=None)
predag = PreDAG()
self.createStatusCache()
self.assertEqual(predag.execute("tail", 6, 1), 0)
self.assertEqual(os.path.exists('RunJobs1.subdag'), False)
def testTailSplittingWithAllProcessed(self):
""" Test that the PreDAG works when there are no tail jobs to create
"""
self.createFakePostjobData(missingLumis=None)
predag = PreDAG()
self.createStatusCache()
self.assertEqual(predag.execute("tail", 6, 1), 0)
self.assertEqual(os.path.exists('RunJobs1.subdag'), False)
def testTailSplittingWithFailedProbesAndAllProcessed(self):
""" Test that the PreDAG works when there are no tail jobs to create, but failed probes
"""
self.createFakePostjobData(missingLumis=None)
predag = PreDAG()
self.createStatusCache(overrides=dict((p, 'failed') for p in self.probes[1:]))
self.assertEqual(predag.execute("tail", 6, 1), 0)
self.assertEqual(os.path.exists('RunJobs1.subdag'), False)
def testTailSplittingWithFailedProbes(self):
""" Test that the PreDAG works when there are tail jobs to create, and failed probes
"""
predag = PreDAG()
self.createStatusCache(overrides=dict((p, 'failed') for p in self.probes[1:]))
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 1)
def testSplitting(self):
""" Test that the PreDAG works when probe jobs finishes
"""
predag = PreDAG()
# Mock the createSubdagSubmission function
self.assertEqual(predag.execute("processing", 5, 0), 0)
with open('RunJobs0.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 50)
def testSplittingWithHighOutput(self):
""" Test that the PreDAG works when probe jobs finishes, with more jobs created due to large output size.
"""
self.createFakePostjobData(eventSize=100000)
predag = PreDAG()
# Mock the createSubdagSubmission function
self.assertEqual(predag.execute("processing", 5, 0), 0)
with open('RunJobs0.subdag') as fd:
self.assertGreater(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 100)
def testTailSplittingWithFailedProbesAndAllProcessed(self):
""" Test that the PreDAG works when there are no tail jobs to create, but failed probes
"""
self.createFakePostjobData(missingLumis=None)
predag = PreDAG()
self.createStatusCache(overrides=dict(
(p, 'failed') for p in self.probes[1:]))
self.assertEqual(predag.execute("tail", 6, 1), 0)
self.assertEqual(os.path.exists('RunJobs1.subdag'), False)
def testTailSplitting(self):
""" Test that the PreDAG works when there are tail jobs to create
"""
predag = PreDAG()
self.createStatusCache()
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 1)
def testSplitting(self):
""" Test that the PreDAG works when probe jobs finishes
"""
predag = PreDAG()
# Mock the createSubdagSubmission function
self.assertEqual(predag.execute("processing", 5, 0), 0)
with open('RunJobs0.subdag') as fd:
self.assertEqual(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 50)
def testTailSplitting(self):
""" Test that the PreDAG works when there are tail jobs to create
"""
predag = PreDAG()
predag.createSubdagSubmission = lambda *args: True
predag.completedJobs = lambda *args, **kwargs: self.done
predag.failedJobs = []
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 1)
def testAllProcessingFailed(self):
""" Check that we don't fail if all processing jobs fail
"""
self.createFakePostjobData(jobs=self.probes, missingLumis=None)
self.createStatusCache(job_status='failed')
predag = PreDAG()
predag.createSubdag = lambda *args: True
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 17)
def testSplittingWithHighOutput(self):
""" Test that the PreDAG works when probe jobs finishes, with more jobs created due to large output size.
"""
self.createFakePostjobData(eventSize=100000)
predag = PreDAG()
# Mock the createSubdagSubmission function
self.assertEqual(predag.execute("processing", 5, 0), 0)
with open('RunJobs0.subdag') as fd:
self.assertGreater(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 100)
def testTailSplittingWithFailedProbes(self):
""" Test that the PreDAG works when there are tail jobs to create, and failed probes
"""
predag = PreDAG()
self.createStatusCache(overrides=dict(
(p, 'failed') for p in self.probes[1:]))
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 1)
def testAllProcessingFailed(self):
""" Check that we don't fail if all processing jobs fail
"""
for p in self.done:
os.unlink(os.path.join(self.throughputdir, p))
predag = PreDAG()
predag.createSubdagSubmission = lambda *args: True
predag.completedJobs = lambda *args, **kwargs: self.probes if kwargs['stage'] == 'processing' else self.done
predag.failedJobs = self.done
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(sum(1 for _ in (line for line in fd if line.startswith('JOB'))), 17)
def testAllProcessingFailed(self):
""" Check that we don't fail if all processing jobs fail
"""
self.createFakePostjobData(jobs=self.probes, missingLumis=None)
self.createStatusCache(job_status='failed')
predag = PreDAG()
predag.createSubdag = lambda *args: True
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 17)
def testAllProcessingFailed(self):
""" Check that we don't fail if all processing jobs fail
"""
for p in self.done:
os.unlink(os.path.join(self.throughputdir, p))
shutil.rmtree(self.missing_lumidir)
self.createStatusCache(job_status='failed')
predag = PreDAG()
predag.createSubdag = lambda *args: True
self.assertEqual(predag.execute("tail", 6, 1), 0)
with open('RunJobs1.subdag') as fd:
self.assertEqual(
sum(1 for _ in (line for line in fd
if line.startswith('JOB'))), 17)
def testSplitting(self):
""" Test that the PreDAG works when probe jobs finishes
"""
predag = PreDAG()
#Mock the createSubdagSubmission function
self.assertEqual(predag.execute("processing", 5, 0), 0)