class SiblingProcessingBasedTest(unittest.TestCase):
"""
_SiblingProcessingBasedTest_
Test SiblingProcessing job splitting.
"""
def setUp(self):
"""
_setUp_
Setup the database connections and schema.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = daofactory(classname = "Locations.New")
locationAction.execute("T2_CH_CERN", pnn = "T2_CH_CERN")
locationAction.execute("T1_US_FNAL", pnn = "T1_US_FNAL_Disk")
self.testFilesetA = Fileset(name = "FilesetA")
self.testFilesetA.create()
self.testFilesetB = Fileset(name = "FilesetB")
self.testFilesetB.create()
self.testFileA = File("testFileA", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileA.create()
self.testFileB = File("testFileB", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileB.create()
self.testFileC = File("testFileC", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileC.create()
self.testFilesetA.addFile(self.testFileA)
self.testFilesetA.addFile(self.testFileB)
self.testFilesetA.addFile(self.testFileC)
self.testFilesetA.commit()
self.testFileD = File("testFileD", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileD.create()
self.testFileE = File("testFileE", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileE.create()
self.testFileF = File("testFileF", size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
self.testFileF.create()
self.testFilesetB.addFile(self.testFileD)
self.testFilesetB.addFile(self.testFileE)
self.testFilesetB.addFile(self.testFileF)
self.testFilesetB.commit()
testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve",
name = "wfA", task = "Test")
testWorkflowA.create()
testWorkflowB = Workflow(spec = "specB.xml", owner = "Steve",
name = "wfB", task = "Test")
testWorkflowB.create()
testWorkflowC = Workflow(spec = "specC.xml", owner = "Steve",
name = "wfC", task = "Test")
testWorkflowC.create()
testWorkflowD = Workflow(spec = "specD.xml", owner = "Steve",
name = "wfD", task = "Test")
testWorkflowD.create()
self.testSubscriptionA = Subscription(fileset = self.testFilesetA,
workflow = testWorkflowA,
split_algo = "FileBased",
type = "Processing")
self.testSubscriptionA.create()
self.testSubscriptionB = Subscription(fileset = self.testFilesetB,
workflow = testWorkflowB,
split_algo = "FileBased",
type = "Processing")
self.testSubscriptionB.create()
self.testSubscriptionC = Subscription(fileset = self.testFilesetB,
workflow = testWorkflowC,
split_algo = "FileBased",
type = "Processing")
self.testSubscriptionC.create()
self.testSubscriptionD = Subscription(fileset = self.testFilesetB,
workflow = testWorkflowD,
split_algo = "FileBased",
type = "Processing")
self.testSubscriptionD.create()
deleteWorkflow = Workflow(spec = "specE.xml", owner = "Steve",
name = "wfE", task = "Test")
deleteWorkflow.create()
self.deleteSubscriptionA = Subscription(fileset = self.testFilesetA,
workflow = deleteWorkflow,
split_algo = "SiblingProcessingBased",
type = "Cleanup")
self.deleteSubscriptionA.create()
self.deleteSubscriptionB = Subscription(fileset = self.testFilesetB,
workflow = deleteWorkflow,
split_algo = "SiblingProcessingBased",
type = "Cleanup")
self.deleteSubscriptionB.create()
return
def tearDown(self):
"""
_tearDown_
Clear out WMBS.
"""
self.testInit.clearDatabase()
return
def testSiblingProcessing(self):
"""
_testSiblingProcessing_
Verify that the sibling processing split works correctly dealing with
failed files and acquiring files correctly.
"""
splitter = SplitterFactory()
deleteFactoryA = splitter(package = "WMCore.WMBS",
subscription = self.deleteSubscriptionA)
deleteFactoryB = splitter(package = "WMCore.WMBS",
subscription = self.deleteSubscriptionB)
result = deleteFactoryA()
assert len(result) == 0, \
"Error: No jobs should be returned."
result = deleteFactoryB()
assert len(result) == 0, \
"Error: No jobs should be returned."
self.testSubscriptionA.completeFiles(self.testFileA)
result = deleteFactoryA(files_per_job = 1)
assert len(result) == 1, \
"Error: Only one jobgroup should be returned."
assert len(result[0].jobs) == 1, \
"Error: There should only be one job in the jobgroup."
assert result[0].jobs[0]["possiblePSN"] == set(["T2_CH_CERN"]), \
"Error: possiblePSN is wrong."
assert len(result[0].jobs[0]["input_files"]) == 1, \
"Error: Job should only have one input file."
assert result[0].jobs[0]["input_files"][0]["lfn"] == "testFileA", \
"Error: Input file for job is wrong."
result = deleteFactoryB(files_per_job = 1)
assert len(result) == 0, \
"Error: Second subscription should have no jobs."
result = deleteFactoryA(files_per_job = 1)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testSubscriptionB.completeFiles(self.testFileD)
self.testSubscriptionC.failFiles(self.testFileD)
result = deleteFactoryA(files_per_job = 1)
assert len(result) == 0, \
"Error: No jobs should have been created."
result = deleteFactoryB(files_per_job = 1)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testSubscriptionD.failFiles(self.testFileD)
result = deleteFactoryA(files_per_job = 1)
assert len(result) == 0, \
"Error: No jobs should have been created."
result = deleteFactoryB(files_per_job = 1)
assert len(result) == 0, \
"Error: No job groups should have been created."
self.testSubscriptionB.completeFiles([self.testFileE, self.testFileF])
self.testSubscriptionC.completeFiles([self.testFileE, self.testFileF])
self.testSubscriptionD.completeFiles([self.testFileE, self.testFileF])
result = deleteFactoryB(files_per_job = 10)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testFilesetB.markOpen(False)
result = deleteFactoryB(files_per_job = 10)
assert len(result) == 1, \
"Error: One jobgroup should have been returned."
assert len(result[0].jobs) == 1, \
"Error: There should only be one job in the jobgroup."
assert len(result[0].jobs[0]["input_files"]) == 2, \
"Error: Job should only have one input file."
lfns = [result[0].jobs[0]["input_files"][0]["lfn"], result[0].jobs[0]["input_files"][1]["lfn"]]
assert "testFileE" in lfns, \
"Error: TestFileE missing from job input."
assert "testFileF" in lfns, \
"Error: TestFileF missing from job input."
self.assertEqual(len(self.deleteSubscriptionB.availableFiles()), 0,
"Error: There should be no available files.")
completeFiles = self.deleteSubscriptionB.filesOfStatus("Completed")
self.assertEqual(len(completeFiles), 1,
"Error: There should only be one complete file.")
self.assertEqual(list(completeFiles)[0]["lfn"], "testFileD",
"Error: Test file D should be complete.")
return
def testMultipleLocations(self):
"""
_testMultipleLocations_
Verify that the sibling processing based algorithm doesn't create jobs
that run over files at multiple sites.
"""
testFile1 = File("testFile1", size = 1000, events = 100,
locations = set(["T1_US_FNAL_Disk"]))
testFile1.create()
testFile2 = File("testFile2", size = 1000, events = 100,
locations = set(["T1_US_FNAL_Disk"]))
testFile2.create()
testFile3 = File("testFile3", size = 1000, events = 100,
locations = set(["T1_US_FNAL_Disk"]))
testFile3.create()
self.testFilesetA.addFile(testFile1)
self.testFilesetA.addFile(testFile2)
self.testFilesetA.addFile(testFile3)
self.testFilesetA.commit()
self.testFilesetA.markOpen(False)
self.testSubscriptionA.completeFiles([testFile1, testFile2, testFile3])
self.testSubscriptionA.completeFiles([self.testFileA, self.testFileB, self.testFileC])
splitter = SplitterFactory()
deleteFactoryA = splitter(package = "WMCore.WMBS",
subscription = self.deleteSubscriptionA)
result = deleteFactoryA(files_per_job = 50)
assert len(result) == 2, \
"Error: Wrong number of jobgroups returned."
goldenFilesA = ["testFileA", "testFileB", "testFileC"]
goldenFilesB = ["testFile1", "testFile2", "testFile3"]
for jobGroup in result:
assert len(jobGroup.jobs) == 1, \
"Error: Wrong number of jobs in jobgroup."
assert len(jobGroup.jobs[0]["input_files"]) == 3, \
"Error: Wrong number of input files in job."
jobSite = jobGroup.jobs[0]["possiblePSN"]
assert (jobSite == set(["T2_CH_CERN"])
or jobSite == set(["T1_US_FNAL"])), \
"Error: Wrong site for job."
if jobSite == set(["T2_CH_CERN"]):
goldenFiles = goldenFilesA
else:
goldenFiles = goldenFilesB
for jobFile in jobGroup.jobs[0]["input_files"]:
goldenFiles.remove(jobFile["lfn"])
assert len(goldenFiles) == 0, \
"Error: Files are missing."
return
def testLargeNumberOfFiles(self):
"""
_testLargeNumberOfFiles_
Setup a subscription with 500 files and verify that the splitting algo
works correctly.
"""
testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve",
name = "wfA", task = "Test")
testWorkflowA.create()
testWorkflowB = Workflow(spec = "specB.xml", owner = "Steve",
name = "wfB", task = "Test")
testWorkflowB.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i), size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(fileset = testFileset,
workflow = testWorkflowA,
split_algo = "FileBased",
type = "Processing")
testSubscriptionA.create()
testSubscriptionB = Subscription(fileset = testFileset,
workflow = testWorkflowB,
split_algo = "SiblingProcessingBased",
type = "Processing")
testSubscriptionB.create()
testSubscriptionA.completeFiles(allFiles)
splitter = SplitterFactory()
deleteFactoryA = splitter(package = "WMCore.WMBS",
subscription = testSubscriptionB)
result = deleteFactoryA(files_per_job = 50)
self.assertEqual(len(result), 1,
"Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10,
"Error: Wrong number of jobs returned.")
return
def testFilesWithoutOtherSubscriptions(self):
"""
_testFilesWithoutOtherSubscriptions_
Test the case where files only in the delete subscription
can happen if cleanup of the other subscriptions is fast
"""
testWorkflowA = Workflow(spec = "specA.xml", owner = "Steve",
name = "wfA", task = "Test")
testWorkflowA.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i), size = 1000, events = 100,
locations = set(["T2_CH_CERN"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(fileset = testFileset,
workflow = testWorkflowA,
split_algo = "SiblingProcessingBased",
type = "Processing")
testSubscriptionA.create()
splitter = SplitterFactory()
deleteFactoryA = splitter(package = "WMCore.WMBS",
subscription = testSubscriptionA)
result = deleteFactoryA(files_per_job = 50)
self.assertEqual(len(result), 1,
"Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10,
"Error: Wrong number of jobs returned.")
return
class FixedDelayTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="Locations.New")
locationAction.execute(siteName="site1", pnn="T2_CH_CERN")
self.multipleFileFileset = Fileset(name="TestFileset1")
self.multipleFileFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(i, *[45 + i]))
newFile.create()
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name="TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleFileLumiset = Fileset(name="TestFileset3")
self.multipleFileLumiset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45 + i / 3]))
newFile.create()
self.multipleFileLumiset.addFile(newFile)
self.multipleFileLumiset.commit()
self.singleLumiFileset = Fileset(name="TestFileset4")
self.singleLumiFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleLumiFileset.addFile(newFile)
self.singleLumiFileset.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="mnorman",
name="wf001",
task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(
fileset=self.multipleFileFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.singleFileSubscription = Subscription(
fileset=self.singleFileFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.multipleLumiSubscription = Subscription(
fileset=self.multipleFileLumiset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.singleLumiSubscription = Subscription(
fileset=self.singleLumiFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription.create()
self.multipleLumiSubscription.create()
self.singleLumiSubscription.create()
return
def tearDown(self):
"""
_tearDown_
Nothing to do...
"""
self.testInit.clearDatabase()
return
def testNone(self):
"""
_testNone_
Since the subscriptions are open, we shouldn't get any jobs back
"""
splitter = SplitterFactory()
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
return
def testClosed(self):
"""
_testClosed_
Since the subscriptions are closed and none of the files have been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.singleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
self.multipleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs), 1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
self.multipleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs), 1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
#self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
def testAllAcquired(self):
"""
_testAllAcquired_
should all return no job groups
"""
splitter = SplitterFactory()
self.singleFileSubscription.acquireFiles(
self.singleFileSubscription.availableFiles())
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.acquireFiles(
self.multipleFileSubscription.availableFiles())
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleLumiSubscription.acquireFiles(
self.multipleLumiSubscription.availableFiles())
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.acquireFiles(
self.singleLumiSubscription.availableFiles())
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
def testClosedSomeAcquired(self):
"""
_testClosedSomeAcquired_
since the subscriptions are closed and none of the files ahve been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.multipleFileSubscription.getFileset().markOpen(False)
self.singleFileSubscription.acquireFiles(
[self.singleFileSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.getFileset().markOpen(False)
self.multipleFileSubscription.acquireFiles(
[self.multipleFileSubscription.availableFiles().pop()])
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.multipleLumiSubscription.getFileset().markOpen(False)
self.multipleLumiSubscription.acquireFiles(
[self.multipleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.singleLumiSubscription.getFileset().markOpen(False)
self.singleLumiSubscription.acquireFiles(
[self.singleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.assertEquals(len(myfiles), 9)
class SiblingProcessingBasedTest(unittest.TestCase):
"""
_SiblingProcessingBasedTest_
Test SiblingProcessing job splitting.
"""
def setUp(self):
"""
_setUp_
Setup the database connections and schema.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="Locations.New")
locationAction.execute("T2_CH_CERN", pnn="T2_CH_CERN")
locationAction.execute("T1_US_FNAL", pnn="T1_US_FNAL_Disk")
self.testFilesetA = Fileset(name="FilesetA")
self.testFilesetA.create()
self.testFilesetB = Fileset(name="FilesetB")
self.testFilesetB.create()
self.testFileA = File("testFileA",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileA.create()
self.testFileB = File("testFileB",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileB.create()
self.testFileC = File("testFileC",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileC.create()
self.testFilesetA.addFile(self.testFileA)
self.testFilesetA.addFile(self.testFileB)
self.testFilesetA.addFile(self.testFileC)
self.testFilesetA.commit()
self.testFileD = File("testFileD",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileD.create()
self.testFileE = File("testFileE",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileE.create()
self.testFileF = File("testFileF",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
self.testFileF.create()
self.testFilesetB.addFile(self.testFileD)
self.testFilesetB.addFile(self.testFileE)
self.testFilesetB.addFile(self.testFileF)
self.testFilesetB.commit()
testWorkflowA = Workflow(spec="specA.xml",
owner="Steve",
name="wfA",
task="Test")
testWorkflowA.create()
testWorkflowB = Workflow(spec="specB.xml",
owner="Steve",
name="wfB",
task="Test")
testWorkflowB.create()
testWorkflowC = Workflow(spec="specC.xml",
owner="Steve",
name="wfC",
task="Test")
testWorkflowC.create()
testWorkflowD = Workflow(spec="specD.xml",
owner="Steve",
name="wfD",
task="Test")
testWorkflowD.create()
self.testSubscriptionA = Subscription(fileset=self.testFilesetA,
workflow=testWorkflowA,
split_algo="FileBased",
type="Processing")
self.testSubscriptionA.create()
self.testSubscriptionB = Subscription(fileset=self.testFilesetB,
workflow=testWorkflowB,
split_algo="FileBased",
type="Processing")
self.testSubscriptionB.create()
self.testSubscriptionC = Subscription(fileset=self.testFilesetB,
workflow=testWorkflowC,
split_algo="FileBased",
type="Processing")
self.testSubscriptionC.create()
self.testSubscriptionD = Subscription(fileset=self.testFilesetB,
workflow=testWorkflowD,
split_algo="FileBased",
type="Processing")
self.testSubscriptionD.create()
deleteWorkflow = Workflow(spec="specE.xml",
owner="Steve",
name="wfE",
task="Test")
deleteWorkflow.create()
self.deleteSubscriptionA = Subscription(
fileset=self.testFilesetA,
workflow=deleteWorkflow,
split_algo="SiblingProcessingBased",
type="Cleanup")
self.deleteSubscriptionA.create()
self.deleteSubscriptionB = Subscription(
fileset=self.testFilesetB,
workflow=deleteWorkflow,
split_algo="SiblingProcessingBased",
type="Cleanup")
self.deleteSubscriptionB.create()
return
def tearDown(self):
"""
_tearDown_
Clear out WMBS.
"""
self.testInit.clearDatabase()
return
def testSiblingProcessing(self):
"""
_testSiblingProcessing_
Verify that the sibling processing split works correctly dealing with
failed files and acquiring files correctly.
"""
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS",
subscription=self.deleteSubscriptionA)
deleteFactoryB = splitter(package="WMCore.WMBS",
subscription=self.deleteSubscriptionB)
result = deleteFactoryA()
assert len(result) == 0, \
"Error: No jobs should be returned."
result = deleteFactoryB()
assert len(result) == 0, \
"Error: No jobs should be returned."
self.testSubscriptionA.completeFiles(self.testFileA)
result = deleteFactoryA(files_per_job=1)
assert len(result) == 1, \
"Error: Only one jobgroup should be returned."
assert len(result[0].jobs) == 1, \
"Error: There should only be one job in the jobgroup."
assert result[0].jobs[0]["possiblePSN"] == set(["T2_CH_CERN"]), \
"Error: possiblePSN is wrong."
assert len(result[0].jobs[0]["input_files"]) == 1, \
"Error: Job should only have one input file."
assert result[0].jobs[0]["input_files"][0]["lfn"] == "testFileA", \
"Error: Input file for job is wrong."
result = deleteFactoryB(files_per_job=1)
assert len(result) == 0, \
"Error: Second subscription should have no jobs."
result = deleteFactoryA(files_per_job=1)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testSubscriptionB.completeFiles(self.testFileD)
self.testSubscriptionC.failFiles(self.testFileD)
result = deleteFactoryA(files_per_job=1)
assert len(result) == 0, \
"Error: No jobs should have been created."
result = deleteFactoryB(files_per_job=1)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testSubscriptionD.failFiles(self.testFileD)
result = deleteFactoryA(files_per_job=1)
assert len(result) == 0, \
"Error: No jobs should have been created."
result = deleteFactoryB(files_per_job=1)
assert len(result) == 0, \
"Error: No job groups should have been created."
self.testSubscriptionB.completeFiles([self.testFileE, self.testFileF])
self.testSubscriptionC.completeFiles([self.testFileE, self.testFileF])
self.testSubscriptionD.completeFiles([self.testFileE, self.testFileF])
result = deleteFactoryB(files_per_job=10)
assert len(result) == 0, \
"Error: No jobs should have been created."
self.testFilesetB.markOpen(False)
result = deleteFactoryB(files_per_job=10)
assert len(result) == 1, \
"Error: One jobgroup should have been returned."
assert len(result[0].jobs) == 1, \
"Error: There should only be one job in the jobgroup."
assert len(result[0].jobs[0]["input_files"]) == 2, \
"Error: Job should only have one input file."
lfns = [
result[0].jobs[0]["input_files"][0]["lfn"],
result[0].jobs[0]["input_files"][1]["lfn"]
]
assert "testFileE" in lfns, \
"Error: TestFileE missing from job input."
assert "testFileF" in lfns, \
"Error: TestFileF missing from job input."
self.assertEqual(len(self.deleteSubscriptionB.availableFiles()), 0,
"Error: There should be no available files.")
completeFiles = self.deleteSubscriptionB.filesOfStatus("Completed")
self.assertEqual(len(completeFiles), 1,
"Error: There should only be one complete file.")
self.assertEqual(
list(completeFiles)[0]["lfn"], "testFileD",
"Error: Test file D should be complete.")
return
def testMultipleLocations(self):
"""
_testMultipleLocations_
Verify that the sibling processing based algorithm doesn't create jobs
that run over files at multiple sites.
"""
testFile1 = File("testFile1",
size=1000,
events=100,
locations=set(["T1_US_FNAL_Disk"]))
testFile1.create()
testFile2 = File("testFile2",
size=1000,
events=100,
locations=set(["T1_US_FNAL_Disk"]))
testFile2.create()
testFile3 = File("testFile3",
size=1000,
events=100,
locations=set(["T1_US_FNAL_Disk"]))
testFile3.create()
self.testFilesetA.addFile(testFile1)
self.testFilesetA.addFile(testFile2)
self.testFilesetA.addFile(testFile3)
self.testFilesetA.commit()
self.testFilesetA.markOpen(False)
self.testSubscriptionA.completeFiles([testFile1, testFile2, testFile3])
self.testSubscriptionA.completeFiles(
[self.testFileA, self.testFileB, self.testFileC])
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS",
subscription=self.deleteSubscriptionA)
result = deleteFactoryA(files_per_job=50)
assert len(result) == 2, \
"Error: Wrong number of jobgroups returned."
goldenFilesA = ["testFileA", "testFileB", "testFileC"]
goldenFilesB = ["testFile1", "testFile2", "testFile3"]
for jobGroup in result:
assert len(jobGroup.jobs) == 1, \
"Error: Wrong number of jobs in jobgroup."
assert len(jobGroup.jobs[0]["input_files"]) == 3, \
"Error: Wrong number of input files in job."
jobSite = jobGroup.jobs[0]["possiblePSN"]
assert (jobSite == set(["T2_CH_CERN"])
or jobSite == set(["T1_US_FNAL"])), \
"Error: Wrong site for job."
if jobSite == set(["T2_CH_CERN"]):
goldenFiles = goldenFilesA
else:
goldenFiles = goldenFilesB
for jobFile in jobGroup.jobs[0]["input_files"]:
goldenFiles.remove(jobFile["lfn"])
assert len(goldenFiles) == 0, \
"Error: Files are missing."
return
def testLargeNumberOfFiles(self):
"""
_testLargeNumberOfFiles_
Setup a subscription with 500 files and verify that the splitting algo
works correctly.
"""
testWorkflowA = Workflow(spec="specA.xml",
owner="Steve",
name="wfA",
task="Test")
testWorkflowA.create()
testWorkflowB = Workflow(spec="specB.xml",
owner="Steve",
name="wfB",
task="Test")
testWorkflowB.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(fileset=testFileset,
workflow=testWorkflowA,
split_algo="FileBased",
type="Processing")
testSubscriptionA.create()
testSubscriptionB = Subscription(fileset=testFileset,
workflow=testWorkflowB,
split_algo="SiblingProcessingBased",
type="Processing")
testSubscriptionB.create()
testSubscriptionA.completeFiles(allFiles)
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS",
subscription=testSubscriptionB)
result = deleteFactoryA(files_per_job=50)
self.assertEqual(len(result), 1,
"Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10,
"Error: Wrong number of jobs returned.")
return
def testFilesWithoutOtherSubscriptions(self):
"""
_testFilesWithoutOtherSubscriptions_
Test the case where files only in the delete subscription
can happen if cleanup of the other subscriptions is fast
"""
testWorkflowA = Workflow(spec="specA.xml",
owner="Steve",
name="wfA",
task="Test")
testWorkflowA.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(fileset=testFileset,
workflow=testWorkflowA,
split_algo="SiblingProcessingBased",
type="Processing")
testSubscriptionA.create()
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS",
subscription=testSubscriptionA)
result = deleteFactoryA(files_per_job=50)
self.assertEqual(len(result), 1,
"Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10,
"Error: Wrong number of jobs returned.")
return
class FixedDelayTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", pnn = "T2_CH_CERN")
self.multipleFileFileset = Fileset(name = "TestFileset1")
self.multipleFileFileset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["T2_CH_CERN"]))
newFile.addRun(Run(i, *[45+i]))
newFile.create()
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name = "TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name", size = 1000, events = 100, locations = set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleFileLumiset = Fileset(name = "TestFileset3")
self.multipleFileLumiset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45+i/3]))
newFile.create()
self.multipleFileLumiset.addFile(newFile)
self.multipleFileLumiset.commit()
self.singleLumiFileset = Fileset(name = "TestFileset4")
self.singleLumiFileset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleLumiFileset.addFile(newFile)
self.singleLumiFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "mnorman", name = "wf001", task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(fileset = self.multipleFileFileset,
workflow = testWorkflow,
split_algo = "FixedDelay",
type = "Processing")
self.singleFileSubscription = Subscription(fileset = self.singleFileFileset,
workflow = testWorkflow,
split_algo = "FixedDelay",
type = "Processing")
self.multipleLumiSubscription = Subscription(fileset = self.multipleFileLumiset,
workflow = testWorkflow,
split_algo = "FixedDelay",
type = "Processing")
self.singleLumiSubscription = Subscription(fileset = self.singleLumiFileset,
workflow = testWorkflow,
split_algo = "FixedDelay",
type = "Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription.create()
self.multipleLumiSubscription.create()
self.singleLumiSubscription.create()
return
def tearDown(self):
"""
_tearDown_
Nothing to do...
"""
self.testInit.clearDatabase()
return
def testNone(self):
"""
_testNone_
Since the subscriptions are open, we shouldn't get any jobs back
"""
splitter = SplitterFactory()
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time = int(time.time())*2)
self.assertEqual(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time = int(time.time())*2)
self.assertEqual(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time = int(time.time())*2)
self.assertEqual(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time = int(time.time())*2)
self.assertEqual(jobGroups, [], "Should have returned a null set")
return
def testClosed(self):
"""
_testClosed_
Since the subscriptions are closed and none of the files have been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.singleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
self.multipleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs),1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 10)
self.multipleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs),1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 10)
#self.assertEqual(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 10)
def testAllAcquired(self):
"""
_testAllAcquired_
should all return no job groups
"""
splitter = SplitterFactory()
self.singleFileSubscription.acquireFiles(
self.singleFileSubscription.availableFiles())
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.acquireFiles(
self.multipleFileSubscription.availableFiles())
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(jobGroups, [], "Should have returned a null set")
self.multipleLumiSubscription.acquireFiles(
self.multipleLumiSubscription.availableFiles())
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.acquireFiles(
self.singleLumiSubscription.availableFiles())
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(jobGroups, [], "Should have returned a null set")
def testClosedSomeAcquired(self):
"""
_testClosedSomeAcquired_
since the subscriptions are closed and none of the files ahve been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.multipleFileSubscription.getFileset().markOpen(False)
self.singleFileSubscription.acquireFiles(
[self.singleFileSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.getFileset().markOpen(False)
self.multipleFileSubscription.acquireFiles(
[self.multipleFileSubscription.availableFiles().pop()])
jobFactory = splitter(package = "WMCore.WMBS", subscription =self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.multipleLumiSubscription.getFileset().markOpen(False)
self.multipleLumiSubscription.acquireFiles(
[self.multipleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.singleLumiSubscription.getFileset().markOpen(False)
self.singleLumiSubscription.acquireFiles(
[self.singleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time = 1)
self.assertEqual(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEqual(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.assertEqual(len(myfiles), 9)
class EndOfRunTest(unittest.TestCase):
"""
_EndOfRun_t_
End of run splitting test
"""
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = daofactory(classname = "Locations.New")
locationAction.execute(siteName = 's1', seName = "somese.cern.ch")
locationAction.execute(siteName = 's2', seName = "otherse.cern.ch")
self.multipleFileFileset = Fileset(name = "TestFileset1")
self.multipleFileFileset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = "somese.cern.ch")
newFile.addRun(Run(i, *[45+i]))
newFile.create()
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name = "TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name", size = 1000, events = 100, locations = "somese.cern.ch")
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleFileLumiset = Fileset(name = "TestFileset3")
self.multipleFileLumiset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = "somese.cern.ch")
newFile.addRun(Run(1, *[45+i/3]))
newFile.create()
self.multipleFileLumiset.addFile(newFile)
self.multipleFileLumiset.commit()
self.singleLumiFileset = Fileset(name = "TestFileset4")
self.singleLumiFileset.create()
for i in range(10):
newFile = File(makeUUID(), size = 1000, events = 100, locations = "somese.cern.ch")
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleLumiFileset.addFile(newFile)
self.singleLumiFileset.commit()
self.multipleSiteFileset = Fileset(name = "TestFileset5")
self.multipleSiteFileset.create()
for i in range(5):
newFile = File(makeUUID(), size = 1000, events = 100, locations = "somese.cern.ch")
newFile.addRun(Run(i, *[45+i]))
newFile.create()
self.multipleSiteFileset.addFile(newFile)
for i in range(5):
newFile = File(makeUUID(), size = 1000, events = 100, locations = "otherse.cern.ch")
newFile.addRun(Run(i, *[45+i]))
newFile.create()
self.multipleSiteFileset.addFile(newFile)
self.multipleSiteFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "mnorman", name = "wf001", task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(fileset = self.multipleFileFileset,
workflow = testWorkflow,
split_algo = "EndOfRun",
type = "Processing")
self.singleFileSubscription = Subscription(fileset = self.singleFileFileset,
workflow = testWorkflow,
split_algo = "EndOfRun",
type = "Processing")
self.multipleLumiSubscription = Subscription(fileset = self.multipleFileLumiset,
workflow = testWorkflow,
split_algo = "EndOfRun",
type = "Processing")
self.singleLumiSubscription = Subscription(fileset = self.singleLumiFileset,
workflow = testWorkflow,
split_algo = "EndOfRun",
type = "Processing")
self.multipleSiteSubscription = Subscription(fileset = self.multipleSiteFileset,
workflow = testWorkflow,
split_algo = "EndOfRun",
type = "Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription.create()
self.multipleLumiSubscription.create()
self.singleLumiSubscription.create()
self.multipleSiteSubscription.create()
return
def tearDown(self):
"""
_tearDown_
Nothing to do...
"""
myThread = threading.currentThread()
if myThread.transaction == None:
myThread.transaction = Transaction(self.dbi)
myThread.transaction.begin()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
destroyworked = destroy.execute(conn = myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
return
def testNone(self):
"""
_testNone_
since the subscriptions are open, we shouldn't get any jobs back
"""
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.singleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
return
def testClosed(self):
"""
_testClosed_
since the subscriptions are closed and none of the files have been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.singleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.singleFileSubscription)
jobGroups = jobFactory()
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
self.multipleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs),1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
self.multipleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs),1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
#self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory()
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
def testAllAcquired(self):
"""
_testAllAcquired_
should all return no job groups
"""
splitter = SplitterFactory()
self.singleFileSubscription.acquireFiles(
self.singleFileSubscription.availableFiles())
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.singleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.acquireFiles(
self.multipleFileSubscription.availableFiles())
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleLumiSubscription.acquireFiles(
self.multipleLumiSubscription.availableFiles())
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.acquireFiles(
self.singleLumiSubscription.availableFiles())
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
def testClosedSomeAcquired(self):
"""
_testClosedSomeAcquired_
since the subscriptions are closed and none of the files ahve been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.multipleFileSubscription.getFileset().markOpen(False)
self.singleFileSubscription.acquireFiles(
[self.singleFileSubscription.availableFiles().pop()])
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.singleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.getFileset().markOpen(False)
self.multipleFileSubscription.acquireFiles(
[self.multipleFileSubscription.availableFiles().pop()])
jobFactory = splitter(package = "WMCore.WMBS",
subscription =self.multipleFileSubscription)
jobGroups = jobFactory()
self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.multipleLumiSubscription.getFileset().markOpen(False)
self.multipleLumiSubscription.acquireFiles(
[self.multipleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.singleLumiSubscription.getFileset().markOpen(False)
self.singleLumiSubscription.acquireFiles(
[self.singleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory()
self.assertEquals(len(jobGroups),1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.assertEquals(len(myfiles), 9)
def testClosed_MultipleJobs(self):
"""
_testClosed_MultipleJobs_
Should break subscription into jobs based on number of files
"""
splitter = SplitterFactory()
self.multipleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.multipleFileSubscription)
jobGroups = jobFactory(files_per_job = 1)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 10)
job = jobGroups[0].jobs[0]
# We should have one file per job
self.assertEqual(len(job['input_files']), 1)
return
def testClosed_MultipleSites(self):
"""
_testClosed_MultipleSites_
Test whether we can handle multiple sites
"""
splitter = SplitterFactory()
self.multipleSiteSubscription.getFileset().markOpen(False)
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.multipleSiteSubscription)
jobGroups = jobFactory(files_per_job = 10)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
for job in jobGroups[0].jobs:
self.assertEqual(len(jobGroups[0].jobs[0]['input_files']), 5)
