def createSubscription(self, nFiles, lumisPerFile, twoSites=False, nEventsPerFile=100):
"""
_createSubscription_
Create a subscription for testing
"""
baseName = makeUUID()
testFileset = Fileset(name=baseName)
testFileset.create()
for i in range(nFiles):
newFile = self.createFile("%s_%i" % (baseName, i), nEventsPerFile, i, lumisPerFile, "somese.cern.ch")
newFile.create()
testFileset.addFile(newFile)
if twoSites:
for i in range(nFiles):
newFile = self.createFile("%s_%i_2" % (baseName, i), nEventsPerFile, i, lumisPerFile, "otherse.cern.ch")
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(
fileset=testFileset, workflow=self.testWorkflow, split_algo="EventAwareLumiBased", type="Processing"
)
testSubscription.create()
return testSubscription
def createLargeFileBlock(self):
"""
_createLargeFileBlock_
Creates a large group of files for testing
"""
testFileset = Fileset(name = "TestFilesetX")
testFileset.create()
for i in range(5000):
newFile = File(makeUUID(), size = 1000, events = 100,
locations = set(["somese.cern.ch"]))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "mnorman",
name = "wf003", task="Test" )
testWorkflow.create()
largeSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
split_algo = "FileBased",
type = "Processing")
largeSubscription.create()
return largeSubscription
def testCreateDeleteExistsNoFiles(self):
"""
_testCreateDeleteExistsNoFiles_
Create and then delete a job but don't add any input files to it.
Use the job class's exists() method to determine if the job has been
written to the database before it is created, after it has been created
and after it has been deleted.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job(name="TestJob")
assert testJob.exists() == False, "ERROR: Job exists before it was created"
testJob.create(group=testJobGroup)
assert testJob.exists() >= 0, "ERROR: Job does not exist after it was created"
testJob.delete()
assert testJob.exists() == False, "ERROR: Job exists after it was delete"
return
def test_AutoIncrementCheck(self):
"""
_AutoIncrementCheck_
Test and see whether we can find and set the auto_increment values
"""
myThread = threading.currentThread()
if not myThread.dialect.lower() == "mysql":
return
testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
incrementDAO = self.daoFactory(classname="Jobs.AutoIncrementCheck")
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 1)
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 2)
incrementDAO.execute(input=10)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 11)
incrementDAO.execute(input=5)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 12)
return
def createTestJob(subscriptionType="Merge"):
"""
_createTestJob_
Create a test job with two files as input. This will also create the
appropriate workflow, jobgroup and subscription.
"""
testWorkflow = Workflow(spec=makeUUID(), owner="Simon", name=makeUUID(), task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type=subscriptionType)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(1, *[46]))
testFileA.create()
testFileB.create()
testJob = Job(name=makeUUID(), files=[testFileA, testFileB])
testJob["couch_record"] = "somecouchrecord"
testJob["location"] = "test.site.ch"
testJob.create(group=testJobGroup)
testJob.associateFiles()
return testJob
def testDifferentSubscritionIDs(self):
"""
_testDifferentSubscriptionIDs_
Make sure that the merge splitting still runs if the subscription ID
is not equal to the workflow ID.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
dummyWorkflow = Workflow(name = "dummyWorkflow", spec = "bunk49",
owner = "Steve", task="Test2")
dummyWorkflow.create()
dummyFileset = Fileset(name = "dummyFileset")
dummyFileset.create()
dummySubscription1 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription2 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription1.create()
dummySubscription2.create()
myThread.transaction.commit()
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.mergeSubscription)
result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999,
max_merge_events = 999999999, merge_across_runs = False)
self.assertEqual(len(result), 1)
jobGroup = result[0]
self.assertEqual(len(jobGroup.jobs), 2)
return
def testFilesWithoutOtherSubscriptions(self):
"""
_testFilesWithoutOtherSubscriptions_
Test the case where files only in the delete subscription
can happen if cleanup of the other subscriptions is fast
"""
testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test")
testWorkflowA.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i), size=1000, events=100, locations=set(["somese.cern.ch"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(
fileset=testFileset, workflow=testWorkflowA, split_algo="SiblingProcessingBased", type="Processing"
)
testSubscriptionA.create()
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionA)
result = deleteFactoryA(files_per_job=50)
self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.")
return
def createSingleJobWorkflow(self):
"""
Create a workflow with one jobs and two files and store the results in instance variables
"""
self.testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test")
self.testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset, workflow=self.testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
self.testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
self.testFileA.addRun(Run(1, *[45]))
self.testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
self.testFileB.addRun(Run(1, *[46]))
self.testFileA.create()
self.testFileB.create()
self.testJob = Job(name="TestJob", files=[self.testFileA, self.testFileB])
self.testJob.create(group=testJobGroup)
self.testJob.associateFiles()
def testByRunAndRunBlacklist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 3)
def createTestSubscription(self, nFiles, nSites=1, closeFileset=False):
"""
_createTestSubscription_
Create a set of test subscriptions for testing purposes.
"""
if nSites > self.nSites:
nSites = self.nSites
testFileset = Fileset(name="TestFileset")
testFileset.create()
# Create a testWorkflow
testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test")
testWorkflow.create()
# Create the files for each site
for s in range(nSites):
for i in range(nFiles):
newFile = File(makeUUID(), size=1024, events=100, locations=set(["site%i.cern.ch" % s]))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(
fileset=testFileset, workflow=testWorkflow, split_algo="MinFileBased", type="Processing"
)
testSubscription.create()
# Close the fileset
if closeFileset:
testFileset.markOpen(isOpen=False)
return testSubscription
def createSubscription(self, nFiles, lumisPerFile, twoSites=False, nEventsPerFile=100):
"""
_createSubscription_
Create a subscription for testing
"""
baseName = makeUUID()
testWorkflow = Workflow(spec="spec.xml", owner="dmwm",
name="testWorkflow_%s" % baseName[:4], task="Test")
testWorkflow.create()
testFileset = Fileset(name=baseName)
for i in range(nFiles):
newFile = self.createFile('%s_%i' % (baseName, i), nEventsPerFile,
i, lumisPerFile, 'T1_US_FNAL_Disk')
testFileset.addFile(newFile)
if twoSites:
for i in range(nFiles):
newFile = self.createFile('%s_%i_2' % (baseName, i), nEventsPerFile,
i, lumisPerFile, 'T2_CH_CERN')
testFileset.addFile(newFile)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
split_algo="EventAwareLumiByWork",
type="Processing")
testSubscription.create()
return testSubscription
def testMask(self):
"""
_testMask_
Test the new mask setup
"""
testWorkflow = Workflow(spec="spec.xml", owner="Steve",
name="wf001", task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job()
testJob['mask'].addRunAndLumis(run=100, lumis=[101, 102])
testJob['mask'].addRunAndLumis(run=200, lumis=[201, 202])
testJob.create(group=testJobGroup)
loadJob = Job(id=testJob.exists())
loadJob.loadData()
runs = loadJob['mask'].getRunAndLumis()
self.assertEqual(len(runs), 2)
self.assertEqual(runs[100], [[101, 102]])
self.assertEqual(runs[200], [[201, 202]])
bigRun = Run(100, *[101, 102, 103, 104])
badRun = Run(300, *[1001, 1002])
result = loadJob['mask'].filterRunLumisByMask([bigRun, badRun])
self.assertEqual(len(result), 1)
alteredRun = result.pop()
self.assertEqual(alteredRun.run, 100)
self.assertEqual(alteredRun.lumis, [101, 102])
run0 = Run(300, *[1001, 1002])
run1 = Run(300, *[1001, 1002])
loadJob['mask'].filterRunLumisByMask([run0, run1])
return
def subscribeWMBS(self, task):
"""
Create a subscription for each task
"""
workFlow, fileSet = self.createWorkflow(task)
workFlow.load()
fileSet.load()
subType = ""
if task.name() == "Processing" or task.name() == "Production":
subType = "Processing"
elif task.name() == "Merge":
subType = "Merge"
newSub = Subscription(fileset=fileSet, workflow=workFlow, split_algo="FileBased", type=subType)
newSub.create()
# Add subscription to dictionary
self.subDict[task.name()] = newSub
# Add subscription id to task
setattr(task.data.input.WMBS, "Subscription", newSub["id"])
if not newSub.exists() >= 0:
raise Exception("ERROR: Subscription does not exist after it was created")
logging.info("Created subscription for task %s" % (task.name()))
return
def subscribeWMBS(self, task):
"""
Create a subscription for each task
"""
workFlow, fileSet = self.createWorkflow(task)
workFlow.load()
fileSet.load()
subType = ''
if task.name() == 'Processing' or task.name() == 'Production':
subType = 'Processing'
elif task.name() == 'Merge':
subType = 'Merge'
newSub = Subscription(fileset=fileSet, workflow=workFlow, split_algo='FileBased', type=subType)
newSub.create()
# Add subscription to dictionary
self.subDict[task.name()] = newSub
# Add subscription id to task
setattr(task.data.input.WMBS, 'Subscription', newSub['id'])
if not newSub.exists() >= 0:
raise Exception("ERROR: Subscription does not exist after it was created")
logging.info('Created subscription for task %s' % (task.name()))
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Create a JobGroup and then delete it. Use the JobGroup's exists()
method to determine if it exists before it is created, after it is
created and after it is deleted.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testFileset = WMBSFileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
self.assertFalse(testJobGroup.exists())
testJobGroup.create()
self.assertTrue(testJobGroup.exists())
testJobGroup.delete()
self.assertFalse(testJobGroup.exists())
testSubscription.delete()
testFileset.delete()
testWorkflow.delete()
return
def generateFakeMCFile(self, numEvents = 100, firstEvent = 1,
lastEvent = 100, firstLumi = 1, lastLumi = 10,
index = 1):
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCTestFileset %i" % index)
singleMCFileset.create()
newFile = File("MCFakeFileTest %i" % index, size = 1000,
events = numEvents,
locations = set(["somese.cern.ch"]))
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
newFile.create()
singleMCFileset.addFile(newFile)
singleMCFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task="Test")
testWorkflow.create()
singleMCFileSubscription = Subscription(fileset = singleMCFileset,
workflow = testWorkflow,
split_algo = "EventBased",
type = "Production")
singleMCFileSubscription.create()
return singleMCFileSubscription
def testCreateTransaction(self):
"""
_testCreateTransaction_
Create a JobGroup and commit it to the database. Rollback the database
transaction and verify that the JobGroup is no longer in the database.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test")
testWorkflow.create()
testFileset = WMBSFileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
assert testJobGroup.exists() == False, "ERROR: Job group exists before it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJobGroup.create()
assert testJobGroup.exists() >= 0, "ERROR: Job group does not exist after it was created"
myThread.transaction.rollback()
assert testJobGroup.exists() == False, "ERROR: Job group exists after transaction was rolled back."
testSubscription.delete()
testFileset.delete()
testWorkflow.delete()
return
def testListRunningJobs(self):
"""
_testListRunningJobs_
Test the ListRunningJobs DAO.
"""
testWorkflow = Workflow(spec = makeUUID(), owner = "Steve",
name = makeUUID(), task="Test")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
type = "Processing")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testJobA = Job(name = makeUUID(), files = [])
testJobA["couch_record"] = makeUUID()
testJobA.create(group = testJobGroup)
testJobA["state"] = "executing"
testJobB = Job(name = makeUUID(), files = [])
testJobB["couch_record"] = makeUUID()
testJobB.create(group = testJobGroup)
testJobB["state"] = "complete"
testJobC = Job(name = makeUUID(), files = [])
testJobC["couch_record"] = makeUUID()
testJobC.create(group = testJobGroup)
testJobC["state"] = "new"
changeStateAction = self.daoFactory(classname = "Jobs.ChangeState")
changeStateAction.execute(jobs = [testJobA, testJobB, testJobC])
runningJobsAction = self.daoFactory(classname = "Monitoring.ListRunningJobs")
runningJobs = runningJobsAction.execute()
assert len(runningJobs) == 2, \
"Error: Wrong number of running jobs returned."
for runningJob in runningJobs:
if runningJob["job_name"] == testJobA["name"]:
assert runningJob["state"] == testJobA["state"], \
"Error: Running job has wrong state."
assert runningJob["couch_record"] == testJobA["couch_record"], \
"Error: Running job has wrong couch record."
else:
assert runningJob["job_name"] == testJobC["name"], \
"Error: Running job has wrong name."
assert runningJob["state"] == testJobC["state"], \
"Error: Running job has wrong state."
assert runningJob["couch_record"] == testJobC["couch_record"], \
"Error: Running job has wrong couch record."
return
def testMultiRunHarvesting(self):
"""
_testMultiRunHarvesting_
Provided a fileset with a couple of files and different runs, create a
single job for all the runs at a specific location, which also adds a
baggage to the job (True) which is later on looked up by SetupCMSSWPSet.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(dqmHarvestUnit="multiRun")
self.assertEqual(len(jobGroups), 1)
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 1)
def testByRunHarvesting(self):
"""
_testByRunHarvesting_
Provided a fileset with a couple of files and 4 different runs, create
one single job per run and location.
The multiRun baggage should be false in this case.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!")
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1, "Should have created 1 job group")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs")
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")
def testD_NonContinuousLumis(self):
"""
_NonContinuousLumis_
Test and see if LumiBased can work when the lumis are non continuous
"""
baseName = makeUUID()
nFiles = 10
testFileset = Fileset(name = baseName)
testFileset.create()
for i in range(nFiles):
newFile = File(lfn = '%s_%i' % (baseName, i), size = 1000,
events = 100, locations = "somese.cern.ch")
# Set to two non-continuous lumi numbers
lumis = [100 + i, 200 + i]
newFile.addRun(Run(i, *lumis))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(fileset = testFileset,
workflow = self.testWorkflow,
split_algo = "LumiBased",
type = "Processing")
testSubscription.create()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = testSubscription)
jobGroups = jobFactory(lumis_per_job = 2,
halt_job_on_file_boundaries = False,
splitOnRun = False,
performance = self.performanceParams)
self.assertEqual(len(jobGroups), 1)
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 10)
for j in jobs:
runs = j['mask'].getRunAndLumis()
for r in runs.keys():
self.assertEqual(len(runs[r]), 2)
for l in runs[r]:
# Each run should have two lumis
# Each lumi should be of form [x, x]
# meaning that the first and last lumis are the same
self.assertEqual(len(l), 2)
self.assertEqual(l[0], l[1])
self.assertEqual(j['estimatedJobTime'], 100 * 12)
self.assertEqual(j['estimatedDiskUsage'], 100 * 400)
self.assertEqual(j['estimatedMemoryUsage'], 2300)
return
def testAddDupsToFilesetBulk(self):
"""
_AddToDupsFilesetBulk_
Same as testAddDupsToFileset() but faster
"""
testWorkflowA = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflowA.create()
testWorkflowB = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production2")
testWorkflowB.create()
testFilesetA = Fileset(name = "inputFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "inputFilesetB")
testFilesetB.create()
testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA)
testSubscriptionA.create()
testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB)
testSubscriptionB.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10, locations = ['SiteA'])
testFileA.addRun(Run( 1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10, locations = ['SiteB'])
testFileB.addRun(Run( 1, *[45]))
addFilesToWMBSInBulk(testFilesetA.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
testFileset2 = Fileset(name = "inputFilesetA")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 2)
for file in testFileset2.files:
self.assertTrue(file in [testFileA, testFileB])
# Check that adding twice doesn't crash
addFilesToWMBSInBulk(testFilesetA.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
# Files should not get added to fileset B because fileset A is associated
# with wf001.
addFilesToWMBSInBulk(testFilesetB.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
testFileset2 = Fileset(name = "inputFilesetB")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 0)
return
def testAddDupsToFileset(self):
"""
_AddToDupsFileset_
Verify the the dups version of the AddToFileset DAO will not add files
to a fileset if they're already associated to another fileset with the
same workflow.
"""
testWorkflowA = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflowA.create()
testWorkflowB = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production2")
testWorkflowB.create()
testFilesetA = Fileset(name = "inputFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "inputFilesetB")
testFilesetB.create()
testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA)
testSubscriptionA.create()
testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB)
testSubscriptionB.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run( 1, *[45]))
testFileA.create()
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run( 1, *[45]))
testFileB.create()
addToFileset = self.daofactory(classname = "Files.AddDupsToFileset")
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetA.id, workflow = "wf001")
testFileset2 = Fileset(name = "inputFilesetA")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 2)
for file in testFileset2.files:
self.assertTrue(file in [testFileA, testFileB])
# Check that adding twice doesn't crash
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetA.id, workflow = "wf001")
# Files should not get added to fileset B because fileset A is associated
# with wf001.
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetB.id, workflow = "wf001")
testFileset2 = Fileset(name = "inputFilesetB")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 0)
return
def _createThisSubscription(self, initialCounter=1):
"""
Private function to create a fileset and subscription with
different fileset and file names
:param initialCounter: just a simple integer to be appended to files
:return: an splitter instance (jobFactory)
"""
splitter = SplitterFactory()
# Create 3 files with 100 events per lumi:
# - file1 with 1 run of 8 lumis
# - file2 with 2 runs of 2 lumis each
# - file3 with 1 run of 5 lumis
testFileset = Fileset(name='Fileset%s' % initialCounter)
fileA = File(lfn="/this/is/file%s" % initialCounter, size=1000, events=800)
lumiListA = []
for lumi in range(8):
lumiListA.append(10 + lumi)
fileA.addRun(Run(1, *lumiListA))
fileA.setLocation("T1_US_FNAL_Disk")
initialCounter = int(initialCounter) + 1
fileB = File(lfn="/this/is/file%s" % initialCounter, size=1000, events=400)
lumiListB1 = []
lumiListB2 = []
for lumi in range(2):
lumiListB1.append(20 + lumi)
lumiListB2.append(30 + lumi)
fileB.addRun(Run(2, *lumiListB1))
fileB.addRun(Run(3, *lumiListB2))
fileB.setLocation("T1_US_FNAL_Disk")
initialCounter = int(initialCounter) + 1
fileC = File(lfn="/this/is/file%s" % initialCounter, size=1000, events=500)
lumiListC = []
for lumi in range(5):
lumiListC.append(40 + lumi)
fileC.addRun(Run(4, *lumiListC))
fileC.setLocation("T1_US_FNAL_Disk")
testFileset.addFile(fileA)
testFileset.addFile(fileB)
testFileset.addFile(fileC)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiByWork",
type="Processing")
testSubscription.create()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
return jobFactory
def testParentageByJob(self):
"""
_testParentageByJob_
Tests the DAO that assigns parentage by Job
"""
testWorkflow = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFileParentA = File(lfn = "/this/is/a/parent/lfnA", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentA.addRun(Run( 1, *[45]))
testFileParentB = File(lfn = "/this/is/a/parent/lfnB", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentB.addRun(Run( 1, *[45]))
testFileParentA.create()
testFileParentB.create()
testFileA = File(lfn = "/this/is/a/lfn", size = 1024, events = 10,
checksums = {'cksum':1})
testFileA.addRun(Run( 1, *[45]))
testFileA.create()
testJobA = Job()
testJobA.create(group = testJobGroup)
testJobA.addFile(testFileParentA)
testJobA.addFile(testFileParentB)
testJobA.associateFiles()
parentAction = self.daofactory(classname = "Files.SetParentageByJob")
parentAction.execute(binds = {'jobid': testJobA.exists(), 'child': testFileA['lfn']})
testFileB = File(id = testFileA["id"])
testFileB.loadData(parentage = 1)
goldenFiles = [testFileParentA, testFileParentB]
for parentFile in testFileB["parents"]:
self.assertEqual(parentFile in goldenFiles, True,
"ERROR: Unknown parent file")
goldenFiles.remove(parentFile)
self.assertEqual(len(goldenFiles), 0,
"ERROR: Some parents are missing")
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site=None, bl=[], wl=[]):
"""
Creates a series of jobGroups for submissions
"""
jobGroupList = []
testWorkflow = Workflow(
spec=workloadSpec,
owner="tapas",
name=makeUUID(),
task="basicWorkload/Production",
owner_vogroup="phgroup",
owner_vorole="cmsrole",
)
testWorkflow.create()
# Create subscriptions
for i in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(
fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased"
)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(
name=name,
task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site,
bl=bl,
wl=wl,
)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
return jobGroupList
def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a new job and commit it to the database. Start a new transaction
and delete the file from the database. Verify that the file has been
deleted. After that, roll back the transaction and verify that the
job is once again in the database.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileA.create()
testFileB.create()
testJob = Job(name="TestJob", files=[testFileA, testFileB])
assert testJob.exists() is False, \
"ERROR: Job exists before it was created"
testJob.create(group=testJobGroup)
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJob.delete()
assert testJob.exists() is False, \
"ERROR: Job exists after it was delete"
myThread.transaction.rollback()
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after transaction was rolled back."
return
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
taskType='Processing', name=None, wfPrio=1, changeState=None):
"""
_createJobGroups_
Creates a series of jobGroups for submissions
changeState is an instance of the ChangeState class to make job status changes
"""
jobGroupList = []
if name is None:
name = makeUUID()
testWorkflow = Workflow(spec=workloadSpec, owner="tapas",
name=name, task="basicWorkload/Production",
priority=wfPrio)
testWorkflow.create()
# Create subscriptions
for _ in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type=taskType,
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(name=name, task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
if changeState:
for group in jobGroupList:
changeState.propagate(group.jobs, 'created', 'new')
return jobGroupList
def stuffWMBS(self):
"""
_stuffWMBS_
Stuff WMBS with workflows
"""
workflow = Workflow(spec = 'spec.xml', name = 'ReRecoTest_v0Emulator',
task = '/ReRecoTest_v0Emulator/Test', priority = 10)
workflow.create()
inputFileset = Fileset(name = 'TestFileset')
inputFileset.create()
subscription = Subscription(inputFileset, workflow)
subscription.create()
def createSubscription(self, nFiles, lumisPerFile, twoSites = False, rand = False):
"""
_createSubscription_
Create a subscription for testing
"""
baseName = makeUUID()
testFileset = Fileset(name = baseName)
testFileset.create()
parentFile = File('%s_parent' % (baseName), size = 1000, events = 100,
locations = set(["somese.cern.ch"]))
parentFile.create()
for i in range(nFiles):
newFile = File(lfn = '%s_%i' % (baseName, i), size = 1000,
events = 100, locations = "somese.cern.ch")
lumis = []
for lumi in range(lumisPerFile):
if rand:
lumis.append(random.randint(1000 * i, 1000 * (i + 1)))
else:
lumis.append((100 * i) + lumi)
newFile.addRun(Run(i, *lumis))
newFile.create()
newFile.addParent(parentFile['lfn'])
testFileset.addFile(newFile)
if twoSites:
for i in range(nFiles):
newFile = File(lfn = '%s_%i_2' % (baseName, i), size = 1000,
events = 100, locations = "otherse.cern.ch")
lumis = []
for lumi in range(lumisPerFile):
if rand:
lumis.append(random.randint(1000 * i, 1000 * (i + 1)))
else:
lumis.append((100 * i) + lumi)
newFile.addRun(Run(i, *lumis))
newFile.create()
newFile.addParent(parentFile['lfn'])
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(fileset = testFileset,
workflow = self.testWorkflow,
split_algo = "LumiBased",
type = "Processing")
testSubscription.create()
return testSubscription
def createTestJob(self, subscriptionType = "Merge"):
"""
_createTestJob_
Create a test job with two files as input. This will also create the
appropriate workflow, jobgroup and subscription.
"""
testWorkflow = Workflow(spec = makeUUID(), owner = "Simon",
name = makeUUID(), task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = subscriptionType)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(1, *[46]))
testFileA.create()
testFileB.create()
testJob = Job(name = makeUUID(), files = [testFileA, testFileB])
testJob["couch_record"] = "somecouchrecord"
testJob["location"] = "test.site.ch"
testJob.create(group = testJobGroup)
testJob.associateFiles()
return testJob
def createSubscription(self,
nFiles,
lumisPerFile,
twoSites=False,
nEventsPerFile=100):
"""
_createSubscription_
Create a subscription for testing
"""
baseName = makeUUID()
testFileset = Fileset(name=baseName)
testFileset.create()
for i in range(nFiles):
newFile = self.createFile('%s_%i' % (baseName, i), nEventsPerFile,
i, lumisPerFile, 'T1_US_FNAL_Disk')
newFile.create()
testFileset.addFile(newFile)
if twoSites:
for i in range(nFiles):
newFile = self.createFile('%s_%i_2' % (baseName, i),
nEventsPerFile, i, lumisPerFile,
'T2_CH_CERN')
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiBased",
type="Processing")
testSubscription.create()
return testSubscription
def subscribeWMBS(self, task):
"""
Create a subscription for each task
"""
workFlow, fileSet = self.createWorkflow(task)
workFlow.load()
fileSet.load()
subType = ''
if task.name() == 'Processing' or task.name() == 'Production':
subType = 'Processing'
elif task.name() == 'Merge':
subType = 'Merge'
newSub = Subscription(fileset=fileSet,
workflow=workFlow,
split_algo='FileBased',
type=subType)
newSub.create()
#Add subscription to dictionary
self.subDict[task.name()] = newSub
#Add subscription id to task
setattr(task.data.input.WMBS, 'Subscription', newSub['id'])
if not newSub.exists() >= 0:
raise Exception(
"ERROR: Subscription does not exist after it was created")
logging.info('Created subscription for task %s' % (task.name()))
return
def stuffWMBS(self, workflowURL, name):
"""
_stuffWMBS_
Insert some dummy jobs, jobgroups, filesets, files and subscriptions
into WMBS to test job creation. Three completed job groups each
containing several files are injected. Another incomplete job group is
also injected. Also files are added to the "Mergeable" subscription as
well as to the output fileset for their jobgroups.
"""
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="s1", seName="somese.cern.ch")
changeStateDAO = self.daoFactory(classname="Jobs.ChangeState")
mergeFileset = Fileset(name="mergeFileset")
mergeFileset.create()
bogusFileset = Fileset(name="bogusFileset")
bogusFileset.create()
mergeWorkflow = Workflow(spec=workflowURL,
owner="mnorman",
name=name,
task="/TestWorkload/ReReco")
mergeWorkflow.create()
mergeSubscription = Subscription(fileset=mergeFileset,
workflow=mergeWorkflow,
split_algo="ParentlessMergeBySize")
mergeSubscription.create()
bogusSubscription = Subscription(fileset=bogusFileset,
workflow=mergeWorkflow,
split_algo="ParentlessMergeBySize")
file1 = File(lfn="file1",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
file1.addRun(Run(1, *[45]))
file1.create()
file2 = File(lfn="file2",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
file2.addRun(Run(1, *[45]))
file2.create()
file3 = File(lfn="file3",
size=1024,
events=1024,
first_event=2048,
locations=set(["somese.cern.ch"]))
file3.addRun(Run(1, *[45]))
file3.create()
file4 = File(lfn="file4",
size=1024,
events=1024,
first_event=3072,
locations=set(["somese.cern.ch"]))
file4.addRun(Run(1, *[45]))
file4.create()
fileA = File(lfn="fileA",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
fileA.addRun(Run(1, *[46]))
fileA.create()
fileB = File(lfn="fileB",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
fileB.addRun(Run(1, *[46]))
fileB.create()
fileC = File(lfn="fileC",
size=1024,
events=1024,
first_event=2048,
locations=set(["somese.cern.ch"]))
fileC.addRun(Run(1, *[46]))
fileC.create()
fileI = File(lfn="fileI",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
fileI.addRun(Run(2, *[46]))
fileI.create()
fileII = File(lfn="fileII",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
fileII.addRun(Run(2, *[46]))
fileII.create()
fileIII = File(lfn="fileIII",
size=1024,
events=102400,
first_event=2048,
locations=set(["somese.cern.ch"]))
fileIII.addRun(Run(2, *[46]))
fileIII.create()
fileIV = File(lfn="fileIV",
size=102400,
events=1024,
first_event=3072,
locations=set(["somese.cern.ch"]))
fileIV.addRun(Run(2, *[46]))
fileIV.create()
for file in [
file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII,
fileIII, fileIV
]:
mergeFileset.addFile(file)
bogusFileset.addFile(file)
mergeFileset.commit()
bogusFileset.commit()
return
class ConditionTest(unittest.TestCase):
"""
_ExpressTest_
Test for Express job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["T0.WMBS"])
self.splitterFactory = SplitterFactory(package = "T0.JobSplitting")
myThread = threading.currentThread()
daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
myThread.dbi.processData("""INSERT INTO wmbs_location
(id, site_name, state)
VALUES (1, 'SomeSite', 1)
""", transaction = False)
myThread.dbi.processData("""INSERT INTO wmbs_location_senames
(location, se_name)
VALUES (1, 'SomeSE')
""", transaction = False)
insertRunDAO = daoFactory(classname = "RunConfig.InsertRun")
insertRunDAO.execute(binds = { 'RUN' : 1,
'TIME' : int(time.time()),
'HLTKEY' : "someHLTKey" },
transaction = False)
insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection")
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 1 },
transaction = False)
insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream")
insertStreamDAO.execute(binds = { 'STREAM' : "Express" },
transaction = False)
insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset")
insertStreamFilesetDAO.execute(1, "Express", "TestFileset1")
insertStreamerDAO = daoFactory(classname = "RunConfig.InsertStreamer")
insertStreamerDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 1,
'STREAM' : "Express",
'TIME' : int(time.time()),
'LFN' : "/streamer",
'FILESIZE' : 0,
'EVENTS' : 0 },
transaction = False)
insertPromptCalibrationDAO = daoFactory(classname = "RunConfig.InsertPromptCalibration")
insertPromptCalibrationDAO.execute( { 'RUN' : 1,
'STREAM' : "Express" },
transaction = False)
self.fileset1 = Fileset(name = "TestFileset1")
self.fileset1.create()
workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset = self.fileset1,
workflow = workflow1,
split_algo = "Condition",
type = "Condition")
self.subscription1.create()
# set parentage chain and sqlite fileset
alcaRecoFile = File("/alcareco", size = 0, events = 0)
alcaRecoFile.addRun(Run(1, *[1]))
alcaRecoFile.setLocation("SomeSE", immediateSave = False)
alcaRecoFile.create()
alcaPromptFile = File("/alcaprompt", size = 0, events = 0)
alcaPromptFile.addRun(Run(1, *[1]))
alcaPromptFile.setLocation("SomeSE", immediateSave = False)
alcaPromptFile.create()
sqliteFile = File("/sqlite", size = 0, events = 0)
sqliteFile.create()
self.fileset1.addFile(sqliteFile)
self.fileset1.commit()
results = myThread.dbi.processData("""SELECT lfn FROM wmbs_file_details
""",
transaction = False)[0].fetchall()
setParentageDAO = wmbsDaoFactory(classname = "Files.SetParentage")
setParentageDAO.execute(binds = [ { 'parent' : "/streamer",
'child' : "/alcareco" },
{ 'parent' : "/alcareco",
'child' : "/alcaprompt" },
{ 'parent' : "/alcaprompt",
'child' : "/sqlite" } ],
transaction = False)
# default split parameters
self.splitArgs = {}
self.splitArgs['runNumber'] = 1
self.splitArgs['streamName'] = "Express"
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def isPromptCalibFinished(self):
"""
_isPromptCalibFinished_
"""
myThread = threading.currentThread()
result = myThread.dbi.processData("""SELECT finished
FROM prompt_calib
""",
transaction = False)[0].fetchall()[0][0]
return result
def countPromptCalibFiles(self):
"""
_deleteSplitLumis_
"""
myThread = threading.currentThread()
result = myThread.dbi.processData("""SELECT COUNT(*)
FROM prompt_calib_file
""",
transaction = False)[0].fetchall()[0][0]
return result
def test00(self):
"""
_test00_
Make sure the job splitter behaves correctly.
Just make sure the job splitter does nothing
when the fileset is open and populates t0ast
data structures when it's closed. In the later
case all input files should be marked as
acquired without creating a job as well.
"""
mySplitArgs = self.splitArgs.copy()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription1)
self.assertEqual(self.isPromptCalibFinished(), 0,
"ERROR: prompt_calib should not be finished")
self.assertEqual(self.countPromptCalibFiles(), 0,
"ERROR: there should be no prompt_calib_file")
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(self.isPromptCalibFinished(), 0,
"ERROR: prompt_calib should not be finished")
self.assertEqual(self.countPromptCalibFiles(), 1,
"ERROR: there should be one prompt_calib_file")
self.fileset1.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
self.assertEqual(self.isPromptCalibFinished(), 1,
"ERROR: prompt_calib should be finished")
self.assertEqual(self.countPromptCalibFiles(), 1,
"ERROR: there should be one prompt_calib_file")
return
def testGetOutputMapDAO(self):
"""
_testGetOutputMapDAO_
Verify the proper behavior of the GetOutputMapDAO for a variety of
different processing chains.
"""
recoOutputFileset = Fileset(name="RECO")
recoOutputFileset.create()
mergedRecoOutputFileset = Fileset(name="MergedRECO")
mergedRecoOutputFileset.create()
alcaOutputFileset = Fileset(name="ALCA")
alcaOutputFileset.create()
mergedAlcaOutputFileset = Fileset(name="MergedALCA")
mergedAlcaOutputFileset.create()
dqmOutputFileset = Fileset(name="DQM")
dqmOutputFileset.create()
mergedDqmOutputFileset = Fileset(name="MergedDQM")
mergedDqmOutputFileset.create()
cleanupFileset = Fileset(name="Cleanup")
cleanupFileset.create()
testWorkflow = Workflow(spec="wf001.xml",
owner="Steve",
name="TestWF",
task="None")
testWorkflow.create()
testWorkflow.addOutput("output", recoOutputFileset,
mergedRecoOutputFileset)
testWorkflow.addOutput("ALCARECOStreamCombined", alcaOutputFileset,
mergedAlcaOutputFileset)
testWorkflow.addOutput("DQM", dqmOutputFileset, mergedDqmOutputFileset)
testWorkflow.addOutput("output", cleanupFileset)
testWorkflow.addOutput("ALCARECOStreamCombined", cleanupFileset)
testWorkflow.addOutput("DQM", cleanupFileset)
testRecoMergeWorkflow = Workflow(spec="wf002.xml",
owner="Steve",
name="TestRecoMergeWF",
task="None")
testRecoMergeWorkflow.create()
testRecoMergeWorkflow.addOutput("anything", mergedRecoOutputFileset,
mergedRecoOutputFileset)
testRecoProcWorkflow = Workflow(spec="wf004.xml",
owner="Steve",
name="TestRecoProcWF",
task="None")
testRecoProcWorkflow.create()
testAlcaChildWorkflow = Workflow(spec="wf003.xml",
owner="Steve",
name="TestAlcaChildWF",
task="None")
testAlcaChildWorkflow.create()
inputFile = File(lfn="/path/to/some/lfn",
size=600000,
events=60000,
locations="cmssrm.fnal.gov")
inputFile.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testFileset.addFile(inputFile)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
split_algo="EventBased",
type="Processing")
testMergeRecoSubscription = Subscription(
fileset=recoOutputFileset,
workflow=testRecoMergeWorkflow,
split_algo="WMBSMergeBySize",
type="Merge")
testProcRecoSubscription = Subscription(fileset=recoOutputFileset,
workflow=testRecoProcWorkflow,
split_algo="FileBased",
type="Processing")
testChildAlcaSubscription = Subscription(
fileset=alcaOutputFileset,
workflow=testAlcaChildWorkflow,
split_algo="FileBased",
type="Processing")
testSubscription.create()
testMergeRecoSubscription.create()
testProcRecoSubscription.create()
testChildAlcaSubscription.create()
testSubscription.acquireFiles()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job(name="SplitJobA", files=[inputFile])
testJob.create(group=testJobGroup)
testJob["state"] = "complete"
testJob.save()
outputMapAction = self.daoFactory(classname="Jobs.GetOutputMap")
outputMap = outputMapAction.execute(jobID=testJob["id"])
assert len(outputMap.keys()) == 3, \
"Error: Wrong number of outputs for primary workflow."
goldenMap = {
"output": (recoOutputFileset.id, mergedRecoOutputFileset.id),
"ALCARECOStreamCombined":
(alcaOutputFileset.id, mergedAlcaOutputFileset.id),
"DQM": (dqmOutputFileset.id, mergedDqmOutputFileset.id)
}
for outputID in outputMap.keys():
for outputFilesets in outputMap[outputID]:
if outputFilesets["merged_output_fileset"] == None:
self.assertEqual(outputFilesets["output_fileset"],
cleanupFileset.id,
"Error: Cleanup fileset is wrong.")
continue
self.assertTrue(outputID in goldenMap.keys(),
"Error: Output identifier is missing.")
self.assertEqual(outputFilesets["output_fileset"],
goldenMap[outputID][0],
"Error: Output fileset is wrong.")
self.assertEqual(outputFilesets["merged_output_fileset"],
goldenMap[outputID][1],
"Error: Merged output fileset is wrong.")
del goldenMap[outputID]
self.assertEqual(len(goldenMap.keys()), 0,
"Error: Missing output maps.")
return
def testC_ACDCTest(self):
"""
_ACDCTest_
Test whether we can get a goodRunList out of ACDC
and process it correctly.
"""
workload = self.createTestWorkload()
dcs = DataCollectionService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
testFileA = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileA.addRun(Run(1, 1, 2))
testFileA.create()
testFileB = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileB.addRun(Run(1, 3))
testFileB.create()
testJobA = getJob(workload)
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileC.addRun(Run(1, 4, 6))
testFileC.create()
testJobB = getJob(workload)
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileD.addRun(Run(1, 7))
testFileD.create()
testJobC = getJob(workload)
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileE.addRun(Run(1, 11, 12))
testFileE.create()
testJobD = getJob(workload)
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileF.addRun(Run(2, 5, 6, 7))
testFileF.create()
testJobE = getJob(workload)
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileG.addRun(Run(2, 10, 11, 12))
testFileG.create()
testJobF = getJob(workload)
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileH.addRun(Run(2, 15))
testFileH.create()
testJobG = getJob(workload)
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileI.addRun(Run(3, 20))
testFileI.create()
testJobH = getJob(workload)
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileJ.addRun(Run(1, 9))
testFileJ.create()
testJobI = getJob(workload)
testJobI.addFile(testFileJ)
# dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE,
# testJobF, testJobG, testJobH, testJobI])
dcs.failedJobs([testJobA, testJobD, testJobH])
baseName = makeUUID()
testFileset = Fileset(name=baseName)
testFileset.create()
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.addFile(testFileC)
testFileset.addFile(testFileD)
testFileset.addFile(testFileE)
testFileset.addFile(testFileF)
testFileset.addFile(testFileG)
testFileset.addFile(testFileH)
testFileset.addFile(testFileI)
testFileset.addFile(testFileJ)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="LumiBased",
type="Processing")
testSubscription.create()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(
lumis_per_job=100,
halt_job_on_file_boundaries=False,
splitOnRun=True,
collectionName=workload.name(),
filesetName=workload.getTask("reco").getPathName(),
owner="evansde77",
group="DMWM",
couchURL=self.testInit.couchUrl,
couchDB=self.testInit.couchDbName,
performance=self.performanceParams)
self.assertEqual(jobGroups[0].jobs[0]['mask'].getRunAndLumis(),
{1: [[1, 2], [3, 3], [11, 12]]})
self.assertEqual(jobGroups[0].jobs[1]['mask'].getRunAndLumis(),
{3: [[20, 20]]})
return
class EventBasedTest(unittest.TestCase):
"""
_EventBasedTest_
Test event based job splitting.
"""
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.couchUrl = os.environ["COUCHURL"]
self.couchDBName = "acdc_event_based_t"
self.testInit.setupCouch(self.couchDBName, "GroupUser", "ACDC")
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
couchSever = CouchServer(dburl=self.couchUrl)
self.couchDB = couchSever.connectDatabase(self.couchDBName)
self.populateWMBS()
self.performanceParams = {
'timePerEvent': 12,
'memoryRequirement': 2300,
'sizePerEvent': 400
}
return
def tearDown(self):
"""
_tearDown_
Clear out WMBS.
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
return
def populateWMBS(self):
"""
_populateWMBS_
Create files and subscriptions in WMBS
"""
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="Locations.New")
locationAction.execute(siteName='s1', seName="somese.cern.ch")
locationAction.execute(siteName='s2', seName="otherse.cern.ch")
self.validLocations = ["somese.cern.ch", "otherse.cern.ch"]
self.multipleFileFileset = Fileset(name="TestFileset1")
self.multipleFileFileset.create()
parentFile = File('/parent/lfn/',
size=1000,
events=100,
locations=set(["somese.cern.ch"]))
parentFile.create()
for _ in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["somese.cern.ch"]))
newFile.create()
newFile.addParent(lfn=parentFile['lfn'])
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name="TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name",
size=1000,
events=100,
locations=set(["somese.cern.ch"]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleSiteFileset = Fileset(name="TestFileset3")
self.multipleSiteFileset.create()
for _ in range(5):
newFile = File(makeUUID(), size=1000, events=100)
newFile.setLocation("somese.cern.ch")
newFile.create()
self.multipleSiteFileset.addFile(newFile)
for _ in range(5):
newFile = File(makeUUID(), size=1000, events=100)
newFile.setLocation(["somese.cern.ch", "otherse.cern.ch"])
newFile.create()
self.multipleSiteFileset.addFile(newFile)
self.multipleSiteFileset.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(
fileset=self.multipleFileFileset,
workflow=testWorkflow,
split_algo="EventBased",
type="Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription = Subscription(
fileset=self.singleFileFileset,
workflow=testWorkflow,
split_algo="EventBased",
type="Processing")
self.singleFileSubscription.create()
self.multipleSiteSubscription = Subscription(
fileset=self.multipleSiteFileset,
workflow=testWorkflow,
split_algo="EventBased",
type="Processing")
self.multipleSiteSubscription.create()
return
def populateACDCCouch(self,
numFiles=2,
lumisPerJob=35,
eventsPerJob=20000):
"""
_populateACDCCouch_
Create production files in couchDB to test the creation
of ACDC jobs for the EventBased algorithm
"""
# Define some constants
workflowName = "ACDC_TestEventBased"
filesetName = "/%s/Production" % workflowName
owner = "*****@*****.**"
group = "unknown"
lumisPerFile = lumisPerJob * 250
for i in range(numFiles):
for j in range(250):
lfn = "MCFakeFile-some-hash-%s" % str(i).zfill(5)
acdcFile = File(lfn=lfn,
size=100,
events=eventsPerJob,
locations=self.validLocations,
merged=False,
first_event=1)
run = Run(
1,
*range(1 + (i * lumisPerFile) + j * lumisPerJob,
(j + 1) * lumisPerJob + (i * lumisPerFile) + 2))
acdcFile.addRun(run)
acdcDoc = {
"collection_name": workflowName,
"collection_type": "ACDC.CollectionTypes.DataCollection",
"files": {
lfn: acdcFile
},
"fileset_name": filesetName,
"owner": {
"user": owner,
"group": group
}
}
self.couchDB.queue(acdcDoc)
self.couchDB.commit()
return
def generateFakeMCFile(self,
numEvents=100,
firstEvent=1,
lastEvent=100,
firstLumi=1,
lastLumi=10,
index=1,
existingSub=None):
"""
_generateFakeMCFile_
Generates a fake MC file for testing production EventBased
creation of jobs, it creates a single file subscription if no
existing subscription is provided.
"""
# MC comes with MCFakeFile(s)
newFile = File("MCFakeFile-some-hash-%s" % str(index).zfill(5),
size=1000,
events=numEvents,
locations=set(["somese.cern.ch"]))
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
newFile.create()
if existingSub is None:
singleMCFileset = Fileset(name="MCTestFileset-%i" % index)
singleMCFileset.create()
singleMCFileset.addFile(newFile)
singleMCFileset.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
testWorkflow.create()
singleMCFileSubscription = Subscription(fileset=singleMCFileset,
workflow=testWorkflow,
split_algo="EventBased",
type="Production")
singleMCFileSubscription.create()
return singleMCFileSubscription
else:
existingSub['fileset'].addFile(newFile)
existingSub['fileset'].commit()
return existingSub
def testExactEvents(self):
"""
_testExactEvents_
Test event based job splitting when the number of events per job is
exactly the same as the number of events in the input file.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(events_per_job=100,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 1)
job = jobGroups[0].jobs.pop()
self.assertEqual(job.getFiles(type="lfn"), ["/some/file/name"])
self.assertEqual(job["mask"].getMaxEvents(), None)
self.assertEqual(job["mask"]["FirstEvent"], 0)
self.assertEqual(job["estimatedJobTime"], 100 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 100)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def testMoreEvents(self):
"""
_testMoreEvents_
Test event based job splitting when the number of events per job is
greater than the number of events in the input file.
Since the file has less events than the splitting, the job goes without a mask.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(events_per_job=1000,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 1)
job = jobGroups[0].jobs.pop()
self.assertEqual(job.getFiles(type="lfn"), ["/some/file/name"])
self.assertEqual(job["mask"].getMaxEvents(), None)
self.assertEqual(job["mask"]["FirstEvent"], None)
self.assertEqual(job["estimatedJobTime"], 100 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 100)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def test50EventSplit(self):
"""
_test50EventSplit_
Test event based job splitting when the number of events per job is
50, this should result in two jobs.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(events_per_job=50,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
for job in jobGroups[0].jobs:
self.assertEqual(job.getFiles(type="lfn"), ["/some/file/name"])
self.assertTrue((job["mask"].getMaxEvents() == 50 and job["mask"]["FirstEvent"] == 0) or \
(job["mask"].getMaxEvents() is None and job["mask"]["FirstEvent"] == 50))
self.assertEqual(job["estimatedJobTime"], 50 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 50)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def test99EventSplit(self):
"""
_test99EventSplit_
Test event based job splitting when the number of events per job is
99, this should result in two jobs. Last job shouldn't have a maximum
number of events, let it run until the end of the file.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(events_per_job=99,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
for job in jobGroups[0].jobs:
self.assertEqual(job.getFiles(type="lfn"), ["/some/file/name"])
if job["mask"].getMaxEvents() == 99:
self.assertEqual(job["mask"]["FirstEvent"], 0)
self.assertEqual(job["estimatedJobTime"], 99 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 99)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
elif job["mask"].getMaxEvents() is None:
self.assertEqual(job["mask"]["FirstEvent"], 99)
self.assertEqual(job["estimatedJobTime"], 1 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 1)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
else:
self.fail("Unexpected splitting was performed")
return
def test100EventMultipleFileSplit(self):
"""
_test100EventMultipleFileSplit_
Test job splitting into 100 event jobs when the input subscription has
more than one file available.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(events_per_job=100,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 10)
for job in jobGroups[0].jobs:
self.assertEqual(len(job.getFiles(type="lfn")), 1)
self.assertEqual(job["mask"].getMaxEvents(), None)
self.assertEqual(job["mask"]["FirstEvent"], 0)
self.assertEqual(job["estimatedJobTime"], 100 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 100)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def test50EventMultipleFileSplit(self):
"""
_test50EventMultipleFileSplit_
Test job splitting into 50 event jobs when the input subscription has
more than one file available.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(events_per_job=50,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 20)
for job in jobGroups[0].jobs:
self.assertEqual(len(job.getFiles(type="lfn")), 1)
if job["mask"].getMaxEvents() == 50:
self.assertEqual(job["mask"]["FirstEvent"], 0)
elif job["mask"].getMaxEvents() is None:
self.assertEqual(job["mask"]["FirstEvent"], 50)
else:
self.fail("Unexpected splitting was performed")
self.assertEqual(job["estimatedJobTime"], 50 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 50)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def test150EventMultipleFileSplit(self):
"""
_test150EventMultipleFileSplit_
Test job splitting into 150 event jobs when the input subscription has
more than one file available. This test verifies that the job splitting
code will put at most one file in a job. Since every job has less events
than the maximum. the job goes without a mask.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(events_per_job=150,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 10)
self.assertEqual(len(jobGroups[0].jobs[0].getFiles(type="lfn")), 1)
self.assertEqual(len(jobGroups[0].jobs[6].getFiles(type="lfn")), 1)
for job in jobGroups[0].jobs:
self.assertEqual(job["mask"].getMaxEvents(), None)
self.assertEqual(job["mask"]["FirstEvent"], None)
self.assertEqual(job["estimatedJobTime"], 100 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 100)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def test100EventMultipleSite(self):
"""
_test100EventMultipleSite_
Test job splitting into 100 event jobs when the input subscription has
more than one file available, at different site combinations.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleSiteSubscription)
jobGroups = jobFactory(events_per_job=100,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 2)
self.assertEqual(len(jobGroups[0].jobs), 5)
self.assertEqual(len(jobGroups[1].jobs), 5)
for job in jobGroups[0].jobs:
self.assertEqual(len(job.getFiles(type="lfn")), 1)
self.assertEqual(job["mask"].getMaxEvents(), None)
self.assertEqual(job["mask"]["FirstEvent"], 0)
self.assertEqual(job["estimatedJobTime"], 100 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 100)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def testMCEventSplitOver32bit(self):
"""
_testMCEventSplitOver32bit_
Make sure that no events will go over a 32 bit unsigned integer
representation, event counter should be reset in that case.
Also test is not over cautious.
"""
firstEvent = 3 * (2**30) + 1
singleMCSubscription = self.generateFakeMCFile(numEvents=2**30,
firstEvent=firstEvent)
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=singleMCSubscription)
jobGroups = jobFactory(events_per_job=2**30 - 1,
events_per_lumi=2**30 - 1,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1,
"Error: JobFactory did not return one JobGroup")
self.assertEqual(
len(jobGroups[0].jobs), 2,
"Error: JobFactory created %s jobs not two" %
len(jobGroups[0].jobs))
for job in jobGroups[0].jobs:
if job["mask"].getMaxEvents() == 2**30 - 1:
self.assertEqual(job["mask"]["FirstLumi"], 1)
self.assertEqual(job["mask"]["FirstEvent"], firstEvent)
self.assertTrue(job["mask"]["LastEvent"] <= 2**32)
self.assertEqual(job["estimatedJobTime"], (2**30 - 1) * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * (2**30 - 1))
self.assertEqual(job["estimatedMemoryUsage"], 2300)
elif job["mask"].getMaxEvents() == 1:
self.assertEqual(job["mask"]["FirstLumi"], 2)
self.assertEqual(job["mask"]["FirstEvent"], 1)
self.assertEqual(job["estimatedJobTime"], 1 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 1)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
else:
self.fail("Unexpected splitting was performed")
def test_addParents(self):
"""
_addParents_
Test our ability to add parents to a job
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(events_per_job=50,
include_parents=True,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 20)
for job in jobGroups[0].jobs:
self.assertEqual(len(job.getFiles(type="lfn")), 1)
self.assertTrue((job["mask"].getMaxEvents() == 50 and job["mask"]["FirstEvent"] == 0) or \
(job["mask"].getMaxEvents() is None and job["mask"]["FirstEvent"] == 50))
self.assertEqual(job["estimatedJobTime"], 50 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 50)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
for f in job['input_files']:
self.assertEqual(len(f['parents']), 1)
self.assertEqual(list(f['parents'])[0]['lfn'], '/parent/lfn/')
return
def testACDCProduction(self):
"""
_testACDCProduction_
Test the ability of the EventBased algorithm of creating
jobs from ACDC correctly
"""
self.populateACDCCouch(numFiles=4)
mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 1, 8750, 0)
mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 8751, 17500,
1, mcSubscription)
mcSubscription = self.generateFakeMCFile(20000, 1, 20001, 17501, 26250,
2, mcSubscription)
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=mcSubscription)
jobGroups = jobFactory(events_per_job=50,
collectionName="ACDC_TestEventBased",
couchURL=self.couchUrl,
couchDB=self.couchDBName,
filesetName="/ACDC_TestEventBased/Production",
owner="*****@*****.**",
group="unknown",
performance=self.performanceParams)
self.assertEqual(1, len(jobGroups))
jobGroup = jobGroups[0]
self.assertEqual(750, len(jobGroup.jobs))
for job in jobGroup.jobs:
self.assertEqual(1, len(job["input_files"]))
mask = job["mask"]
self.assertEqual(35, mask["LastLumi"] - mask["FirstLumi"])
self.assertEqual(20000, mask["LastEvent"] - mask["FirstEvent"])
self.assertFalse(mask["runAndLumis"])
self.assertEqual(job["estimatedJobTime"], 20000 * 12)
self.assertEqual(job["estimatedDiskUsage"], 400 * 20000)
self.assertEqual(job["estimatedMemoryUsage"], 2300)
return
def testB_NoFileSplitNoHardLimit(self):
"""
_testB_NoFileSplitNoHardLimit_
In this case we don't split on file boundaries, check different combination of files
make sure we make the most of the splitting, e.g. include many zero event files in
a single job.
"""
splitter = SplitterFactory()
# Create 100 files with 7 lumi per file and 0 events per lumi on average.
testSubscription = self.createSubscription(nFiles=100,
lumisPerFile=7,
twoSites=False,
nEventsPerFile=0)
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
# First test, the optimal settings are 360 events per job
# As we have files with 0 events per lumi, this will configure the splitting to
# a single job containing all files
jobGroups = jobFactory(halt_job_on_file_boundaries=False,
splitOnRun=False,
events_per_job=360,
performance=self.performanceParams)
self.assertEqual(
len(jobGroups), 0,
"There aren't enough events, so it should have 0 job groups")
# we close this fileset to get it moving
fileset = testSubscription.getFileset()
fileset.markOpen(False)
jobGroups = jobFactory(halt_job_on_file_boundaries=False,
splitOnRun=False,
events_per_job=360,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1,
"There should be only one job group")
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 1, "There should be 1 job")
self.assertEqual(len(jobs[0]['input_files']), 100,
"All 100 files must be in the job")
self.assertEqual(jobs[0]['estimatedMemoryUsage'], 2300)
self.assertEqual(jobs[0]['estimatedDiskUsage'], 0)
self.assertEqual(jobs[0]['estimatedJobTime'], 0)
# Create 7 files, each one with different lumi/event distributions
testFileset = Fileset(name="FilesetA")
testFileset.create()
testFileA = self.createFile("/this/is/file1", 250, 0, 5, "T2_CH_CERN")
testFileB = self.createFile("/this/is/file2", 600, 1, 1, "T2_CH_CERN")
testFileC = self.createFile("/this/is/file3", 1200, 2, 2, "T2_CH_CERN")
testFileD = self.createFile("/this/is/file4", 100, 3, 1, "T2_CH_CERN")
testFileE = self.createFile("/this/is/file5", 30, 4, 1, "T2_CH_CERN")
testFileF = self.createFile("/this/is/file6", 10, 5, 1, "T2_CH_CERN")
testFileG = self.createFile("/this/is/file7", 151, 6, 3, "T2_CH_CERN")
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.addFile(testFileC)
testFileset.addFile(testFileD)
testFileset.addFile(testFileE)
testFileset.addFile(testFileF)
testFileset.addFile(testFileG)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiBased",
type="Processing")
testSubscription.create()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
# Optimal settings are: jobs with 150 events per job
# This means, the first file must be splitted in 3 lumis per job which would leave room
# for another lumi in the second job, but the second file has a lumi too big for that
# The 3rd job only contains the second file, the fourth and fifth job split the third file
jobGroups = jobFactory(halt_job_on_file_boundaries=False,
splitOnRun=False,
events_per_job=150,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1,
"There should be only one job group")
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 7, "7 jobs must be in the jobgroup")
self.assertEqual(jobs[0]["mask"].getRunAndLumis(), {0: [[0, 2]]},
"Wrong mask for the first job")
self.assertEqual(jobs[0]["estimatedJobTime"], 150 * 12)
self.assertEqual(jobs[0]["estimatedDiskUsage"], 150 * 400)
self.assertEqual(jobs[1]["mask"].getRunAndLumis(), {0: [[3, 4]]},
"Wrong mask for the second job")
self.assertEqual(jobs[1]["estimatedJobTime"], 100 * 12)
self.assertEqual(jobs[1]["estimatedDiskUsage"], 100 * 400)
self.assertEqual(jobs[2]["mask"].getRunAndLumis(), {1: [[1, 1]]},
"Wrong mask for the third job")
self.assertEqual(jobs[2]["estimatedJobTime"], 600 * 12)
self.assertEqual(jobs[2]["estimatedDiskUsage"], 600 * 400)
self.assertEqual(jobs[3]["mask"].getRunAndLumis(), {2: [[4, 4]]},
"Wrong mask for the fourth job")
self.assertEqual(jobs[3]["estimatedJobTime"], 600 * 12)
self.assertEqual(jobs[3]["estimatedDiskUsage"], 600 * 400)
self.assertEqual(jobs[4]["mask"].getRunAndLumis(), {2: [[5, 5]]},
"Wrong mask for the fifth job")
self.assertEqual(jobs[4]["estimatedJobTime"], 600 * 12)
self.assertEqual(jobs[4]["estimatedDiskUsage"], 600 * 400)
self.assertEqual(jobs[5]["mask"].getRunAndLumis(), {
3: [[3, 3]],
4: [[4, 4]],
5: [[5, 5]]
}, "Wrong mask for the sixth job")
self.assertEqual(jobs[5]["estimatedJobTime"], 140 * 12)
self.assertEqual(jobs[5]["estimatedDiskUsage"], 140 * 400)
self.assertEqual(jobs[6]["mask"].getRunAndLumis(), {6: [[18, 20]]},
"Wrong mask for the seventh job")
self.assertEqual(jobs[6]["estimatedJobTime"], 150 * 12)
self.assertEqual(jobs[6]["estimatedDiskUsage"], 150 * 400)
for job in jobs:
self.assertEqual(job["estimatedMemoryUsage"], 2300)
# Test interactions of this algorithm with splitOnRun = True
# Make 2 files, one with 3 runs and a second one with the last run of the first
fileA = File(lfn="/this/is/file1a", size=1000, events=2400)
lumiListA = []
lumiListB = []
lumiListC = []
for lumi in range(8):
lumiListA.append(1 + lumi)
lumiListB.append(1 + lumi)
lumiListC.append(1 + lumi)
fileA.addRun(Run(1, *lumiListA))
fileA.addRun(Run(2, *lumiListA))
fileA.addRun(Run(3, *lumiListA))
fileA.setLocation("T1_US_FNAL_Disk")
fileB = self.createFile('/this/is/file2a', 200, 3, 5,
"T1_US_FNAL_Disk")
testFileset = Fileset(name='FilesetB')
testFileset.create()
testFileset.addFile(fileA)
testFileset.addFile(fileB)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiBased",
type="Processing")
testSubscription.create()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
# The settings for this splitting are 700 events per job
jobGroups = jobFactory(splitOnRun=True,
halt_job_on_file_boundaries=False,
events_per_job=700,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1,
"There should be only one job group")
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 6, "Six jobs must be in the jobgroup")
self.assertEqual(jobs[0]["estimatedJobTime"], 700 * 12)
self.assertEqual(jobs[0]["estimatedDiskUsage"], 700 * 400)
self.assertEqual(jobs[1]["estimatedJobTime"], 100 * 12)
self.assertEqual(jobs[1]["estimatedDiskUsage"], 100 * 400)
self.assertEqual(jobs[2]["estimatedJobTime"], 700 * 12)
self.assertEqual(jobs[2]["estimatedDiskUsage"], 700 * 400)
self.assertEqual(jobs[3]["estimatedJobTime"], 100 * 12)
self.assertEqual(jobs[3]["estimatedDiskUsage"], 100 * 400)
self.assertEqual(jobs[4]["estimatedJobTime"], 700 * 12)
self.assertEqual(jobs[4]["estimatedDiskUsage"], 700 * 400)
self.assertEqual(jobs[5]["estimatedJobTime"], 300 * 12)
self.assertEqual(jobs[5]["estimatedDiskUsage"], 300 * 400)
def test_AutoIncrementCheck(self):
"""
_AutoIncrementCheck_
Test and see whether we can find and set the auto_increment values
"""
myThread = threading.currentThread()
if not myThread.dialect.lower() == 'mysql':
return
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
incrementDAO = self.daoFactory(classname="Jobs.AutoIncrementCheck")
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 1)
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 2)
incrementDAO.execute(input=10)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 11)
incrementDAO.execute(input=5)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 12)
return
def createTestJobGroup(self,
name="TestWorkthrough",
specLocation="spec.xml",
error=False,
task="/TestWorkload/ReReco",
nJobs=10):
"""
_createTestJobGroup_
Generate a test WMBS JobGroup with real FWJRs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=specLocation,
owner="Simon",
name=name,
task=task)
testWorkflow.create()
testWMBSFileset = Fileset(name=name)
testWMBSFileset.create()
testFileA = File(lfn=makeUUID(), size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn=makeUUID(), size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
report = Report()
if error:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"badBackfillJobReport.pkl")
else:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"PerformanceReport2.pkl")
report.load(filename=path)
self.changeState.propagate(testJobGroup.jobs, 'created', 'new')
self.changeState.propagate(testJobGroup.jobs, 'executing', 'created')
self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing')
for job in testJobGroup.jobs:
job['fwjr'] = report
self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed')
self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
class ReportIntegrationTest(unittest.TestCase):
"""
_ReportIntegrationTest_
"""
def setUp(self):
"""
_setUp_
Setup the database and WMBS for the test.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer",
"WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", pnn = "T1_US_FNAL_Disk")
inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10,
locations = "T1_US_FNAL_Disk")
inputFile.create()
inputFileset = Fileset(name = "InputFileset")
inputFileset.create()
inputFileset.addFile(inputFile)
inputFileset.commit()
unmergedFileset = Fileset(name = "UnmergedFileset")
unmergedFileset.create()
mergedFileset = Fileset(name = "MergedFileset")
mergedFileset.create()
procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve",
name = "TestWF", task = "/TestWF/None")
procWorkflow.create()
procWorkflow.addOutput("outputRECORECO", unmergedFileset)
mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve",
name = "MergeWF", task = "/MergeWF/None")
mergeWorkflow.create()
mergeWorkflow.addOutput("Merged", mergedFileset)
insertWorkflow = self.dbsfactory(classname = "InsertWorkflow")
insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0)
insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0)
self.procSubscription = Subscription(fileset = inputFileset,
workflow = procWorkflow,
split_algo = "FileBased",
type = "Processing")
self.procSubscription.create()
self.procSubscription.acquireFiles()
self.mergeSubscription = Subscription(fileset = unmergedFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize",
type = "Merge")
self.mergeSubscription.create()
self.procJobGroup = JobGroup(subscription = self.procSubscription)
self.procJobGroup.create()
self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription)
self.mergeJobGroup.create()
self.testJob = Job(name = "testJob", files = [inputFile])
self.testJob.create(group = self.procJobGroup)
self.testJob["state"] = "complete"
myThread = threading.currentThread()
self.daofactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState")
self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
locationAction = self.daofactory(classname = "Locations.New")
locationAction.execute(siteName = "cmssrm.fnal.gov")
self.stateChangeAction.execute(jobs = [self.testJob])
self.tempDir = tempfile.mkdtemp()
return
def tearDown(self):
"""
_tearDown_
Clear out the database and the pickled report file.
"""
self.testInit.clearDatabase()
try:
os.remove(os.path.join(self.tempDir, "ProcReport.pkl"))
os.remove(os.path.join(self.tempDir, "MergeReport.pkl"))
except Exception as ex:
pass
try:
os.rmdir(self.tempDir)
except Exception as ex:
pass
return
def createConfig(self, workerThreads):
"""
_createConfig_
Create a config for the JobAccountant with the given number of worker
threads. This config needs to include information for connecting to the
database as the component will create it's own database connections.
These parameters are still pulled from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "report_integration_t"
config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.workerThreads = workerThreads
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
return config
def verifyJobSuccess(self, jobID):
"""
_verifyJobSuccess_
Verify that the metadata for a successful job is correct. This will
check the outcome, retry count and state.
"""
testJob = Job(id = jobID)
testJob.load()
assert testJob["state"] == "success", \
"Error: test job in wrong state: %s" % testJob["state"]
assert testJob["retry_count"] == 0, \
"Error: test job has wrong retry count: %s" % testJob["retry_count"]
assert testJob["outcome"] == "success", \
"Error: test job has wrong outcome: %s" % testJob["outcome"]
return
def verifyFileMetaData(self, jobID, fwkJobReportFiles):
"""
_verifyFileMetaData_
Verify that all the files that were output by a job made it into WMBS
correctly. Compare the contents of WMBS to the files in the frameworks
job report.
Note that fwkJobReportFiles is a list of DataStructs File objects.
"""
testJob = Job(id = jobID)
testJob.loadData()
inputLFNs = []
for inputFile in testJob["input_files"]:
inputLFNs.append(inputFile["lfn"])
for fwkJobReportFile in fwkJobReportFiles:
outputFile = File(lfn = fwkJobReportFile["lfn"])
outputFile.loadData(parentage = 1)
assert outputFile["events"] == int(fwkJobReportFile["events"]), \
"Error: Output file has wrong events: %s, %s" % \
(outputFile["events"], fwkJobReportFile["events"])
assert outputFile["size"] == int(fwkJobReportFile["size"]), \
"Error: Output file has wrong size: %s, %s" % \
(outputFile["size"], fwkJobReportFile["size"])
for ckType in fwkJobReportFile["checksums"]:
assert ckType in outputFile["checksums"], \
"Error: Output file is missing checksums: %s" % ckType
assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \
"Error: Checksums don't match."
assert len(fwkJobReportFile["checksums"]) == \
len(outputFile["checksums"]), \
"Error: Wrong number of checksums."
jobType = self.getJobTypeAction.execute(jobID = jobID)
if jobType == "Merge":
assert str(outputFile["merged"]) == "True", \
"Error: Merge jobs should output merged files."
else:
assert outputFile["merged"] == fwkJobReportFile["merged"], \
"Error: Output file merged output is wrong: %s, %s" % \
(outputFile["merged"], fwkJobReportFile["merged"])
assert len(outputFile["locations"]) == 1, \
"Error: outputfile should have one location: %s" % outputFile["locations"]
assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \
"Error: wrong location for file."
assert len(outputFile["parents"]) == len(inputLFNs), \
"Error: Output file has wrong number of parents."
for outputParent in outputFile["parents"]:
assert outputParent["lfn"] in inputLFNs, \
"Error: Unknown parent file: %s" % outputParent["lfn"]
fwjrRuns = {}
for run in fwkJobReportFile["runs"]:
fwjrRuns[run.run] = run.lumis
for run in outputFile["runs"]:
assert run.run in fwjrRuns, \
"Error: Extra run in output: %s" % run.run
for lumi in run:
assert lumi in fwjrRuns[run.run], \
"Error: Extra lumi: %s" % lumi
fwjrRuns[run.run].remove(lumi)
if len(fwjrRuns[run.run]) == 0:
del fwjrRuns[run.run]
assert len(fwjrRuns) == 0, \
"Error: Missing runs, lumis: %s" % fwjrRuns
testJobGroup = JobGroup(id = testJob["jobgroup"])
testJobGroup.loadData()
jobGroupFileset = testJobGroup.output
jobGroupFileset.loadData()
assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \
"Error: output file not in jobgroup fileset."
if testJob["mask"]["FirstEvent"] == None:
assert outputFile["first_event"] == 0, \
"Error: first event not set correctly: 0, %s" % \
outputFile["first_event"]
else:
assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \
"Error: last event not set correctly: %s, %s" % \
(testJob["mask"]["FirstEvent"], outputFile["first_event"])
return
def testReportHandling(self):
"""
_testReportHandling_
Verify that we're able to parse a CMSSW report, convert it to a Report()
style report, pickle it and then have the accountant process it.
"""
self.procPath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(self.procPath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl")
cmsRunStep = myReport.retrieveStep("cmsRun1")
cmsRunStep.status = 0
myReport.setTaskName('/TestWF/None')
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath)
pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000)
pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
pFile.setDatasetPath("/bogus/dataset/path")
#pFile.addRun(Run(1, *[45]))
pFile.create()
config = self.createConfig(workerThreads = 1)
accountant = JobAccountantPoller(config)
accountant.setup()
accountant.algorithm()
self.verifyJobSuccess(self.testJob["id"])
self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1"))
inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root")
inputFile.load()
self.testMergeJob = Job(name = "testMergeJob", files = [inputFile])
self.testMergeJob.create(group = self.mergeJobGroup)
self.testMergeJob["state"] = "complete"
self.stateChangeAction.execute(jobs = [self.testMergeJob])
self.mergePath = os.path.join(WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml")
myReport = Report("mergeReco")
myReport.parse(self.mergePath)
# Fake some metadata that should be added by the stageout scripts.
for fileRef in myReport.getAllFileRefsFromStep("mergeReco"):
fileRef.size = 1024
fileRef.location = "cmssrm.fnal.gov"
fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1",
"primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1",
"dataTier": "RECO"}
fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl")
myReport.setTaskName('/MergeWF/None')
cmsRunStep = myReport.retrieveStep("mergeReco")
cmsRunStep.status = 0
myReport.persist(fwjrPath)
self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath)
accountant.algorithm()
self.verifyJobSuccess(self.testMergeJob["id"])
self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco"))
return
def testMask(self):
"""
_testMask_
Test the new mask setup
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job()
testJob['mask'].addRunAndLumis(run=100, lumis=[101, 102])
testJob['mask'].addRunAndLumis(run=200, lumis=[201, 202])
testJob.create(group=testJobGroup)
loadJob = Job(id=testJob.exists())
loadJob.loadData()
runs = loadJob['mask'].getRunAndLumis()
self.assertEqual(len(runs), 2)
self.assertEqual(runs[100], [[101, 102]])
self.assertEqual(runs[200], [[201, 202]])
bigRun = Run(100, *[101, 102, 103, 104])
badRun = Run(300, *[1001, 1002])
result = loadJob['mask'].filterRunLumisByMask([bigRun, badRun])
self.assertEqual(len(result), 1)
alteredRun = result.pop()
self.assertEqual(alteredRun.run, 100)
self.assertEqual(alteredRun.lumis, [101, 102])
run0 = Run(300, *[1001, 1002])
run1 = Run(300, *[1001, 1002])
loadJob['mask'].filterRunLumisByMask([run0, run1])
return
class EventBasedTest(unittest.TestCase):
"""
_EventBasedTest_
Test event based job splitting.
"""
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="Locations.New")
locationAction.execute(siteName="site1", pnn="T2_CH_CERN")
self.multipleFileFileset = Fileset(name="TestFileset1")
self.multipleFileFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations="T2_CH_CERN")
newFile.addRun(Run(i, *[45 + i]))
newFile.create()
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name="TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name",
size=1000,
events=100,
locations="T2_CH_CERN")
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleFileRunset = Fileset(name="TestFileset3")
self.multipleFileRunset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations="T2_CH_CERN")
newFile.addRun(Run(i / 3, *[45]))
newFile.create()
self.multipleFileRunset.addFile(newFile)
self.multipleFileRunset.commit()
self.singleRunFileset = Fileset(name="TestFileset4")
self.singleRunFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations="T2_CH_CERN")
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleRunFileset.addFile(newFile)
self.singleRunFileset.commit()
self.singleRunMultipleLumi = Fileset(name="TestFileset5")
self.singleRunMultipleLumi.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations="T2_CH_CERN")
newFile.addRun(Run(1, *[45 + i]))
newFile.create()
self.singleRunMultipleLumi.addFile(newFile)
self.singleRunMultipleLumi.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="mnorman",
name="wf001",
task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(
fileset=self.multipleFileFileset,
workflow=testWorkflow,
split_algo="RunBased",
type="Processing")
self.singleFileSubscription = Subscription(
fileset=self.singleFileFileset,
workflow=testWorkflow,
split_algo="RunBased",
type="Processing")
self.multipleRunSubscription = Subscription(
fileset=self.multipleFileRunset,
workflow=testWorkflow,
split_algo="RunBased",
type="Processing")
self.singleRunSubscription = Subscription(
fileset=self.singleRunFileset,
workflow=testWorkflow,
split_algo="RunBased",
type="Processing")
self.singleRunMultipleLumiSubscription = Subscription(
fileset=self.singleRunMultipleLumi,
workflow=testWorkflow,
split_algo="RunBased",
type="Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription.create()
self.multipleRunSubscription.create()
self.singleRunSubscription.create()
self.singleRunMultipleLumiSubscription.create()
return
def tearDown(self):
"""
_tearDown_
Tear down WMBS architechture.
"""
self.testInit.clearDatabase()
return
def testExactRuns(self):
"""
_testExactRuns_
Test run based job splitting when the number of events per job is
exactly the same as the number of events in the input file.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(files_per_job=1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
return
def testMoreRuns(self):
"""
_testMoreEvents_
Test run based job splitting when the number of runs per job is
greater than the number of runs in the input file.
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleFileSubscription)
jobGroups = jobFactory(files_per_job=2)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
return
def testMultipleRuns(self):
"""
_testMultipleRuns_
Test run based job splitting when the number of runs is
equal to the number in each input file, with multiple files
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(files_per_job=1)
assert len(jobGroups) == 10, \
"ERROR: JobFactory didn't return one JobGroup per run."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't put each run in a file."
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 1)
return
def testMultipleRunsCombine(self):
"""
_testMultipleRunsCombine_
Test run based job splitting when the number of jobs is
less then the number of files, with multiple files
"""
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleRunSubscription)
jobGroups = jobFactory(files_per_job=2)
assert len(jobGroups) == 4, \
"ERROR: JobFactory didn't return one JobGroup per run."
assert len(jobGroups[1].jobs) == 2, \
"ERROR: JobFactory didn't put only one job in the first job"
#Last one in the queue should have one job, previous two (three files per run)
self.assertEqual(len(jobGroups[1].jobs.pop().getFiles(type="lfn")), 1)
self.assertEqual(len(jobGroups[1].jobs.pop().getFiles(type="lfn")), 2)
return
def testSingleRunsCombineUneven(self):
"""
_testSingleRunsCombineUneven_
Test run based job splitting when the number of jobs is
less then and indivisible by the number of files, with multiple files.
"""
#This should return two jobs, one with 8 and one with 2 files
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleRunSubscription)
jobGroups = jobFactory(files_per_job=8)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 8)
return
def testPersistSingleRunsCombineUneven(self):
"""
_testPerisistSingleRunsCombineUneven_
Test run based job splitting when the number of jobs is
less then and indivisible by the number of files, with multiple files.
"""
#This should return two jobs, one with 8 and one with 2 files
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.singleRunSubscription)
jobGroups = jobFactory(files_per_job=8)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 8)
return
def testSingleRunsMultipleLumiCombineUneven(self):
"""
_testSingleRunsMultipeLumiCombineUneven_
Test run based job splitting when the number of jobs is
less then and indivisible by the number of files, with multiple files.
"""
#This should return two jobs, one with 8 and one with 2 files
splitter = SplitterFactory()
jobFactory = splitter(
package="WMCore.WMBS",
subscription=self.singleRunMultipleLumiSubscription)
jobGroups = jobFactory(files_per_job=8)
self.assertEqual(len(jobGroups), 1)
self.assertEqual(len(jobGroups[0].jobs), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 2)
self.assertEqual(len(jobGroups[0].jobs.pop().getFiles(type="lfn")), 8)
return
class ParentlessMergeBySizeTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Boiler plate DB setup.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
return
def tearDown(self):
"""
_tearDown_
Clear out WMBS.
"""
self.testInit.clearDatabase()
return
def stuffWMBS(self):
"""
_stuffWMBS_
Insert some dummy jobs, jobgroups, filesets, files and subscriptions
into WMBS to test job creation. Three completed job groups each
containing several files are injected. Another incomplete job group is
also injected. Also files are added to the "Mergeable" subscription as
well as to the output fileset for their jobgroups.
"""
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="s1", seName="somese.cern.ch")
locationAction.execute(siteName="s1", seName="somese2.cern.ch")
changeStateDAO = self.daoFactory(classname="Jobs.ChangeState")
self.mergeFileset = Fileset(name="mergeFileset")
self.mergeFileset.create()
self.bogusFileset = Fileset(name="bogusFileset")
self.bogusFileset.create()
mergeWorkflow = Workflow(name="mergeWorkflow",
spec="bunk2",
owner="Steve",
task="Test")
mergeWorkflow.create()
markWorkflow = self.daoFactory(
classname="Workflow.MarkInjectedWorkflows")
markWorkflow.execute(names=[mergeWorkflow.name], injected=True)
self.mergeSubscription = Subscription(
fileset=self.mergeFileset,
workflow=mergeWorkflow,
split_algo="ParentlessMergeBySize")
self.mergeSubscription.create()
self.bogusSubscription = Subscription(
fileset=self.bogusFileset,
workflow=mergeWorkflow,
split_algo="ParentlessMergeBySize")
file1 = File(lfn="file1",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
file1.addRun(Run(1, *[45]))
file1.create()
file2 = File(lfn="file2",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
file2.addRun(Run(1, *[45]))
file2.create()
file3 = File(lfn="file3",
size=1024,
events=1024,
first_event=2048,
locations=set(["somese.cern.ch"]))
file3.addRun(Run(1, *[45]))
file3.create()
file4 = File(lfn="file4",
size=1024,
events=1024,
first_event=3072,
locations=set(["somese.cern.ch"]))
file4.addRun(Run(1, *[45]))
file4.create()
fileA = File(lfn="fileA",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
fileA.addRun(Run(1, *[46]))
fileA.create()
fileB = File(lfn="fileB",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
fileB.addRun(Run(1, *[46]))
fileB.create()
fileC = File(lfn="fileC",
size=1024,
events=1024,
first_event=2048,
locations=set(["somese.cern.ch"]))
fileC.addRun(Run(1, *[46]))
fileC.create()
fileI = File(lfn="fileI",
size=1024,
events=1024,
first_event=0,
locations=set(["somese.cern.ch"]))
fileI.addRun(Run(2, *[46]))
fileI.create()
fileII = File(lfn="fileII",
size=1024,
events=1024,
first_event=1024,
locations=set(["somese.cern.ch"]))
fileII.addRun(Run(2, *[46]))
fileII.create()
fileIII = File(lfn="fileIII",
size=1024,
events=102400,
first_event=2048,
locations=set(["somese.cern.ch"]))
fileIII.addRun(Run(2, *[46]))
fileIII.create()
fileIV = File(lfn="fileIV",
size=102400,
events=1024,
first_event=3072,
locations=set(["somese.cern.ch"]))
fileIV.addRun(Run(2, *[46]))
fileIV.create()
for file in [
file1, file2, file3, file4, fileA, fileB, fileC, fileI, fileII,
fileIII, fileIV
]:
self.mergeFileset.addFile(file)
self.bogusFileset.addFile(file)
self.mergeFileset.commit()
self.bogusFileset.commit()
return
def testMinMergeSize1(self):
"""
_testMinMergeSize1_
Set the minimum merge size to be 20,000 bytes which is more than the
sum of all file sizes in the WMBS instance. Verify that no merge jobs
will be produced.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=200000,
max_merge_size=2000000000,
max_merge_events=200000000)
assert len(result) == 0, \
"ERROR: No job groups should be returned."
return
def testMinMergeSize1(self):
"""
_testMinMergeSize1_
Set the minimum merge size to be 20,000 bytes which is more than the
sum of all file sizes in the WMBS instance. Verify that no merge jobs
will be produced.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=200000,
max_merge_size=2000000000,
max_merge_events=200000000)
assert len(result) == 0, \
"ERROR: No job groups should be returned."
return
def testMinMergeSize1a(self):
"""
_testMinMergeSize1a_
Set the minimum merge size to be 20,000 bytes which is more than the
sum of all file sizes in the WMBS instance and mark the fileset as
closed. Verify that one job containing all files is pushed out.
"""
self.stuffWMBS()
self.mergeFileset.markOpen(False)
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=200000,
max_merge_size=2000000,
max_merge_events=2000000)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned: %s" % len(result)
assert len(result[0].jobs) == 1, \
"Error: One job should have been returned: %s" % len(result[0].jobs)
goldenFiles = [
"file1", "file2", "file3", "file4", "fileA", "fileB", "fileC",
"fileI", "fileII", "fileIII", "fileIV"
]
jobFiles = result[0].jobs[0].getFiles()
currentRun = 0
currentLumi = 0
currentEvent = 0
for file in jobFiles:
file.loadData()
assert file["lfn"] in goldenFiles, \
"Error: Unknown file: %s" % file["lfn"]
self.assertTrue(
file["locations"] == set(["somese.cern.ch",
"somese2.cern.ch"]),
"Error: File is missing a location.")
goldenFiles.remove(file["lfn"])
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
assert fileRun >= currentRun, \
"ERROR: Files not sorted by run."
if fileRun == currentRun:
assert fileLumi >= currentLumi, \
"ERROR: Files not ordered by lumi"
if fileLumi == currentLumi:
assert fileEvent >= currentEvent, \
"ERROR: Files not ordered by first event"
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
return
def testMaxMergeSize(self):
"""
_testMaxMergeSize_
Set the maximum merge size to be 100000 bytes. Verify that two merge
jobs are created, one for the one large file and another for the rest of
the files. Verify that each merge job contains the expected files and
that we merge across runs.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=1,
max_merge_size=100000,
max_merge_events=200000)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned: %s" % result
assert len(result[0].jobs) == 2, \
"ERROR: Two jobs should have been returned."
goldenFilesA = [
"file1", "file2", "file3", "file4", "fileA", "fileB", "fileC",
"fileI", "fileII", "fileIII"
]
goldenFilesB = ["fileIV"]
for job in result[0].jobs:
jobFiles = job.getFiles()
if jobFiles[0]["lfn"] in goldenFilesA:
goldenFiles = goldenFilesA
elif jobFiles[0]["lfn"] in goldenFilesB:
goldenFiles = goldenFilesB
currentRun = 0
currentLumi = 0
currentEvent = 0
for file in jobFiles:
assert file["lfn"] in goldenFiles, \
"Error: Unknown file in merge jobs."
assert file["locations"] == set(["somese.cern.ch"]), \
"Error: File is missing a location."
goldenFiles.remove(file["lfn"])
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
assert fileRun >= currentRun, \
"ERROR: Files not sorted by run."
if fileRun == currentRun:
assert fileLumi >= currentLumi, \
"ERROR: Files not ordered by lumi"
if fileLumi == currentLumi:
assert fileEvent >= currentEvent, \
"ERROR: Files not ordered by first event"
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0, \
"ERROR: Files missing from merge jobs."
return
def testMaxEvents(self):
"""
_testMaxEvents_
Verify the the max_merge_events parameter works and that we correctly
merge across runs.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=1,
max_merge_size=20000000,
max_merge_events=100000)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned: %s" % result
assert len(result[0].jobs) == 2, \
"ERROR: Two jobs should have been returned: %s" % len(result[0].jobs)
goldenFilesA = [
"file1", "file2", "file3", "file4", "fileA", "fileB", "fileC",
"fileI", "fileII", "fileIV"
]
goldenFilesB = ["fileIII"]
for job in result[0].jobs:
jobFiles = job.getFiles()
if jobFiles[0]["lfn"] in goldenFilesA:
goldenFiles = goldenFilesA
elif jobFiles[0]["lfn"] in goldenFilesB:
goldenFiles = goldenFilesB
currentRun = 0
currentLumi = 0
currentEvent = 0
for file in jobFiles:
assert file["lfn"] in goldenFiles, \
"Error: Unknown file in merge jobs."
assert file["locations"] == set(["somese.cern.ch"]), \
"Error: File is missing a location: %s" % file["locations"]
goldenFiles.remove(file["lfn"])
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
assert fileRun >= currentRun, \
"ERROR: Files not sorted by run: %s, %s" % (fileRun, currentRun)
if fileRun == currentRun:
assert fileLumi >= currentLumi, \
"ERROR: Files not ordered by lumi"
if fileLumi == currentLumi:
assert fileEvent >= currentEvent, \
"ERROR: Files not ordered by first event"
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
assert len(goldenFilesA) == 0 and len(goldenFilesB) == 0 and \
"ERROR: Files missing from merge jobs."
return
def testMinMergeSize1aNoRunMerge(self):
"""
_testMinMergeSize1aNoRunMerge_
Set the minimum merge size to be 20,000 bytes which is more than the
sum of all file sizes in the WMBS instance and mark the fileset as
closed. Verify that two jobs are pushed out and that we don't merge
accross run boundaries.
"""
self.stuffWMBS()
self.mergeFileset.markOpen(False)
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=200000,
max_merge_size=2000000,
max_merge_events=2000000,
merge_across_runs=False)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned: %s" % len(result)
assert len(result[0].jobs) == 2, \
"Error: Two jobs should have been returned: %s" % len(result[0].jobs)
goldenFilesA = [
"file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"
]
goldenFilesB = ["fileI", "fileII", "fileIII", "fileIV"]
goldenFilesA.sort()
goldenFilesB.sort()
for job in result[0].jobs:
currentRun = 0
currentLumi = 0
currentEvent = 0
jobLFNs = []
for file in job.getFiles():
file.loadData()
jobLFNs.append(file["lfn"])
self.assertTrue(
file["locations"] == set(
["somese.cern.ch", "somese2.cern.ch"]),
"Error: File is missing a location.")
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
assert fileRun >= currentRun, \
"ERROR: Files not sorted by run."
if fileRun == currentRun:
assert fileLumi >= currentLumi, \
"ERROR: Files not ordered by lumi"
if fileLumi == currentLumi:
assert fileEvent >= currentEvent, \
"ERROR: Files not ordered by first event"
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
jobLFNs.sort()
if jobLFNs == goldenFilesA:
goldenFilesA = []
else:
self.assertEqual(jobLFNs, goldenFilesB,
"Error: LFNs do not match.")
goldenFilesB = []
return
def testMaxMergeSizeNoRunMerge(self):
"""
_testMaxMergeSizeNoRunMerge_
Set the maximum merge size to be 100000 bytes. Verify that two merge
jobs are created, one for the one large file and another for the rest of
the files. Verify that each merge job contains the expected files and
that we don't merge across run boundaries.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=1,
max_merge_size=100000,
max_merge_events=200000,
merge_across_runs=False)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned: %s" % result
assert len(result[0].jobs) == 3, \
"ERROR: Three jobs should have been returned."
goldenFilesA = [
"file1", "file2", "file3", "file4", "fileA", "fileB", "fileC"
]
goldenFilesB = ["fileI", "fileII", "fileIII"]
goldenFilesC = ["fileIV"]
for job in result[0].jobs:
jobFiles = job.getFiles()
if jobFiles[0]["lfn"] in goldenFilesA:
goldenFiles = goldenFilesA
elif jobFiles[0]["lfn"] in goldenFilesB:
goldenFiles = goldenFilesB
else:
goldenFiles = goldenFilesC
currentRun = 0
currentLumi = 0
currentEvent = 0
for file in jobFiles:
self.assertTrue(file["lfn"] in goldenFiles,
"Error: Unknown file in merge jobs.")
self.assertTrue(file["locations"] == set(["somese.cern.ch"]),
"Error: File is missing a location.")
goldenFiles.remove(file["lfn"])
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
self.assertTrue(fileRun >= currentRun,
"ERROR: Files not sorted by run.")
if fileRun == currentRun:
self.assertTrue(fileLumi >= currentLumi,
"ERROR: Files not ordered by lumi")
if fileLumi == currentLumi:
self.assertTrue(fileEvent >= currentEvent,
"ERROR: Files not ordered by first event")
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
self.assertTrue(
len(goldenFilesA) == 0 and len(goldenFilesB) == 0,
"ERROR: Files missing from merge jobs.")
return
def testMaxEventsNoRunMerge(self):
"""
_testMaxEventsNoRunMerge_
Verify that the max events merge parameter works correctly and that we
don't merge accross run boundaries.
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=1,
max_merge_size=20000000,
max_merge_events=100000,
merge_across_runs=False)
self.assertTrue(
len(result) == 1,
"ERROR: More than one JobGroup returned: %s" % result)
self.assertTrue(
len(result[0].jobs) == 3,
"ERROR: Three jobs should have been returned: %s" %
len(result[0].jobs))
goldenFilesA = [
"file1",
"file2",
"file3",
"file4",
"fileA",
"fileB",
"fileC",
]
goldenFilesB = ["fileI", "fileII", "fileIV"]
goldenFilesC = ["fileIII"]
for job in result[0].jobs:
jobFiles = job.getFiles()
if jobFiles[0]["lfn"] in goldenFilesA:
goldenFiles = goldenFilesA
elif jobFiles[0]["lfn"] in goldenFilesB:
goldenFiles = goldenFilesB
else:
goldenFiles = goldenFilesC
currentRun = 0
currentLumi = 0
currentEvent = 0
for file in jobFiles:
self.assertTrue(file["lfn"] in goldenFiles,
"Error: Unknown file in merge jobs.")
self.assertTrue(
file["locations"] == set(["somese.cern.ch"]),
"Error: File is missing a location: %s" %
file["locations"])
goldenFiles.remove(file["lfn"])
fileRun = list(file["runs"])[0].run
fileLumi = min(list(file["runs"])[0])
fileEvent = file["first_event"]
if currentRun == 0:
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
continue
self.assertTrue(
fileRun >= currentRun,
"ERROR: Files not sorted by run: %s, %s" %
(fileRun, currentRun))
if fileRun == currentRun:
self.assertTrue(fileLumi >= currentLumi,
"ERROR: Files not ordered by lumi")
if fileLumi == currentLumi:
self.assertTrue(
fileEvent >= currentEvent,
"ERROR: Files not ordered by first event")
currentRun = fileRun
currentLumi = fileLumi
currentEvent = fileEvent
self.assertTrue(
len(goldenFilesA) == 0 and len(goldenFilesB) == 0
and len(goldenFilesC) == 0,
"ERROR: Files missing from merge jobs.")
return
def testLocationMerging(self):
"""
_testLocationMerging_
Verify that files residing on different SEs are not merged together in
the same job.
"""
self.stuffWMBS()
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="s2", seName="somese2.cern.ch")
fileSite2 = File(lfn="fileSite2",
size=4098,
events=1024,
first_event=0,
locations=set(["somese2.cern.ch"]))
fileSite2.addRun(Run(1, *[46]))
fileSite2.create()
self.mergeFileset.addFile(fileSite2)
self.mergeFileset.commit()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=4097,
max_merge_size=99999999,
max_merge_events=999999999,
merge_across_runs=False)
assert len(result) == 1, \
"ERROR: More than one JobGroup returned."
assert len(result[0].jobs) == 3, \
"ERROR: Three jobs should have been returned."
for job in result[0].jobs:
firstInputFile = job.getFiles()[0]
baseLocation = list(firstInputFile["locations"])[0]
for inputFile in job.getFiles():
assert len(inputFile["locations"]) == 1, \
"Error: Wrong number of locations"
assert list(inputFile["locations"])[0] == baseLocation, \
"Error: Wrong location."
return
def testMaxWaitTime(self):
"""
_testMaxWaitTime_
Set the max wait times to be negative - this should force all files to merge
out immediately
Using the first setup as the first merge test which should normally produce
no jobGroups
"""
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=200000,
max_merge_size=2000000000,
max_merge_events=200000000,
max_wait_time=-10)
# Everything should be in one, small jobGroup
self.assertEqual(len(result), 1)
self.assertEqual(len(result[0].jobs), 1)
job = result[0].jobs[0]
# All files should be in one job
self.assertEqual(len(job.getFiles()), 11)
return
def testDifferentSubscritionIDs(self):
"""
_testDifferentSubscriptionIDs_
Make sure that the merge splitting still runs if the subscription ID
is not equal to the workflow ID.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
dummyWorkflow = Workflow(name="dummyWorkflow",
spec="bunk49",
owner="Steve",
task="Test2")
dummyWorkflow.create()
dummyFileset = Fileset(name="dummyFileset")
dummyFileset.create()
dummySubscription1 = Subscription(fileset=dummyFileset,
workflow=dummyWorkflow,
split_algo="ParentlessMergeBySize")
dummySubscription2 = Subscription(fileset=dummyFileset,
workflow=dummyWorkflow,
split_algo="ParentlessMergeBySize")
dummySubscription1.create()
dummySubscription2.create()
myThread.transaction.commit()
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.mergeSubscription)
result = jobFactory(min_merge_size=4097,
max_merge_size=99999999,
max_merge_events=999999999,
merge_across_runs=False)
self.assertEqual(len(result), 1)
jobGroup = result[0]
self.assertEqual(len(jobGroup.jobs), 2)
return
def test_NotEnoughEvents(self):
"""
_test_NotEnoughEvents_
Checks whether jobs are not created when there are not enough files (actually, events)
according to the events_per_job requested to the splitter algorithm
"""
splitter = SplitterFactory()
# Very small fileset (single file) without enough events
testSubscription = self.createSubscription(nFiles=1,
lumisPerFile=2,
nEventsPerFile=200)
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(events_per_job=500,
performance=self.performanceParams,
splitOnRun=False)
self.assertEqual(len(jobGroups), 0)
# Still a small fileset (two files) without enough events
testSubscription = self.createSubscription(nFiles=2,
lumisPerFile=2,
nEventsPerFile=200)
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(events_per_job=500,
performance=self.performanceParams,
splitOnRun=False)
self.assertEqual(len(jobGroups), 0)
# Finally an acceptable fileset size (three files) with enough events
testSubscription = self.createSubscription(nFiles=3,
lumisPerFile=2,
nEventsPerFile=200)
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(events_per_job=500,
performance=self.performanceParams,
splitOnRun=False)
self.assertEqual(len(jobGroups), 1)
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 2)
self.assertEqual(len(jobs[0]['input_files']), 3)
self.assertEqual(len(jobs[1]['input_files']), 1)
self.assertEqual(jobs[0]['mask'].getRunAndLumis(), {
0: [[0, 1]],
1: [[2, 3]],
2: [[4, 4]]
})
self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {2: [[5, 5]]})
# Test fileset with a single run and splitOnRun=True
testFileset = Fileset(name="FilesetA")
testFileA = self.createFile("/this/is/file1",
200,
1,
2,
"T1_US_FNAL_Disk",
lumiMultiplier=0)
testFileB = self.createFile("/this/is/file2",
200,
1,
2,
"T1_US_FNAL_Disk",
lumiMultiplier=1)
testFileC = self.createFile("/this/is/file3",
200,
1,
2,
"T1_US_FNAL_Disk",
lumiMultiplier=2)
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.addFile(testFileC)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="EventAwareLumiBased",
type="Processing")
testSubscription.create()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(events_per_job=500,
performance=self.performanceParams)
self.assertEqual(len(jobGroups), 1)
jobs = jobGroups[0].jobs
self.assertEqual(len(jobs), 2)
self.assertEqual(len(jobs[0]['input_files']), 3)
self.assertEqual(len(jobs[1]['input_files']), 1)
self.assertEqual(jobs[0]['mask'].getRunAndLumis(),
{1: [[0, 1], [2, 3], [4, 4]]})
self.assertEqual(jobs[1]['mask'].getRunAndLumis(), {1: [[5, 5]]})
return
class FixedDelayTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Create two subscriptions: One that contains a single file and one that
contains multiple files.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daofactory(classname="Locations.New")
locationAction.execute(siteName="site1", pnn="T2_CH_CERN")
self.multipleFileFileset = Fileset(name="TestFileset1")
self.multipleFileFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(i, *[45 + i]))
newFile.create()
self.multipleFileFileset.addFile(newFile)
self.multipleFileFileset.commit()
self.singleFileFileset = Fileset(name="TestFileset2")
self.singleFileFileset.create()
newFile = File("/some/file/name",
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleFileFileset.addFile(newFile)
self.singleFileFileset.commit()
self.multipleFileLumiset = Fileset(name="TestFileset3")
self.multipleFileLumiset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45 + i / 3]))
newFile.create()
self.multipleFileLumiset.addFile(newFile)
self.multipleFileLumiset.commit()
self.singleLumiFileset = Fileset(name="TestFileset4")
self.singleLumiFileset.create()
for i in range(10):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
newFile.addRun(Run(1, *[45]))
newFile.create()
self.singleLumiFileset.addFile(newFile)
self.singleLumiFileset.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="mnorman",
name="wf001",
task="Test")
testWorkflow.create()
self.multipleFileSubscription = Subscription(
fileset=self.multipleFileFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.singleFileSubscription = Subscription(
fileset=self.singleFileFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.multipleLumiSubscription = Subscription(
fileset=self.multipleFileLumiset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.singleLumiSubscription = Subscription(
fileset=self.singleLumiFileset,
workflow=testWorkflow,
split_algo="FixedDelay",
type="Processing")
self.multipleFileSubscription.create()
self.singleFileSubscription.create()
self.multipleLumiSubscription.create()
self.singleLumiSubscription.create()
return
def tearDown(self):
"""
_tearDown_
Nothing to do...
"""
self.testInit.clearDatabase()
return
def testNone(self):
"""
_testNone_
Since the subscriptions are open, we shouldn't get any jobs back
"""
splitter = SplitterFactory()
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=int(time.time()) * 2)
self.assertEquals(jobGroups, [], "Should have returned a null set")
return
def testClosed(self):
"""
_testClosed_
Since the subscriptions are closed and none of the files have been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.singleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
job = jobGroups[0].jobs.pop()
assert job.getFiles(type = "lfn") == ["/some/file/name"], \
"ERROR: Job contains unknown files."
self.multipleFileSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs), 1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
self.multipleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1)
self.assertEquals(len(jobGroups[0].jobs), 1)
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
#self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.getFileset().markOpen(False)
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
assert len(jobGroups) == 1, \
"ERROR: JobFactory didn't return one JobGroup."
assert len(jobGroups[0].jobs) == 1, \
"ERROR: JobFactory didn't create a single job."
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 10)
def testAllAcquired(self):
"""
_testAllAcquired_
should all return no job groups
"""
splitter = SplitterFactory()
self.singleFileSubscription.acquireFiles(
self.singleFileSubscription.availableFiles())
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.acquireFiles(
self.multipleFileSubscription.availableFiles())
jobFactory = splitter(self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleLumiSubscription.acquireFiles(
self.multipleLumiSubscription.availableFiles())
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.singleLumiSubscription.acquireFiles(
self.singleLumiSubscription.availableFiles())
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
def testClosedSomeAcquired(self):
"""
_testClosedSomeAcquired_
since the subscriptions are closed and none of the files ahve been
acquired, all of the files should show up
"""
splitter = SplitterFactory()
self.multipleFileSubscription.getFileset().markOpen(False)
self.singleFileSubscription.acquireFiles(
[self.singleFileSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(jobGroups, [], "Should have returned a null set")
self.multipleFileSubscription.getFileset().markOpen(False)
self.multipleFileSubscription.acquireFiles(
[self.multipleFileSubscription.availableFiles().pop()])
jobFactory = splitter(package="WMCore.WMBS",
subscription=self.multipleFileSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.multipleLumiSubscription.getFileset().markOpen(False)
self.multipleLumiSubscription.acquireFiles(
[self.multipleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.multipleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.singleLumiSubscription.getFileset().markOpen(False)
self.singleLumiSubscription.acquireFiles(
[self.singleLumiSubscription.availableFiles().pop()])
jobFactory = splitter(self.singleLumiSubscription)
jobGroups = jobFactory(trigger_time=1)
self.assertEquals(len(jobGroups), 1, "Should have gotten one jobGroup")
self.assertEquals(len(jobGroups[0].jobs), 1, \
"JobFactory should have made one job")
myfiles = jobGroups[0].jobs[0].getFiles()
self.assertEquals(len(myfiles), 9, \
"JobFactory should have provides us with 9 files")
self.assertEquals(len(myfiles), 9)
def testGetOutputParentLFNs(self):
"""
_testGetOutputParentLFNs_
Verify that the getOutputDBSParentLFNs() method returns the correct
parent LFNs.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA",
size=1024,
events=10,
merged=True)
testFileB = File(lfn="/this/is/a/lfnB",
size=1024,
events=10,
merged=True)
testFileC = File(lfn="/this/is/a/lfnC",
size=1024,
events=10,
merged=False)
testFileD = File(lfn="/this/is/a/lfnD",
size=1024,
events=10,
merged=False)
testFileE = File(lfn="/this/is/a/lfnE",
size=1024,
events=10,
merged=True)
testFileF = File(lfn="/this/is/a/lfnF",
size=1024,
events=10,
merged=True)
testFileA.create()
testFileB.create()
testFileC.create()
testFileD.create()
testFileE.create()
testFileF.create()
testFileE.addChild(testFileC["lfn"])
testFileF.addChild(testFileD["lfn"])
testJobA = Job(name="TestJob", files=[testFileA, testFileB])
testJobA["couch_record"] = "somecouchrecord"
testJobA["location"] = "test.site.ch"
testJobA.create(group=testJobGroup)
testJobA.associateFiles()
testJobB = Job(name="TestJobB", files=[testFileC, testFileD])
testJobB["couch_record"] = "somecouchrecord"
testJobB["location"] = "test.site.ch"
testJobB.create(group=testJobGroup)
testJobB.associateFiles()
goldenLFNs = ["/this/is/a/lfnA", "/this/is/a/lfnB"]
parentLFNs = testJobA.getOutputDBSParentLFNs()
for parentLFN in parentLFNs:
assert parentLFN in goldenLFNs, \
"ERROR: Unknown lfn: %s" % parentLFN
goldenLFNs.remove(parentLFN)
assert len(goldenLFNs) == 0, \
"ERROR: LFNs are missing: %s" % goldenLFNs
goldenLFNs = ["/this/is/a/lfnE", "/this/is/a/lfnF"]
parentLFNs = testJobB.getOutputDBSParentLFNs()
for parentLFN in parentLFNs:
assert parentLFN in goldenLFNs, \
"ERROR: Unknown lfn: %s" % parentLFN
goldenLFNs.remove(parentLFN)
assert len(goldenLFNs) == 0, \
"ERROR: LFNs are missing..."
return
def notestCreateDeleteExists(self):
"""
Create and then delete a job and workflow. Use the workunit class's exists() method to
determine if the workunit has been written to the database before the job is
created, after the job has been created, and after the workflow has been deleted.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(1, *[46]))
testFileA.create()
testFileB.create()
testJob = Job(name="TestJob", files=[testFileA, testFileB])
testWU1 = WorkUnit(taskID=testWorkflow.id,
fileid=testFileA['id'],
runLumi=Run(1, *[45]))
testWU2 = WorkUnit(taskID=testWorkflow.id,
fileid=testFileB['id'],
runLumi=Run(1, *[46]))
self.assertFalse(testWU1.exists(),
"WorkUnit exists before job was created")
self.assertFalse(testWU2.exists(),
"WorkUnit exists before job was created")
testJob.create(group=testJobGroup)
self.assertTrue(testWU1.exists(),
"WorkUnit does not exist after job was created")
self.assertTrue(testWU2.exists(),
"WorkUnit does not exist after job was created")
testJob.delete()
self.assertTrue(testWU1.exists(),
"WorkUnit does not exist after job is deleted")
self.assertTrue(testWU2.exists(),
"WorkUnit does not exist after job is deleted")
testWorkflow.delete()
self.assertFalse(testWU1.exists(),
"WorkUnit exists after workflow is deleted")
self.assertFalse(testWU2.exists(),
"WorkUnit exists after workflow is deleted")
return
def testFailJobInput(self):
"""
_testFailJobInput_
Test the Jobs.FailInput DAO and verify that it doesn't affect other
jobs/subscriptions that run over the same files.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
bogusWorkflow = Workflow(spec="spec1.xml",
owner="Steve",
name="wf002",
task="Test")
testWorkflow.create()
bogusWorkflow.create()
testFileset = Fileset(name="TestFileset")
bogusFileset = Fileset(name="BogusFileset")
testFileset.create()
bogusFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
bogusSubscription = Subscription(fileset=bogusFileset,
workflow=bogusWorkflow)
testSubscription.create()
bogusSubscription.create()
testFileA = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileB = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileC = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileA.create()
testFileB.create()
testFileC.create()
testFileset.addFile([testFileA, testFileB, testFileC])
bogusFileset.addFile([testFileA, testFileB, testFileC])
testFileset.commit()
bogusFileset.commit()
testSubscription.completeFiles([testFileA, testFileB, testFileC])
bogusSubscription.acquireFiles([testFileA, testFileB, testFileC])
testJobGroup = JobGroup(subscription=testSubscription)
bogusJobGroup = JobGroup(subscription=bogusSubscription)
testJobGroup.create()
bogusJobGroup.create()
testJobA = Job(name="TestJobA",
files=[testFileA, testFileB, testFileC])
testJobB = Job(name="TestJobB",
files=[testFileA, testFileB, testFileC])
bogusJob = Job(name="BogusJob",
files=[testFileA, testFileB, testFileC])
testJobA.create(group=testJobGroup)
testJobB.create(group=testJobGroup)
bogusJob.create(group=bogusJobGroup)
testJobA.failInputFiles()
testJobB.failInputFiles()
self.assertEqual(len(testSubscription.filesOfStatus("Available")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Failed")), 3)
self.assertEqual(len(testSubscription.filesOfStatus("Completed")), 0)
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
testJobB["state"] = "cleanout"
changeStateAction.execute([testJobB])
# Try again
testJobA.failInputFiles()
# Should now be failed
self.assertEqual(len(testSubscription.filesOfStatus("Available")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Acquired")), 0)
self.assertEqual(len(testSubscription.filesOfStatus("Failed")), 3)
self.assertEqual(len(testSubscription.filesOfStatus("Completed")), 0)
# bogus should be unchanged
self.assertEqual(len(bogusSubscription.filesOfStatus("Available")), 0)
self.assertEqual(len(bogusSubscription.filesOfStatus("Acquired")), 3)
self.assertEqual(len(bogusSubscription.filesOfStatus("Failed")), 0)
self.assertEqual(len(bogusSubscription.filesOfStatus("Completed")), 0)
return
def testList(self):
"""
_testList_
Test the functions that list thresholds for creating jobs and submitting
jobs.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1",
"T1_US_FNAL", "LsfPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
testWorkflow = Workflow(spec=makeUUID(),
owner="Steve",
name=makeUUID(),
task="Test")
testWorkflow.create()
testFilesetA = Fileset(name="TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name="TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name="TestFilesetC")
testFilesetC.create()
testFileA = File(lfn="testFileA",
locations=set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset=testFilesetA,
workflow=testWorkflow,
type="Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{
"site_name": "testSite1",
"valid": True
}])
testSubscriptionB = Subscription(fileset=testFilesetB,
workflow=testWorkflow,
type="Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{
"site_name": "testSite1",
"valid": False
}])
testSubscriptionC = Subscription(fileset=testFilesetC,
workflow=testWorkflow,
type="Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription=testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription=testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name="testJobA", files=[testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group=testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name="testJobB", files=[testFileA])
testJobB["couch_record"] = makeUUID()
testJobB.create(group=testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name="testJobC", files=[testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group=testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name="testJobD", files=[testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group=testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name="testJobE", files=[testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group=testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name="testJobF", files=[testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group=testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name="testJobG", files=[testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group=testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name="testJobH", files=[testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group=testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name="testJobI", files=[testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group=testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name="testJobJ", files=[testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group=testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
changeStateAction.execute(jobs=[
testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
testJobG, testJobH, testJobI, testJobJ
])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite1")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
createThresholds = myResourceControl.listThresholdsForCreate()
submitThresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of sites in create thresholds")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of slots for site 1")
self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
"Error: Wrong number of slots for site 2")
# We should have two running jobs with locations at site one,
# two running jobs without locations at site two, and one running
# job without a location at site one and two.
self.assertEqual(createThresholds["testSite1"]["pending_jobs"], 4,
"Error: Wrong number of pending jobs for site 1")
# We should have one running job with a location at site 2 and
# another running job without a location.
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 2,
"Error: Wrong number of pending jobs for site 2")
# We should also have a phedex_name
self.assertEqual(createThresholds["testSite1"]["cms_name"],
"T1_US_FNAL")
self.assertEqual(createThresholds["testSite2"]["cms_name"], None)
mergeThreshold1 = None
mergeThreshold2 = None
procThreshold1 = None
procThreshold2 = None
self.assertEqual(submitThresholds["testSite1"]['cms_name'],
'T1_US_FNAL')
for threshold in submitThresholds["testSite1"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold1 = threshold
elif threshold['task_type'] == "Processing":
procThreshold1 = threshold
self.assertEqual(submitThresholds["testSite2"]['cms_name'], None)
for threshold in submitThresholds["testSite2"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold2 = threshold
elif threshold['task_type'] == "Processing":
procThreshold2 = threshold
self.assertEqual(
submitThresholds["testSite1"]["total_running_jobs"], 1,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_running_jobs"], 0,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite1"]["total_pending_jobs"], 1,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_pending_jobs"], 0,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_running_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_pending_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
return
def testCompleteJobInput(self):
"""
_testCompleteJobInput_
Verify the correct output of the CompleteInput DAO. This should mark
the input for a job as complete once all the jobs that run over a
particular file have complete successfully.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task="Test")
bogusWorkflow = Workflow(spec="spec1.xml",
owner="Steve",
name="wf002",
task="Test")
testWorkflow.create()
bogusWorkflow.create()
testFileset = Fileset(name="TestFileset")
bogusFileset = Fileset(name="BogusFileset")
testFileset.create()
bogusFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
bogusSubscription = Subscription(fileset=bogusFileset,
workflow=bogusWorkflow)
testSubscription.create()
bogusSubscription.create()
testFileA = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileB = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileC = File(lfn=makeUUID(), locations="T2_CH_CERN")
testFileA.create()
testFileB.create()
testFileC.create()
testFileset.addFile([testFileA, testFileB, testFileC])
bogusFileset.addFile([testFileA, testFileB, testFileC])
testFileset.commit()
bogusFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB, testFileC])
bogusSubscription.acquireFiles([testFileA, testFileB, testFileC])
testJobGroup = JobGroup(subscription=testSubscription)
bogusJobGroup = JobGroup(subscription=bogusSubscription)
testJobGroup.create()
bogusJobGroup.create()
testJobA = Job(name="TestJobA", files=[testFileA])
testJobB = Job(name="TestJobB", files=[testFileA, testFileB])
testJobC = Job(name="TestJobC", files=[testFileC])
bogusJob = Job(name="BogusJob",
files=[testFileA, testFileB, testFileC])
testJobA.create(group=testJobGroup)
testJobB.create(group=testJobGroup)
testJobC.create(group=testJobGroup)
bogusJob.create(group=bogusJobGroup)
testJobA["outcome"] = "success"
testJobB["outcome"] = "failure"
testJobC["outcome"] = "success"
testJobA.save()
testJobB.save()
testJobC.save()
testJobA.completeInputFiles()
compFiles = len(testSubscription.filesOfStatus("Completed"))
assert compFiles == 0, \
"Error: test sub has wrong number of complete files: %s" % compFiles
testJobB["outcome"] = "success"
testJobB.save()
testJobB.completeInputFiles(skipFiles=[testFileB["lfn"]])
availFiles = len(testSubscription.filesOfStatus("Available"))
assert availFiles == 0, \
"Error: test sub has wrong number of available files: %s" % availFiles
acqFiles = len(testSubscription.filesOfStatus("Acquired"))
assert acqFiles == 1, \
"Error: test sub has wrong number of acquired files: %s" % acqFiles
compFiles = len(testSubscription.filesOfStatus("Completed"))
assert compFiles == 1, \
"Error: test sub has wrong number of complete files: %s" % compFiles
failFiles = len(testSubscription.filesOfStatus("Failed"))
assert failFiles == 1, \
"Error: test sub has wrong number of failed files: %s" % failFiles
availFiles = len(bogusSubscription.filesOfStatus("Available"))
assert availFiles == 0, \
"Error: test sub has wrong number of available files: %s" % availFiles
acqFiles = len(bogusSubscription.filesOfStatus("Acquired"))
assert acqFiles == 3, \
"Error: test sub has wrong number of acquired files: %s" % acqFiles
compFiles = len(bogusSubscription.filesOfStatus("Completed"))
assert compFiles == 0, \
"Error: test sub has wrong number of complete files: %s" % compFiles
failFiles = len(bogusSubscription.filesOfStatus("Failed"))
assert failFiles == 0, \
"Error: test sub has wrong number of failed files: %s" % failFiles
return
class ExpressMergeTest(unittest.TestCase):
"""
_ExpressMergeTest_
Test for ExpressMerge job splitter
"""
def setUp(self):
"""
_setUp_
"""
import WMQuality.TestInit
WMQuality.TestInit.deleteDatabaseAfterEveryTest("I'm Serious")
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", "T0.WMBS"])
self.splitterFactory = SplitterFactory(package="T0.JobSplitting")
myThread = threading.currentThread()
daoFactory = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=myThread.dbi)
myThread.dbi.processData("""INSERT INTO wmbs_location
(id, site_name, state, state_time)
VALUES (1, 'SomeSite', 1, 1)
""",
transaction=False)
myThread.dbi.processData("""INSERT INTO wmbs_pnns
(id, pnn)
VALUES (2, 'SomePNN')
""",
transaction=False)
myThread.dbi.processData("""INSERT INTO wmbs_location_pnns
(location, pnn)
VALUES (1, 2)
""",
transaction=False)
insertRunDAO = daoFactory(classname="RunConfig.InsertRun")
insertRunDAO.execute(binds={
'RUN': 1,
'HLTKEY': "someHLTKey"
},
transaction=False)
insertLumiDAO = daoFactory(classname="RunConfig.InsertLumiSection")
for lumi in range(1, 5):
insertLumiDAO.execute(binds={
'RUN': 1,
'LUMI': lumi
},
transaction=False)
insertStreamDAO = daoFactory(classname="RunConfig.InsertStream")
insertStreamDAO.execute(binds={'STREAM': "Express"}, transaction=False)
insertStreamFilesetDAO = daoFactory(
classname="RunConfig.InsertStreamFileset")
insertStreamFilesetDAO.execute(1, "Express", "TestFileset1")
fileset1 = Fileset(name="TestFileset1")
self.fileset2 = Fileset(name="TestFileset2")
fileset1.load()
self.fileset2.create()
workflow1 = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow1",
task="Test")
workflow2 = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow2",
task="Test")
workflow1.create()
workflow2.create()
self.subscription1 = Subscription(fileset=fileset1,
workflow=workflow1,
split_algo="Express",
type="Express")
self.subscription2 = Subscription(fileset=self.fileset2,
workflow=workflow2,
split_algo="ExpressMerge",
type="ExpressMerge")
self.subscription1.create()
self.subscription2.create()
myThread.dbi.processData("""INSERT INTO wmbs_workflow_output
(WORKFLOW_ID, OUTPUT_IDENTIFIER, OUTPUT_FILESET)
VALUES (%d, 'SOMEOUTPUT', %d)
""" % (workflow1.id, self.fileset2.id),
transaction=False)
# keep for later
self.insertSplitLumisDAO = daoFactory(
classname="JobSplitting.InsertSplitLumis")
# default split parameters
self.splitArgs = {}
self.splitArgs['maxInputSize'] = 2 * 1024 * 1024 * 1024
self.splitArgs['maxInputFiles'] = 500,
self.splitArgs['maxLatency'] = 15 * 23
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def deleteSplitLumis(self):
"""
_deleteSplitLumis_
"""
myThread = threading.currentThread()
myThread.dbi.processData("""DELETE FROM lumi_section_split_active
""",
transaction=False)
return
def test00(self):
"""
_test00_
Test that the job name prefix feature works
Test latency trigger (wait and 0)
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxLatency'] = 0
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertTrue(job['name'].startswith("ExpressMerge-"),
"ERROR: Job has wrong name")
return
def test01(self):
"""
_test01_
Test size and event triggers for single lumis (they are ignored)
Test latency trigger (timed out)
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
mySplitArgs['maxInputSize'] = 1
mySplitArgs['maxInputFiles'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
time.sleep(1)
mySplitArgs['maxLatency'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
return
def test02(self):
"""
_test02_
Test input files threshold on multi lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
mySplitArgs['maxInputFiles'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
time.sleep(1)
mySplitArgs['maxLatency'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
return
def test03(self):
"""
_test03_
Test input size threshold on multi lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
mySplitArgs['maxInputSize'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
time.sleep(1)
mySplitArgs['maxLatency'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
return
def test04(self):
"""
_test04_
Test multi lumis express merges
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
time.sleep(1)
mySplitArgs['maxLatency'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
return
def test05(self):
"""
_test05_
Test multi lumis express merges with holes
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 4]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
time.sleep(1)
mySplitArgs['maxLatency'] = 1
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
return
def test06(self):
"""
_test06_
Test active split lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1]:
for i in range(2):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave=False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription2)
self.insertSplitLumisDAO.execute(binds={
'SUB': self.subscription1['id'],
'LUMI': 1,
'NFILES': 5
})
mySplitArgs['maxLatency'] = 0
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
self.deleteSplitLumis()
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
return
def _createSubscriptionsInWMBS(self,
task,
fileset,
alternativeFilesetClose=False):
"""
__createSubscriptionsInWMBS_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
# create runtime sandbox for workflow
self.createSandbox()
# FIXME: Let workflow put in values if spec is missing them
workflow = Workflow(
spec=self.wmSpec.specUrl(),
owner=self.wmSpec.getOwner()["name"],
dn=self.wmSpec.getOwner().get("dn", "unknown"),
group=self.wmSpec.getOwner().get("group", "unknown"),
owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"),
name=self.wmSpec.name(),
task=task.getPathName(),
wfType=self.wmSpec.getDashboardActivity(),
alternativeFilesetClose=alternativeFilesetClose,
priority=self.wmSpec.priority())
workflow.create()
subscription = Subscription(fileset=fileset,
workflow=workflow,
split_algo=task.jobSplittingAlgorithm(),
type=task.getPrimarySubType())
if subscription.exists():
subscription.load()
msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)"
self.logger.info(msg % (subscription['id'], task.getPathName()))
else:
subscription.create()
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": True
}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": False
}])
if self.topLevelSubscription is None:
self.topLevelSubscription = subscription
logging.info("Top level subscription created: %s",
subscription["id"])
else:
logging.info("Child subscription created: %s", subscription["id"])
outputModules = task.getOutputModulesForTask()
ignoredOutputModules = task.getIgnoredOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
if outputModuleName in ignoredOutputModules:
logging.info(
"IgnoredOutputModule set for %s, skipping fileset creation.",
outputModuleName)
continue
outputFileset = Fileset(
self.outputFilesetName(task, outputModuleName))
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
if childTask.taskType() == "Merge":
mergedOutputFileset = Fileset(
self.outputFilesetName(childTask, "Merged"))
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
primaryDataset = getattr(
getattr(outputModule, outputModuleName),
"primaryDataset", None)
if primaryDataset != None:
self.mergeOutputMapping[
mergedOutputFileset.id] = primaryDataset
self._createSubscriptionsInWMBS(
childTask, outputFileset, alternativeFilesetClose)
if mergedOutputFileset is None:
workflow.addOutput(outputModuleName, outputFileset,
outputFileset)
else:
workflow.addOutput(outputModuleName, outputFileset,
mergedOutputFileset)
return self.topLevelSubscription
class WMBSHelperTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump")
self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump")
self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache")
os.environ["COUCHDB"] = "wmbshelper_t"
self.testInit.setSchema(customModules=[
"WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir",
"WMCore.ResourceControl"
],
useDefault=False)
self.workDir = self.testInit.generateWorkDir()
self.wmspec = self.createWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = MockDBSReader(self.inputDataset.dbsurl)
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=threading.currentThread().logger,
dbinterface=threading.currentThread().dbi)
return
def tearDown(self):
"""
_tearDown_
Clear out the database.
"""
self.testInit.clearDatabase()
self.testInit.tearDownCouch()
self.testInit.delWorkDir()
return
def setupForKillTest(self, baAPI=None):
"""
_setupForKillTest_
Inject a workflow into WMBS that has a processing task, a merge task and
a cleanup task. Inject files into the various tasks at various
processing states (acquired, complete, available...). Also create jobs
for each subscription in various states.
"""
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
locationAction = daoFactory(classname="Locations.New")
changeStateAction = daoFactory(classname="Jobs.ChangeState")
resourceControl = ResourceControl()
resourceControl.insertSite(siteName='site1',
seName='goodse.cern.ch',
ceName='site1',
plugin="TestPlugin")
resourceControl.insertThreshold(siteName = 'site1', taskType = 'Processing', \
maxSlots = 10000, pendingSlots = 10000)
userDN = 'someDN'
userAction = daoFactory(classname="Users.New")
userAction.execute(dn=userDN,
group_name='DEFAULT',
role_name='DEFAULT')
inputFileset = Fileset("input")
inputFileset.create()
inputFileA = File("lfnA", locations="goodse.cern.ch")
inputFileB = File("lfnB", locations="goodse.cern.ch")
inputFileC = File("lfnC", locations="goodse.cern.ch")
inputFileA.create()
inputFileB.create()
inputFileC.create()
inputFileset.addFile(inputFileA)
inputFileset.addFile(inputFileB)
inputFileset.addFile(inputFileC)
inputFileset.commit()
unmergedOutputFileset = Fileset("unmerged")
unmergedOutputFileset.create()
unmergedFileA = File("ulfnA", locations="goodse.cern.ch")
unmergedFileB = File("ulfnB", locations="goodse.cern.ch")
unmergedFileC = File("ulfnC", locations="goodse.cern.ch")
unmergedFileA.create()
unmergedFileB.create()
unmergedFileC.create()
unmergedOutputFileset.addFile(unmergedFileA)
unmergedOutputFileset.addFile(unmergedFileB)
unmergedOutputFileset.addFile(unmergedFileC)
unmergedOutputFileset.commit()
mainProcWorkflow = Workflow(spec="spec1",
owner="Steve",
name="Main",
task="Proc")
mainProcWorkflow.create()
mainProcMergeWorkflow = Workflow(spec="spec1",
owner="Steve",
name="Main",
task="ProcMerge")
mainProcMergeWorkflow.create()
mainCleanupWorkflow = Workflow(spec="spec1",
owner="Steve",
name="Main",
task="Cleanup")
mainCleanupWorkflow.create()
self.mainProcSub = Subscription(fileset=inputFileset,
workflow=mainProcWorkflow,
type="Processing")
self.mainProcSub.create()
self.mainProcSub.acquireFiles(inputFileA)
self.mainProcSub.completeFiles(inputFileB)
procJobGroup = JobGroup(subscription=self.mainProcSub)
procJobGroup.create()
self.procJobA = Job(name="ProcJobA")
self.procJobA["state"] = "new"
self.procJobA["location"] = "site1"
self.procJobB = Job(name="ProcJobB")
self.procJobB["state"] = "executing"
self.procJobB["location"] = "site1"
self.procJobC = Job(name="ProcJobC")
self.procJobC["state"] = "complete"
self.procJobC["location"] = "site1"
self.procJobA.create(procJobGroup)
self.procJobB.create(procJobGroup)
self.procJobC.create(procJobGroup)
self.mainMergeSub = Subscription(fileset=unmergedOutputFileset,
workflow=mainProcMergeWorkflow,
type="Merge")
self.mainMergeSub.create()
self.mainMergeSub.acquireFiles(unmergedFileA)
self.mainMergeSub.failFiles(unmergedFileB)
mergeJobGroup = JobGroup(subscription=self.mainMergeSub)
mergeJobGroup.create()
self.mergeJobA = Job(name="MergeJobA")
self.mergeJobA["state"] = "exhausted"
self.mergeJobA["location"] = "site1"
self.mergeJobB = Job(name="MergeJobB")
self.mergeJobB["state"] = "cleanout"
self.mergeJobB["location"] = "site1"
self.mergeJobC = Job(name="MergeJobC")
self.mergeJobC["state"] = "new"
self.mergeJobC["location"] = "site1"
self.mergeJobA.create(mergeJobGroup)
self.mergeJobB.create(mergeJobGroup)
self.mergeJobC.create(mergeJobGroup)
self.mainCleanupSub = Subscription(fileset=unmergedOutputFileset,
workflow=mainCleanupWorkflow,
type="Cleanup")
self.mainCleanupSub.create()
self.mainCleanupSub.acquireFiles(unmergedFileA)
self.mainCleanupSub.completeFiles(unmergedFileB)
cleanupJobGroup = JobGroup(subscription=self.mainCleanupSub)
cleanupJobGroup.create()
self.cleanupJobA = Job(name="CleanupJobA")
self.cleanupJobA["state"] = "new"
self.cleanupJobA["location"] = "site1"
self.cleanupJobB = Job(name="CleanupJobB")
self.cleanupJobB["state"] = "executing"
self.cleanupJobB["location"] = "site1"
self.cleanupJobC = Job(name="CleanupJobC")
self.cleanupJobC["state"] = "complete"
self.cleanupJobC["location"] = "site1"
self.cleanupJobA.create(cleanupJobGroup)
self.cleanupJobB.create(cleanupJobGroup)
self.cleanupJobC.create(cleanupJobGroup)
jobList = [
self.procJobA, self.procJobB, self.procJobC, self.mergeJobA,
self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB,
self.cleanupJobC
]
changeStateAction.execute(jobList)
if baAPI:
for job in jobList:
job['plugin'] = 'TestPlugin'
job['userdn'] = userDN
job['usergroup'] = 'DEFAULT'
job['userrole'] = 'DEFAULT'
job['custom']['location'] = 'site1'
baAPI.createNewJobs(wmbsJobs=jobList)
# We'll create an unrelated workflow to verify that it isn't affected
# by the killing code.
bogusFileset = Fileset("dontkillme")
bogusFileset.create()
bogusFileA = File("bogus/lfnA", locations="goodse.cern.ch")
bogusFileA.create()
bogusFileset.addFile(bogusFileA)
bogusFileset.commit()
bogusWorkflow = Workflow(spec="spec2",
owner="Steve",
name="Bogus",
task="Proc")
bogusWorkflow.create()
self.bogusSub = Subscription(fileset=bogusFileset,
workflow=bogusWorkflow,
type="Processing")
self.bogusSub.create()
self.bogusSub.acquireFiles(bogusFileA)
return
def verifyFileKillStatus(self):
"""
_verifyFileKillStatus_
Verify that all files were killed correctly. The status of files in
Cleanup and LogCollect subscriptions isn't modified. Status of
already completed and failed files is not modified. Also verify that
the bogus subscription is untouched.
"""
failedFiles = self.mainProcSub.filesOfStatus("Failed")
acquiredFiles = self.mainProcSub.filesOfStatus("Acquired")
completedFiles = self.mainProcSub.filesOfStatus("Completed")
availableFiles = self.mainProcSub.filesOfStatus("Available")
bogusAcquiredFiles = self.bogusSub.filesOfStatus("Acquired")
self.assertEqual(len(availableFiles), 0, \
"Error: There should be no available files.")
self.assertEqual(len(acquiredFiles), 0, \
"Error: There should be no acquired files.")
self.assertEqual(len(bogusAcquiredFiles), 1, \
"Error: There should be one acquired file.")
self.assertEqual(len(completedFiles), 3, \
"Error: There should be only one completed file.")
goldenLFNs = ["lfnA", "lfnB", "lfnC"]
for completedFile in completedFiles:
self.assertTrue(completedFile["lfn"] in goldenLFNs, \
"Error: Extra completed file.")
goldenLFNs.remove(completedFile["lfn"])
self.assertEqual(len(failedFiles), 0, \
"Error: There should be no failed files.")
self.assertEqual(len(goldenLFNs), 0, \
"Error: Missing LFN")
failedFiles = self.mainMergeSub.filesOfStatus("Failed")
acquiredFiles = self.mainMergeSub.filesOfStatus("Acquired")
completedFiles = self.mainMergeSub.filesOfStatus("Completed")
availableFiles = self.mainMergeSub.filesOfStatus("Available")
self.assertEqual(len(acquiredFiles), 0, \
"Error: Merge subscription should have 0 acq files.")
self.assertEqual(len(availableFiles), 0, \
"Error: Merge subscription should have 0 avail files.")
self.assertEqual(len(failedFiles), 1, \
"Error: Merge subscription should have 1 failed files.")
self.assertEqual(
list(failedFiles)[0]["lfn"], "ulfnB", "Error: Wrong failed file.")
self.assertEqual(len(completedFiles), 2, \
"Error: Merge subscription should have 2 compl files.")
goldenLFNs = ["ulfnA", "ulfnC"]
for completedFile in completedFiles:
self.assertTrue(completedFile["lfn"] in goldenLFNs, \
"Error: Extra complete file.")
goldenLFNs.remove(completedFile["lfn"])
self.assertEqual(len(goldenLFNs), 0, \
"Error: Missing LFN")
failedFiles = self.mainCleanupSub.filesOfStatus("Failed")
acquiredFiles = self.mainCleanupSub.filesOfStatus("Acquired")
completedFiles = self.mainCleanupSub.filesOfStatus("Completed")
availableFiles = self.mainCleanupSub.filesOfStatus("Available")
self.assertEqual(len(failedFiles), 0, \
"Error: Cleanup subscription should have 0 fai files.")
self.assertEqual(len(acquiredFiles), 1, \
"Error: There should be only one acquired file.")
self.assertEqual(list(acquiredFiles)[0]["lfn"], "ulfnA", \
"Error: Wrong acquired LFN.")
self.assertEqual(len(completedFiles), 1, \
"Error: There should be only one completed file.")
self.assertEqual(list(completedFiles)[0]["lfn"], "ulfnB", \
"Error: Wrong completed LFN.")
self.assertEqual(len(availableFiles), 1, \
"Error: There should be only one available file.")
self.assertEqual(list(availableFiles)[0]["lfn"], "ulfnC", \
"Error: Wrong completed LFN.")
return
def verifyJobKillStatus(self):
"""
_verifyJobKillStatus_
Verify that jobs are killed correctly. Jobs belonging to Cleanup and
LogCollect subscriptions are not killed. The status of jobs that have
already finished running is not changed.
"""
self.procJobA.load()
self.procJobB.load()
self.procJobC.load()
self.assertEqual(self.procJobA["state"], "killed", \
"Error: Proc job A should be killed.")
self.assertEqual(self.procJobB["state"], "killed", \
"Error: Proc job B should be killed.")
self.assertEqual(self.procJobC["state"], "complete", \
"Error: Proc job C should be complete.")
self.mergeJobA.load()
self.mergeJobB.load()
self.mergeJobC.load()
self.assertEqual(self.mergeJobA["state"], "exhausted", \
"Error: Merge job A should be exhausted.")
self.assertEqual(self.mergeJobB["state"], "cleanout", \
"Error: Merge job B should be cleanout.")
self.assertEqual(self.mergeJobC["state"], "killed", \
"Error: Merge job C should be killed.")
self.cleanupJobA.load()
self.cleanupJobB.load()
self.cleanupJobC.load()
self.assertEqual(self.cleanupJobA["state"], "new", \
"Error: Cleanup job A should be new.")
self.assertEqual(self.cleanupJobB["state"], "executing", \
"Error: Cleanup job B should be executing.")
self.assertEqual(self.cleanupJobC["state"], "complete", \
"Error: Cleanup job C should be complete.")
return
def createTestWMSpec(self):
"""
_createTestWMSpec_
Create a WMSpec that has a processing, merge, cleanup and skims tasks that
can be used by the subscription creation test.
"""
testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload"))
testWorkload.setDashboardActivity("TestReReco")
testWorkload.setSpecUrl("/path/to/workload")
testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'})
procTask = testWorkload.newTask("ProcessingTask")
procTask.setTaskType("Processing")
procTask.setSplittingAlgorithm("FileBased", files_per_job=1)
procTaskCMSSW = procTask.makeStep("cmsRun1")
procTaskCMSSW.setStepType("CMSSW")
procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper()
procTask.setTaskType("Processing")
procTask.setSiteWhitelist(["site1"])
procTask.setSiteBlacklist(["site2"])
procTask.applyTemplates()
procTaskCMSSWHelper.addOutputModule("OutputA",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
mergeTask = procTask.addTask("MergeTask")
mergeTask.setInputReference(procTaskCMSSW, outputModule="OutputA")
mergeTask.setTaskType("Merge")
mergeTask.setSplittingAlgorithm("WMBSMergeBySize",
min_merge_size=1,
max_merge_size=2,
max_merge_events=3)
mergeTaskCMSSW = mergeTask.makeStep("cmsRun1")
mergeTaskCMSSW.setStepType("CMSSW")
mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper()
mergeTask.setTaskType("Merge")
mergeTask.applyTemplates()
mergeTaskCMSSWHelper.addOutputModule("Merged",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
cleanupTask = procTask.addTask("CleanupTask")
cleanupTask.setInputReference(procTaskCMSSW, outputModule="OutputA")
cleanupTask.setTaskType("Merge")
cleanupTask.setSplittingAlgorithm("SiblingProcessingBased",
files_per_job=50)
cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1")
cleanupTaskCMSSW.setStepType("CMSSW")
cleanupTaskCMSSWHelper = cleanupTaskCMSSW.getTypeHelper()
cleanupTask.setTaskType("Cleanup")
cleanupTask.applyTemplates()
skimTask = mergeTask.addTask("SkimTask")
skimTask.setTaskType("Skim")
skimTask.setInputReference(mergeTaskCMSSW, outputModule="Merged")
skimTask.setSplittingAlgorithm("FileBased",
files_per_job=1,
include_parents=True)
skimTaskCMSSW = skimTask.makeStep("cmsRun1")
skimTaskCMSSW.setStepType("CMSSW")
skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper()
skimTask.setTaskType("Skim")
skimTask.applyTemplates()
skimTaskCMSSWHelper.addOutputModule("SkimOutputA",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
skimTaskCMSSWHelper.addOutputModule("SkimOutputB",
primaryDataset="bogusPrimary",
processedDataset="bogusProcessed",
dataTier="DataTierA",
lfnBase="bogusUnmerged",
mergedLFNBase="bogusMerged",
filterName=None)
return testWorkload
def setupMCWMSpec(self):
"""Setup MC workflow"""
self.wmspec = self.createMCWMSpec()
self.topLevelTask = getFirstTask(self.wmspec)
self.inputDataset = self.topLevelTask.inputDataset()
self.dataset = self.topLevelTask.getInputDatasetPath()
self.dbs = None
self.siteDB = fakeSiteDB()
# add sites that would normally be added by operator via resource_control
locationDAO = self.daoFactory(classname="Locations.New")
self.ses = []
for site in ['T2_XX_SiteA', 'T2_XX_SiteB']:
locationDAO.execute(siteName=site,
seName=self.siteDB.cmsNametoSE(site)[0])
self.ses.append(self.siteDB.cmsNametoSE(site)[0])
def createWMSpec(self, name='ReRecoWorkload'):
factory = ReRecoWorkloadFactory()
rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"])
wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs)
wmspec.setSpecUrl("/path/to/workload")
wmspec.setSubscriptionInformation(custodialSites=[],
nonCustodialSites=[],
autoApproveSites=[],
priority="Low",
custodialSubType="Move")
return wmspec
def createMCWMSpec(self, name='MonteCarloWorkload'):
wmspec = monteCarloWorkload(name, mcArgs)
wmspec.setSpecUrl("/path/to/workload")
getFirstTask(wmspec).addProduction(totalevents=10000)
return wmspec
def getDBS(self, wmspec):
topLevelTask = getFirstTask(wmspec)
inputDataset = topLevelTask.inputDataset()
dbs = MockDBSReader(inputDataset.dbsurl)
#dbsDict = {self.inputDataset.dbsurl : self.dbs}
return dbs
def createWMBSHelperWithTopTask(self,
wmspec,
block,
mask=None,
parentFlag=False,
detail=False):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec,
topLevelTask.name(),
block,
mask,
cachepath=self.workDir)
if block:
if parentFlag:
block = self.dbs.getFileBlockWithParents(block)[block]
else:
block = self.dbs.getFileBlock(block)[block]
sub, files = wmbs.createSubscriptionAndAddFiles(block=block)
if detail:
return wmbs, sub, files
else:
return wmbs
def testKillWorkflow(self):
"""
_testKillWorkflow_
Verify that workflow killing works correctly.
"""
configFile = EmulatorSetup.setupWMAgentConfig()
config = loadConfigurationFile(configFile)
baAPI = BossAirAPI(config=config)
# Create nine jobs
self.setupForKillTest(baAPI=baAPI)
self.assertEqual(len(baAPI._listRunJobs()), 9)
killWorkflow("Main", config, config)
self.verifyFileKillStatus()
self.verifyJobKillStatus()
self.assertEqual(len(baAPI._listRunJobs()), 8)
EmulatorSetup.deleteConfig(configFile)
return
def testCreateSubscription(self):
"""
_testCreateSubscription_
Verify that the subscription creation code works correctly.
"""
resourceControl = ResourceControl()
resourceControl.insertSite(siteName='site1',
seName='goodse.cern.ch',
ceName='site1',
plugin="TestPlugin")
resourceControl.insertSite(siteName='site2',
seName='goodse2.cern.ch',
ceName='site2',
plugin="TestPlugin")
testWorkload = self.createTestWMSpec()
testTopLevelTask = getFirstTask(testWorkload)
testWMBSHelper = WMBSHelper(testWorkload,
testTopLevelTask.name(),
"SomeBlock",
cachepath=self.workDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testTopLevelTask, testWMBSHelper.topLevelFileset)
procWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/ProcessingTask")
procWorkflow.load()
self.assertEqual(procWorkflow.owner, "sfoulkes",
"Error: Wrong owner: %s" % procWorkflow.owner)
self.assertEqual(procWorkflow.group, "DMWM",
"Error: Wrong group: %s" % procWorkflow.group)
self.assertEqual(procWorkflow.wfType, "TestReReco",
"Error: Wrong type.")
self.assertEqual(
procWorkflow.spec,
os.path.join(self.workDir, procWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(procWorkflow.outputMap.keys()), 1,
"Error: Wrong number of WF outputs.")
mergedProcOutput = procWorkflow.outputMap["OutputA"][0][
"merged_output_fileset"]
unmergedProcOutput = procWorkflow.outputMap["OutputA"][0][
"output_fileset"]
mergedProcOutput.loadData()
unmergedProcOutput.loadData()
self.assertEqual(
mergedProcOutput.name,
"/TestWorkload/ProcessingTask/MergeTask/merged-Merged",
"Error: Merged output fileset is wrong.")
self.assertEqual(unmergedProcOutput.name,
"/TestWorkload/ProcessingTask/unmerged-OutputA",
"Error: Unmerged output fileset is wrong.")
mergeWorkflow = Workflow(name="TestWorkload",
task="/TestWorkload/ProcessingTask/MergeTask")
mergeWorkflow.load()
self.assertEqual(mergeWorkflow.owner, "sfoulkes",
"Error: Wrong owner.")
self.assertEqual(
mergeWorkflow.spec,
os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1,
"Error: Wrong number of WF outputs.")
cleanupWorkflow = Workflow(
name="TestWorkload",
task="/TestWorkload/ProcessingTask/CleanupTask")
cleanupWorkflow.load()
self.assertEqual(cleanupWorkflow.owner, "sfoulkes",
"Error: Wrong owner.")
self.assertEqual(
cleanupWorkflow.spec,
os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0,
"Error: Wrong number of WF outputs.")
unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0][
"output_fileset"]
unmergedMergeOutput.loadData()
self.assertEqual(
unmergedMergeOutput.name,
"/TestWorkload/ProcessingTask/MergeTask/merged-Merged",
"Error: Unmerged output fileset is wrong.")
skimWorkflow = Workflow(
name="TestWorkload",
task="/TestWorkload/ProcessingTask/MergeTask/SkimTask")
skimWorkflow.load()
self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.")
self.assertEqual(
skimWorkflow.spec,
os.path.join(self.workDir, skimWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(skimWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][
"merged_output_fileset"]
unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][
"output_fileset"]
mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][
"merged_output_fileset"]
unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][
"output_fileset"]
mergedSkimOutputA.loadData()
mergedSkimOutputB.loadData()
unmergedSkimOutputA.loadData()
unmergedSkimOutputB.loadData()
self.assertEqual(
mergedSkimOutputA.name,
"/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA",
"Error: Merged output fileset is wrong: %s" %
mergedSkimOutputA.name)
self.assertEqual(
unmergedSkimOutputA.name,
"/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA",
"Error: Unmerged output fileset is wrong.")
self.assertEqual(
mergedSkimOutputB.name,
"/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB",
"Error: Merged output fileset is wrong.")
self.assertEqual(
unmergedSkimOutputB.name,
"/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB",
"Error: Unmerged output fileset is wrong.")
topLevelFileset = Fileset(name="TestWorkload-ProcessingTask-SomeBlock")
topLevelFileset.loadData()
procSubscription = Subscription(fileset=topLevelFileset,
workflow=procWorkflow)
procSubscription.loadData()
self.assertEqual(len(procSubscription.getWhiteBlackList()), 2,
"Error: Wrong site white/black list for proc sub.")
for site in procSubscription.getWhiteBlackList():
if site["site_name"] == "site1":
self.assertEqual(site["valid"], 1,
"Error: Site should be white listed.")
else:
self.assertEqual(site["valid"], 0,
"Error: Site should be black listed.")
self.assertEqual(procSubscription["type"], "Processing",
"Error: Wrong subscription type.")
self.assertEqual(procSubscription["split_algo"], "FileBased",
"Error: Wrong split algo.")
mergeSubscription = Subscription(fileset=unmergedProcOutput,
workflow=mergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0,
"Error: Wrong white/black list for merge sub.")
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize",
"Error: Wrong split algo.")
skimSubscription = Subscription(fileset=unmergedMergeOutput,
workflow=skimWorkflow)
skimSubscription.loadData()
self.assertEqual(skimSubscription["type"], "Skim",
"Error: Wrong subscription type.")
self.assertEqual(skimSubscription["split_algo"], "FileBased",
"Error: Wrong split algo.")
return
def testTruncatedWFInsertion(self):
"""
_testTruncatedWFInsertion_
"""
resourceControl = ResourceControl()
resourceControl.insertSite(siteName='site1',
seName='goodse.cern.ch',
ceName='site1',
plugin="TestPlugin")
resourceControl.insertSite(siteName='site2',
seName='goodse2.cern.ch',
ceName='site2',
plugin="TestPlugin")
testWorkload = self.createTestWMSpec()
testTopLevelTask = getFirstTask(testWorkload)
testWMBSHelper = WMBSHelper(testWorkload,
testTopLevelTask.name(),
"SomeBlock",
cachepath=self.workDir)
testWMBSHelper.createTopLevelFileset()
testWMBSHelper._createSubscriptionsInWMBS(
testTopLevelTask, testWMBSHelper.topLevelFileset)
testWorkload.truncate("ResubmitTestWorkload",
"/TestWorkload/ProcessingTask/MergeTask",
"someserver", "somedatabase")
# create the subscription for multiple top task (MergeTask and CleanupTask for the same block)
for task in testWorkload.getTopLevelTask():
testResubmitWMBSHelper = WMBSHelper(testWorkload,
task.name(),
"SomeBlock2",
cachepath=self.workDir)
testResubmitWMBSHelper.createTopLevelFileset()
testResubmitWMBSHelper._createSubscriptionsInWMBS(
task, testResubmitWMBSHelper.topLevelFileset)
mergeWorkflow = Workflow(name="ResubmitTestWorkload",
task="/ResubmitTestWorkload/MergeTask")
mergeWorkflow.load()
self.assertEqual(mergeWorkflow.owner, "sfoulkes",
"Error: Wrong owner.")
self.assertEqual(
mergeWorkflow.spec,
os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1,
"Error: Wrong number of WF outputs.")
unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0][
"output_fileset"]
unmergedMergeOutput.loadData()
self.assertEqual(unmergedMergeOutput.name,
"/ResubmitTestWorkload/MergeTask/merged-Merged",
"Error: Unmerged output fileset is wrong.")
skimWorkflow = Workflow(
name="ResubmitTestWorkload",
task="/ResubmitTestWorkload/MergeTask/SkimTask")
skimWorkflow.load()
self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.")
self.assertEqual(
skimWorkflow.spec,
os.path.join(self.workDir, skimWorkflow.name, "WMSandbox",
"WMWorkload.pkl"), "Error: Wrong spec URL")
self.assertEqual(len(skimWorkflow.outputMap.keys()), 2,
"Error: Wrong number of WF outputs.")
mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][
"merged_output_fileset"]
unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][
"output_fileset"]
mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][
"merged_output_fileset"]
unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][
"output_fileset"]
mergedSkimOutputA.loadData()
mergedSkimOutputB.loadData()
unmergedSkimOutputA.loadData()
unmergedSkimOutputB.loadData()
self.assertEqual(
mergedSkimOutputA.name,
"/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA",
"Error: Merged output fileset is wrong: %s" %
mergedSkimOutputA.name)
self.assertEqual(
unmergedSkimOutputA.name,
"/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA",
"Error: Unmerged output fileset is wrong.")
self.assertEqual(
mergedSkimOutputB.name,
"/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB",
"Error: Merged output fileset is wrong.")
self.assertEqual(
unmergedSkimOutputB.name,
"/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB",
"Error: Unmerged output fileset is wrong.")
topLevelFileset = Fileset(
name="ResubmitTestWorkload-MergeTask-SomeBlock2")
topLevelFileset.loadData()
mergeSubscription = Subscription(fileset=topLevelFileset,
workflow=mergeWorkflow)
mergeSubscription.loadData()
self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0,
"Error: Wrong white/black list for merge sub.")
self.assertEqual(mergeSubscription["type"], "Merge",
"Error: Wrong subscription type.")
self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize",
"Error: Wrong split algo.")
skimSubscription = Subscription(fileset=unmergedMergeOutput,
workflow=skimWorkflow)
skimSubscription.loadData()
self.assertEqual(skimSubscription["type"], "Skim",
"Error: Wrong subscription type.")
self.assertEqual(skimSubscription["split_algo"], "FileBased",
"Error: Wrong split algo.")
return
def testReReco(self):
"""ReReco workflow"""
# create workflow
block = self.dataset + "#1"
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block))
self.assertEqual(len(files), 1)
def testReRecoBlackRunRestriction(self):
"""ReReco workflow with Run restrictions"""
block = self.dataset + "#2"
#add run blacklist
self.topLevelTask.setInputRunBlacklist([1, 2, 3, 4])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), 0)
def testReRecoWhiteRunRestriction(self):
block = self.dataset + "#2"
# Run Whitelist
self.topLevelTask.setInputRunWhitelist([2])
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testLumiMaskRestrictionsOK(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['1']
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['1,1']
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), GlobalParams.numOfFilesPerBlock())
def testLumiMaskRestrictionsKO(self):
block = self.dataset + "#1"
self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = [
'123454321'
]
self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = [
'123,123'
]
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
files = wmbs.validFiles(self.dbs.getFileBlock(block)[block]['Files'])
self.assertEqual(len(files), 0)
def testDuplicateFileInsert(self):
# using default wmspec
block = self.dataset + "#1"
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
wmbs.topLevelFileset.loadData()
numOfFiles = len(wmbs.topLevelFileset.files)
# check initially inserted files.
dbsFiles = self.dbs.getFileBlock(block)[block]['Files']
self.assertEqual(numOfFiles, len(dbsFiles))
firstFileset = wmbs.topLevelFileset
wmbsDao = wmbs.daofactory(classname="Files.InFileset")
numOfFiles = len(wmbsDao.execute(firstFileset.id))
self.assertEqual(numOfFiles, len(dbsFiles))
# use the new spec with same inputdataset
block = self.dataset + "#1"
wmspec = self.createWMSpec("TestSpec1")
dbs = self.getDBS(wmspec)
wmbs = self.createWMBSHelperWithTopTask(wmspec, block)
# check duplicate insert
dbsFiles = dbs.getFileBlock(block)[block]['Files']
numOfFiles = wmbs.addFiles(dbs.getFileBlock(block)[block])
self.assertEqual(numOfFiles, 0)
secondFileset = wmbs.topLevelFileset
wmbsDao = wmbs.daofactory(classname="Files.InFileset")
numOfFiles = len(wmbsDao.execute(secondFileset.id))
self.assertEqual(numOfFiles, len(dbsFiles))
self.assertNotEqual(firstFileset.id, secondFileset.id)
def testDuplicateSubscription(self):
"""Can't duplicate subscriptions"""
# using default wmspec
block = self.dataset + "#1"
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
wmbs.topLevelFileset.loadData()
numOfFiles = len(wmbs.topLevelFileset.files)
filesetId = wmbs.topLevelFileset.id
subId = wmbs.topLevelSubscription['id']
# check initially inserted files.
dbsFiles = self.dbs.getFileBlock(block)[block]['Files']
self.assertEqual(numOfFiles, len(dbsFiles))
firstFileset = wmbs.topLevelFileset
self.assertEqual(numOfFiles, len(dbsFiles))
# reinsert subscription - shouldn't create anything new
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block)
wmbs.topLevelFileset.loadData()
self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files))
self.assertEqual(filesetId, wmbs.topLevelFileset.id)
self.assertEqual(subId, wmbs.topLevelSubscription['id'])
# now do a montecarlo workflow
self.setupMCWMSpec()
mask = Mask(FirstRun=12,
FirstLumi=1234,
FirstEvent=12345,
LastEvent=999995,
LastLumi=12345,
LastRun=12)
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask)
wmbs.topLevelFileset.loadData()
numOfFiles = len(wmbs.topLevelFileset.files)
filesetId = wmbs.topLevelFileset.id
subId = wmbs.topLevelSubscription['id']
# check initially inserted files.
numDbsFiles = 1
self.assertEqual(numOfFiles, numDbsFiles)
firstFileset = wmbs.topLevelFileset
self.assertEqual(numOfFiles, numDbsFiles)
# reinsert subscription - shouldn't create anything new
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask)
wmbs.topLevelFileset.loadData()
self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files))
self.assertEqual(filesetId, wmbs.topLevelFileset.id)
self.assertEqual(subId, wmbs.topLevelSubscription['id'])
def testParentage(self):
"""
1. check whether parent files are created in wmbs.
2. check parent files are associated to child.
3. When 2 specs with the same input data (one with parent processing, one without it)
is inserted, if one without parent processing inserted first then the other with
parent processing insert, it still needs to create parent files although child files
are duplicate
"""
block = self.dataset + "#1"
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(
self.wmspec, block, parentFlag=False, detail=True)
# file creation without parents
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# no parent per child
self.assertEqual(len(child["parents"]), 0)
wmbs, sub, numFiles = self.createWMBSHelperWithTopTask(self.wmspec,
block,
parentFlag=True,
detail=True)
self.assertEqual(GlobalParams.numOfFilesPerBlock(), numFiles)
wmbs.topLevelFileset.loadData()
for child in wmbs.topLevelFileset.files:
# one parent per child
self.assertEqual(len(child["parents"]), 1)
def testMCFakeFileInjection(self):
"""Inject fake Monte Carlo files into WMBS"""
self.setupMCWMSpec()
mask = Mask(FirstRun=12,
FirstLumi=1234,
FirstEvent=12345,
LastEvent=999995,
LastLumi=12345,
LastRun=12)
wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask)
subscription = wmbs.topLevelSubscription
self.assertEqual(1, subscription.exists())
fileset = subscription['fileset']
self.assertEqual(1, fileset.exists())
fileset.loadData() # need to refresh from database
self.assertEqual(len(fileset.files), 1)
self.assertEqual(len(fileset.parents), 0)
self.assertFalse(fileset.open)
file = list(fileset.files)[0]
self.assertEqual(file['events'], mask['LastEvent'] -
mask['FirstEvent'] + 1) # inclusive range
self.assertEqual(file['merged'],
False) # merged files get added to dbs
self.assertEqual(len(file['parents']), 0)
#file.loadData()
self.assertEqual(sorted(file['locations']), sorted(self.ses))
self.assertEqual(len(file.getParentLFNs()), 0)
self.assertEqual(len(file.getRuns()), 1)
run = file.getRuns()[0]
self.assertEqual(run.run, mask['FirstRun'])
self.assertEqual(run.lumis[0], mask['FirstLumi'])
self.assertEqual(run.lumis[-1], mask['LastLumi'])
self.assertEqual(len(run.lumis),
mask['LastLumi'] - mask['FirstLumi'] + 1)
class RepackMergeTest(unittest.TestCase):
"""
_RepackMergeTest_
Test for RepackMerge job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "T0.WMBS"])
self.splitterFactory = SplitterFactory(package = "T0.JobSplitting")
myThread = threading.currentThread()
daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
wmbsDaoFactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
myThread.dbi.processData("""INSERT INTO wmbs_location
(id, site_name, state, state_time)
VALUES (1, 'SomeSite', 1, 1)
""", transaction = False)
myThread.dbi.processData("""INSERT INTO wmbs_pnns
(id, pnn)
VALUES (2, 'SomePNN')
""", transaction = False)
myThread.dbi.processData("""INSERT INTO wmbs_location_pnns
(location, pnn)
VALUES (1, 2)
""", transaction = False)
insertRunDAO = daoFactory(classname = "RunConfig.InsertRun")
insertRunDAO.execute(binds = { 'RUN' : 1,
'HLTKEY' : "someHLTKey" },
transaction = False)
insertLumiDAO = daoFactory(classname = "RunConfig.InsertLumiSection")
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 1 },
transaction = False)
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 2 },
transaction = False)
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 3 },
transaction = False)
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 4 },
transaction = False)
insertLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 5 },
transaction = False)
insertStreamDAO = daoFactory(classname = "RunConfig.InsertStream")
insertStreamDAO.execute(binds = { 'STREAM' : "A" },
transaction = False)
insertCMSSVersionDAO = daoFactory(classname = "RunConfig.InsertCMSSWVersion")
insertCMSSVersionDAO.execute(binds = { 'VERSION' : "CMSSW_4_2_7" },
transaction = False)
insertStreamCMSSWVersionDAO = daoFactory(classname = "RunConfig.InsertStreamCMSSWVersion")
insertStreamCMSSWVersionDAO.execute(binds = { 'RUN' : 1,
'STREAM' : 'A',
'VERSION' : "CMSSW_4_2_7" },
transaction = False)
insertStreamerDAO = daoFactory(classname = "RunConfig.InsertStreamer")
insertStreamerDAO.execute(streamerPNN = "SomePNN",
binds = { 'RUN' : 1,
'P5_ID' : 1,
'LUMI' : 4,
'STREAM' : "A",
'LFN' : "/testLFN/A",
'FILESIZE' : 100,
'EVENTS' : 100,
'TIME' : int(time.time()) },
transaction = False)
insertStreamFilesetDAO = daoFactory(classname = "RunConfig.InsertStreamFileset")
insertStreamFilesetDAO.execute(1, "A", "TestFileset1")
self.fileset1 = Fileset(name = "TestFileset1")
self.fileset2 = Fileset(name = "TestFileset2")
self.fileset1.load()
self.fileset2.create()
workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test")
workflow2 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow2", task="Test")
workflow1.create()
workflow2.create()
self.subscription1 = Subscription(fileset = self.fileset1,
workflow = workflow1,
split_algo = "Repack",
type = "Repack")
self.subscription2 = Subscription(fileset = self.fileset2,
workflow = workflow2,
split_algo = "RepackMerge",
type = "RepackMerge")
self.subscription1.create()
self.subscription2.create()
myThread.dbi.processData("""INSERT INTO wmbs_workflow_output
(WORKFLOW_ID, OUTPUT_IDENTIFIER, OUTPUT_FILESET)
VALUES (%d, 'SOMEOUTPUT', %d)
""" % (workflow1.id, self.fileset2.id),
transaction = False)
# keep for later
self.insertSplitLumisDAO = daoFactory(classname = "JobSplitting.InsertSplitLumis")
self.insertClosedLumiDAO = daoFactory(classname = "RunLumiCloseout.InsertClosedLumi")
self.feedStreamersDAO = daoFactory(classname = "Tier0Feeder.FeedStreamers")
self.acquireFilesDAO = wmbsDaoFactory(classname = "Subscriptions.AcquireFiles")
self.completeFilesDAO = wmbsDaoFactory(classname = "Subscriptions.CompleteFiles")
self.currentTime = int(time.time())
# default split parameters
self.splitArgs = {}
self.splitArgs['minInputSize'] = 2.1 * 1024 * 1024 * 1024
self.splitArgs['maxInputSize'] = 4.0 * 1024 * 1024 * 1024
self.splitArgs['maxInputEvents'] = 100000000
self.splitArgs['maxInputFiles'] = 1000
self.splitArgs['maxEdmSize'] = 20 * 1024 * 1024 * 1024
self.splitArgs['maxOverSize'] = 10 * 1024 * 1024 * 1024
self.SplitArgs['maxLatency'] = 50000
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def deleteSplitLumis(self):
"""
_deleteSplitLumis_
"""
myThread = threading.currentThread()
myThread.dbi.processData("""DELETE FROM lumi_section_split_active
""",
transaction = False)
return
def test00(self):
"""
_test00_
Test that the job name prefix feature works
Test max edm size threshold for single lumi
small lumi, followed by over-large lumi
expect 1 job for small lumi and 4 jobs for over-large
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2 * lumi):
newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxEdmSize'] = 13000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 3,
"ERROR: JobFactory didn't create three jobs")
job = jobGroups[0].jobs[0]
self.assertTrue(job['name'].startswith("RepackMerge-"),
"ERROR: Job has wrong name")
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 3,
"ERROR: Job does not process 3 files")
job = jobGroups[0].jobs[2]
self.assertEqual(len(job.getFiles()), 1,
"ERROR: Job does not process 1 file")
return
def test01(self):
"""
_test01_
Test max size threshold for single lumi
small lumi, followed by large lumi
expect 1 job for small lumi and 1 job for large
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size = 1000 * lumi, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputSize'] = 3000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test02(self):
"""
_test02_
Test max event threshold for single lumi
small lumi, followed by large lumi
expect 1 job for small lumi and 1 job for large
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size = 1000, events = 100 * lumi)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputEvents'] = 300
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test03(self):
"""
_test03_
Test max input files threshold for single lumi
small lumi, followed by large lumi
expect 1 job for small lumi and 1 job for large
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(lumi * 2):
newFile = File(makeUUID(), size = 1000, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputFiles'] = 3
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
return
def test04(self):
"""
_test04_
Test max size threshold for multi lumi
3 same size lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 3]:
for i in range(2):
newFile = File(makeUUID(), size = 1000, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 3000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputSize'] = 5000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset2.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test05(self):
"""
_test05_
Test max event threshold for multi lumi
3 same size lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 3]:
for i in range(2):
newFile = File(makeUUID(), size = 1000, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 3000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputEvents'] = 500
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset2.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test06(self):
"""
_test06_
Test max input files threshold for multi lumi
3 same size lumis
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 3]:
for i in range(2):
newFile = File(makeUUID(), size = 1000, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 3000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputFiles'] = 5
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset2.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test07(self):
"""
_test07_
Test over merge
one small lumi, one large lumi (small below min size,
large below max size, but both together above max size)
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 3000
mySplitArgs['maxInputSize'] = 9000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
return
def test08(self):
"""
_test08_
Test under merge (over merge size threshold)
one small lumi, one large lumi (small below min size,
large below max size, but both together above max size)
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 3000
mySplitArgs['maxInputSize'] = 9000
mySplitArgs['maxOverSize'] = 9500
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
self.fileset2.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test09(self):
"""
_test09_
Test under merge (over merge event threshold)
one small lumi, one large lumi (small below min size,
large below max size, but both together above max size)
It was changed due to maxinputevents not being used anymore.
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2]:
for i in range(2):
newFile = File(makeUUID(), size = 1000 * lumi * lumi, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 1500
mySplitArgs['maxInputSize'] = 9000
mySplitArgs['maxOverSize'] = 9500
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
self.fileset2.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
return
def test10(self):
"""
_test10_
Test merging of multiple lumis with holes in the lumi sequence
Hole is due to no streamer files for the lumi
Multi lumi input
It only works with a single hole, as it creates a merged file even with it being of a smaller size than the mininputsize.
It was changed due to the maxinputevents not being used anymore
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 4]:
for i in range(2):
newFile = File(makeUUID(), size = 1000, events = 100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomePNN", immediateSave = False)
newFile.create()
self.fileset2.addFile(newFile)
self.fileset2.commit()
jobFactory = self.splitterFactory(package = "WMCore.WMBS",
subscription = self.subscription2)
mySplitArgs['minInputSize'] = 100000
mySplitArgs['maxInputSize'] = 200000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
self.insertClosedLumiDAO.execute(binds = { 'RUN' : 1,
'LUMI' : 3,
'STREAM' : "A",
'FILECOUNT' : 0,
'INSERT_TIME' : self.currentTime,
'CLOSE_TIME' : self.currentTime },
transaction = False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
return
def injectJobs(self):
"""
_injectJobs_
Inject two workflows into WMBS and save the job objects to disk.
"""
testWorkflowA = Workflow(spec="specA.pkl",
owner="Steve",
name="wf001",
task="TestTaskA")
testWorkflowA.create()
testWorkflowB = Workflow(spec="specB.pkl",
owner="Steve",
name="wf002",
task="TestTaskB")
testWorkflowB.create()
testFileset = Fileset("testFileset")
testFileset.create()
testSubA = Subscription(fileset=testFileset, workflow=testWorkflowA)
testSubA.create()
testSubB = Subscription(fileset=testFileset, workflow=testWorkflowB)
testSubB.create()
testGroupA = JobGroup(subscription=testSubA)
testGroupA.create()
testGroupB = JobGroup(subscription=testSubB)
testGroupB.create()
stateChanger = ChangeState(self.createConfig(),
"jobsubmittercaching_t")
for i in range(10):
newFile = File(lfn="testFile%s" % i,
locations=set(["se.T1_US_FNAL", "se.T1_UK_RAL"]))
newFile.create()
newJobA = Job(name="testJobA-%s" % i, files=[newFile])
newJobA["workflow"] = "wf001"
newJobA["possiblePSN"] = ["T1_US_FNAL"]
newJobA["sandbox"] = "%s/somesandbox" % self.testDir
newJobA["owner"] = "Steve"
jobCacheDir = os.path.join(self.testDir, "jobA-%s" % i)
os.mkdir(jobCacheDir)
newJobA["cache_dir"] = jobCacheDir
newJobA["type"] = "Processing"
newJobA['requestType'] = 'ReReco'
newJobA.create(testGroupA)
jobHandle = open(os.path.join(jobCacheDir, "job.pkl"), "wb")
pickle.dump(newJobA, jobHandle)
jobHandle.close()
stateChanger.propagate([newJobA], "created", "new")
newJobB = Job(name="testJobB-%s" % i, files=[newFile])
newJobB["workflow"] = "wf001"
newJobB["possiblePSN"] = ["T1_UK_RAL"]
newJobB["sandbox"] = "%s/somesandbox" % self.testDir
newJobB["owner"] = "Steve"
jobCacheDir = os.path.join(self.testDir, "jobB-%s" % i)
os.mkdir(jobCacheDir)
newJobB["cache_dir"] = jobCacheDir
newJobB["type"] = "Processing"
newJobB['requestType'] = 'ReReco'
newJobB.create(testGroupB)
jobHandle = open(os.path.join(jobCacheDir, "job.pkl"), "wb")
pickle.dump(newJobB, jobHandle)
jobHandle.close()
stateChanger.propagate([newJobB], "created", "new")
return
def createTestJobGroup(self,
config,
name="TestWorkthrough",
filesetName="TestFileset",
specLocation="spec.xml",
error=False,
task="/TestWorkload/ReReco",
multicore=False):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=specLocation,
owner=self.OWNERDN,
name=name,
task=task,
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
self.inject.execute(names=[name], injected=True)
testWMBSFileset = Fileset(name=filesetName)
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
outputWMBSFileset = Fileset(name='%sOutput' % filesetName)
outputWMBSFileset.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation('malpaquet')
testFileC.create()
outputWMBSFileset.addFile(testFileC)
outputWMBSFileset.commit()
outputWMBSFileset.markOpen(0)
testWorkflow.addOutput('output', outputWMBSFileset)
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"badBackfillJobReport.pkl")
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
elif multicore:
path1 = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/MulticoreReport.pkl")
path2 = path1
else:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'mergeReport1.pkl')
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
report1.load(filename=path1)
report2.load(filename=path2)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
for i in range(self.nJobs):
if i < self.nJobs / 2:
testJobGroup.jobs[i]['fwjr'] = report1
else:
testJobGroup.jobs[i]['fwjr'] = report2
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
class RepackTest(unittest.TestCase):
"""
_RepackTest_
Test for Repack job splitter
"""
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["T0.WMBS"])
self.splitterFactory = SplitterFactory(package="T0.JobSplitting")
myThread = threading.currentThread()
daoFactory = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=myThread.dbi)
myThread.dbi.processData("""INSERT INTO wmbs_location
(id, site_name, state)
VALUES (1, 'SomeSite', 1)
""",
transaction=False)
myThread.dbi.processData("""INSERT INTO wmbs_location_senames
(location, se_name)
VALUES (1, 'SomeSE')
""",
transaction=False)
myThread.dbi.processData("""INSERT INTO wmbs_location_senames
(location, se_name)
VALUES (1, 'SomeSE2')
""",
transaction=False)
insertRunDAO = daoFactory(classname="RunConfig.InsertRun")
insertRunDAO.execute(binds={
'RUN': 1,
'TIME': int(time.time()),
'HLTKEY': "someHLTKey"
},
transaction=False)
insertLumiDAO = daoFactory(classname="RunConfig.InsertLumiSection")
for lumi in [1, 2, 3, 4]:
insertLumiDAO.execute(binds={
'RUN': 1,
'LUMI': lumi
},
transaction=False)
insertStreamDAO = daoFactory(classname="RunConfig.InsertStream")
insertStreamDAO.execute(binds={'STREAM': "A"}, transaction=False)
insertStreamFilesetDAO = daoFactory(
classname="RunConfig.InsertStreamFileset")
insertStreamFilesetDAO.execute(1, "A", "TestFileset1")
self.fileset1 = Fileset(name="TestFileset1")
self.fileset1.load()
workflow1 = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow1",
task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset=self.fileset1,
workflow=workflow1,
split_algo="Repack",
type="Repack")
self.subscription1.create()
# keep for later
self.insertClosedLumiDAO = daoFactory(
classname="RunLumiCloseout.InsertClosedLumi")
self.currentTime = int(time.time())
# default split parameters
self.splitArgs = {}
self.splitArgs['maxSizeSingleLumi'] = 20 * 1024 * 1024 * 1024
self.splitArgs['maxSizeMultiLumi'] = 10 * 1024 * 1024 * 1024
self.splitArgs['maxInputEvents'] = 500000
self.splitArgs['maxInputFiles'] = 1000
return
def tearDown(self):
"""
_tearDown_
"""
self.testInit.clearDatabase()
return
def getNumActiveSplitLumis(self):
"""
_getNumActiveSplitLumis_
helper function that counts the number of active split lumis
"""
myThread = threading.currentThread()
results = myThread.dbi.processData("""SELECT COUNT(*)
FROM lumi_section_split_active
""",
transaction=False)[0].fetchall()
return results[0][0]
def test00(self):
"""
_test00_
Test that the job name prefix feature works
Test multi lumi size threshold
Multi lumi input
"""
mySplitArgs = self.splitArgs.copy()
for lumi in [1, 2, 3, 4]:
filecount = 2
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
mySplitArgs['maxSizeMultiLumi'] = self.splitArgs['maxSizeMultiLumi']
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxSizeMultiLumi'] = 5000
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertTrue(job['name'].startswith("Repack-"),
"ERROR: Job has wrong name")
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset1.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertTrue(job['name'].startswith("Repack-"),
"ERROR: Job has wrong name")
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.assertEqual(self.getNumActiveSplitLumis(), 0,
"ERROR: Split lumis were created")
return
def test01(self):
"""
_test01_
Test multi lumi event threshold
Multi lumi input
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1, 2, 3, 4]:
filecount = 2
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputEvents'] = 500
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset1.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.assertEqual(self.getNumActiveSplitLumis(), 0,
"ERROR: Split lumis were created")
return
def test02(self):
"""
_test02_
Test single lumi size threshold
Single lumi input
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1]:
filecount = 8
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxSizeSingleLumi'] = 6500
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 6,
"ERROR: Job does not process 6 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
self.assertEqual(self.getNumActiveSplitLumis(), 1,
"ERROR: Split lumis were not created")
return
def test03(self):
"""
_test03_
Test single lumi event threshold
Single lumi input
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1]:
filecount = 8
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputEvents'] = 650
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 2,
"ERROR: JobFactory didn't create two jobs")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 6,
"ERROR: Job does not process 6 files")
job = jobGroups[0].jobs[1]
self.assertEqual(len(job.getFiles()), 2,
"ERROR: Job does not process 2 files")
self.assertEqual(self.getNumActiveSplitLumis(), 1,
"ERROR: Split lumis were not created")
return
def test04(self):
"""
_test04_
Test streamer count threshold (only multi lumi)
Multi lumi input
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1, 2, 3, 4]:
filecount = 2
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
mySplitArgs['maxInputFiles'] = 5
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.fileset1.markOpen(False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create a single job")
job = jobGroups[0].jobs[0]
self.assertEqual(len(job.getFiles()), 4,
"ERROR: Job does not process 4 files")
self.assertEqual(self.getNumActiveSplitLumis(), 0,
"ERROR: Split lumis were created")
return
def test05(self):
"""
_test05_
Test repacking of multiple lumis with holes in the lumi sequence
Multi lumi input
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1, 2, 4]:
filecount = 2
for i in range(filecount):
newFile = File(makeUUID(), size=1000, events=100)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
mySplitArgs['maxInputFiles'] = 5
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 0,
"ERROR: JobFactory should have returned no JobGroup")
self.insertClosedLumiDAO.execute(binds={
'RUN': 1,
'LUMI': 3,
'STREAM': "A",
'FILECOUNT': 0,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
},
transaction=False)
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 1,
"ERROR: JobFactory didn't create one job")
self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4,
"ERROR: first job does not process 4 files")
return
def test06(self):
"""
_test06_
Test repacking of 3 lumis
2 small lumis (single job), followed by a big one (multiple jobs)
files for lumi 1 and 2 are below multi-lumi thresholds
files for lumi 3 are above single-lumi threshold
"""
mySplitArgs = self.splitArgs.copy()
insertClosedLumiBinds = []
for lumi in [1, 2, 3]:
filecount = 2
for i in range(filecount):
if lumi == 3:
nevents = 500
else:
nevents = 100
newFile = File(makeUUID(), size=1000, events=nevents)
newFile.addRun(Run(1, *[lumi]))
newFile.setLocation("SomeSE", immediateSave=False)
newFile.create()
self.fileset1.addFile(newFile)
insertClosedLumiBinds.append({
'RUN': 1,
'LUMI': lumi,
'STREAM': "A",
'FILECOUNT': filecount,
'INSERT_TIME': self.currentTime,
'CLOSE_TIME': self.currentTime
})
self.fileset1.commit()
jobFactory = self.splitterFactory(package="WMCore.WMBS",
subscription=self.subscription1)
self.insertClosedLumiDAO.execute(binds=insertClosedLumiBinds,
transaction=False)
mySplitArgs['maxInputEvents'] = 900
jobGroups = jobFactory(**mySplitArgs)
self.assertEqual(len(jobGroups), 1,
"ERROR: JobFactory didn't return one JobGroup")
self.assertEqual(len(jobGroups[0].jobs), 3,
"ERROR: JobFactory didn't create three jobs")
self.assertEqual(len(jobGroups[0].jobs[0].getFiles()), 4,
"ERROR: first job does not process 4 files")
self.assertEqual(len(jobGroups[0].jobs[1].getFiles()), 1,
"ERROR: second job does not process 1 file")
self.assertEqual(len(jobGroups[0].jobs[2].getFiles()), 1,
"ERROR: third job does not process 1 file")
return
def createGiantJobSet(self,
name,
config,
nSubs=10,
nJobs=10,
nFiles=1,
spec="spec.xml"):
"""
Creates a massive set of jobs
"""
jobList = []
for i in range(0, nSubs):
# Make a bunch of subscriptions
localName = '%s-%i' % (name, i)
testWorkflow = Workflow(spec=spec,
owner=self.OWNERDN,
name=localName,
task="Test",
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
testWMBSFileset = Fileset(name=localName)
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
filesToComplete = []
for j in range(0, nJobs):
# Create jobs for each subscription
testFileA = File(lfn="%s-%i-lfnA" % (localName, j),
size=1024,
events=10)
testFileA.addRun(
Run(
10, *[
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40
]))
testFileA.setLocation('malpaquet')
testFileA.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.commit()
filesToComplete.append(testFileA)
testJob = Job(name='%s-%i' % (localName, j))
testJob.addFile(testFileA)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
jobList.append(testJob)
for k in range(0, nFiles):
# Create output files
testFile = File(lfn="%s-%i-output" % (localName, k),
size=1024,
events=10)
testFile.addRun(Run(10, *[12312]))
testFile.setLocation('malpaquet')
testFile.create()
testJobGroup.output.addFile(testFile)
testJobGroup.output.commit()
testJobGroup.commit()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'success', 'complete')
changer.propagate(testJobGroup.jobs, 'cleanout', 'success')
testWMBSFileset.markOpen(0)
testSubscription.completeFiles(filesToComplete)
return jobList
