def createTestSubscription(self, nFiles, nSites=1, closeFileset=False):
"""
_createTestSubscription_
Create a set of test subscriptions for testing purposes.
"""
if nSites > self.nSites:
nSites = self.nSites
testFileset = Fileset(name="TestFileset")
testFileset.create()
# Create a testWorkflow
testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test")
testWorkflow.create()
# Create the files for each site
for s in range(nSites):
for i in range(nFiles):
newFile = File(makeUUID(), size=1024, events=100, locations=set(["site%i.cern.ch" % s]))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testSubscription = Subscription(
fileset=testFileset, workflow=testWorkflow, split_algo="MinFileBased", type="Processing"
)
testSubscription.create()
# Close the fileset
if closeFileset:
testFileset.markOpen(isOpen=False)
return testSubscription
def createTestJob(subscriptionType="Merge"):
"""
_createTestJob_
Create a test job with two files as input. This will also create the
appropriate workflow, jobgroup and subscription.
"""
testWorkflow = Workflow(spec=makeUUID(), owner="Simon", name=makeUUID(), task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type=subscriptionType)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(1, *[46]))
testFileA.create()
testFileB.create()
testJob = Job(name=makeUUID(), files=[testFileA, testFileB])
testJob["couch_record"] = "somecouchrecord"
testJob["location"] = "test.site.ch"
testJob.create(group=testJobGroup)
testJob.associateFiles()
return testJob
def testWorkflowInjectMarking(self):
"""
_testWorkflowInjectMarking_
Test whether or not we can mark a workflow as injected or not.
"""
owner = "moron"
workflows = []
for i in range(0, 10):
testWorkflow = Workflow(spec="sp00%i" % i, owner=owner,
name="wf00%i" % i, task="task%i" % i)
testWorkflow.create()
workflows.append(testWorkflow)
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
dbinterface=myThread.dbi)
getAction = daoFactory(classname="Workflow.GetInjectedWorkflows")
markAction = daoFactory(classname="Workflow.MarkInjectedWorkflows")
result = getAction.execute(injected=True)
self.assertEqual(len(result), 0)
result = getAction.execute(injected=False)
self.assertEqual(len(result), 10)
names = ['wf002', 'wf004', 'wf006', 'wf008']
markAction.execute(names=names, injected=True)
result = getAction.execute(injected=True)
self.assertEqual(result, names)
result = getAction.execute(injected=False)
self.assertEqual(len(result), 6)
return
def createLargeFileBlock(self):
"""
_createLargeFileBlock_
Creates a large group of files for testing
"""
testFileset = Fileset(name = "TestFilesetX")
testFileset.create()
for i in range(5000):
newFile = File(makeUUID(), size = 1000, events = 100,
locations = set(["somese.cern.ch"]))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "mnorman",
name = "wf003", task="Test" )
testWorkflow.create()
largeSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
split_algo = "FileBased",
type = "Processing")
largeSubscription.create()
return largeSubscription
def generateFakeMCFile(self, numEvents = 100, firstEvent = 1,
lastEvent = 100, firstLumi = 1, lastLumi = 10,
index = 1):
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCTestFileset %i" % index)
singleMCFileset.create()
newFile = File("MCFakeFileTest %i" % index, size = 1000,
events = numEvents,
locations = set(["somese.cern.ch"]))
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
newFile.create()
singleMCFileset.addFile(newFile)
singleMCFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task="Test")
testWorkflow.create()
singleMCFileSubscription = Subscription(fileset = singleMCFileset,
workflow = testWorkflow,
split_algo = "EventBased",
type = "Production")
singleMCFileSubscription.create()
return singleMCFileSubscription
def testA(self):
"""
_testA_
Handle AddWorkflowToManage events
"""
myThread = threading.currentThread()
config = self.getConfig()
testWorkflowManager = WorkflowManager(config)
testWorkflowManager.prepareToStart()
for i in xrange(0, WorkflowManagerTest._maxMessage):
workflow = Workflow(spec = "testSpec.xml", owner = "riahi", \
name = "testWorkflow" + str(i), task = "testTask")
workflow.create()
for j in xrange(0, 3):
workflowManagerdict = {'payload':{'WorkflowId' : workflow.id \
, 'FilesetMatch': 'FILESET_' + str(j) ,'SplitAlgo':'NO SPLITALGO', 'Type':'NO TYPE' }}
testWorkflowManager.handleMessage( \
type = 'AddWorkflowToManage' , payload = workflowManagerdict )
time.sleep(30)
myThread.workerThreadManager.terminateWorkers()
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
def testCreateDeleteExistsNoFiles(self):
"""
_testCreateDeleteExistsNoFiles_
Create and then delete a job but don't add any input files to it.
Use the job class's exists() method to determine if the job has been
written to the database before it is created, after it has been created
and after it has been deleted.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job(name="TestJob")
assert testJob.exists() == False, "ERROR: Job exists before it was created"
testJob.create(group=testJobGroup)
assert testJob.exists() >= 0, "ERROR: Job does not exist after it was created"
testJob.delete()
assert testJob.exists() == False, "ERROR: Job exists after it was delete"
return
def testFilesWithoutOtherSubscriptions(self):
"""
_testFilesWithoutOtherSubscriptions_
Test the case where files only in the delete subscription
can happen if cleanup of the other subscriptions is fast
"""
testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test")
testWorkflowA.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i), size=1000, events=100, locations=set(["somese.cern.ch"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(
fileset=testFileset, workflow=testWorkflowA, split_algo="SiblingProcessingBased", type="Processing"
)
testSubscriptionA.create()
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionA)
result = deleteFactoryA(files_per_job=50)
self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.")
return
def testByRunAndRunBlacklist(self):
"""
_testByRunAndRunWhitelist_
Create harvesting jobs by run for the runs provided in the RunWhitelist
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro",
name="TestWorkflow", task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow,
split_algo="Harvest", type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3])
self.assertEqual(len(jobGroups), 1, "One jobgroup per location")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 3)
def testLoadFromTask(self):
"""
_testLoadFromTask_
Verify that Workflow.LoadFromTask DAO correct loads the workflow by
task.
"""
testWorkflow1 = Workflow(spec="spec1.xml",
owner="Hassen",
name="wf001",
task="sometask")
testWorkflow1.create()
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
loadFromTaskDAO = daoFactory(classname="Workflow.LoadFromTask")
listFromTask = loadFromTaskDAO.execute(task=testWorkflow1.task)
self.assertEqual(len(listFromTask), 1,
"ERROR: listFromTask should be 1.")
self.assertEqual(listFromTask[0]["task"], "sometask",
"ERROR: task should be sometask.")
return
def setUp(self):
"""
_setUp_
Create a single subscription with one file.
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules = ["WMCore.WMBS"],
useDefault = False)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute(siteName = "site1", seName = "somese.cern.ch")
locationAction.execute(siteName = "site2", seName = "otherse.cern.ch")
self.testFileset = Fileset(name = "TestFileset1")
self.testFileset.create()
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task="Test" )
testWorkflow.create()
self.testSubscription = Subscription(fileset = self.testFileset,
workflow = testWorkflow,
split_algo = "Periodic",
type = "Processing")
self.testSubscription.create()
return
def testCreateTransaction(self):
"""
_testCreateTransaction_
Create a JobGroup and commit it to the database. Rollback the database
transaction and verify that the JobGroup is no longer in the database.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test")
testWorkflow.create()
testFileset = WMBSFileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
assert testJobGroup.exists() == False, "ERROR: Job group exists before it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJobGroup.create()
assert testJobGroup.exists() >= 0, "ERROR: Job group does not exist after it was created"
myThread.transaction.rollback()
assert testJobGroup.exists() == False, "ERROR: Job group exists after transaction was rolled back."
testSubscription.delete()
testFileset.delete()
testWorkflow.delete()
return
def testCreateTransaction(self):
"""
_testCreateTransaction_
Create a workflow and commit it to the database and then roll the
transaction back. Use the workflow's exists() method to verify that the
workflow does not exist before create() is called, exists after create()
is called and does not exist after the transaction is rolled back.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task='Test')
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists before it was created")
testWorkflow.create()
self.assertTrue(
testWorkflow.exists() > 0,
"ERROR: Workflow does not exist after it has been created")
myThread.transaction.rollback()
self.assertEqual(
testWorkflow.exists(), False,
"ERROR: Workflow exists after the transaction was rolled back.")
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Create a JobGroup and then delete it. Use the JobGroup's exists()
method to determine if it exists before it is created, after it is
created and after it is deleted.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testFileset = WMBSFileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
self.assertFalse(testJobGroup.exists())
testJobGroup.create()
self.assertTrue(testJobGroup.exists())
testJobGroup.delete()
self.assertFalse(testJobGroup.exists())
testSubscription.delete()
testFileset.delete()
testWorkflow.delete()
return
def generateFakeMCFile(self, numEvents = 100, firstEvent = 1,
lastEvent = 100, firstLumi = 1, lastLumi = 10,
index = 1):
#MC comes with only one MCFakeFile
singleMCFileset = Fileset(name = "MCTestFileset %i" % index)
singleMCFileset.create()
newFile = File("MCFakeFileTest %i" % index, size = 1000,
events = numEvents,
locations = set(["somese.cern.ch"]))
newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1)))
newFile["first_event"] = firstEvent
newFile["last_event"] = lastEvent
newFile.create()
singleMCFileset.addFile(newFile)
singleMCFileset.commit()
testWorkflow = Workflow(spec = "spec.xml", owner = "Steve",
name = "wf001", task="Test")
testWorkflow.create()
singleMCFileSubscription = Subscription(fileset = singleMCFileset,
workflow = testWorkflow,
split_algo = "EventBased",
type = "Production")
singleMCFileSubscription.create()
return singleMCFileSubscription
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Test the create(), delete() and exists() methods of the workflow class
by creating and deleting a workflow. The exists() method will be
called before and after creation and after deletion.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task='Test', wfType="ReReco")
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists before it was created")
testWorkflow.create()
self.assertTrue(testWorkflow.exists() > 0,
"ERROR: Workflow does not exist after it has been created")
testWorkflow.create()
testWorkflow.delete()
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists after it has been deleted")
return
def testMask(self):
"""
_testMask_
Test the new mask setup
"""
testWorkflow = Workflow(spec="spec.xml", owner="Steve",
name="wf001", task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testJob = Job()
testJob['mask'].addRunAndLumis(run=100, lumis=[101, 102])
testJob['mask'].addRunAndLumis(run=200, lumis=[201, 202])
testJob.create(group=testJobGroup)
loadJob = Job(id=testJob.exists())
loadJob.loadData()
runs = loadJob['mask'].getRunAndLumis()
self.assertEqual(len(runs), 2)
self.assertEqual(runs[100], [[101, 102]])
self.assertEqual(runs[200], [[201, 202]])
bigRun = Run(100, *[101, 102, 103, 104])
badRun = Run(300, *[1001, 1002])
result = loadJob['mask'].filterRunLumisByMask([bigRun, badRun])
self.assertEqual(len(result), 1)
alteredRun = result.pop()
self.assertEqual(alteredRun.run, 100)
self.assertEqual(alteredRun.lumis, [101, 102])
run0 = Run(300, *[1001, 1002])
run1 = Run(300, *[1001, 1002])
loadJob['mask'].filterRunLumisByMask([run0, run1])
return
def testCountWorkflow(self):
"""
_testCountWorkflow_
"""
spec = "spec.py"
owner = "moron"
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
countSpecDAO = daoFactory(classname="Workflow.CountWorkflowBySpec")
workflows = []
for i in range(0, 10):
testWorkflow = Workflow(spec=spec,
owner=owner,
name="wf00%i" % i,
task="task%i" % i)
testWorkflow.create()
workflows.append(testWorkflow)
self.assertEqual(countSpecDAO.execute(spec=spec), 10)
for i in range(0, 10):
wf = workflows.pop()
wf.delete()
self.assertEqual(countSpecDAO.execute(spec=spec), 10 - (i + 1))
return
def testWorkflowInjectMarking(self):
"""
_testWorkflowInjectMarking_
Test whether or not we can mark a workflow as injected or not.
"""
owner = "moron"
workflows = []
for i in range(0, 10):
testWorkflow = Workflow(spec="sp00%i" % i, owner=owner,
name="wf00%i" % i, task="task%i" % i)
testWorkflow.create()
workflows.append(testWorkflow)
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
dbinterface=myThread.dbi)
getAction = daoFactory(classname="Workflow.GetInjectedWorkflows")
markAction = daoFactory(classname="Workflow.MarkInjectedWorkflows")
result = getAction.execute(injected=True)
self.assertEqual(len(result), 0)
result = getAction.execute(injected=False)
self.assertEqual(len(result), 10)
names = ['wf002', 'wf004', 'wf006', 'wf008']
markAction.execute(names=names, injected=True)
result = getAction.execute(injected=True)
self.assertEqual(result, names)
result = getAction.execute(injected=False)
self.assertEqual(len(result), 6)
return
def testListRunningJobs(self):
"""
_testListRunningJobs_
Test the ListRunningJobs DAO.
"""
testWorkflow = Workflow(spec = makeUUID(), owner = "Steve",
name = makeUUID(), task="Test")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset,
workflow = testWorkflow,
type = "Processing")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testJobA = Job(name = makeUUID(), files = [])
testJobA["couch_record"] = makeUUID()
testJobA.create(group = testJobGroup)
testJobA["state"] = "executing"
testJobB = Job(name = makeUUID(), files = [])
testJobB["couch_record"] = makeUUID()
testJobB.create(group = testJobGroup)
testJobB["state"] = "complete"
testJobC = Job(name = makeUUID(), files = [])
testJobC["couch_record"] = makeUUID()
testJobC.create(group = testJobGroup)
testJobC["state"] = "new"
changeStateAction = self.daoFactory(classname = "Jobs.ChangeState")
changeStateAction.execute(jobs = [testJobA, testJobB, testJobC])
runningJobsAction = self.daoFactory(classname = "Monitoring.ListRunningJobs")
runningJobs = runningJobsAction.execute()
assert len(runningJobs) == 2, \
"Error: Wrong number of running jobs returned."
for runningJob in runningJobs:
if runningJob["job_name"] == testJobA["name"]:
assert runningJob["state"] == testJobA["state"], \
"Error: Running job has wrong state."
assert runningJob["couch_record"] == testJobA["couch_record"], \
"Error: Running job has wrong couch record."
else:
assert runningJob["job_name"] == testJobC["name"], \
"Error: Running job has wrong name."
assert runningJob["state"] == testJobC["state"], \
"Error: Running job has wrong state."
assert runningJob["couch_record"] == testJobC["couch_record"], \
"Error: Running job has wrong couch record."
return
def test_AutoIncrementCheck(self):
"""
_AutoIncrementCheck_
Test and see whether we can find and set the auto_increment values
"""
myThread = threading.currentThread()
if not myThread.dialect.lower() == "mysql":
return
testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow)
testSubscription.create()
testFileA = File(lfn=makeUUID(), locations="test.site.ch")
testFileB = File(lfn=makeUUID(), locations="test.site.ch")
testFileA.create()
testFileB.create()
testFileset.addFile([testFileA, testFileB])
testFileset.commit()
testSubscription.acquireFiles([testFileA, testFileB])
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
incrementDAO = self.daoFactory(classname="Jobs.AutoIncrementCheck")
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 1)
incrementDAO.execute()
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 2)
incrementDAO.execute(input=10)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 11)
incrementDAO.execute(input=5)
testJob = Job()
testJob.create(group=testJobGroup)
self.assertEqual(testJob.exists(), 12)
return
def testMultiRunHarvesting(self):
"""
_testMultiRunHarvesting_
Provided a fileset with a couple of files and different runs, create a
single job for all the runs at a specific location, which also adds a
baggage to the job (True) which is later on looked up by SetupCMSSWPSet.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True)
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory(dqmHarvestUnit="multiRun")
self.assertEqual(len(jobGroups), 1)
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 1)
def createWorkflow(self, task):
"""
Register job into WMBS for each task through Workflows
"""
specURL = self.getWorkflowURL(task)
fileSet = Fileset(name=self.getFilesetName(task), is_open=True)
fileSet.create()
taskFlow = Workflow(spec=specURL, owner=self.owner, dn=self.owner_dn,
name=self.getWorkflowName(task), task=task.name())
taskFlow.create()
self.workflowDict[task.name()] = taskFlow
# Insert workflow into task
setattr(task.data.input.WMBS, 'WorkflowSpecURL', specURL)
# If the job is a merge job
# Find the task it merges from
# Then find the workflow for that task and assign it an output
if hasattr(task.inputReference(), 'outputModule'):
dummyStepName = task.inputReference().inputStep.split('/')[-1]
taskName = task.inputReference().inputStep.split('/')[-2]
outputModule = task.inputReference().outputModule
if taskName not in self.workflowDict.keys():
raise Exception('I am being asked to chain output for a task %s which does not yet exist' % taskName)
outputWorkflow = self.workflowDict[taskName]
outputWorkflow.addOutput(outputModule, fileSet)
logging.info('Registered workflow for step %s' % (task.name()))
return taskFlow, fileSet
def createSubscriptionWithFileABC(self):
""""
_createSubscriptionWithFileABC_
Create a subscription where the input fileset has three files. Also
create a second subscription that has acquired two of the files.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWorkflow2 = Workflow(spec="specBOGUS.xml",
owner="Simon",
name="wfBOGUS",
task="Test")
testWorkflow2.create()
testFileA = File(lfn="/this/is/a/lfnA",
size=1024,
events=20,
locations=set(["test.site.ch"]))
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn="/this/is/a/lfnB",
size=1024,
events=20,
locations=set(["test.site.ch"]))
testFileB.addRun(Run(1, *[46]))
testFileC = File(lfn="/this/is/a/lfnC",
size=1024,
events=20,
locations=set(["test.site.ch"]))
testFileC.addRun(Run(2, *[48]))
testFileA.create()
testFileB.create()
testFileC.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.addFile(testFileC)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testSubscription2 = Subscription(fileset=testFileset,
workflow=testWorkflow2)
testSubscription2.create()
testSubscription2.acquireFiles([testFileA, testFileB])
#return (testSubscription, testFileset, testWorkflow, testFileA,
# testFileB, testFileC)
return (testSubscription, testFileA, testFileB, testFileC)
def createSubscription(self, nFiles, lumisPerFile, twoSites=False, nEventsPerFile=100):
"""
_createSubscription_
Create a subscription for testing
"""
baseName = makeUUID()
testWorkflow = Workflow(spec="spec.xml", owner="dmwm",
name="testWorkflow_%s" % baseName[:4], task="Test")
testWorkflow.create()
testFileset = Fileset(name=baseName)
for i in range(nFiles):
newFile = self.createFile('%s_%i' % (baseName, i), nEventsPerFile,
i, lumisPerFile, 'T1_US_FNAL_Disk')
testFileset.addFile(newFile)
if twoSites:
for i in range(nFiles):
newFile = self.createFile('%s_%i_2' % (baseName, i), nEventsPerFile,
i, lumisPerFile, 'T2_CH_CERN')
testFileset.addFile(newFile)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
split_algo="EventAwareLumiByWork",
type="Processing")
testSubscription.create()
return testSubscription
def createLargeFileBlock(self):
"""
_createLargeFileBlock_
Creates a large group of files for testing
"""
testFileset = Fileset(name="TestFilesetX")
testFileset.create()
for _ in range(5000):
newFile = File(makeUUID(),
size=1000,
events=100,
locations=set(["T1_US_FNAL_Disk"]))
newFile.create()
testFileset.addFile(newFile)
testFileset.commit()
testWorkflow = Workflow(spec="spec.xml",
owner="mnorman",
name="wf003",
task="Test")
testWorkflow.create()
largeSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
split_algo="FileBased",
type="Processing")
largeSubscription.create()
return largeSubscription
def testDifferentSubscritionIDs(self):
"""
_testDifferentSubscriptionIDs_
Make sure that the merge splitting still runs if the subscription ID
is not equal to the workflow ID.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
dummyWorkflow = Workflow(name = "dummyWorkflow", spec = "bunk49",
owner = "Steve", task="Test2")
dummyWorkflow.create()
dummyFileset = Fileset(name = "dummyFileset")
dummyFileset.create()
dummySubscription1 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription2 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription1.create()
dummySubscription2.create()
myThread.transaction.commit()
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.mergeSubscription)
result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999,
max_merge_events = 999999999, merge_across_runs = False)
self.assertEqual(len(result), 1)
jobGroup = result[0]
self.assertEqual(len(jobGroup.jobs), 2)
return
def testByRunHarvesting(self):
"""
_testByRunHarvesting_
Provided a fileset with a couple of files and 4 different runs, create
one single job per run and location.
The multiRun baggage should be false in this case.
"""
multipleFilesFileset = createCommonFileset()
self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!")
harvestingWorkflow = Workflow(spec="spec.xml",
owner="amaltaro",
name="TestWorkflow",
task="Test")
harvestingWorkflow.create()
harvestSub = Subscription(fileset=multipleFilesFileset,
workflow=harvestingWorkflow,
split_algo="Harvest",
type="Harvesting")
harvestSub.create()
multipleFilesFileset.markOpen(False)
self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed")
jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub)
jobGroups = jobFactory()
self.assertEqual(len(jobGroups), 1, "Should have created 1 job group")
for jobGroup in jobGroups:
self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs")
for job in jobGroup.jobs:
baggage = job.getBaggage()
self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")
def testDifferentSubscritionIDs(self):
"""
_testDifferentSubscriptionIDs_
Make sure that the merge splitting still runs if the subscription ID
is not equal to the workflow ID.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
dummyWorkflow = Workflow(name = "dummyWorkflow", spec = "bunk49",
owner = "Steve", task="Test2")
dummyWorkflow.create()
dummyFileset = Fileset(name = "dummyFileset")
dummyFileset.create()
dummySubscription1 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription2 = Subscription(fileset = dummyFileset,
workflow = dummyWorkflow,
split_algo = "ParentlessMergeBySize")
dummySubscription1.create()
dummySubscription2.create()
myThread.transaction.commit()
self.stuffWMBS()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = self.mergeSubscription)
result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999,
max_merge_events = 999999999, merge_across_runs = False)
self.assertEqual(len(result), 1)
jobGroup = result[0]
self.assertEqual(len(jobGroup.jobs), 2)
return
def testCountWorkflow(self):
"""
_testCountWorkflow_
"""
spec = "spec.py"
owner = "moron"
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger,
dbinterface=myThread.dbi)
countSpecDAO = daoFactory(classname="Workflow.CountWorkflowBySpec")
workflows = []
for i in range(0, 10):
testWorkflow = Workflow(spec=spec, owner=owner,
name="wf00%i" % i, task="task%i" % i)
testWorkflow.create()
workflows.append(testWorkflow)
self.assertEqual(countSpecDAO.execute(spec=spec), 10)
for i in range(0, 10):
wf = workflows.pop()
wf.delete()
self.assertEqual(countSpecDAO.execute(spec=spec), 10 - (i + 1))
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Test the create(), delete() and exists() methods of the workflow class
by creating and deleting a workflow. The exists() method will be
called before and after creation and after deletion.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task='Test',
wfType="ReReco")
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists before it was created")
testWorkflow.create()
self.assertTrue(
testWorkflow.exists() > 0,
"ERROR: Workflow does not exist after it has been created")
testWorkflow.create()
testWorkflow.delete()
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists after it has been deleted")
return
def createJobGroups(self,
nSubs,
nJobs,
task,
workloadSpec,
site,
bl=[],
wl=[],
taskType='Processing',
name=None):
"""
_createJobGroups_
Creates a series of jobGroups for submissions
"""
jobGroupList = []
if name is None:
name = makeUUID()
testWorkflow = Workflow(spec=workloadSpec,
owner="tapas",
name=name,
task="basicWorkload/Production")
testWorkflow.create()
# Create subscriptions
for _ in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type=taskType,
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(name=name,
task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site,
bl=bl,
wl=wl)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
return jobGroupList
def testCreateTransaction(self):
"""
_testCreateTransaction_
Create a workflow and commit it to the database and then roll the
transaction back. Use the workflow's exists() method to verify that the
workflow does not exist before create() is called, exists after create()
is called and does not exist after the transaction is rolled back.
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task='Test')
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists before it was created")
testWorkflow.create()
self.assertTrue(testWorkflow.exists() > 0,
"ERROR: Workflow does not exist after it has been created")
myThread.transaction.rollback()
self.assertEqual(testWorkflow.exists(), False,
"ERROR: Workflow exists after the transaction was rolled back.")
return
def testAddDupsToFilesetBulk(self):
"""
_AddToDupsFilesetBulk_
Same as testAddDupsToFileset() but faster
"""
testWorkflowA = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflowA.create()
testWorkflowB = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production2")
testWorkflowB.create()
testFilesetA = Fileset(name = "inputFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "inputFilesetB")
testFilesetB.create()
testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA)
testSubscriptionA.create()
testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB)
testSubscriptionB.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10, locations = ['SiteA'])
testFileA.addRun(Run( 1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10, locations = ['SiteB'])
testFileB.addRun(Run( 1, *[45]))
addFilesToWMBSInBulk(testFilesetA.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
testFileset2 = Fileset(name = "inputFilesetA")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 2)
for file in testFileset2.files:
self.assertTrue(file in [testFileA, testFileB])
# Check that adding twice doesn't crash
addFilesToWMBSInBulk(testFilesetA.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
# Files should not get added to fileset B because fileset A is associated
# with wf001.
addFilesToWMBSInBulk(testFilesetB.id, "wf001",
[testFileA, testFileB],
conn = testFileA.getDBConn(),
transaction = testFileA.existingTransaction())
testFileset2 = Fileset(name = "inputFilesetB")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 0)
return
def testLoad(self):
"""
_testLoad_
Create a workflow and then try to load it from the database using the
following load methods:
Workflow.LoadFromName
Workflow.LoadFromID
Workflow.LoadFromSpecOwner
"""
testWorkflowA = Workflow(
spec="spec.xml", owner="Simon", name="wf001", task="Test", wfType="ReReco", priority=1000
)
testWorkflowA.create()
testWorkflowB = Workflow(name="wf001", task="Test")
testWorkflowB.load()
self.assertTrue(
(testWorkflowA.id == testWorkflowB.id)
and (testWorkflowA.name == testWorkflowB.name)
and (testWorkflowA.spec == testWorkflowB.spec)
and (testWorkflowA.task == testWorkflowB.task)
and (testWorkflowA.wfType == testWorkflowB.wfType)
and (testWorkflowA.owner == testWorkflowB.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromName Failed",
)
testWorkflowC = Workflow(id=testWorkflowA.id)
testWorkflowC.load()
self.assertTrue(
(testWorkflowA.id == testWorkflowC.id)
and (testWorkflowA.name == testWorkflowC.name)
and (testWorkflowA.spec == testWorkflowC.spec)
and (testWorkflowA.task == testWorkflowC.task)
and (testWorkflowA.wfType == testWorkflowC.wfType)
and (testWorkflowA.owner == testWorkflowC.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromID Failed",
)
testWorkflowD = Workflow(spec="spec.xml", owner="Simon", task="Test")
testWorkflowD.load()
self.assertTrue(
(testWorkflowA.id == testWorkflowD.id)
and (testWorkflowA.name == testWorkflowD.name)
and (testWorkflowA.spec == testWorkflowD.spec)
and (testWorkflowA.task == testWorkflowD.task)
and (testWorkflowA.wfType == testWorkflowD.wfType)
and (testWorkflowA.owner == testWorkflowD.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromSpecOwner Failed",
)
testWorkflowA.delete()
return
def testAddDupsToFileset(self):
"""
_AddToDupsFileset_
Verify the the dups version of the AddToFileset DAO will not add files
to a fileset if they're already associated to another fileset with the
same workflow.
"""
testWorkflowA = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflowA.create()
testWorkflowB = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production2")
testWorkflowB.create()
testFilesetA = Fileset(name = "inputFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "inputFilesetB")
testFilesetB.create()
testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA)
testSubscriptionA.create()
testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB)
testSubscriptionB.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run( 1, *[45]))
testFileA.create()
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run( 1, *[45]))
testFileB.create()
addToFileset = self.daofactory(classname = "Files.AddDupsToFileset")
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetA.id, workflow = "wf001")
testFileset2 = Fileset(name = "inputFilesetA")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 2)
for file in testFileset2.files:
self.assertTrue(file in [testFileA, testFileB])
# Check that adding twice doesn't crash
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetA.id, workflow = "wf001")
# Files should not get added to fileset B because fileset A is associated
# with wf001.
addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']],
fileset = testFilesetB.id, workflow = "wf001")
testFileset2 = Fileset(name = "inputFilesetB")
testFileset2.loadData()
self.assertEqual(len(testFileset2.files), 0)
return
def setUp(self):
"""
_setUp_
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"])
self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting")
myThread = threading.currentThread()
self.myThread = myThread
daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.WMBSFactory = daoFactory
config = self.getConfig()
self.changer = ChangeState(config)
myResourceControl = ResourceControl()
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk",
"T1_US_FNAL")
myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC",
"T1_US_FNAL")
myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN",
"T2_CH_CERN")
self.fileset1 = Fileset(name="TestFileset1")
for fileNum in range(11):
newFile = File("/some/file/name%d" % fileNum,
size=1000,
events=100)
newFile.addRun(Run(1, *[1]))
newFile.setLocation('T1_US_FNAL_Disk')
self.fileset1.addFile(newFile)
self.fileset1.create()
workflow1 = Workflow(spec="spec.xml",
owner="hufnagel",
name="TestWorkflow1",
task="Test")
workflow1.create()
self.subscription1 = Subscription(fileset=self.fileset1,
workflow=workflow1,
split_algo="Harvest",
type="Harvesting")
self.subscription1.create()
self.configFile = EmulatorSetup.setupWMAgentConfig()
return
def testCreateDeleteExists(self):
"""
_testCreateDeleteExists_
Create and then delete a job. Use the job class's exists() method to
determine if the job has been written to the database before it is
created, after it has been created and after it has been deleted.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileA.create()
testFileB.create()
testJob = Job(name="TestJob", files=[testFileA, testFileB])
testWU = WorkUnit(taskID=testWorkflow.id,
fileid=testFileA['id'],
runLumi=Run(1, *[44]))
self.assertFalse(testJob.exists(), "Job exists before it was created")
self.assertFalse(testWU.exists(),
"WorkUnit exists before it was created")
testJob.create(group=testJobGroup)
self.assertTrue(testJob.exists(),
"Job does not exist after it was created")
self.assertFalse(testWU.exists(),
"WorkUnit exists when there is no work")
# Test the getWorkflow method
workflow = testJob.getWorkflow()
self.assertEqual(workflow['task'], 'Test')
self.assertEqual(workflow['name'], 'wf001')
testJob.delete()
self.assertFalse(testJob.exists(), "Job exists after it was deleted")
self.assertFalse(testWU.exists(),
"WorkUnit exists after job is deleted")
return
def notestTwoJobGroups(self):
"""
Test two job groups with a shared fileset. (Minimal part of testGetLocations which was failing)
"""
testWorkflow1 = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test1")
testWorkflow1.create()
testWMBSFileset1 = WMBSFileset(name="TestFileset1")
testWMBSFileset1.create()
testSubscription1 = Subscription(fileset=testWMBSFileset1,
workflow=testWorkflow1)
testSubscription1.create()
testJobGroup1 = JobGroup(subscription=testSubscription1)
testJobGroup1.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.create()
testJobA = Job(name="TestJobA")
testJobA.addFile(testFileA)
testJobGroup1.add(testJobA)
testJobGroup1.commit()
testWorkflow2 = Workflow(spec="spec.xml",
owner="Simon",
name="wf002",
task="Test2")
testWorkflow2.create()
testWMBSFileset2 = WMBSFileset(name="TestFileset1")
testWMBSFileset2.create()
testSubscription2 = Subscription(fileset=testWMBSFileset2,
workflow=testWorkflow2)
testSubscription2.create()
testJobGroup2 = JobGroup(subscription=testSubscription2)
testJobGroup2.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.create()
testJobA1 = Job(name="TestJobA1")
testJobA1.addFile(testFileC)
testJobGroup2.add(testJobA1)
testJobGroup2.commit()
def testLargeNumberOfFiles(self):
"""
_testLargeNumberOfFiles_
Setup a subscription with 500 files and verify that the splitting algo
works correctly.
"""
testWorkflowA = Workflow(spec="specA.xml",
owner="Steve",
name="wfA",
task="Test")
testWorkflowA.create()
testWorkflowB = Workflow(spec="specB.xml",
owner="Steve",
name="wfB",
task="Test")
testWorkflowB.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
allFiles = []
for i in range(500):
testFile = File(str(i),
size=1000,
events=100,
locations=set(["T2_CH_CERN"]))
testFile.create()
allFiles.append(testFile)
testFileset.addFile(testFile)
testFileset.commit()
testSubscriptionA = Subscription(fileset=testFileset,
workflow=testWorkflowA,
split_algo="FileBased",
type="Processing")
testSubscriptionA.create()
testSubscriptionB = Subscription(fileset=testFileset,
workflow=testWorkflowB,
split_algo="SiblingProcessingBased",
type="Processing")
testSubscriptionB.create()
testSubscriptionA.completeFiles(allFiles)
splitter = SplitterFactory()
deleteFactoryA = splitter(package="WMCore.WMBS",
subscription=testSubscriptionB)
result = deleteFactoryA(files_per_job=50)
self.assertEqual(len(result), 1,
"Error: Wrong number of job groups returned.")
self.assertEqual(len(result[0].jobs), 10,
"Error: Wrong number of jobs returned.")
return
def createJobGroups(self,
nSubs,
nJobs,
task,
workloadSpec,
site=None,
bl=[],
wl=[]):
"""
Creates a series of jobGroups for submissions
"""
jobGroupList = []
testWorkflow = Workflow(spec=workloadSpec,
owner="mnorman",
name=makeUUID(),
task="basicWorkload/Production",
owner_vogroup='phgroup',
owner_vorole='cmsrole')
testWorkflow.create()
# Create subscriptions
for i in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(name=name,
task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site,
bl=bl,
wl=wl)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
return jobGroupList
def testLoad(self):
"""
_testLoad_
Create a workflow and then try to load it from the database using the
following load methods:
Workflow.LoadFromName
Workflow.LoadFromID
Workflow.LoadFromSpecOwner
"""
testWorkflowA = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task='Test',
wfType="ReReco",
priority=1000)
testWorkflowA.create()
testWorkflowB = Workflow(name="wf001", task='Test')
testWorkflowB.load()
self.assertTrue((testWorkflowA.id == testWorkflowB.id)
and (testWorkflowA.name == testWorkflowB.name)
and (testWorkflowA.spec == testWorkflowB.spec)
and (testWorkflowA.task == testWorkflowB.task)
and (testWorkflowA.wfType == testWorkflowB.wfType)
and (testWorkflowA.owner == testWorkflowB.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromName Failed")
testWorkflowC = Workflow(id=testWorkflowA.id)
testWorkflowC.load()
self.assertTrue((testWorkflowA.id == testWorkflowC.id)
and (testWorkflowA.name == testWorkflowC.name)
and (testWorkflowA.spec == testWorkflowC.spec)
and (testWorkflowA.task == testWorkflowC.task)
and (testWorkflowA.wfType == testWorkflowC.wfType)
and (testWorkflowA.owner == testWorkflowC.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromID Failed")
testWorkflowD = Workflow(spec="spec.xml", owner="Simon", task='Test')
testWorkflowD.load()
self.assertTrue((testWorkflowA.id == testWorkflowD.id)
and (testWorkflowA.name == testWorkflowD.name)
and (testWorkflowA.spec == testWorkflowD.spec)
and (testWorkflowA.task == testWorkflowD.task)
and (testWorkflowA.wfType == testWorkflowD.wfType)
and (testWorkflowA.owner == testWorkflowD.owner)
and (testWorkflowA.priority == testWorkflowB.priority),
"ERROR: Workflow.LoadFromSpecOwner Failed")
testWorkflowA.delete()
return
def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a new job and commit it to the database. Start a new transaction
and delete the file from the database. Verify that the file has been
deleted. After that, roll back the transaction and verify that the
job is once again in the database.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileA.create()
testFileB.create()
testJob = Job(name="TestJob", files=[testFileA, testFileB])
assert testJob.exists() == False, \
"ERROR: Job exists before it was created"
testJob.create(group=testJobGroup)
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJob.delete()
assert testJob.exists() == False, \
"ERROR: Job exists after it was delete"
myThread.transaction.rollback()
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after transaction was rolled back."
return
def testParentageByJob(self):
"""
_testParentageByJob_
Tests the DAO that assigns parentage by Job
"""
testWorkflow = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFileParentA = File(lfn = "/this/is/a/parent/lfnA", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentA.addRun(Run( 1, *[45]))
testFileParentB = File(lfn = "/this/is/a/parent/lfnB", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentB.addRun(Run( 1, *[45]))
testFileParentA.create()
testFileParentB.create()
testFileA = File(lfn = "/this/is/a/lfn", size = 1024, events = 10,
checksums = {'cksum':1})
testFileA.addRun(Run( 1, *[45]))
testFileA.create()
testJobA = Job()
testJobA.create(group = testJobGroup)
testJobA.addFile(testFileParentA)
testJobA.addFile(testFileParentB)
testJobA.associateFiles()
parentAction = self.daofactory(classname = "Files.SetParentageByJob")
parentAction.execute(binds = {'jobid': testJobA.exists(), 'child': testFileA['lfn']})
testFileB = File(id = testFileA["id"])
testFileB.loadData(parentage = 1)
goldenFiles = [testFileParentA, testFileParentB]
for parentFile in testFileB["parents"]:
self.assertEqual(parentFile in goldenFiles, True,
"ERROR: Unknown parent file")
goldenFiles.remove(parentFile)
self.assertEqual(len(goldenFiles), 0,
"ERROR: Some parents are missing")
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
taskType='Processing', name=None, wfPrio=1, changeState=None):
"""
_createJobGroups_
Creates a series of jobGroups for submissions
changeState is an instance of the ChangeState class to make job status changes
"""
jobGroupList = []
if name is None:
name = makeUUID()
testWorkflow = Workflow(spec=workloadSpec, owner="tapas",
name=name, task="basicWorkload/Production",
priority=wfPrio)
testWorkflow.create()
# Create subscriptions
for _ in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type=taskType,
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(name=name, task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
if changeState:
for group in jobGroupList:
changeState.propagate(group.jobs, 'created', 'new')
return jobGroupList
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site=None, bl=[], wl=[]):
"""
Creates a series of jobGroups for submissions
"""
jobGroupList = []
testWorkflow = Workflow(
spec=workloadSpec,
owner="tapas",
name=makeUUID(),
task="basicWorkload/Production",
owner_vogroup="phgroup",
owner_vorole="cmsrole",
)
testWorkflow.create()
# Create subscriptions
for i in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(
fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased"
)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(
name=name,
task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site,
bl=bl,
wl=wl,
)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
return jobGroupList
def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a new job and commit it to the database. Start a new transaction
and delete the file from the database. Verify that the file has been
deleted. After that, roll back the transaction and verify that the
job is once again in the database.
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileA.create()
testFileB.create()
testJob = Job(name="TestJob", files=[testFileA, testFileB])
assert testJob.exists() is False, \
"ERROR: Job exists before it was created"
testJob.create(group=testJobGroup)
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJob.delete()
assert testJob.exists() is False, \
"ERROR: Job exists after it was delete"
myThread.transaction.rollback()
assert testJob.exists() >= 0, \
"ERROR: Job does not exist after transaction was rolled back."
return
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site,
taskType='Processing', name=None, wfPrio=1, changeState=None):
"""
_createJobGroups_
Creates a series of jobGroups for submissions
changeState is an instance of the ChangeState class to make job status changes
"""
jobGroupList = []
if name is None:
name = makeUUID()
testWorkflow = Workflow(spec=workloadSpec, owner="tapas",
name=name, task="basicWorkload/Production",
priority=wfPrio)
testWorkflow.create()
# Create subscriptions
for _ in range(nSubs):
name = makeUUID()
# Create Fileset, Subscription, jobGroup
testFileset = Fileset(name=name)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type=taskType,
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create jobs
self.makeNJobs(name=name, task=task,
nJobs=nJobs,
jobGroup=testJobGroup,
fileset=testFileset,
sub=testSubscription.exists(),
site=site)
testFileset.commit()
testJobGroup.commit()
jobGroupList.append(testJobGroup)
if changeState:
for group in jobGroupList:
changeState.propagate(group.jobs, 'created', 'new')
return jobGroupList
def createLargerTestJobGroup(self, commitFlag=True):
"""
_createTestJobGroup_
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation("goodse.cern.ch")
testFileC.setLocation("malpaquet")
testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10)
testFileD.addRun(Run(10, *[12312]))
testFileD.setLocation("goodse.cern.ch")
testFileD.setLocation("malpaquet")
testFileC.create()
testFileD.create()
testJobA = Job(name="TestJobA1")
testJobA.addFile(testFileC)
testJobB = Job(name="TestJobB1")
testJobB.addFile(testFileD)
testJobGroup.add(testJobA)
testJobGroup.add(testJobB)
for i in range(0, 100):
testJob = Job(name="TestJob%i" % (i))
testJob.addFile(testFileC)
testJobGroup.add(testJob)
if commitFlag:
testJobGroup.commit()
return testJobGroup
def testGetGroupsByJobStateDAO(self):
"""
_testGetGroupsByJobStateDAO_
Verify that the GetGrounsByJobState DAO does what it is supposed to do.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroupA = JobGroup(subscription=testSubscription)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscription)
testJobGroupB.create()
testJobA = Job(name="TestJobA")
testJobB = Job(name="TestJobB")
testJobGroupA.add(testJobA)
testJobGroupB.add(testJobB)
testJobGroupA.commit()
testJobGroupB.commit()
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
stateChangeAction = daofactory(classname="Jobs.ChangeState")
testJobA["state"] = "complete"
testJobB["state"] = "executing"
stateChangeAction.execute(jobs=[testJobA, testJobB])
jobGroupAction = daofactory(classname="JobGroup.GetGroupsByJobState")
jobGroups = jobGroupAction.execute(jobState="complete")
assert len(jobGroups) == 1, \
"Error: Wrong number of job groups returned."
assert jobGroups[0] == testJobGroupA.id, \
"Error: Wrong job group returned."
return
def stuffWMBS(self):
"""
_stuffWMBS_
Stuff WMBS with workflows
"""
workflow = Workflow(spec = 'spec.xml', name = 'ReRecoTest_v0Emulator',
task = '/ReRecoTest_v0Emulator/Test', priority = 10)
workflow.create()
inputFileset = Fileset(name = 'TestFileset')
inputFileset.create()
subscription = Subscription(inputFileset, workflow)
subscription.create()
def createTestJobs(self, nJobs, cacheDir):
"""
_createTestJobs_
Create several jobs
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create a file
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
baseName = makeUUID()
# Now create a job
for i in range(nJobs):
testJob = Job(name='%s-%i' % (baseName, i))
testJob.addFile(testFileA)
testJob['location'] = 'malpaquet'
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob.create(testJobGroup)
testJob.save()
testJobGroup.add(testJob)
testJobGroup.commit()
# Set test job caches
for job in testJobGroup.jobs:
job.setCache(cacheDir)
return testJobGroup
def stuffWMBS(self):
"""
_stuffWMBS_
Inject the workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def testDeleteTransaction(self):
"""
_testDeleteTransaction_
Create a JobGroup and then commit it to the database. Begin a
transaction and the delete the JobGroup from the database. Using the
exists() method verify that the JobGroup is not in the database.
Finally, roll back the transaction and verify that the JobGroup is
in the database.
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testFileset = WMBSFileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
assert testJobGroup.exists() == False, \
"ERROR: Job group exists before it was created"
testJobGroup.create()
assert testJobGroup.exists() >= 0, \
"ERROR: Job group does not exist after it was created"
myThread = threading.currentThread()
myThread.transaction.begin()
testJobGroup.delete()
assert testJobGroup.exists() == False, \
"ERROR: Job group exists after it was deleted"
myThread.transaction.rollback()
assert testJobGroup.exists() >= 0, \
"ERROR: Job group does not exist after transaction was rolled back."
testSubscription.delete()
testFileset.delete()
testWorkflow.delete()
return
def testUpdateFailedDoc(self):
"""
_testUpdateFailedDoc_
Verify that the update function will work correctly and not throw a 500
error if the doc didn't make it into the database for some reason.
"""
change = ChangeState(self.config, "changestate_t")
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute("site1", seName="somese.cern.ch")
testWorkflow = Workflow(spec="spec.xml",
owner="Steve",
name="wf001",
task=self.taskName)
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
split_algo="FileBased")
testSubscription.create()
testFileA = File(lfn="SomeLFNA",
events=1024,
size=2048,
locations=set(["somese.cern.ch"]))
testFileA.create()
testFileset.addFile(testFileA)
testFileset.commit()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroup = jobFactory(files_per_job=1)[0]
testJobA = jobGroup.jobs[0]
testJobA["user"] = "******"
testJobA["group"] = "DMWM"
testJobA["taskType"] = "Merge"
testJobA["couch_record"] = str(testJobA["id"])
change.propagate([testJobA], "new", "none")
testJobADoc = change.jobsdatabase.document(testJobA["couch_record"])
self.assertTrue(testJobADoc.has_key("states"))
self.assertTrue(testJobADoc["states"].has_key("1"))
return
def createFileCollection(self,
name,
nSubs,
nFiles,
workflowURL='test',
site=None):
"""
_createFileCollection_
Create a collection of files for splitting into jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=workflowURL,
owner="mnorman",
name=name,
task="/TestWorkload/ReReco")
testWorkflow.create()
for sub in range(nSubs):
nameStr = '%s-%i' % (name, sub)
testFileset = Fileset(name=nameStr)
testFileset.create()
for f in range(nFiles):
# pick a random site
if not site:
tmpSite = 'se.%s' % (random.choice(self.sites))
else:
tmpSite = 'se.%s' % (site)
testFile = File(lfn="/lfn/%s/%i" % (nameStr, f),
size=1024,
events=10)
testFile.setLocation(tmpSite)
testFile.create()
testFileset.addFile(testFile)
testFileset.commit()
testFileset.markOpen(isOpen=0)
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
return
def createDummyJobs(self, nJobs, location=None):
"""
_createDummyJobs_
Create some dummy jobs
"""
if not location:
location = self.sites[0]
nameStr = makeUUID()
testWorkflow = Workflow(spec=nameStr,
owner="tapas",
name=nameStr,
task="basicWorkload/Production",
owner_vogroup='phgroup',
owner_vorole='cmsrole')
testWorkflow.create()
testFileset = Fileset(name=nameStr)
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
jobList = []
for i in range(nJobs):
testJob = Job(name='%s-%i' % (nameStr, i))
testJob['location'] = location
testJob['custom']['location'] = location
testJob['userdn'] = 'tapas'
testJob['owner'] = 'tapas'
testJob['userrole'] = 'cmsrole'
testJob['usergroup'] = 'phgroup'
testJob.create(testJobGroup)
jobList.append(testJob)
return jobList
