def createTestSubscription(self, nFiles, nSites=1, closeFileset=False): """ _createTestSubscription_ Create a set of test subscriptions for testing purposes. """ if nSites > self.nSites: nSites = self.nSites testFileset = Fileset(name="TestFileset") testFileset.create() # Create a testWorkflow testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() # Create the files for each site for s in range(nSites): for i in range(nFiles): newFile = File(makeUUID(), size=1024, events=100, locations=set(["site%i.cern.ch" % s])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testSubscription = Subscription( fileset=testFileset, workflow=testWorkflow, split_algo="MinFileBased", type="Processing" ) testSubscription.create() # Close the fileset if closeFileset: testFileset.markOpen(isOpen=False) return testSubscription
def createTestJob(subscriptionType="Merge"): """ _createTestJob_ Create a test job with two files as input. This will also create the appropriate workflow, jobgroup and subscription. """ testWorkflow = Workflow(spec=makeUUID(), owner="Simon", name=makeUUID(), task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type=subscriptionType) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(1, *[45])) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileB.addRun(Run(1, *[46])) testFileA.create() testFileB.create() testJob = Job(name=makeUUID(), files=[testFileA, testFileB]) testJob["couch_record"] = "somecouchrecord" testJob["location"] = "test.site.ch" testJob.create(group=testJobGroup) testJob.associateFiles() return testJob
def testWorkflowInjectMarking(self): """ _testWorkflowInjectMarking_ Test whether or not we can mark a workflow as injected or not. """ owner = "moron" workflows = [] for i in range(0, 10): testWorkflow = Workflow(spec="sp00%i" % i, owner=owner, name="wf00%i" % i, task="task%i" % i) testWorkflow.create() workflows.append(testWorkflow) myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) getAction = daoFactory(classname="Workflow.GetInjectedWorkflows") markAction = daoFactory(classname="Workflow.MarkInjectedWorkflows") result = getAction.execute(injected=True) self.assertEqual(len(result), 0) result = getAction.execute(injected=False) self.assertEqual(len(result), 10) names = ['wf002', 'wf004', 'wf006', 'wf008'] markAction.execute(names=names, injected=True) result = getAction.execute(injected=True) self.assertEqual(result, names) result = getAction.execute(injected=False) self.assertEqual(len(result), 6) return
def createLargeFileBlock(self): """ _createLargeFileBlock_ Creates a large group of files for testing """ testFileset = Fileset(name = "TestFilesetX") testFileset.create() for i in range(5000): newFile = File(makeUUID(), size = 1000, events = 100, locations = set(["somese.cern.ch"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testWorkflow = Workflow(spec = "spec.xml", owner = "mnorman", name = "wf003", task="Test" ) testWorkflow.create() largeSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, split_algo = "FileBased", type = "Processing") largeSubscription.create() return largeSubscription
def generateFakeMCFile(self, numEvents = 100, firstEvent = 1, lastEvent = 100, firstLumi = 1, lastLumi = 10, index = 1): #MC comes with only one MCFakeFile singleMCFileset = Fileset(name = "MCTestFileset %i" % index) singleMCFileset.create() newFile = File("MCFakeFileTest %i" % index, size = 1000, events = numEvents, locations = set(["somese.cern.ch"])) newFile.addRun(Run(1, *range(firstLumi, lastLumi + 1))) newFile["first_event"] = firstEvent newFile["last_event"] = lastEvent newFile.create() singleMCFileset.addFile(newFile) singleMCFileset.commit() testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task="Test") testWorkflow.create() singleMCFileSubscription = Subscription(fileset = singleMCFileset, workflow = testWorkflow, split_algo = "EventBased", type = "Production") singleMCFileSubscription.create() return singleMCFileSubscription
def testA(self): """ _testA_ Handle AddWorkflowToManage events """ myThread = threading.currentThread() config = self.getConfig() testWorkflowManager = WorkflowManager(config) testWorkflowManager.prepareToStart() for i in xrange(0, WorkflowManagerTest._maxMessage): workflow = Workflow(spec = "testSpec.xml", owner = "riahi", \ name = "testWorkflow" + str(i), task = "testTask") workflow.create() for j in xrange(0, 3): workflowManagerdict = {'payload':{'WorkflowId' : workflow.id \ , 'FilesetMatch': 'FILESET_' + str(j) ,'SplitAlgo':'NO SPLITALGO', 'Type':'NO TYPE' }} testWorkflowManager.handleMessage( \ type = 'AddWorkflowToManage' , payload = workflowManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
def testCreateDeleteExistsNoFiles(self): """ _testCreateDeleteExistsNoFiles_ Create and then delete a job but don't add any input files to it. Use the job class's exists() method to determine if the job has been written to the database before it is created, after it has been created and after it has been deleted. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testJob = Job(name="TestJob") assert testJob.exists() == False, "ERROR: Job exists before it was created" testJob.create(group=testJobGroup) assert testJob.exists() >= 0, "ERROR: Job does not exist after it was created" testJob.delete() assert testJob.exists() == False, "ERROR: Job exists after it was delete" return
def testFilesWithoutOtherSubscriptions(self): """ _testFilesWithoutOtherSubscriptions_ Test the case where files only in the delete subscription can happen if cleanup of the other subscriptions is fast """ testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test") testWorkflowA.create() testFileset = Fileset(name="TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size=1000, events=100, locations=set(["somese.cern.ch"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription( fileset=testFileset, workflow=testWorkflowA, split_algo="SiblingProcessingBased", type="Processing" ) testSubscriptionA.create() splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionA) result = deleteFactoryA(files_per_job=50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return
def testByRunAndRunBlacklist(self): """ _testByRunAndRunWhitelist_ Create harvesting jobs by run for the runs provided in the RunWhitelist """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3]) self.assertEqual(len(jobGroups), 1, "One jobgroup per location") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 3)
def testLoadFromTask(self): """ _testLoadFromTask_ Verify that Workflow.LoadFromTask DAO correct loads the workflow by task. """ testWorkflow1 = Workflow(spec="spec1.xml", owner="Hassen", name="wf001", task="sometask") testWorkflow1.create() myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) loadFromTaskDAO = daoFactory(classname="Workflow.LoadFromTask") listFromTask = loadFromTaskDAO.execute(task=testWorkflow1.task) self.assertEqual(len(listFromTask), 1, "ERROR: listFromTask should be 1.") self.assertEqual(listFromTask[0]["task"], "sometask", "ERROR: task should be sometask.") return
def setUp(self): """ _setUp_ Create a single subscription with one file. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "site1", seName = "somese.cern.ch") locationAction.execute(siteName = "site2", seName = "otherse.cern.ch") self.testFileset = Fileset(name = "TestFileset1") self.testFileset.create() testWorkflow = Workflow(spec = "spec.xml", owner = "Steve", name = "wf001", task="Test" ) testWorkflow.create() self.testSubscription = Subscription(fileset = self.testFileset, workflow = testWorkflow, split_algo = "Periodic", type = "Processing") self.testSubscription.create() return
def testCreateTransaction(self): """ _testCreateTransaction_ Create a JobGroup and commit it to the database. Rollback the database transaction and verify that the JobGroup is no longer in the database. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testFileset = WMBSFileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) assert testJobGroup.exists() == False, "ERROR: Job group exists before it was created" myThread = threading.currentThread() myThread.transaction.begin() testJobGroup.create() assert testJobGroup.exists() >= 0, "ERROR: Job group does not exist after it was created" myThread.transaction.rollback() assert testJobGroup.exists() == False, "ERROR: Job group exists after transaction was rolled back." testSubscription.delete() testFileset.delete() testWorkflow.delete() return
def testCreateTransaction(self): """ _testCreateTransaction_ Create a workflow and commit it to the database and then roll the transaction back. Use the workflow's exists() method to verify that the workflow does not exist before create() is called, exists after create() is called and does not exist after the transaction is rolled back. """ myThread = threading.currentThread() myThread.transaction.begin() testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task='Test') self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists before it was created") testWorkflow.create() self.assertTrue( testWorkflow.exists() > 0, "ERROR: Workflow does not exist after it has been created") myThread.transaction.rollback() self.assertEqual( testWorkflow.exists(), False, "ERROR: Workflow exists after the transaction was rolled back.") return
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Create a JobGroup and then delete it. Use the JobGroup's exists() method to determine if it exists before it is created, after it is created and after it is deleted. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testFileset = WMBSFileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) self.assertFalse(testJobGroup.exists()) testJobGroup.create() self.assertTrue(testJobGroup.exists()) testJobGroup.delete() self.assertFalse(testJobGroup.exists()) testSubscription.delete() testFileset.delete() testWorkflow.delete() return
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Test the create(), delete() and exists() methods of the workflow class by creating and deleting a workflow. The exists() method will be called before and after creation and after deletion. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task='Test', wfType="ReReco") self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists before it was created") testWorkflow.create() self.assertTrue(testWorkflow.exists() > 0, "ERROR: Workflow does not exist after it has been created") testWorkflow.create() testWorkflow.delete() self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists after it has been deleted") return
def testMask(self): """ _testMask_ Test the new mask setup """ testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() testFileset = Fileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testFileA = File(lfn=makeUUID(), locations="test.site.ch") testFileB = File(lfn=makeUUID(), locations="test.site.ch") testFileA.create() testFileB.create() testFileset.addFile([testFileA, testFileB]) testFileset.commit() testSubscription.acquireFiles([testFileA, testFileB]) testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testJob = Job() testJob['mask'].addRunAndLumis(run=100, lumis=[101, 102]) testJob['mask'].addRunAndLumis(run=200, lumis=[201, 202]) testJob.create(group=testJobGroup) loadJob = Job(id=testJob.exists()) loadJob.loadData() runs = loadJob['mask'].getRunAndLumis() self.assertEqual(len(runs), 2) self.assertEqual(runs[100], [[101, 102]]) self.assertEqual(runs[200], [[201, 202]]) bigRun = Run(100, *[101, 102, 103, 104]) badRun = Run(300, *[1001, 1002]) result = loadJob['mask'].filterRunLumisByMask([bigRun, badRun]) self.assertEqual(len(result), 1) alteredRun = result.pop() self.assertEqual(alteredRun.run, 100) self.assertEqual(alteredRun.lumis, [101, 102]) run0 = Run(300, *[1001, 1002]) run1 = Run(300, *[1001, 1002]) loadJob['mask'].filterRunLumisByMask([run0, run1]) return
def testCountWorkflow(self): """ _testCountWorkflow_ """ spec = "spec.py" owner = "moron" myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) countSpecDAO = daoFactory(classname="Workflow.CountWorkflowBySpec") workflows = [] for i in range(0, 10): testWorkflow = Workflow(spec=spec, owner=owner, name="wf00%i" % i, task="task%i" % i) testWorkflow.create() workflows.append(testWorkflow) self.assertEqual(countSpecDAO.execute(spec=spec), 10) for i in range(0, 10): wf = workflows.pop() wf.delete() self.assertEqual(countSpecDAO.execute(spec=spec), 10 - (i + 1)) return
def testListRunningJobs(self): """ _testListRunningJobs_ Test the ListRunningJobs DAO. """ testWorkflow = Workflow(spec = makeUUID(), owner = "Steve", name = makeUUID(), task="Test") testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing") testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() testJobA = Job(name = makeUUID(), files = []) testJobA["couch_record"] = makeUUID() testJobA.create(group = testJobGroup) testJobA["state"] = "executing" testJobB = Job(name = makeUUID(), files = []) testJobB["couch_record"] = makeUUID() testJobB.create(group = testJobGroup) testJobB["state"] = "complete" testJobC = Job(name = makeUUID(), files = []) testJobC["couch_record"] = makeUUID() testJobC.create(group = testJobGroup) testJobC["state"] = "new" changeStateAction = self.daoFactory(classname = "Jobs.ChangeState") changeStateAction.execute(jobs = [testJobA, testJobB, testJobC]) runningJobsAction = self.daoFactory(classname = "Monitoring.ListRunningJobs") runningJobs = runningJobsAction.execute() assert len(runningJobs) == 2, \ "Error: Wrong number of running jobs returned." for runningJob in runningJobs: if runningJob["job_name"] == testJobA["name"]: assert runningJob["state"] == testJobA["state"], \ "Error: Running job has wrong state." assert runningJob["couch_record"] == testJobA["couch_record"], \ "Error: Running job has wrong couch record." else: assert runningJob["job_name"] == testJobC["name"], \ "Error: Running job has wrong name." assert runningJob["state"] == testJobC["state"], \ "Error: Running job has wrong state." assert runningJob["couch_record"] == testJobC["couch_record"], \ "Error: Running job has wrong couch record." return
def test_AutoIncrementCheck(self): """ _AutoIncrementCheck_ Test and see whether we can find and set the auto_increment values """ myThread = threading.currentThread() if not myThread.dialect.lower() == "mysql": return testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task="Test") testWorkflow.create() testFileset = Fileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testFileA = File(lfn=makeUUID(), locations="test.site.ch") testFileB = File(lfn=makeUUID(), locations="test.site.ch") testFileA.create() testFileB.create() testFileset.addFile([testFileA, testFileB]) testFileset.commit() testSubscription.acquireFiles([testFileA, testFileB]) testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() incrementDAO = self.daoFactory(classname="Jobs.AutoIncrementCheck") incrementDAO.execute() testJob = Job() testJob.create(group=testJobGroup) self.assertEqual(testJob.exists(), 1) incrementDAO.execute() testJob = Job() testJob.create(group=testJobGroup) self.assertEqual(testJob.exists(), 2) incrementDAO.execute(input=10) testJob = Job() testJob.create(group=testJobGroup) self.assertEqual(testJob.exists(), 11) incrementDAO.execute(input=5) testJob = Job() testJob.create(group=testJobGroup) self.assertEqual(testJob.exists(), 12) return
def testMultiRunHarvesting(self): """ _testMultiRunHarvesting_ Provided a fileset with a couple of files and different runs, create a single job for all the runs at a specific location, which also adds a baggage to the job (True) which is later on looked up by SetupCMSSWPSet. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(dqmHarvestUnit="multiRun") self.assertEqual(len(jobGroups), 1) for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 1)
def createWorkflow(self, task): """ Register job into WMBS for each task through Workflows """ specURL = self.getWorkflowURL(task) fileSet = Fileset(name=self.getFilesetName(task), is_open=True) fileSet.create() taskFlow = Workflow(spec=specURL, owner=self.owner, dn=self.owner_dn, name=self.getWorkflowName(task), task=task.name()) taskFlow.create() self.workflowDict[task.name()] = taskFlow # Insert workflow into task setattr(task.data.input.WMBS, 'WorkflowSpecURL', specURL) # If the job is a merge job # Find the task it merges from # Then find the workflow for that task and assign it an output if hasattr(task.inputReference(), 'outputModule'): dummyStepName = task.inputReference().inputStep.split('/')[-1] taskName = task.inputReference().inputStep.split('/')[-2] outputModule = task.inputReference().outputModule if taskName not in self.workflowDict.keys(): raise Exception('I am being asked to chain output for a task %s which does not yet exist' % taskName) outputWorkflow = self.workflowDict[taskName] outputWorkflow.addOutput(outputModule, fileSet) logging.info('Registered workflow for step %s' % (task.name())) return taskFlow, fileSet
def createSubscriptionWithFileABC(self): """" _createSubscriptionWithFileABC_ Create a subscription where the input fileset has three files. Also create a second subscription that has acquired two of the files. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWorkflow2 = Workflow(spec="specBOGUS.xml", owner="Simon", name="wfBOGUS", task="Test") testWorkflow2.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=20, locations=set(["test.site.ch"])) testFileA.addRun(Run(1, *[45])) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=20, locations=set(["test.site.ch"])) testFileB.addRun(Run(1, *[46])) testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=20, locations=set(["test.site.ch"])) testFileC.addRun(Run(2, *[48])) testFileA.create() testFileB.create() testFileC.create() testFileset = Fileset(name="TestFileset") testFileset.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.addFile(testFileC) testFileset.commit() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testSubscription2 = Subscription(fileset=testFileset, workflow=testWorkflow2) testSubscription2.create() testSubscription2.acquireFiles([testFileA, testFileB]) #return (testSubscription, testFileset, testWorkflow, testFileA, # testFileB, testFileC) return (testSubscription, testFileA, testFileB, testFileC)
def createSubscription(self, nFiles, lumisPerFile, twoSites=False, nEventsPerFile=100): """ _createSubscription_ Create a subscription for testing """ baseName = makeUUID() testWorkflow = Workflow(spec="spec.xml", owner="dmwm", name="testWorkflow_%s" % baseName[:4], task="Test") testWorkflow.create() testFileset = Fileset(name=baseName) for i in range(nFiles): newFile = self.createFile('%s_%i' % (baseName, i), nEventsPerFile, i, lumisPerFile, 'T1_US_FNAL_Disk') testFileset.addFile(newFile) if twoSites: for i in range(nFiles): newFile = self.createFile('%s_%i_2' % (baseName, i), nEventsPerFile, i, lumisPerFile, 'T2_CH_CERN') testFileset.addFile(newFile) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, split_algo="EventAwareLumiByWork", type="Processing") testSubscription.create() return testSubscription
def createLargeFileBlock(self): """ _createLargeFileBlock_ Creates a large group of files for testing """ testFileset = Fileset(name="TestFilesetX") testFileset.create() for _ in range(5000): newFile = File(makeUUID(), size=1000, events=100, locations=set(["T1_US_FNAL_Disk"])) newFile.create() testFileset.addFile(newFile) testFileset.commit() testWorkflow = Workflow(spec="spec.xml", owner="mnorman", name="wf003", task="Test") testWorkflow.create() largeSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, split_algo="FileBased", type="Processing") largeSubscription.create() return largeSubscription
def testDifferentSubscritionIDs(self): """ _testDifferentSubscriptionIDs_ Make sure that the merge splitting still runs if the subscription ID is not equal to the workflow ID. """ myThread = threading.currentThread() myThread.transaction.begin() dummyWorkflow = Workflow(name = "dummyWorkflow", spec = "bunk49", owner = "Steve", task="Test2") dummyWorkflow.create() dummyFileset = Fileset(name = "dummyFileset") dummyFileset.create() dummySubscription1 = Subscription(fileset = dummyFileset, workflow = dummyWorkflow, split_algo = "ParentlessMergeBySize") dummySubscription2 = Subscription(fileset = dummyFileset, workflow = dummyWorkflow, split_algo = "ParentlessMergeBySize") dummySubscription1.create() dummySubscription2.create() myThread.transaction.commit() self.stuffWMBS() splitter = SplitterFactory() jobFactory = splitter(package = "WMCore.WMBS", subscription = self.mergeSubscription) result = jobFactory(min_merge_size = 4097, max_merge_size = 99999999, max_merge_events = 999999999, merge_across_runs = False) self.assertEqual(len(result), 1) jobGroup = result[0] self.assertEqual(len(jobGroup.jobs), 2) return
def testByRunHarvesting(self): """ _testByRunHarvesting_ Provided a fileset with a couple of files and 4 different runs, create one single job per run and location. The multiRun baggage should be false in this case. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!") harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1, "Should have created 1 job group") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs") for job in jobGroup.jobs: baggage = job.getBaggage() self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job")
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Test the create(), delete() and exists() methods of the workflow class by creating and deleting a workflow. The exists() method will be called before and after creation and after deletion. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task='Test', wfType="ReReco") self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists before it was created") testWorkflow.create() self.assertTrue( testWorkflow.exists() > 0, "ERROR: Workflow does not exist after it has been created") testWorkflow.create() testWorkflow.delete() self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists after it has been deleted") return
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site, bl=[], wl=[], taskType='Processing', name=None): """ _createJobGroups_ Creates a series of jobGroups for submissions """ jobGroupList = [] if name is None: name = makeUUID() testWorkflow = Workflow(spec=workloadSpec, owner="tapas", name=name, task="basicWorkload/Production") testWorkflow.create() # Create subscriptions for _ in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name=name) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type=taskType, split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create jobs self.makeNJobs(name=name, task=task, nJobs=nJobs, jobGroup=testJobGroup, fileset=testFileset, sub=testSubscription.exists(), site=site, bl=bl, wl=wl) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) return jobGroupList
def testCreateTransaction(self): """ _testCreateTransaction_ Create a workflow and commit it to the database and then roll the transaction back. Use the workflow's exists() method to verify that the workflow does not exist before create() is called, exists after create() is called and does not exist after the transaction is rolled back. """ myThread = threading.currentThread() myThread.transaction.begin() testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task='Test') self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists before it was created") testWorkflow.create() self.assertTrue(testWorkflow.exists() > 0, "ERROR: Workflow does not exist after it has been created") myThread.transaction.rollback() self.assertEqual(testWorkflow.exists(), False, "ERROR: Workflow exists after the transaction was rolled back.") return
def testAddDupsToFilesetBulk(self): """ _AddToDupsFilesetBulk_ Same as testAddDupsToFileset() but faster """ testWorkflowA = Workflow(spec = 'hello', owner = "mnorman", name = "wf001", task="basicWorkload/Production") testWorkflowA.create() testWorkflowB = Workflow(spec = 'hello', owner = "mnorman", name = "wf001", task="basicWorkload/Production2") testWorkflowB.create() testFilesetA = Fileset(name = "inputFilesetA") testFilesetA.create() testFilesetB = Fileset(name = "inputFilesetB") testFilesetB.create() testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA) testSubscriptionA.create() testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB) testSubscriptionB.create() testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10, locations = ['SiteA']) testFileA.addRun(Run( 1, *[45])) testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10, locations = ['SiteB']) testFileB.addRun(Run( 1, *[45])) addFilesToWMBSInBulk(testFilesetA.id, "wf001", [testFileA, testFileB], conn = testFileA.getDBConn(), transaction = testFileA.existingTransaction()) testFileset2 = Fileset(name = "inputFilesetA") testFileset2.loadData() self.assertEqual(len(testFileset2.files), 2) for file in testFileset2.files: self.assertTrue(file in [testFileA, testFileB]) # Check that adding twice doesn't crash addFilesToWMBSInBulk(testFilesetA.id, "wf001", [testFileA, testFileB], conn = testFileA.getDBConn(), transaction = testFileA.existingTransaction()) # Files should not get added to fileset B because fileset A is associated # with wf001. addFilesToWMBSInBulk(testFilesetB.id, "wf001", [testFileA, testFileB], conn = testFileA.getDBConn(), transaction = testFileA.existingTransaction()) testFileset2 = Fileset(name = "inputFilesetB") testFileset2.loadData() self.assertEqual(len(testFileset2.files), 0) return
def testLoad(self): """ _testLoad_ Create a workflow and then try to load it from the database using the following load methods: Workflow.LoadFromName Workflow.LoadFromID Workflow.LoadFromSpecOwner """ testWorkflowA = Workflow( spec="spec.xml", owner="Simon", name="wf001", task="Test", wfType="ReReco", priority=1000 ) testWorkflowA.create() testWorkflowB = Workflow(name="wf001", task="Test") testWorkflowB.load() self.assertTrue( (testWorkflowA.id == testWorkflowB.id) and (testWorkflowA.name == testWorkflowB.name) and (testWorkflowA.spec == testWorkflowB.spec) and (testWorkflowA.task == testWorkflowB.task) and (testWorkflowA.wfType == testWorkflowB.wfType) and (testWorkflowA.owner == testWorkflowB.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromName Failed", ) testWorkflowC = Workflow(id=testWorkflowA.id) testWorkflowC.load() self.assertTrue( (testWorkflowA.id == testWorkflowC.id) and (testWorkflowA.name == testWorkflowC.name) and (testWorkflowA.spec == testWorkflowC.spec) and (testWorkflowA.task == testWorkflowC.task) and (testWorkflowA.wfType == testWorkflowC.wfType) and (testWorkflowA.owner == testWorkflowC.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromID Failed", ) testWorkflowD = Workflow(spec="spec.xml", owner="Simon", task="Test") testWorkflowD.load() self.assertTrue( (testWorkflowA.id == testWorkflowD.id) and (testWorkflowA.name == testWorkflowD.name) and (testWorkflowA.spec == testWorkflowD.spec) and (testWorkflowA.task == testWorkflowD.task) and (testWorkflowA.wfType == testWorkflowD.wfType) and (testWorkflowA.owner == testWorkflowD.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromSpecOwner Failed", ) testWorkflowA.delete() return
def testAddDupsToFileset(self): """ _AddToDupsFileset_ Verify the the dups version of the AddToFileset DAO will not add files to a fileset if they're already associated to another fileset with the same workflow. """ testWorkflowA = Workflow(spec = 'hello', owner = "mnorman", name = "wf001", task="basicWorkload/Production") testWorkflowA.create() testWorkflowB = Workflow(spec = 'hello', owner = "mnorman", name = "wf001", task="basicWorkload/Production2") testWorkflowB.create() testFilesetA = Fileset(name = "inputFilesetA") testFilesetA.create() testFilesetB = Fileset(name = "inputFilesetB") testFilesetB.create() testSubscriptionA = Subscription(workflow = testWorkflowA, fileset = testFilesetA) testSubscriptionA.create() testSubscriptionB = Subscription(workflow = testWorkflowB, fileset = testFilesetB) testSubscriptionB.create() testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10) testFileA.addRun(Run( 1, *[45])) testFileA.create() testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run( 1, *[45])) testFileB.create() addToFileset = self.daofactory(classname = "Files.AddDupsToFileset") addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']], fileset = testFilesetA.id, workflow = "wf001") testFileset2 = Fileset(name = "inputFilesetA") testFileset2.loadData() self.assertEqual(len(testFileset2.files), 2) for file in testFileset2.files: self.assertTrue(file in [testFileA, testFileB]) # Check that adding twice doesn't crash addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']], fileset = testFilesetA.id, workflow = "wf001") # Files should not get added to fileset B because fileset A is associated # with wf001. addToFileset.execute(file = [testFileA['lfn'], testFileB['lfn']], fileset = testFilesetB.id, workflow = "wf001") testFileset2 = Fileset(name = "inputFilesetB") testFileset2.loadData() self.assertEqual(len(testFileset2.files), 0) return
def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN") self.fileset1 = Fileset(name="TestFileset1") for fileNum in range(11): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Harvest", type="Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return
def testCreateDeleteExists(self): """ _testCreateDeleteExists_ Create and then delete a job. Use the job class's exists() method to determine if the job has been written to the database before it is created, after it has been created and after it has been deleted. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileA.create() testFileB.create() testJob = Job(name="TestJob", files=[testFileA, testFileB]) testWU = WorkUnit(taskID=testWorkflow.id, fileid=testFileA['id'], runLumi=Run(1, *[44])) self.assertFalse(testJob.exists(), "Job exists before it was created") self.assertFalse(testWU.exists(), "WorkUnit exists before it was created") testJob.create(group=testJobGroup) self.assertTrue(testJob.exists(), "Job does not exist after it was created") self.assertFalse(testWU.exists(), "WorkUnit exists when there is no work") # Test the getWorkflow method workflow = testJob.getWorkflow() self.assertEqual(workflow['task'], 'Test') self.assertEqual(workflow['name'], 'wf001') testJob.delete() self.assertFalse(testJob.exists(), "Job exists after it was deleted") self.assertFalse(testWU.exists(), "WorkUnit exists after job is deleted") return
def notestTwoJobGroups(self): """ Test two job groups with a shared fileset. (Minimal part of testGetLocations which was failing) """ testWorkflow1 = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test1") testWorkflow1.create() testWMBSFileset1 = WMBSFileset(name="TestFileset1") testWMBSFileset1.create() testSubscription1 = Subscription(fileset=testWMBSFileset1, workflow=testWorkflow1) testSubscription1.create() testJobGroup1 = JobGroup(subscription=testSubscription1) testJobGroup1.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.create() testJobA = Job(name="TestJobA") testJobA.addFile(testFileA) testJobGroup1.add(testJobA) testJobGroup1.commit() testWorkflow2 = Workflow(spec="spec.xml", owner="Simon", name="wf002", task="Test2") testWorkflow2.create() testWMBSFileset2 = WMBSFileset(name="TestFileset1") testWMBSFileset2.create() testSubscription2 = Subscription(fileset=testWMBSFileset2, workflow=testWorkflow2) testSubscription2.create() testJobGroup2 = JobGroup(subscription=testSubscription2) testJobGroup2.create() testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10) testFileC.addRun(Run(10, *[12312])) testFileC.create() testJobA1 = Job(name="TestJobA1") testJobA1.addFile(testFileC) testJobGroup2.add(testJobA1) testJobGroup2.commit()
def testLargeNumberOfFiles(self): """ _testLargeNumberOfFiles_ Setup a subscription with 500 files and verify that the splitting algo works correctly. """ testWorkflowA = Workflow(spec="specA.xml", owner="Steve", name="wfA", task="Test") testWorkflowA.create() testWorkflowB = Workflow(spec="specB.xml", owner="Steve", name="wfB", task="Test") testWorkflowB.create() testFileset = Fileset(name="TestFileset") testFileset.create() allFiles = [] for i in range(500): testFile = File(str(i), size=1000, events=100, locations=set(["T2_CH_CERN"])) testFile.create() allFiles.append(testFile) testFileset.addFile(testFile) testFileset.commit() testSubscriptionA = Subscription(fileset=testFileset, workflow=testWorkflowA, split_algo="FileBased", type="Processing") testSubscriptionA.create() testSubscriptionB = Subscription(fileset=testFileset, workflow=testWorkflowB, split_algo="SiblingProcessingBased", type="Processing") testSubscriptionB.create() testSubscriptionA.completeFiles(allFiles) splitter = SplitterFactory() deleteFactoryA = splitter(package="WMCore.WMBS", subscription=testSubscriptionB) result = deleteFactoryA(files_per_job=50) self.assertEqual(len(result), 1, "Error: Wrong number of job groups returned.") self.assertEqual(len(result[0].jobs), 10, "Error: Wrong number of jobs returned.") return
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site=None, bl=[], wl=[]): """ Creates a series of jobGroups for submissions """ jobGroupList = [] testWorkflow = Workflow(spec=workloadSpec, owner="mnorman", name=makeUUID(), task="basicWorkload/Production", owner_vogroup='phgroup', owner_vorole='cmsrole') testWorkflow.create() # Create subscriptions for i in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name=name) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create jobs self.makeNJobs(name=name, task=task, nJobs=nJobs, jobGroup=testJobGroup, fileset=testFileset, sub=testSubscription.exists(), site=site, bl=bl, wl=wl) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) return jobGroupList
def testLoad(self): """ _testLoad_ Create a workflow and then try to load it from the database using the following load methods: Workflow.LoadFromName Workflow.LoadFromID Workflow.LoadFromSpecOwner """ testWorkflowA = Workflow(spec="spec.xml", owner="Simon", name="wf001", task='Test', wfType="ReReco", priority=1000) testWorkflowA.create() testWorkflowB = Workflow(name="wf001", task='Test') testWorkflowB.load() self.assertTrue((testWorkflowA.id == testWorkflowB.id) and (testWorkflowA.name == testWorkflowB.name) and (testWorkflowA.spec == testWorkflowB.spec) and (testWorkflowA.task == testWorkflowB.task) and (testWorkflowA.wfType == testWorkflowB.wfType) and (testWorkflowA.owner == testWorkflowB.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromName Failed") testWorkflowC = Workflow(id=testWorkflowA.id) testWorkflowC.load() self.assertTrue((testWorkflowA.id == testWorkflowC.id) and (testWorkflowA.name == testWorkflowC.name) and (testWorkflowA.spec == testWorkflowC.spec) and (testWorkflowA.task == testWorkflowC.task) and (testWorkflowA.wfType == testWorkflowC.wfType) and (testWorkflowA.owner == testWorkflowC.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromID Failed") testWorkflowD = Workflow(spec="spec.xml", owner="Simon", task='Test') testWorkflowD.load() self.assertTrue((testWorkflowA.id == testWorkflowD.id) and (testWorkflowA.name == testWorkflowD.name) and (testWorkflowA.spec == testWorkflowD.spec) and (testWorkflowA.task == testWorkflowD.task) and (testWorkflowA.wfType == testWorkflowD.wfType) and (testWorkflowA.owner == testWorkflowD.owner) and (testWorkflowA.priority == testWorkflowB.priority), "ERROR: Workflow.LoadFromSpecOwner Failed") testWorkflowA.delete() return
def testDeleteTransaction(self): """ _testDeleteTransaction_ Create a new job and commit it to the database. Start a new transaction and delete the file from the database. Verify that the file has been deleted. After that, roll back the transaction and verify that the job is once again in the database. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileA.create() testFileB.create() testJob = Job(name="TestJob", files=[testFileA, testFileB]) assert testJob.exists() == False, \ "ERROR: Job exists before it was created" testJob.create(group=testJobGroup) assert testJob.exists() >= 0, \ "ERROR: Job does not exist after it was created" myThread = threading.currentThread() myThread.transaction.begin() testJob.delete() assert testJob.exists() == False, \ "ERROR: Job exists after it was delete" myThread.transaction.rollback() assert testJob.exists() >= 0, \ "ERROR: Job does not exist after transaction was rolled back." return
def testParentageByJob(self): """ _testParentageByJob_ Tests the DAO that assigns parentage by Job """ testWorkflow = Workflow(spec = 'hello', owner = "mnorman", name = "wf001", task="basicWorkload/Production") testWorkflow.create() testFileset = Fileset(name = "TestFileset") testFileset.create() testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased") testSubscription.create() testJobGroup = JobGroup(subscription = testSubscription) testJobGroup.create() testFileParentA = File(lfn = "/this/is/a/parent/lfnA", size = 1024, events = 20, checksums = {'cksum': 1}) testFileParentA.addRun(Run( 1, *[45])) testFileParentB = File(lfn = "/this/is/a/parent/lfnB", size = 1024, events = 20, checksums = {'cksum': 1}) testFileParentB.addRun(Run( 1, *[45])) testFileParentA.create() testFileParentB.create() testFileA = File(lfn = "/this/is/a/lfn", size = 1024, events = 10, checksums = {'cksum':1}) testFileA.addRun(Run( 1, *[45])) testFileA.create() testJobA = Job() testJobA.create(group = testJobGroup) testJobA.addFile(testFileParentA) testJobA.addFile(testFileParentB) testJobA.associateFiles() parentAction = self.daofactory(classname = "Files.SetParentageByJob") parentAction.execute(binds = {'jobid': testJobA.exists(), 'child': testFileA['lfn']}) testFileB = File(id = testFileA["id"]) testFileB.loadData(parentage = 1) goldenFiles = [testFileParentA, testFileParentB] for parentFile in testFileB["parents"]: self.assertEqual(parentFile in goldenFiles, True, "ERROR: Unknown parent file") goldenFiles.remove(parentFile) self.assertEqual(len(goldenFiles), 0, "ERROR: Some parents are missing")
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site, taskType='Processing', name=None, wfPrio=1, changeState=None): """ _createJobGroups_ Creates a series of jobGroups for submissions changeState is an instance of the ChangeState class to make job status changes """ jobGroupList = [] if name is None: name = makeUUID() testWorkflow = Workflow(spec=workloadSpec, owner="tapas", name=name, task="basicWorkload/Production", priority=wfPrio) testWorkflow.create() # Create subscriptions for _ in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name=name) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type=taskType, split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create jobs self.makeNJobs(name=name, task=task, nJobs=nJobs, jobGroup=testJobGroup, fileset=testFileset, sub=testSubscription.exists(), site=site) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) if changeState: for group in jobGroupList: changeState.propagate(group.jobs, 'created', 'new') return jobGroupList
def createJobGroups(self, nSubs, nJobs, task, workloadSpec, site=None, bl=[], wl=[]): """ Creates a series of jobGroups for submissions """ jobGroupList = [] testWorkflow = Workflow( spec=workloadSpec, owner="tapas", name=makeUUID(), task="basicWorkload/Production", owner_vogroup="phgroup", owner_vorole="cmsrole", ) testWorkflow.create() # Create subscriptions for i in range(nSubs): name = makeUUID() # Create Fileset, Subscription, jobGroup testFileset = Fileset(name=name) testFileset.create() testSubscription = Subscription( fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased" ) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create jobs self.makeNJobs( name=name, task=task, nJobs=nJobs, jobGroup=testJobGroup, fileset=testFileset, sub=testSubscription.exists(), site=site, bl=bl, wl=wl, ) testFileset.commit() testJobGroup.commit() jobGroupList.append(testJobGroup) return jobGroupList
def testDeleteTransaction(self): """ _testDeleteTransaction_ Create a new job and commit it to the database. Start a new transaction and delete the file from the database. Verify that the file has been deleted. After that, roll back the transaction and verify that the job is once again in the database. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10) testFileA.create() testFileB.create() testJob = Job(name="TestJob", files=[testFileA, testFileB]) assert testJob.exists() is False, \ "ERROR: Job exists before it was created" testJob.create(group=testJobGroup) assert testJob.exists() >= 0, \ "ERROR: Job does not exist after it was created" myThread = threading.currentThread() myThread.transaction.begin() testJob.delete() assert testJob.exists() is False, \ "ERROR: Job exists after it was delete" myThread.transaction.rollback() assert testJob.exists() >= 0, \ "ERROR: Job does not exist after transaction was rolled back." return
def createLargerTestJobGroup(self, commitFlag=True): """ _createTestJobGroup_ """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = WMBSFileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10) testFileC.addRun(Run(10, *[12312])) testFileC.setLocation("goodse.cern.ch") testFileC.setLocation("malpaquet") testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10) testFileD.addRun(Run(10, *[12312])) testFileD.setLocation("goodse.cern.ch") testFileD.setLocation("malpaquet") testFileC.create() testFileD.create() testJobA = Job(name="TestJobA1") testJobA.addFile(testFileC) testJobB = Job(name="TestJobB1") testJobB.addFile(testFileD) testJobGroup.add(testJobA) testJobGroup.add(testJobB) for i in range(0, 100): testJob = Job(name="TestJob%i" % (i)) testJob.addFile(testFileC) testJobGroup.add(testJob) if commitFlag: testJobGroup.commit() return testJobGroup
def testGetGroupsByJobStateDAO(self): """ _testGetGroupsByJobStateDAO_ Verify that the GetGrounsByJobState DAO does what it is supposed to do. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = WMBSFileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow) testSubscription.create() testJobGroupA = JobGroup(subscription=testSubscription) testJobGroupA.create() testJobGroupB = JobGroup(subscription=testSubscription) testJobGroupB.create() testJobA = Job(name="TestJobA") testJobB = Job(name="TestJobB") testJobGroupA.add(testJobA) testJobGroupB.add(testJobB) testJobGroupA.commit() testJobGroupB.commit() myThread = threading.currentThread() daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) stateChangeAction = daofactory(classname="Jobs.ChangeState") testJobA["state"] = "complete" testJobB["state"] = "executing" stateChangeAction.execute(jobs=[testJobA, testJobB]) jobGroupAction = daofactory(classname="JobGroup.GetGroupsByJobState") jobGroups = jobGroupAction.execute(jobState="complete") assert len(jobGroups) == 1, \ "Error: Wrong number of job groups returned." assert jobGroups[0] == testJobGroupA.id, \ "Error: Wrong job group returned." return
def stuffWMBS(self): """ _stuffWMBS_ Stuff WMBS with workflows """ workflow = Workflow(spec = 'spec.xml', name = 'ReRecoTest_v0Emulator', task = '/ReRecoTest_v0Emulator/Test', priority = 10) workflow.create() inputFileset = Fileset(name = 'TestFileset') inputFileset.create() subscription = Subscription(inputFileset, workflow) subscription.create()
def createTestJobs(self, nJobs, cacheDir): """ _createTestJobs_ Create several jobs """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testWMBSFileset = Fileset(name="TestFileset") testWMBSFileset.create() testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() # Create a file testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileA.create() baseName = makeUUID() # Now create a job for i in range(nJobs): testJob = Job(name='%s-%i' % (baseName, i)) testJob.addFile(testFileA) testJob['location'] = 'malpaquet' testJob['retry_count'] = 1 testJob['retry_max'] = 10 testJob.create(testJobGroup) testJob.save() testJobGroup.add(testJob) testJobGroup.commit() # Set test job caches for job in testJobGroup.jobs: job.setCache(cacheDir) return testJobGroup
def stuffWMBS(self): """ _stuffWMBS_ Inject the workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge)
def testDeleteTransaction(self): """ _testDeleteTransaction_ Create a JobGroup and then commit it to the database. Begin a transaction and the delete the JobGroup from the database. Using the exists() method verify that the JobGroup is not in the database. Finally, roll back the transaction and verify that the JobGroup is in the database. """ testWorkflow = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test") testWorkflow.create() testFileset = WMBSFileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow) testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) assert testJobGroup.exists() == False, \ "ERROR: Job group exists before it was created" testJobGroup.create() assert testJobGroup.exists() >= 0, \ "ERROR: Job group does not exist after it was created" myThread = threading.currentThread() myThread.transaction.begin() testJobGroup.delete() assert testJobGroup.exists() == False, \ "ERROR: Job group exists after it was deleted" myThread.transaction.rollback() assert testJobGroup.exists() >= 0, \ "ERROR: Job group does not exist after transaction was rolled back." testSubscription.delete() testFileset.delete() testWorkflow.delete() return
def testUpdateFailedDoc(self): """ _testUpdateFailedDoc_ Verify that the update function will work correctly and not throw a 500 error if the doc didn't make it into the database for some reason. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname="Locations.New") locationAction.execute("site1", seName="somese.cern.ch") testWorkflow = Workflow(spec="spec.xml", owner="Steve", name="wf001", task=self.taskName) testWorkflow.create() testFileset = Fileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, split_algo="FileBased") testSubscription.create() testFileA = File(lfn="SomeLFNA", events=1024, size=2048, locations=set(["somese.cern.ch"])) testFileA.create() testFileset.addFile(testFileA) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroup = jobFactory(files_per_job=1)[0] testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobA["couch_record"] = str(testJobA["id"]) change.propagate([testJobA], "new", "none") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) self.assertTrue(testJobADoc.has_key("states")) self.assertTrue(testJobADoc["states"].has_key("1")) return
def createFileCollection(self, name, nSubs, nFiles, workflowURL='test', site=None): """ _createFileCollection_ Create a collection of files for splitting into jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) testFileset = Fileset(name=nameStr) testFileset.create() for f in range(nFiles): # pick a random site if not site: tmpSite = 'se.%s' % (random.choice(self.sites)) else: tmpSite = 'se.%s' % (site) testFile = File(lfn="/lfn/%s/%i" % (nameStr, f), size=1024, events=10) testFile.setLocation(tmpSite) testFile.create() testFileset.addFile(testFile) testFileset.commit() testFileset.markOpen(isOpen=0) testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() return
def createDummyJobs(self, nJobs, location=None): """ _createDummyJobs_ Create some dummy jobs """ if not location: location = self.sites[0] nameStr = makeUUID() testWorkflow = Workflow(spec=nameStr, owner="tapas", name=nameStr, task="basicWorkload/Production", owner_vogroup='phgroup', owner_vorole='cmsrole') testWorkflow.create() testFileset = Fileset(name=nameStr) testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() testJobGroup = JobGroup(subscription=testSubscription) testJobGroup.create() jobList = [] for i in range(nJobs): testJob = Job(name='%s-%i' % (nameStr, i)) testJob['location'] = location testJob['custom']['location'] = location testJob['userdn'] = 'tapas' testJob['owner'] = 'tapas' testJob['userrole'] = 'cmsrole' testJob['usergroup'] = 'phgroup' testJob.create(testJobGroup) jobList.append(testJob) return jobList