def testInstantiation(self): """ _testInstantiation_ Verify that the WMTask and the WMTaskHelper classes can be instantiated. """ WMTask("task1") makeWMTask("task2") return
def testTrustSitelists(self): """ _testTrustSitelists_ Verify that we can set/get the proper TrustSitelists and TrustPUSitelists flag. Also make sure they are retrievable through job splitting parameters. """ testTask = makeWMTask("TestTask") testTask.setJobResourceInformation(timePerEvent=50, memoryReq=4000, sizePerEvent=10) splitArgs = testTask.jobSplittingParameters(performance=False) self.assertFalse(splitArgs['trustSitelists']) self.assertFalse(splitArgs['trustPUSitelists']) testTask.setTrustSitelists(True, True) trustlists = testTask.getTrustSitelists() self.assertTrue(trustlists['trustlists']) self.assertTrue(trustlists['trustPUlists']) splitArgs = testTask.jobSplittingParameters(performance=False) self.assertTrue(splitArgs['trustSitelists']) self.assertTrue(splitArgs['trustPUSitelists']) testTask.setTrustSitelists(False, False) trustlists = testTask.getTrustSitelists() self.assertFalse(trustlists['trustlists']) self.assertFalse(trustlists['trustPUlists']) return
def testSetGetPriority(self): """ _testSetGetPriority_ Test and see whether we can set and get the priorities in a task. """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") # Should be null to start self.assertEqual(testTask.getTaskPriority(), None) # Should not accept bad values testTask.setTaskPriority(priority = 'fail') self.assertEqual(testTask.getTaskPriority(), None) # Should cast strings testTask.setTaskPriority(priority = '1') self.assertEqual(testTask.getTaskPriority(), 1) # Should handle ints testTask.setTaskPriority(priority = 2) self.assertEqual(testTask.getTaskPriority(), 2) return
def testJobSplittingArgs(self): """ _testJobSplittingArgs_ Verify that setting job splitting arguments for a task works correctly. When pulling the job splitting arguments out of a task the site white list and black list should also be included as they need to be passed to the job splitting code. """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") assert testTask.taskType() == "Processing", "Error: Wrong task type." testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job=100, max_job_size=24, one_more_param="Hello") testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"]) testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"]) assert testTask.jobSplittingAlgorithm() == "MadeUpAlgo", "Error: Wrong job splitting algorithm name." algoParams = testTask.jobSplittingParameters() assert len(algoParams.keys()) == 6, "Error: Wrong number of algo parameters." assert "algorithm" in algoParams.keys(), "Error: Missing algo parameter." assert algoParams["algorithm"] == "MadeUpAlgo", "Error: Parameter has wrong value." assert "events_per_job" in algoParams.keys(), "Error: Missing algo parameter." assert algoParams["events_per_job"] == 100, "Error: Parameter has wrong value." assert "max_job_size" in algoParams.keys(), "Error: Missing algo parameter." assert algoParams["max_job_size"] == 24, "Error: Parameter has wrong value." assert "one_more_param" in algoParams.keys(), "Error: Missing algo parameter." assert algoParams["one_more_param"] == "Hello", "Error: Parameter has wrong value." return
def testPresetSeeder(self): """ _testPresetSeeder_ Test whether the PresetSeeder works """ task1 = makeWMTask("task2") seederDict = {"generator.initialSeed": 1001, "evtgenproducer.initialSeed": 1001} task1.addGenerator("PresetSeeder", **seederDict) manager = GeneratorManager(task = task1) jobs = self.oneHundredFiles() for jobGrp in jobs: manager(jobGrp) for jobGrp in jobs: count = 0 for job in jobGrp.jobs: conf = job.getBaggage() self.assertEqual(conf.PresetSeeder.evtgenproducer.initialSeed, 1001) self.assertEqual(conf.PresetSeeder.generator.initialSeed, 1001) return
def testSiteWhiteBlacklists(self): """ _testSiteWhiteBlacklists_ Verify that methods for setting and retrieving site white and black lists work correctly. """ testTask = makeWMTask("TestTask") testTask.setSiteWhitelist(["T1_US_FNAL", "T1_DE_KIT"]) testTask.setSiteBlacklist( ["T1_IT_IN2P3", "T1_CH_CERN", "T2_US_NEBRASKA"]) taskWhitelist = testTask.siteWhitelist() assert len(taskWhitelist) == 2, \ "Error: Wrong number of sites in white list." assert "T1_US_FNAL" in taskWhitelist, \ "Error: Site missing from white list." assert "T1_DE_KIT" in taskWhitelist, \ "Error: Site missing from white list." taskBlacklist = testTask.siteBlacklist() assert len(taskBlacklist) == 3, \ "Error: Wrong number if sites in black list." assert "T1_IT_IN2P3" in taskBlacklist, \ "Error: Site missing from black list." assert "T1_CH_CERN" in taskBlacklist, \ "Error: Site missing from black list." assert "T2_US_NEBRASKA" in taskBlacklist, \ "Error: Site missing from black list." return
def testSubscriptionInformation(self): """ _testSubscriptionInformation_ Check the three methods related to the subscription information in a task Make sure that we can set the subscription information for all datasets produced by this task and we can select only some primaryDatasets/DataTiers, and check that we can update the dataset in a subscription information section. """ testTask = makeWMTask("TestTask") cmsswStep = testTask.makeStep("cmsRun1") cmsswStep.setStepType("CMSSW") testTask.applyTemplates() cmsswHelper = cmsswStep.getTypeHelper() cmsswHelper.addOutputModule( "outputRECO", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="RECO" ) cmsswHelper.addOutputModule( "outputDQM", primaryDataset="TwoParticles", processedDataset="DawnOfAnEra-v1", dataTier="DQM" ) cmsswHelper.addOutputModule( "outputAOD", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="AOD" ) self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info") testTask.setSubscriptionInformation(["mercury"], ["mars", "earth"], ["earth"], "High", "Replica", "OneParticle") subInfo = testTask.getSubscriptionInformation() outputRecoSubInfo = { "CustodialSites": ["mercury"], "NonCustodialSites": ["mars", "earth"], "AutoApproveSites": ["earth"], "Priority": "High", "CustodialSubType": "Replica", } self.assertEqual( subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong" ) self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") testTask.setSubscriptionInformation(["jupiter"], primaryDataset="TwoParticles") subInfo = testTask.getSubscriptionInformation() self.assertEqual( subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong" ) self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") recoutOutputModule = cmsswHelper.getOutputModule("outputRECO") setattr(recoutOutputModule, "primaryDataset", "ThreeParticles") testTask.updateSubscriptionDataset("outputRECO", recoutOutputModule) subInfo = testTask.getSubscriptionInformation() self.assertEqual( subInfo["/ThreeParticles/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong", ) self.assertFalse("/OneParticle/DawnOfAnEra-v1/RECO" in subInfo, "The RECO subscription information is wrong")
def testGetSwVersionAndScramArch(self): """ _testGetSwVersionAndScramArch_ Test whether we can fetch the CMSSW release and ScramArch being used in a task """ testTask = makeWMTask("MultiTask") taskCmssw = testTask.makeStep("cmsRun1") taskCmssw.setStepType("CMSSW") taskCmsswStageOut = taskCmssw.addStep("stageOut1") taskCmsswStageOut.setStepType("StageOut") taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1") taskCmsswLogArch.setStepType("LogArchive") testTask.applyTemplates() taskCmsswHelper = taskCmssw.getTypeHelper() taskCmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123") self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3") self.assertEqual(testTask.getSwVersion(allSteps=True), ["CMSSW_1_2_3"]) self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123") self.assertEqual(testTask.getScramArch(allSteps=True), ["slc7_amd64_gcc123"]) return
def testJobSplittingArgs(self): """ _testJobSplittingArgs_ Verify that setting job splitting arguments for a task works correctly. When pulling the job splitting arguments out of a task the site white list and black list should also be included as they need to be passed to the job splitting code. """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") assert testTask.taskType() == "Processing", \ "Error: Wrong task type." testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job = 100, max_job_size = 24, one_more_param = "Hello") testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"]) testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"]) testTask.addInputDataset(primary = "PrimaryDataset", processed = "ProcessedDataset", tier = "DataTier", dbsurl = "DBSURL", block_whitelist = ["Block1", "Block2"], block_blacklist = ["Block3", "Block4", "Block5"], run_whitelist = [1, 2, 3], run_blacklist = [4, 5]) assert testTask.jobSplittingAlgorithm() == "MadeUpAlgo", \ "Error: Wrong job splitting algorithm name." algoParams = testTask.jobSplittingParameters() assert len(algoParams.keys()) == 8, \ "Error: Wrong number of algo parameters." assert "algorithm" in algoParams.keys(), \ "Error: Missing algo parameter." assert algoParams["algorithm"] == "MadeUpAlgo", \ "Error: Parameter has wrong value." assert "events_per_job" in algoParams.keys(), \ "Error: Missing algo parameter." assert algoParams["events_per_job"] == 100, \ "Error: Parameter has wrong value." assert "max_job_size" in algoParams.keys(), \ "Error: Missing algo parameter." assert algoParams["max_job_size"] == 24, \ "Error: Parameter has wrong value." assert "one_more_param" in algoParams.keys(), \ "Error: Missing algo parameter." assert algoParams["one_more_param"] == "Hello", \ "Error: Parameter has wrong value." assert len(algoParams["runWhitelist"]) == 3, \ "Error: Wrong number of runs in whitelist." return
def testAddingSteps(self): """ _testAddingSteps_ Verify that adding steps to a task works correctly. """ task1 = makeWMTask("task1") task2a = task1.addTask("task2a") task2b = task1.addTask("task2b") task2c = task1.addTask("task2c") task3 = task2a.addTask("task3") step1 = makeWMStep("step1") step1.addStep("step1a") step1.addStep("step1b") step1.addStep("step1c") step2 = makeWMStep("step2") step2.addStep("step2a") step2.addStep("step2b") step2.addStep("step2c") step3 = makeWMStep("step3") step3.addStep("step3a") step3.addStep("step3b") step3.addStep("step3c") step4 = makeWMStep("step4") step4.addStep("step4a") step4.addStep("step4b") step4.addStep("step4c") task1.setStep(step1) task2a.setStep(step2) task2b.setStep(step3) task2c.setStep(step4) self.assertEqual(task1.getStep("step1a").name(), "step1a") self.assertEqual(task2a.getStep("step2b").name(), "step2b") self.assertEqual(task2b.getStep("step3c").name(), "step3c") self.assertEqual(task2c.getStep("step4a").name(), "step4a") self.assertEqual(task1.getStep("step2"), None) self.assertEqual(task2a.getStep("step4"), None) self.assertEqual(task2b.getStep("step2"), None) self.assertEqual(task2c.getStep("step1"), None) self.assertEqual(task1.listNodes(), ['task1', 'task2a', 'task3', 'task2b', 'task2c']) return
def testMaxRSS(self): """ _testMaxRSS_ Test whether we can properly add MaxRSS performance monitor to this task. """ testTask = makeWMTask("TestTask") testTask.setMaxRSS(123) self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor']) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxRSS, 123) return
def testProcessingVerAndAcquisitionEra(self): """ _testProcessingVerAndAcquisitionEra_ Test that we can add a processing version and acquisition era, and then get it back. """ testTask = makeWMTask("TestTask") testTask.setAcquisitionEra("StoneAge") testTask.setProcessingVersion("vLast") self.assertEqual(testTask.getAcquisitionEra(), "StoneAge") self.assertEqual(testTask.getProcessingVersion(), "vLast") return
def testGetSwVersionAndScramArchMulti(self): """ _testGetSwVersionAndScramArchMulti_ Test whether we can fetch the CMSSW release and ScramArch being used in a task """ testTask = makeWMTask("MultiTask") taskCmssw = testTask.makeStep("cmsRun1") taskCmssw.setStepType("CMSSW") taskCmsswStageOut = taskCmssw.addStep("stageOut1") taskCmsswStageOut.setStepType("StageOut") taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1") taskCmsswLogArch.setStepType("LogArchive") testTask.applyTemplates() taskCmsswHelper = taskCmssw.getTypeHelper() taskCmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123") # setup step2/cmsRun2 step1Cmssw = testTask.getStep("cmsRun1") step2Cmssw = step1Cmssw.addTopStep("cmsRun2") step2Cmssw.setStepType("CMSSW") template = StepFactory.getStepTemplate("CMSSW") template(step2Cmssw.data) step2CmsswHelper = step2Cmssw.getTypeHelper() step2CmsswHelper.setupChainedProcessing("cmsRun1", "RAWSIMoutput") step2CmsswHelper.cmsswSetup("CMSSW_2_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc223") # setup step3/cmsRun3 --> duplicate CMSSW and ScramArch step3Cmssw = step2Cmssw.addTopStep("cmsRun3") step3Cmssw.setStepType("CMSSW") template = StepFactory.getStepTemplate("CMSSW") template(step3Cmssw.data) step3CmsswHelper = step3Cmssw.getTypeHelper() step3CmsswHelper.setupChainedProcessing("cmsRun2", "AODoutput") step3CmsswHelper.cmsswSetup("CMSSW_1_2_3", softwareEnvironment="", scramArch="slc7_amd64_gcc123") self.assertEqual(testTask.getSwVersion(), "CMSSW_1_2_3") self.assertEqual(testTask.getSwVersion(allSteps=True), ["CMSSW_1_2_3", "CMSSW_2_2_3", "CMSSW_1_2_3"]) self.assertEqual(testTask.getScramArch(), "slc7_amd64_gcc123") self.assertEqual(testTask.getScramArch(allSteps=True), ["slc7_amd64_gcc123", "slc7_amd64_gcc223", "slc7_amd64_gcc123"]) return
def testPerformanceMonitor(self): """ _testPerformanceMonitor_ Test automated adding of performanceMonitor Really you shouldn't be using this, so don't add it. """ testTask = makeWMTask("TestTask") testTask.setPerformanceMonitor(maxRSS = 100, maxVSize = 101) self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor']) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxRSS, 100) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.maxVSize, 101) return
def testAddNotifications(self): """ _testAddNotifications_ Test whether we can add notification addresses """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") targetList = ['*****@*****.**', '*****@*****.**'] self.assertEqual(testTask.getNotifications(), []) for x in targetList: testTask.addNotification(target=x) self.assertEqual(testTask.getNotifications(), targetList)
def testProcessedDatasetElements(self): """ _testProcessedDatasetElements_ Test that we can add a processing version and acquisition era, and then get it back. """ testTask = makeWMTask("TestTask") testTask.setAcquisitionEra("StoneAge") testTask.setProcessingVersion(2) testTask.setProcessingString("Test") self.assertEqual(testTask.getAcquisitionEra(), "StoneAge", "Wrong acquisition era in the task") self.assertEqual(testTask.getProcessingVersion(), 2, "Wrong processing version in the task") self.assertEqual(testTask.getProcessingString(), "Test", "Wrong processing string in the task") return
def testTreeBuilding(self): """ _testTreeBuilding_ Verify that tasks can be created and arranged in a hierarchy. """ task1 = makeWMTask("task1") task2a = task1.addTask("task2a") task2b = task1.addTask("task2b") task2c = task1.addTask("task2c") goldenTasks = ["task2a", "task2b", "task2c"] for childTask in task1.childTaskIterator(): assert childTask.name() in goldenTasks, "Error: Unknown child task: %s" % childTask.name() goldenTasks.remove(childTask.name()) assert len(goldenTasks) == 0, "Error: Missing tasks." return
def testPerformanceMonitor(self): """ _testPerformanceMonitor_ Test automated adding of performanceMonitor Really you shouldn't be using this, so don't add it. """ testTask = makeWMTask("TestTask") testTask.setPerformanceMonitor(softTimeout=100, gracePeriod=1) self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor']) self.assertFalse(hasattr(testTask.data.watchdog.PerformanceMonitor, "maxRSS")) self.assertFalse(hasattr(testTask.data.watchdog.PerformanceMonitor, "maxVSize")) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.softTimeout, 100) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.hardTimeout, 101) return
def testAddLumiMask(self): """ _testAddLumiMask_ Verify that setting and getting the lumiMask objects for a task works correctly. Do a round trip of a typical lumi mask """ testTask = makeWMTask("TestTask") lumiMask = LumiList(compactList = { '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]], '2':[[1,45]], '3':[[1,45],[50,80]], }) testTask.setLumiMask(lumiMask = lumiMask.getCompactList()) outMask = LumiList(compactList = testTask.getLumiMask()) self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString()) return
def testPerformanceMonitor(self): """ _testPerformanceMonitor_ Test automated adding of performanceMonitor Really you shouldn't be using this, so don't add it. """ testTask = makeWMTask("TestTask") testTask.setPerformanceMonitor(softTimeout=100, gracePeriod=1) self.assertEqual(testTask.data.watchdog.monitors, ['PerformanceMonitor']) self.assertFalse( hasattr(testTask.data.watchdog.PerformanceMonitor, "maxPSS")) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.softTimeout, 100) self.assertEqual(testTask.data.watchdog.PerformanceMonitor.hardTimeout, 101) return
def testInputPileup(self): """ _testInputPileup_ Verify that the input pileup dataset getter/setter methods work correctly """ testTask = makeWMTask("TestTask") self.assertEqual(testTask.getInputPileupDatasets(), []) pileupConfig = parsePileupConfig("/MC/ProcessedDataset/DataTier", "/Data/ProcessedDataset/DataTier") # then mimic the setupPileup method thesePU = [] for puType, puList in viewitems(pileupConfig): # there should be only one type and one PU dataset testTask.setInputPileupDatasets(puList) thesePU.extend(puList) self.assertItemsEqual(testTask.getInputPileupDatasets(), thesePU) with self.assertRaises(ValueError): testTask.setInputPileupDatasets(None)
def testAddEnvironmentVariables(self): """ _testAddEnvironmentVariables_ Verify that methods for setting and retrieving task environment variables. """ testTask = makeWMTask("TestTask") testDict = { "VAR0": "Value0", "VAR1": "Value1", "VAR2": "Value2", "VAR3": "Value3", } testTask.addEnvironmentVariables(testDict) retrievedDict = testTask.getEnvironmentVariables() self.assertEqual( retrievedDict, testDict, "Error: Env variables dict doesn't match with test dict") return
def testTreeBuilding(self): """ _testTreeBuilding_ Verify that tasks can be created and arranged in a hierarchy. """ task1 = makeWMTask("task1") task1.addTask("task2a") task1.addTask("task2b") task1.addTask("task2c") goldenTasks = ["task2a", "task2b", "task2c"] for childTask in task1.childTaskIterator(): assert childTask.name() in goldenTasks, \ "Error: Unknown child task: %s" % childTask.name() goldenTasks.remove(childTask.name()) assert len(goldenTasks) == 0, \ "Error: Missing tasks." return
def testParameters(self): """ _testParameters_ Test any random junk that we throw into parameters """ testTask = makeWMTask("TestTask") # Test the primarySubType first # Before we set anything, the subType should be none self.assertEqual(testTask.getPrimarySubType(), None) # After we set the task Type, but before the subType, it # should be the taskType testTask.setTaskType("SillyTask") self.assertEqual(testTask.getPrimarySubType(), "SillyTask") testTask.setPrimarySubType(subType="subType") self.assertEqual(testTask.getPrimarySubType(), "subType") return
def testAddLumiMask(self): """ _testAddLumiMask_ Verify that setting and getting the lumiMask objects for a task works correctly. Do a round trip of a typical lumi mask """ testTask = makeWMTask("TestTask") lumiMask = LumiList( compactList={ '1': [[1, 33], [35, 35], [37, 47], [49, 75], [77, 130], [133, 136]], '2': [[1, 45]], '3': [[1, 45], [50, 80]], }) testTask.setLumiMask(lumiMask=lumiMask.getCompactList()) outMask = testTask.getLumiMask() self.assertEqual(lumiMask.getCMSSWString(), outMask.getCMSSWString()) return
def testDeleteChild(self): """ _testDeleteChild_ Test that we can remove all reference from a child and other children are left intact """ task1 = makeWMTask("task1") task1.addTask("task2a") task1.addTask("task2b") task1.addTask("task2c") task1.deleteChild("task2a") childrenNumber = 0 for childTask in task1.childTaskIterator(): if childTask.name() == "task2a": self.fail("Error: It was possible to find the deleted child") childrenNumber += 1 self.assertEqual(childrenNumber, 2, "Error: Wrong number of children tasks") return
def testWMTask(self): """ _testWMTask_ Test whether or not we can read the seeder parameters out of a WMTask. Also tests RandomSeeder and RunAndLumiSeeder """ task1 = makeWMTask("task1") randomDict = {"generator.initialSeed": None, "evtgenproducer.initialSeed": None, "MAXINT": 1} lumiDict = {"lumi_per_run": 5} task1.addGenerator("RandomSeeder", **randomDict) task1.addGenerator("RunAndLumiSeeder", **lumiDict) manager = GeneratorManager(task = task1) jobs = self.oneHundredFiles() for jobGrp in jobs: manager(jobGrp) for jobGrp in jobs: count = 0 for job in jobGrp.jobs: conf = job.getBaggage() self.assertTrue(hasattr(conf.RandomSeeder.evtgenproducer, 'initialSeed')) self.assertTrue(hasattr(conf.RandomSeeder.generator, 'initialSeed')) #self.assertEqual(job["mask"]["FirstLumi"], count%6) #self.assertEqual(job["mask"]["FirstRun"], (count/6) + 1) x = conf.RandomSeeder.generator.initialSeed self.assertTrue( x > 0, "ERROR: producing negative random numbers") self.assertTrue( x <= 1, "ERROR: MAXINT tag failed; producing bad random number %i" %(x)) count += 1 return
def createMultiTaskObject(self): """ Creates a multi task object in the following structure: Task1: cmsRun1 stageOut1 logArch1 Task2: cmsRun1 cmsRun2 stageOut1 logArch1 :return: a task helper object for the first task """ task1 = makeWMTask("Taskname_1") task1.setTaskType("Production") task1Cmssw = task1.makeStep("cmsRun1") task1Cmssw.setStepType("CMSSW") taskCmsswStageOut = task1Cmssw.addStep("stageOut1") taskCmsswStageOut.setStepType("StageOut") taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1") taskCmsswLogArch.setStepType("LogArchive") task1.applyTemplates() task2 = task1.addTask("Taskname_2") task1.setTaskType("Processing") task2ParentCmssw = task2.makeStep("cmsRun1") task2ParentCmssw.setStepType("CMSSW") task2Cmssw = task2ParentCmssw.addTopStep("cmsRun2") task2Cmssw.setStepType("CMSSW") taskCmsswStageOut = task2Cmssw.addStep("stageOut1") taskCmsswStageOut.setStepType("StageOut") taskCmsswLogArch = taskCmsswStageOut.addStep("logArch1") taskCmsswLogArch.setStepType("LogArchive") task2.applyTemplates() return task1
def testSubscriptionInformation(self): """ _testSubscriptionInformation_ Check the three methods related to the subscription information in a task Make sure that we can set the subscription information for all datasets produced by this task and we can select only some primaryDatasets/DataTiers. Since subscriptions are defined during request assignment, there is no more need to update them, they are set once only. """ testTask = makeWMTask("TestTask") cmsswStep = testTask.makeStep("cmsRun1") cmsswStep.setStepType("CMSSW") testTask.applyTemplates() cmsswHelper = cmsswStep.getTypeHelper() cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="RECO") cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles", processedDataset="DawnOfAnEra-v1", dataTier="DQM") cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="AOD") childStep = cmsswHelper.addTopStep("cmsRun2") childStep.setStepType("CMSSW") template = StepFactory.getStepTemplate("CMSSW") template(childStep.data) childStep = childStep.getTypeHelper() childStep.addOutputModule("outputAOD", primaryDataset="ThreeParticles", processedDataset="DawnOfAnEra-v1", dataTier="MINIAOD") self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info") testTask.setSubscriptionInformation(custodialSites=["mercury"], nonCustodialSites=["mars", "earth"], autoApproveSites=["earth"], custodialSubType="Replica", nonCustodialSubType="Move", custodialGroup="DataOps", nonCustodialGroup="AnalysisOps", priority="Normal", deleteFromSource=True, primaryDataset="OneParticle") subInfo = testTask.getSubscriptionInformation() outputRecoSubInfo = {"CustodialSites": ["mercury"], "NonCustodialSites": ["mars", "earth"], "AutoApproveSites": ["earth"], "CustodialSubType": "Replica", "NonCustodialSubType": "Move", "CustodialGroup": "DataOps", "NonCustodialGroup": "AnalysisOps", "Priority": "Normal", "DeleteFromSource": True} self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo) testTask.setSubscriptionInformation(custodialSites=["jupiter"], primaryDataset="TwoParticles") subInfo = testTask.getSubscriptionInformation() outputDQMSubInfo = {"CustodialSites": ["jupiter"], "NonCustodialSites": [], "AutoApproveSites": [], "CustodialSubType": "Replica", "NonCustodialSubType": "Replica", "CustodialGroup": "DataOps", "NonCustodialGroup": "DataOps", "Priority": "Low", "DeleteFromSource": False} self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo, "The DQM subscription information is wrong") self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") self.assertFalse("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo) testTask.setSubscriptionInformation(nonCustodialSites=["jupiter"], primaryDataset="ThreeParticles") subInfo = testTask.getSubscriptionInformation() outputAODSubInfo = {"CustodialSites": [], "NonCustodialSites": ["jupiter"], "AutoApproveSites": [], "CustodialSubType": "Replica", "NonCustodialSubType": "Replica", "CustodialGroup": "DataOps", "NonCustodialGroup": "DataOps", "Priority": "Low", "DeleteFromSource": False} self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertEqual(subInfo["/TwoParticles/DawnOfAnEra-v1/DQM"], outputDQMSubInfo, "The DQM subscription information is wrong") self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/MINIAOD"], outputAODSubInfo, "The AOD subscription information is wrong") self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") self.assertTrue("/ThreeParticles/DawnOfAnEra-v1/MINIAOD" in subInfo)
def testInputDataset(self): """ _testInputDataset_ Verify that the addInputDataset() method works correctly and that the run/block black and white lists can be changed after calling addInputDataset(). """ testTask = makeWMTask("TestTask") assert testTask.getInputDatasetPath() == None, \ "Error: Input dataset path should be None." assert testTask.inputDatasetDBSURL() == None, \ "Error: Input DBS URL should be None." assert testTask.inputBlockWhitelist() == None, \ "Error: Input block white list should be None." assert testTask.inputBlockBlacklist() == None, \ "Error: Input block black list should be None." assert testTask.inputRunWhitelist() == None, \ "Error: Input run white list should be None." assert testTask.inputRunBlacklist() == None, \ "Error: Input run black list should be None." testTask.addInputDataset( primary="PrimaryDataset", processed="ProcessedDataset", tier="DataTier", dbsurl="DBSURL", block_whitelist=["Block1", "Block2"], block_blacklist=["Block3", "Block4", "Block5"], run_whitelist=[1, 2, 3], run_blacklist=[4, 5]) assert testTask.getInputDatasetPath() == "/PrimaryDataset/ProcessedDataset/DataTier", \ "Error: Input dataset path is wrong." assert testTask.inputDatasetDBSURL() == "DBSURL", \ "Error: Input DBS URL is wrong." assert len(testTask.inputBlockWhitelist()) == 2, \ "Error: Wrong number of blocks in white list." assert "Block1" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." assert "Block2" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." assert len(testTask.inputBlockBlacklist()) == 3, \ "Error: Wrong number of blocks in black list." assert "Block3" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block4" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block5" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert len(testTask.inputRunWhitelist()) == 3, \ "Error: Wrong number of runs in white list." assert 1 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert 2 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert 3 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert len(testTask.inputRunBlacklist()) == 2, \ "Error: Wrong number of runs in black list." assert 4 in testTask.inputRunBlacklist(), \ "Error: Run is missing from black list." assert 5 in testTask.inputRunBlacklist(), \ "Error: Run is missing from black list." testTask.setInputBlockWhitelist(["Block6"]) assert len(testTask.inputBlockWhitelist()) == 1, \ "Error: Wrong number of blocks in white list." assert "Block6" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." testTask.setInputBlockBlacklist(["Block7", "Block8"]) assert len(testTask.inputBlockBlacklist()) == 2, \ "Error: Wrong number of blocks in black list." assert "Block7" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block8" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." testTask.setInputRunWhitelist([6]) assert len(testTask.inputRunWhitelist()) == 1, \ "Error: Wrong number of runs in white list." assert 6 in testTask.inputRunWhitelist(), \ "Error: Run missing from white list." testTask.setInputRunBlacklist([7, 8]) assert len(testTask.inputRunBlacklist()) == 2, \ "Error: Wrong number of runs in black list." assert 7 in testTask.inputRunBlacklist(), \ "Error: Run missing from black list." assert 8 in testTask.inputRunBlacklist(), \ "Error: Run missing from black list." return
def testJobSplittingArgs(self): """ _testJobSplittingArgs_ Verify that setting job splitting arguments for a task works correctly. When pulling the job splitting arguments out of a task the site white list and black list should also be included as they need to be passed to the job splitting code. """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") self.assertEqual(testTask.taskType(), "Processing", "Error: Wrong task type.") testTask.setJobResourceInformation( timePerEvent=12, memoryReq=2300000, sizePerEvent=512) testTask.setSplittingAlgorithm( "MadeUpAlgo", events_per_job=100, max_job_size=24, one_more_param="Hello") testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"]) testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"]) testTask.addInputDataset( primary="PrimaryDataset", processed="ProcessedDataset", tier="DataTier", dbsurl="DBSURL", block_whitelist=["Block1", "Block2"], block_blacklist=["Block3", "Block4", "Block5"], run_whitelist=[1, 2, 3], run_blacklist=[4, 5]) # Make sure we can set individual performance parameters without affecting the others testTask.setJobResourceInformation(timePerEvent=14) self.assertEqual(testTask.jobSplittingAlgorithm(), "MadeUpAlgo", "Error: Wrong job splitting algorithm name.") algoParams = testTask.jobSplittingParameters(performance=False) self.assertEqual( len(algoParams), 9, "Error: Wrong number of algo parameters.") algoParams = testTask.jobSplittingParameters() self.assertEqual( len(algoParams), 10, "Error: Wrong number of algo parameters.") self.assertTrue("algorithm" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["algorithm"], "MadeUpAlgo", "Error: Parameter has wrong value.") self.assertTrue("events_per_job" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["events_per_job"], 100, "Error: Parameter has wrong value.") self.assertTrue("max_job_size" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["max_job_size"], 24, "Error: Parameter has wrong value.") self.assertTrue("one_more_param" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["one_more_param"], "Hello", "Error: Parameter has wrong value.") self.assertTrue("runWhitelist" in algoParams, "Error: Missing algo parameter.") self.assertEqual( len(algoParams["runWhitelist"]), 3, "Error: Wrong number of runs in whitelist.") self.assertTrue("performance" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["performance"]["timePerEvent"], 14, "Error: Wrong time per event") self.assertEqual(algoParams["performance"]["memoryRequirement"], 2300000, "Error: Wrong memory requirement") self.assertEqual(algoParams["performance"]["sizePerEvent"], 512, "Error: Wrong size per event") return
def executeCMSSWStack(opts, scram): def getOutputModules(): pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\ "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\ "tweakJson = makeTweak(config.process).jsondictionary();"+\ "print tweakJson[\"process\"][\"outputModules_\"]" with tempSetLogLevel(logger=logging.getLogger(), level=logging.ERROR): ret = scram("python -c '%s'" % pythonScript, runtimeDir=os.getcwd()) if ret > 0: msg = 'Error getting output modules from the pset.\n\tScram Diagnostic %s' % scram.diagnostic( ) handleException("FAILED", EC_CMSRunWrapper, msg) mintime() sys.exit(EC_CMSRunWrapper) output = literal_eval(scram.getStdout()) return output cmssw = CMSSW() cmssw.stepName = "cmsRun" cmssw.step = WMStep(cmssw.stepName) CMSSWTemplate().install(cmssw.step) cmssw.task = makeWMTask(cmssw.stepName) cmssw.workload = newWorkload(cmssw.stepName) cmssw.step.application.setup.softwareEnvironment = '' cmssw.step.application.setup.scramArch = opts.scramArch cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion cmssw.step.application.configuration.section_("arguments") cmssw.step.application.configuration.arguments.globalTag = "" for output in getOutputModules(): cmssw.step.output.modules.section_(output) getattr(cmssw.step.output.modules, output).primaryDataset = '' getattr(cmssw.step.output.modules, output).processedDataset = '' getattr(cmssw.step.output.modules, output).dataTier = '' #cmssw.step.application.command.arguments = '' #TODO cmssw.step.user.inputSandboxes = [opts.archiveJob] cmssw.step.user.userFiles = opts.userFiles or '' #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles. #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime.. cmssw.job = {'counter': 0, 'workflow': 'unused'} cmssw.step.user.lfnBase = '/store/temp/user/' cmssw.step.section_("builder") cmssw.step.builder.workingDir = os.getcwd() cmssw.step.runtime.invokeCommand = 'python' cmssw.step.runtime.scramPreDir = os.getcwd() cmssw.step.runtime.preScripts = [] cmssw.step.runtime.scramPreScripts = [ ('%s/TweakPSet.py --location=%s ' + '--inputFile=\'%s\' ' + '--runAndLumis=\'%s\' ' + '--firstEvent=%s ' + '--lastEvent=%s ' + '--firstLumi=%s ' + '--firstRun=%s ' + '--seeding=%s ' + '--lheInputFiles=%s ' + '--oneEventMode=%s ' + '--eventsPerLumi=%s ' + '--maxRuntime=%s') % (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis, opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun, opts.seeding, opts.lheInputFiles, opts.oneEventMode, opts.eventsPerLumi, opts.maxRuntime) ] cmssw.step.section_("execution") #exitStatus of cmsRun is set here cmssw.report = Report("cmsRun") #report is loaded and put here cmssw.execute() return cmssw
def testJobSplittingArgs(self): """ _testJobSplittingArgs_ Verify that setting job splitting arguments for a task works correctly. When pulling the job splitting arguments out of a task the site white list and black list should also be included as they need to be passed to the job splitting code. """ testTask = makeWMTask("TestTask") testTask.setTaskType("Processing") self.assertEqual(testTask.taskType(), "Processing", "Error: Wrong task type.") testTask.setJobResourceInformation(timePerEvent=12, memoryReq=2300000, sizePerEvent=512) testTask.setSplittingAlgorithm("MadeUpAlgo", events_per_job=100, max_job_size=24, one_more_param="Hello") testTask.setSiteWhitelist(["T1_US_FNAL", "T1_CH_CERN"]) testTask.setSiteBlacklist(["T2_US_PERDUE", "T2_US_UCSD", "T1_TW_ASGC"]) testTask.addInputDataset(name="/PrimaryDataset/ProcessedDataset/DataTier", primary="PrimaryDataset", processed="ProcessedDataset", tier="DataTier", dbsurl="DBSURL", block_whitelist=["Block1", "Block2"], block_blacklist=["Block3", "Block4", "Block5"], run_whitelist=[1, 2, 3], run_blacklist=[4, 5]) # Make sure we can set individual performance parameters without affecting the others testTask.setJobResourceInformation(timePerEvent=14) self.assertEqual(testTask.jobSplittingAlgorithm(), "MadeUpAlgo", "Error: Wrong job splitting algorithm name.") algoParams = testTask.jobSplittingParameters(performance=False) self.assertEqual(len(algoParams), 10, "Error: Wrong number of algo parameters.") algoParams = testTask.jobSplittingParameters() self.assertEqual(len(algoParams), 11, "Error: Wrong number of algo parameters.") self.assertTrue("algorithm" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["algorithm"], "MadeUpAlgo", "Error: Parameter has wrong value.") self.assertTrue("events_per_job" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["events_per_job"], 100, "Error: Parameter has wrong value.") self.assertTrue("max_job_size" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["max_job_size"], 24, "Error: Parameter has wrong value.") self.assertTrue("one_more_param" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["one_more_param"], "Hello", "Error: Parameter has wrong value.") self.assertTrue("runWhitelist" in algoParams, "Error: Missing algo parameter.") self.assertEqual(len(algoParams["runWhitelist"]), 3, "Error: Wrong number of runs in whitelist.") self.assertTrue("performance" in algoParams, "Error: Missing algo parameter.") self.assertEqual(algoParams["performance"]["timePerEvent"], 14, "Error: Wrong time per event") self.assertEqual(algoParams["performance"]["memoryRequirement"], 2300000, "Error: Wrong memory requirement") self.assertEqual(algoParams["performance"]["sizePerEvent"], 512, "Error: Wrong size per event") return
def executeCMSSWStack(opts, scram): def getOutputModules(): pythonScript = "from PSetTweaks.WMTweak import makeTweak;"+\ "config = __import__(\"WMTaskSpace.cmsRun.PSet\", globals(), locals(), [\"process\"], -1);"+\ "tweakJson = makeTweak(config.process).jsondictionary();"+\ "print tweakJson[\"process\"][\"outputModules_\"]" ret = scram("python -c '%s'" % pythonScript, logName=subprocess.PIPE, runtimeDir=os.getcwd()) if ret > 0: msg = scram.diagnostic() handleException("FAILED", EC_CMSRunWrapper, 'Error getting output modules from the pset.\n\tScram Env %s\n\tCommand:%s' % (msg, pythonScript)) mintime() sys.exit(EC_CMSRunWrapper) return literal_eval(scram.stdout) cmssw = CMSSW() cmssw.stepName = "cmsRun" cmssw.step = WMStep(cmssw.stepName) CMSSWTemplate().install(cmssw.step) cmssw.task = makeWMTask(cmssw.stepName) cmssw.workload = newWorkload(cmssw.stepName) cmssw.step.application.setup.softwareEnvironment = '' cmssw.step.application.setup.scramArch = opts.scramArch cmssw.step.application.setup.cmsswVersion = opts.cmsswVersion cmssw.step.application.configuration.section_("arguments") cmssw.step.application.configuration.arguments.globalTag = "" for output in getOutputModules(): cmssw.step.output.modules.section_(output) getattr(cmssw.step.output.modules, output).primaryDataset = '' getattr(cmssw.step.output.modules, output).processedDataset = '' getattr(cmssw.step.output.modules, output).dataTier = '' #cmssw.step.application.command.arguments = '' #TODO cmssw.step.user.inputSandboxes = [opts.archiveJob] cmssw.step.user.userFiles = opts.userFiles or '' #Setting the following job attribute is required because in the CMSSW executor there is a call to analysisFileLFN to set up some attributes for TFiles. #Same for lfnbase. We actually don't use these information so I am setting these to dummy values. Next: fix and use this lfn or drop WMCore runtime.. cmssw.job = {'counter' : 0, 'workflow' : 'unused'} cmssw.step.user.lfnBase = '/store/temp/user/' cmssw.step.section_("builder") cmssw.step.builder.workingDir = os.getcwd() cmssw.step.runtime.invokeCommand = 'python' cmssw.step.runtime.scramPreDir = os.getcwd() cmssw.step.runtime.preScripts = [] cmssw.step.runtime.scramPreScripts = [('%s/TweakPSet.py --location=%s '+ '--inputFile=\'%s\' '+ '--runAndLumis=\'%s\' '+ '--firstEvent=%s '+ '--lastEvent=%s '+ '--firstLumi=%s '+ '--firstRun=%s '+ '--seeding=%s '+ '--lheInputFiles=%s '+ '--oneEventMode=%s ' + '--eventsPerLumi=%s') % (os.getcwd(), os.getcwd(), opts.inputFile, opts.runAndLumis, opts.firstEvent, opts.lastEvent, opts.firstLumi, opts.firstRun, opts.seeding, opts.lheInputFiles, opts.oneEventMode, opts.eventsPerLumi)] cmssw.step.section_("execution") #exitStatus of cmsRun is set here cmssw.report = Report("cmsRun") #report is loaded and put here cmssw.execute() return cmssw
def testSubscriptionInformation(self): """ _testSubscriptionInformation_ Check the three methods related to the subscription information in a task Make sure that we can set the subscription information for all datasets produced by this task and we can select only some primaryDatasets/DataTiers, and check that we can update the dataset in a subscription information section. """ testTask = makeWMTask("TestTask") cmsswStep = testTask.makeStep("cmsRun1") cmsswStep.setStepType("CMSSW") testTask.applyTemplates() cmsswHelper = cmsswStep.getTypeHelper() cmsswHelper.addOutputModule("outputRECO", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="RECO") cmsswHelper.addOutputModule("outputDQM", primaryDataset="TwoParticles", processedDataset="DawnOfAnEra-v1", dataTier="DQM") cmsswHelper.addOutputModule("outputAOD", primaryDataset="OneParticle", processedDataset="DawnOfAnEra-v1", dataTier="AOD") self.assertEqual(testTask.getSubscriptionInformation(), {}, "There should not be any subscription info") testTask.setSubscriptionInformation(["mercury"], ["mars", "earth"], ["earth"], "High", "OneParticle") subInfo = testTask.getSubscriptionInformation() outputRecoSubInfo = { "CustodialSites": ["mercury"], "NonCustodialSites": ["mars", "earth"], "AutoApproveSites": ["earth"], "Priority": "High" } self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertFalse("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") testTask.setSubscriptionInformation(["jupiter"], primaryDataset="TwoParticles") subInfo = testTask.getSubscriptionInformation() self.assertEqual(subInfo["/OneParticle/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertTrue("/OneParticle/DawnOfAnEra-v1/AOD" in subInfo, "The AOD subscription information is wrong") self.assertTrue("/TwoParticles/DawnOfAnEra-v1/DQM" in subInfo, "The DQM subscription information is wrong") recoutOutputModule = cmsswHelper.getOutputModule("outputRECO") setattr(recoutOutputModule, "primaryDataset", "ThreeParticles") testTask.updateSubscriptionDataset("outputRECO", recoutOutputModule) subInfo = testTask.getSubscriptionInformation() self.assertEqual(subInfo["/ThreeParticles/DawnOfAnEra-v1/RECO"], outputRecoSubInfo, "The RECO subscription information is wrong") self.assertFalse("/OneParticle/DawnOfAnEra-v1/RECO" in subInfo, "The RECO subscription information is wrong")
def testInputDataset(self): """ _testInputDataset_ Verify that the addInputDataset() method works correctly and that the run/block black and white lists can be changed after calling addInputDataset(). """ testTask = makeWMTask("TestTask") assert testTask.getInputDatasetPath() is None, \ "Error: Input dataset path should be None." assert testTask.dbsUrl() is None, \ "Error: Input DBS URL should be None." assert testTask.inputBlockWhitelist() is None, \ "Error: Input block white list should be None." assert testTask.inputBlockBlacklist() is None, \ "Error: Input block black list should be None." assert testTask.inputRunWhitelist() is None, \ "Error: Input run white list should be None." assert testTask.inputRunBlacklist() is None, \ "Error: Input run black list should be None." testTask.addInputDataset(name="/PrimaryDataset/ProcessedDataset/DataTier", primary="PrimaryDataset", processed="ProcessedDataset", tier="DataTier", dbsurl="DBSURL", block_whitelist=["Block1", "Block2"], block_blacklist=["Block3", "Block4", "Block5"], run_whitelist=[1, 2, 3], run_blacklist=[4, 5]) assert testTask.getInputDatasetPath() == "/PrimaryDataset/ProcessedDataset/DataTier", \ "Error: Input dataset path is wrong." assert testTask.dbsUrl() == "DBSURL", \ "Error: Input DBS URL is wrong." assert len(testTask.inputBlockWhitelist()) == 2, \ "Error: Wrong number of blocks in white list." assert "Block1" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." assert "Block2" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." assert len(testTask.inputBlockBlacklist()) == 3, \ "Error: Wrong number of blocks in black list." assert "Block3" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block4" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block5" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert len(testTask.inputRunWhitelist()) == 3, \ "Error: Wrong number of runs in white list." assert 1 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert 2 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert 3 in testTask.inputRunWhitelist(), \ "Error: Run is missing from white list." assert len(testTask.inputRunBlacklist()) == 2, \ "Error: Wrong number of runs in black list." assert 4 in testTask.inputRunBlacklist(), \ "Error: Run is missing from black list." assert 5 in testTask.inputRunBlacklist(), \ "Error: Run is missing from black list." testTask.setInputBlockWhitelist(["Block6"]) assert len(testTask.inputBlockWhitelist()) == 1, \ "Error: Wrong number of blocks in white list." assert "Block6" in testTask.inputBlockWhitelist(), \ "Error: Block missing from white list." testTask.setInputBlockBlacklist(["Block7", "Block8"]) assert len(testTask.inputBlockBlacklist()) == 2, \ "Error: Wrong number of blocks in black list." assert "Block7" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." assert "Block8" in testTask.inputBlockBlacklist(), \ "Error: Block missing from black list." testTask.setInputRunWhitelist([6]) assert len(testTask.inputRunWhitelist()) == 1, \ "Error: Wrong number of runs in white list." assert 6 in testTask.inputRunWhitelist(), \ "Error: Run missing from white list." testTask.setInputRunBlacklist([7, 8]) assert len(testTask.inputRunBlacklist()) == 2, \ "Error: Wrong number of runs in black list." assert 7 in testTask.inputRunBlacklist(), \ "Error: Run missing from black list." assert 8 in testTask.inputRunBlacklist(), \ "Error: Run missing from black list." return