예제 #1
0
    def test3StepsMemCoresSettingsB(self):
        """
        _test3StepsMemCoresSettingsB_

        Mix Multicore settings at both step and request level and make sure they
        are properly propagated to each step. Three steps in the task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        configDocs = injectStepChainConfigMC(self.configDatabase)
        testArguments.update(getThreeStepsOverride())
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
        testArguments['Step2']['KeepOutput'] = False

        # Test Multicore/Memory settings at step level **only**
        testArguments['Step1']["Multicore"] = 2
        testArguments['Step3']["Multicore"] = 4
        testArguments['Step1']["Memory"] = 2200.0
        testArguments['Step3']["Memory"] = 4400.0
        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun2', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
        self.assertEqual(prodTask.getStepHelper('cmsRun1').getNumberOfCores(), testArguments['Step1']["Multicore"])
        self.assertEqual(prodTask.getStepHelper('cmsRun3').getNumberOfCores(), testArguments['Step3']["Multicore"])

        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])

        # Test mix of Multicore/Memory settings at both top and step level
        testArguments["Multicore"] = 3
        testArguments['Step1']["Multicore"] = 2
        testArguments['Step3']["Multicore"] = 4
        testArguments["Memory"] = 3300.0
        testArguments['Step1']["Memory"] = 2200.0
        testArguments['Step3']["Memory"] = 4400.0
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
        self.assertEqual(prodTask.getStepHelper('cmsRun1').getNumberOfCores(), testArguments['Step1']["Multicore"])
        self.assertEqual(prodTask.getStepHelper('cmsRun2').getNumberOfCores(), testArguments["Multicore"])
        self.assertEqual(prodTask.getStepHelper('cmsRun3').getNumberOfCores(), testArguments['Step3']["Multicore"])

        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])

        return
예제 #2
0
    def test1StepMemCoresSettings(self):
        """
        _test1StepMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Single step in a task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(getSingleStepOverride())
        testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
            self.configDatabase),
        if isinstance(testArguments['Step1']['ConfigCacheID'], tuple):
            testArguments['Step1']['ConfigCacheID'] = testArguments['Step1'][
                'ConfigCacheID'][0]

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
            self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        testArguments["Multicore"] = 6
        testArguments["Memory"] = 4600.0
        testArguments["EventStreams"] = 3
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            if step in ('stageOut1', 'logArch1'):
                self.assertEqual(stepHelper.getNumberOfCores(), 1,
                                 "%s should have 1 core" % step)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)
            else:
                self.assertEqual(stepHelper.getNumberOfCores(),
                                 testArguments["Multicore"])
                self.assertEqual(stepHelper.getNumberOfStreams(),
                                 testArguments["EventStreams"])
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'],
                         testArguments["Memory"])

        return
예제 #3
0
    def test3StepsMemCoresSettingsA(self):
        """
        _test3StepsMemCoresSettingsA_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Three steps in the task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        configDocs = injectStepChainConfigMC(self.configDatabase)
        testArguments.update(getThreeStepsOverride())
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
        testArguments['Step2']['KeepOutput'] = False

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
            self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        # Test Multicore/Memory settings at TOP level **only**
        testArguments["Multicore"] = 6
        testArguments["Memory"] = 4600.0
        testArguments["EventStreams"] = 3
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            if step in ('stageOut1', 'logArch1'):
                self.assertEqual(stepHelper.getNumberOfCores(), 1,
                                 "%s should have 1 core" % step)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)
            else:
                self.assertEqual(stepHelper.getNumberOfCores(),
                                 testArguments["Multicore"])
                self.assertEqual(stepHelper.getNumberOfStreams(),
                                 testArguments["EventStreams"])
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'],
                         testArguments["Memory"])
예제 #4
0
    def test3StepsMemCoresSettingsA(self):
        """
        _test3StepsMemCoresSettingsA_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Three steps in the task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        configDocs = injectStepChainConfigMC(self.configDatabase)
        testArguments.update(getThreeStepsOverride())
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
        testArguments['Step2']['KeepOutput'] = False

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
        # then test Memory requirements
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        # Test Multicore/Memory settings at TOP level **only**
        testArguments["Multicore"] = 6
        testArguments["Memory"] = 4600.0
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'cmsRun2', 'cmsRun3', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            if step in ('stageOut1', 'logArch1'):
                self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
            else:
                self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"])
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])
예제 #5
0
    def test1StepMemCoresSettings(self):
        """
        _test1StepMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Single step in a task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(getSingleStepOverride())
        testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(self.configDatabase),
        if isinstance(testArguments['Step1']['ConfigCacheID'], tuple):
            testArguments['Step1']['ConfigCacheID'] = testArguments['Step1']['ConfigCacheID'][0]

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
        # then test Memory requirements
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        testArguments["Multicore"] = 6
        testArguments["Memory"] = 4600.0
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            if step in ('stageOut1', 'logArch1'):
                self.assertEqual(stepHelper.getNumberOfCores(), 1, "%s should have 1 core" % step)
            else:
                self.assertEqual(stepHelper.getNumberOfCores(), testArguments["Multicore"])
        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], testArguments["Memory"])

        return
예제 #6
0
    def testStepChainSingleStep(self):
        """
        Build a StepChain single step, reading AODSIM and producing MINIAODSIM
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        request = {
            "Campaign": "TaskForceUnitTest",
            "CMSSWVersion": "CMSSW_7_5_0",
            "ScramArch": "slc6_amd64_gcc491",
            "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
            "GlobalTag": "PHYS14_25_V3",
            "AcquisitionEra": "SingleStep",
            "ProcessingString": "UnitTest_StepChain",
            "ProcessingVersion": 3,
            "PrepID": "MainStep",
            "CouchURL": os.environ["COUCHURL"],
            "CouchDBName": "stepchain_t",
            "Memory": 3500,
            "SizePerEvent": 2600,
            "TimePerEvent": 26.5,
            "Step1": {
                "ConfigCacheID":
                injectStepChainConfigSingle(self.configDatabase),
                "GlobalTag": "PHYS14_25_V44",
                "InputDataset":
                "/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM",
                "SplittingAlgo": "EventAwareLumiBased",
                "EventsPerJob": 500,
                "StepName": "StepMini"
            },
            "StepChain": 1
        }
        testArguments.update(request)

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "StepMini",
                                    "GravWhatever",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "processing")
        self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
        self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep")
        self.assertEqual(testWorkload.getProcessingString(),
                         "UnitTest_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 3)
        self.assertEqual(testWorkload.getPrepID(), "MainStep")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
                         ['CMSSW_7_5_0'])
        self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 4)
        self.assertTrue('StepMiniMergeMINIAODSIMoutput' in tasks)

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.taskType(), "Processing", "Wrong task type")
        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased",
                         "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 500)
        self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4)
        self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599)
        self.assertTrue(
            splitParams['performance']['memoryRequirement'] == 3500)
        # test workload step stuff
        self.assertEqual(sorted(task.listAllStepNames()),
                         ['cmsRun1', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()[0]
        self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
        self.assertEqual(
            outModsAndDsets['outputDataset'],
            '/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM'
        )
        self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0')
        self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491')
        step = task.getStep("cmsRun1")
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            'PHYS14_25_V44')

        return
예제 #7
0
    def testStepChainMC(self):
        """
        Build a StepChain workload starting from scratch
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "stepchain_t"
            })
        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "ProdMinBias", "MCFakeBlock", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "production")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11")
        self.assertEqual(testWorkload.getProcessingString(), "START70_V4")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "Step-00")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12'])

        # test workload attributes
        self.assertEqual(testWorkload.processingString, "START70_V4")
        self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11")
        self.assertEqual(testWorkload.processingVersion, 1)
        self.assertFalse(testWorkload.lumiList, "Wrong lumiList")
        self.assertEqual(testWorkload.data.policies.start.policyName, "MonteCarlo")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 8)
        for t in ['ProdMinBias', 'ProdMinBiasMergeAODSIMoutput', 'ProdMinBiasMergeRECOSIMoutput']:
            self.assertTrue(t in tasks, "Wrong task name")
        self.assertFalse('ProdMinBiasMergeRAWSIMoutput' in tasks, "Wrong task name")

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "ProdMinBias")
        self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias")
        self.assertEqual(task.taskType(), "Production", "Wrong task type")

        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventBased", "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 150)
        self.assertEqual(splitParams['events_per_lumi'], 50)
        self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
        self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75)
        self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233)
        self.assertTrue(splitParams['performance']['memoryRequirement'] == 2400)

        self.assertFalse(task.getTrustSitelists(), "Wrong input location flag")
        self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")

        # test workload step stuff
        self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()
        outMods = sorted([elem['outputModule'] for elem in outModsAndDsets])
        outDsets = sorted([elem['outputDataset'] for elem in outModsAndDsets])
        self.assertEqual(outMods, ['AODSIMoutput', 'RECOSIMoutput'], "Wrong output modules")
        self.assertTrue('/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM' in outDsets)
        self.assertTrue('/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO' in outDsets)
        self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12')
        self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481')

        step = task.getStep("cmsRun1")
        self.assertFalse(step.data.tree.parent)
        self.assertFalse(getattr(step.data.input, 'inputStepName', None))
        self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterA')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM')
        self.assertFalse(step.data.output.keep)
        self.assertEqual(sorted(step.data.tree.childNames), ['cmsRun2', 'logArch1', 'stageOut1'])
        self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre12')
        self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
        self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')

        step = task.getStep("cmsRun2")
        self.assertEqual(step.data.tree.parent, "cmsRun1")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
        self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName, 'FilterB')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier, 'GEN-SIM-RAW')
        self.assertFalse(step.data.output.keep)
        self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
        self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11')
        self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
        self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')

        step = task.getStep("cmsRun3")
        self.assertEqual(step.data.tree.parent, "cmsRun2")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
        self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName, 'FilterC')
        self.assertEqual(step.data.output.modules.AODSIMoutput.filterName, 'FilterD')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier, 'GEN-SIM-RECO')
        self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier, 'AODSIM')
        self.assertTrue(step.data.output.keep)
        self.assertFalse(step.data.tree.childNames)
        self.assertEqual(step.data.application.setup.cmsswVersion, 'CMSSW_7_0_0_pre11')
        self.assertEqual(step.data.application.setup.scramArch, 'slc5_amd64_gcc481')
        self.assertEqual(step.data.application.configuration.arguments.globalTag, 'START70_V4::All')

        return
예제 #8
0
    def testStepChainSingleStep(self):
        """
        Build a StepChain single step, reading AODSIM and producing MINIAODSIM
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        request = {
            "Campaign": "TaskForceUnitTest",
            "CMSSWVersion": "CMSSW_7_5_0",
            "ScramArch": "slc6_amd64_gcc491",
            "DbsUrl": "https://cmsweb.cern.ch/dbs/prod/global/DBSReader",
            "GlobalTag": "PHYS14_25_V3",
            "AcquisitionEra": "SingleStep",
            "ProcessingString": "UnitTest_StepChain",
            "ProcessingVersion": 3,
            "PrepID": "MainStep",
            "CouchURL": os.environ["COUCHURL"],
            "CouchDBName": "stepchain_t",
            "Memory": 3500,
            "SizePerEvent": 2600,
            "TimePerEvent": 26.5,
            "Step1": {
                "ConfigCacheID": injectStepChainConfigSingle(self.configDatabase),
                "GlobalTag": "PHYS14_25_V44",
                "InputDataset": "/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/RunIISpring15DR74-Asympt50ns_MCRUN2_74_V9A-v1/AODSIM",
                "SplittingAlgo": "EventAwareLumiBased",
                "EventsPerJob": 500,
                "StepName": "StepMini"},
            "StepChain": 1
            }
        testArguments.update(request)

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "StepMini", "GravWhatever", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "processing")
        self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
        self.assertEqual(testWorkload.getAcquisitionEra(), "SingleStep")
        self.assertEqual(testWorkload.getProcessingString(), "UnitTest_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 3)
        self.assertEqual(testWorkload.getPrepID(), "MainStep")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()), ['CMSSW_7_5_0'])
        self.assertEqual(testWorkload.data.policies.start.policyName, "Block")
        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 2)
        self.assertEqual(sorted(tasks), ['LogCollectForStepMini', 'StepMini'])

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.taskType(), "Processing", "Wrong task type")
        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased", "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 500)
        self.assertTrue(splitParams['performance']['timePerEvent'] > 26.4)
        self.assertTrue(splitParams['performance']['sizePerEvent'] > 2599)
        self.assertTrue(splitParams['performance']['memoryRequirement'] == 3500)
        # test workload step stuff
        self.assertEqual(sorted(task.listAllStepNames()), ['cmsRun1', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()[0]
        self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
        self.assertEqual(outModsAndDsets['outputDataset'], 
                         '/RSGravToGG_kMpl-01_M-5000_TuneCUEP8M1_13TeV-pythia8/SingleStep-UnitTest_StepChain-v3/MINIAODSIM')
        self.assertEqual(task.getSwVersion(), 'CMSSW_7_5_0')
        self.assertEqual(task.getScramArch(), 'slc6_amd64_gcc491')
        step = task.getStep("cmsRun1")
        self.assertEqual(step.data.application.configuration.arguments.globalTag, 'PHYS14_25_V44')

        return
예제 #9
0
    def testStepMapping(self):
        """
        Build a mapping of steps, input and output modules
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(deepcopy(REQUEST))
        testArguments['StepChain'] = 1
        testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
            self.configDatabase)
        testArguments.update({
            "CMSSWVersion": "CMSSW_8_0_17",
            "ScramArch": "slc6_amd64_gcc530",
            "StepChain": 4
        })

        # Create a new DIGI step in Step3 and shift Step3 to Step4
        testArguments['Step4'] = deepcopy(testArguments['Step3'])
        testArguments['Step3'] = {
            "GlobalTag": "GT-Step3",
            "InputFromOutputModule": "RAWSIMoutput",
            "InputStep": "GENSIM",
            "StepName": "DIGI2"
        }

        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3', 'Step4']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
            testArguments[s]['KeepOutput'] = False
        # these are the inverse...
        testArguments['Step3']['ConfigCacheID'] = configDocs['Step4']
        testArguments['Step4']['ConfigCacheID'] = configDocs['Step3']

        expectedTasks = set([
            'GENSIM', 'RECOMergeAODSIMoutput', 'RECOMergeRECOSIMoutput',
            'RECOAODSIMoutputMergeLogCollect',
            'RECORECOSIMoutputMergeLogCollect',
            'RECOCleanupUnmergedAODSIMoutput',
            'RECOCleanupUnmergedRECOSIMoutput'
        ])
        expectedSteps = set([
            'cmsRun1', 'cmsRun2', 'cmsRun3', 'cmsRun4', 'stageOut1', 'logArch1'
        ])

        factory = StepChainWorkloadFactory()
        # Raise exception because the last Step cannot have KeepOutput=False
        self.assertRaises(WMSpecFactoryException,
                          factory.factoryWorkloadConstruction, "TestWorkload",
                          testArguments)
        testArguments['Step4']['KeepOutput'] = True

        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)
        self.assertEqual(len(testWorkload.listAllTaskNames()),
                         len(expectedTasks))
        self.assertEqual(set(testWorkload.listAllTaskNames()), expectedTasks)
        task = testWorkload.getTask('GENSIM')
        self.assertEqual(set(task.listAllStepNames()), expectedSteps)

        step1 = task.getStep('cmsRun1')
        stepInputSection = step1.data.input.dictionary_()
        self.assertFalse('inputStepName' in stepInputSection)
        self.assertEqual(set(step1.data.output.modules.dictionary_().keys()),
                         set(['RAWSIMoutput']))
        self.assertEqual(
            step1.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
            'GEN-SIM')

        step2 = task.getStep('cmsRun2')
        stepInputSection = step2.data.input.dictionary_()
        self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
        self.assertTrue(set(stepInputSection['inputOutputModule']),
                        'RAWSIMoutput')
        self.assertEqual(set(step2.data.output.modules.dictionary_().keys()),
                         set(['RAWSIMoutput']))
        self.assertEqual(
            step2.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
            'GEN-SIM-RAW')

        step3 = task.getStep('cmsRun3')
        stepInputSection = step3.data.input.dictionary_()
        self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun1')
        self.assertTrue(set(stepInputSection['inputOutputModule']),
                        'RAWSIMoutput')
        self.assertEqual(set(step3.data.output.modules.dictionary_().keys()),
                         set(['RAWSIMoutput']))
        self.assertEqual(
            step3.data.output.modules.RAWSIMoutput.dictionary_()['dataTier'],
            'GEN-SIM-RAW')

        step4 = task.getStep('cmsRun4')
        stepInputSection = step4.data.input.dictionary_()
        self.assertTrue(set(stepInputSection['inputStepName']), 'cmsRun2')
        self.assertTrue(set(stepInputSection['inputOutputModule']),
                        'RAWSIMoutput')
        self.assertEqual(set(step4.data.output.modules.dictionary_().keys()),
                         set(['AODSIMoutput', 'RECOSIMoutput']))
        self.assertEqual(
            step4.data.output.modules.AODSIMoutput.dictionary_()['dataTier'],
            'AODSIM')
예제 #10
0
    def testStepChainOutput(self):
        """
        Build a StepChain workload defining different processed dataset name among the steps
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(deepcopy(REQUEST))

        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
            testArguments[s]['AcquisitionEra'] = 'AcqEra_' + s
            testArguments[s]['ProcessingString'] = 'ProcStr_' + s
        testArguments['Step2']['KeepOutput'] = False

        factory = StepChainWorkloadFactory()

        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        # test workload properties
        self.assertEqual(testWorkload.getAcquisitionEra(),
                         "AcquisitionEra_StepChain")
        self.assertEqual(testWorkload.getProcessingString(),
                         "ProcessingString_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 10)
        for t in [
                'GENSIM', 'GENSIMMergeRAWSIMoutput', 'RECOMergeAODSIMoutput',
                'RECOMergeRECOSIMoutput', 'GENSIMRAWSIMoutputMergeLogCollect',
                'RECOAODSIMoutputMergeLogCollect',
                'RECORECOSIMoutputMergeLogCollect'
        ]:
            self.assertTrue(t in tasks, "Wrong task name")
        self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
                         "Wrong task name")

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "GENSIM")
        self.assertEqual(task.getPathName(), "/TestWorkload/GENSIM")
        self.assertEqual(task.taskType(), "Production", "Wrong task type")

        # test workload and task output data stuff
        self.assertEqual(
            sorted(task.listAllStepNames()),
            ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
        outModsAndDsets = task.listOutputDatasetsAndModules()
        outMods = [elem['outputModule'] for elem in outModsAndDsets]
        outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
        self.assertItemsEqual(
            outMods, ['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput'],
            "Wrong output modules")
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcqEra_Step1-FilterA-ProcStr_Step1-v1/GEN-SIM'
            in outDsets)
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcqEra_Step3-FilterD-ProcStr_Step3-v1/AODSIM'
            in outDsets)
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcqEra_Step3-FilterC-ProcStr_Step3-v1/GEN-SIM-RECO'
            in outDsets)

        step = task.getStepHelper("cmsRun1")
        self.assertEqual(step.listOutputModules(), [u'RAWSIMoutput'])
        outputData = step.getOutputModule(u'RAWSIMoutput')
        self.assertEqual(outputData.dataTier, 'GEN-SIM')
        self.assertTrue(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step1-FilterA-ProcStr_Step1-v1')

        step = task.getStepHelper("cmsRun2")
        self.assertEqual(step.listOutputModules(), [u'RAWSIMoutput'])
        outputData = step.getOutputModule(u'RAWSIMoutput')
        self.assertEqual(outputData.dataTier, 'GEN-SIM-RAW')
        self.assertTrue(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step2-FilterB-ProcStr_Step2-v1')

        step = task.getStepHelper("cmsRun3")
        self.assertEqual(step.listOutputModules(),
                         ['AODSIMoutput', 'RECOSIMoutput'])
        outputData = step.getOutputModule('AODSIMoutput')
        self.assertEqual(outputData.dataTier, 'AODSIM')
        self.assertTrue(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step3-FilterD-ProcStr_Step3-v1')
        outputData = step.getOutputModule('RECOSIMoutput')
        self.assertEqual(outputData.dataTier, 'GEN-SIM-RECO')
        self.assertTrue(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/unmerged'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step3-FilterC-ProcStr_Step3-v1')

        # test merge tasks now
        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput')
        self.assertEqual(task.taskType(), "Merge")
        step = task.getStepHelper("cmsRun1")
        self.assertEqual(step.listOutputModules(), ['Merged'])
        outputData = step.getOutputModule('Merged')
        self.assertEqual(outputData.dataTier, 'GEN-SIM')
        self.assertFalse(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/data'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step1-FilterA-ProcStr_Step1-v1')

        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/RECOMergeAODSIMoutput')
        self.assertEqual(task.taskType(), "Merge")
        step = task.getStepHelper("cmsRun1")
        self.assertEqual(step.listOutputModules(), ['Merged'])
        outputData = step.getOutputModule('Merged')
        self.assertEqual(outputData.dataTier, 'AODSIM')
        self.assertFalse(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/data'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step3-FilterD-ProcStr_Step3-v1')

        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/RECOMergeRECOSIMoutput')
        self.assertEqual(task.taskType(), "Merge")
        step = task.getStepHelper("cmsRun1")
        self.assertEqual(step.listOutputModules(), ['Merged'])
        outputData = step.getOutputModule('Merged')
        self.assertEqual(outputData.dataTier, 'GEN-SIM-RECO')
        self.assertFalse(outputData.transient)
        self.assertTrue(outputData.lfnBase.startswith('/store/data'))
        self.assertEqual(outputData.processedDataset,
                         'AcqEra_Step3-FilterC-ProcStr_Step3-v1')

        return
예제 #11
0
    def testStepChainReDigi(self):
        """
        Build a StepChain workload with input dataset in the first step
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(deepcopy(REQUEST))

        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]

        testArguments['Step1'] = deepcopy(testArguments.pop('Step2'))
        testArguments['Step2'] = deepcopy(testArguments.pop('Step3'))
        testArguments['StepChain'] = 2

        testArguments['Step1'].pop('InputFromOutputModule')
        testArguments['Step1'].pop('InputStep')
        testArguments['Step1'].update({
            'KeepOutput':
            False,
            'InputDataset':
            '/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM',
            'BlockWhitelist': [
                "/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a09df90-5593-11e4-bd05-003048f0e3f4",
                "/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a8b15e6-54e0-11e4-afc7-003048f0e3f4"
            ]
        })
        factory = StepChainWorkloadFactory()

        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "DIGI",
                                    "Block",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "processing")
        self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
        self.assertEqual(testWorkload.getAcquisitionEra(),
                         "AcquisitionEra_StepChain")
        self.assertEqual(testWorkload.getProcessingString(),
                         "ProcessingString_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
                         ['CMSSW_8_0_21'])
        self.assertFalse(testWorkload.getLumiList(), "Wrong lumiList")
        self.assertEqual(testWorkload.data.policies.start.policyName, "Block")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 7)
        for t in [
                'DIGI', 'RECOMergeAODSIMoutput', 'RECOMergeRECOSIMoutput',
                'RECOAODSIMoutputMergeLogCollect',
                'RECORECOSIMoutputMergeLogCollect',
                'RECOCleanupUnmergedAODSIMoutput',
                'RECOCleanupUnmergedRECOSIMoutput'
        ]:
            self.assertTrue(t in tasks, "Wrong task name")
        self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
                         "Wrong task name")

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "DIGI")
        self.assertEqual(task.getPathName(), "/TestWorkload/DIGI")
        self.assertEqual(task.taskType(), "Processing", "Wrong task type")

        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventAwareLumiBased",
                         "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 200)
        self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
        self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
        self.assertTrue(
            splitParams['performance']['memoryRequirement'] == 3500)

        self.assertEqual(
            task.getInputDatasetPath(),
            '/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM'
        )
        self.assertTrue(
            '/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a09df90-5593-11e4-bd05-003048f0e3f4'
            in task.inputBlockWhitelist())
        self.assertTrue(
            '/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/Fall13-POSTLS162_V1-v1/GEN-SIM#3a8b15e6-54e0-11e4-afc7-003048f0e3f4'
            in task.inputBlockWhitelist())

        # test workload step stuff
        self.assertEqual(sorted(task.listAllStepNames()),
                         ['cmsRun1', 'cmsRun2', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()
        outMods = set([elem['outputModule'] for elem in outModsAndDsets])
        outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
        self.assertEqual(outMods, set(['AODSIMoutput', 'RECOSIMoutput']),
                         "Wrong output modules")
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterD-ProcessingString_StepChain-v1/AODSIM'
            in outDsets)
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterC-ProcessingString_StepChain-v1/GEN-SIM-RECO'
            in outDsets)
        return
예제 #12
0
    def testStepChainMC(self):
        """
        Build a StepChain workload starting from scratch
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(deepcopy(REQUEST))

        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
            testArguments[s]['KeepOutput'] = True

        factory = StepChainWorkloadFactory()

        # test that we cannot stage out different samples with the same output module
        self.assertRaises(WMSpecFactoryException,
                          factory.factoryWorkloadConstruction, "TestWorkload",
                          testArguments)

        testArguments['Step2']['KeepOutput'] = False
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "GENSIM",
                                    "MCFakeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "production")
        self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
        self.assertEqual(testWorkload.getAcquisitionEra(),
                         "AcquisitionEra_StepChain")
        self.assertEqual(testWorkload.getProcessingString(),
                         "ProcessingString_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
        self.assertItemsEqual(testWorkload.getCMSSWVersions(),
                              ['CMSSW_7_1_25_patch2', 'CMSSW_8_0_21'])
        self.assertFalse(testWorkload.getLumiList(), "Wrong lumiList")
        self.assertEqual(testWorkload.data.policies.start.policyName,
                         "MonteCarlo")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 10)
        for t in [
                'GENSIM', 'GENSIMMergeRAWSIMoutput', 'RECOMergeAODSIMoutput',
                'RECOMergeRECOSIMoutput', 'GENSIMRAWSIMoutputMergeLogCollect',
                'RECOAODSIMoutputMergeLogCollect',
                'RECORECOSIMoutputMergeLogCollect'
        ]:
            self.assertTrue(t in tasks, "Wrong task name")
        self.assertFalse('ProdMinBiasMergeAODSIMoutput' in tasks,
                         "Wrong task name")

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "GENSIM")
        self.assertEqual(task.getPathName(), "/TestWorkload/GENSIM")
        self.assertEqual(task.taskType(), "Production", "Wrong task type")

        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventBased",
                         "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 200)
        self.assertEqual(splitParams['events_per_lumi'], 100)
        self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
        self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
        self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
        self.assertTrue(
            splitParams['performance']['memoryRequirement'] == 3500)

        self.assertFalse(task.getTrustSitelists().get('trustlists'),
                         "Wrong input location flag")
        self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")

        # test workload step stuff
        self.assertEqual(
            sorted(task.listAllStepNames()),
            ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()
        outMods = set([elem['outputModule'] for elem in outModsAndDsets])
        outDsets = [elem['outputDataset'] for elem in outModsAndDsets]
        self.assertEqual(
            outMods, set(['RAWSIMoutput', 'AODSIMoutput', 'RECOSIMoutput']),
            "Wrong output modules")
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterA-ProcessingString_StepChain-v1/GEN-SIM'
            in outDsets)
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterD-ProcessingString_StepChain-v1/AODSIM'
            in outDsets)
        self.assertTrue(
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-FilterC-ProcessingString_StepChain-v1/GEN-SIM-RECO'
            in outDsets)
        self.assertEqual(task.getSwVersion(),
                         testArguments['Step1']["CMSSWVersion"])
        self.assertEqual(task.getScramArch(),
                         testArguments['Step1']["ScramArch"])

        step = task.getStep("cmsRun1")
        self.assertFalse(step.data.tree.parent)
        self.assertFalse(getattr(step.data.input, 'inputStepName', None))
        self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
                         'FilterA')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
                         'GEN-SIM')
        self.assertTrue(step.data.output.keep)
        self.assertEqual(sorted(step.data.tree.childNames),
                         ['cmsRun2', 'logArch1', 'stageOut1'])
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         testArguments['Step1']["CMSSWVersion"])
        self.assertEqual(step.data.application.setup.scramArch,
                         testArguments['Step1']["ScramArch"])
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            testArguments['Step1']["GlobalTag"])

        step = task.getStep("cmsRun2")
        self.assertEqual(step.data.tree.parent, "cmsRun1")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
        self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
                         'FilterB')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
                         'GEN-SIM-RAW')
        self.assertFalse(step.data.output.keep)
        self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         testArguments['Step2']["CMSSWVersion"])
        self.assertEqual(step.data.application.setup.scramArch,
                         testArguments['Step2']["ScramArch"])
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            testArguments['Step2']["GlobalTag"])

        step = task.getStep("cmsRun3")
        self.assertEqual(step.data.tree.parent, "cmsRun2")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
        self.assertEqual(step.data.input.inputOutputModule, 'PREMIXRAWoutput')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName,
                         'FilterC')
        self.assertEqual(step.data.output.modules.AODSIMoutput.filterName,
                         'FilterD')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier,
                         'GEN-SIM-RECO')
        self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier,
                         'AODSIM')
        self.assertTrue(step.data.output.keep)
        self.assertFalse(step.data.tree.childNames)
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         testArguments['Step3']["CMSSWVersion"])
        self.assertEqual(step.data.application.setup.scramArch,
                         testArguments['Step3']["ScramArch"])
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            testArguments['Step3']["GlobalTag"])

        # test merge stuff
        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput')
        self.assertEqual(task.taskType(), "Merge")
        self.assertEqual(task.getSwVersion(),
                         testArguments['Step1']["CMSSWVersion"])
        self.assertEqual(task.getScramArch(),
                         testArguments['Step1']["ScramArch"])

        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/RECOMergeAODSIMoutput')
        self.assertEqual(task.taskType(), "Merge")
        self.assertEqual(task.getSwVersion(),
                         testArguments['Step3']["CMSSWVersion"])
        self.assertEqual(task.getScramArch(),
                         testArguments['Step3']["ScramArch"])

        # test logCollect stuff
        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/GENSIMMergeRAWSIMoutput/GENSIMRAWSIMoutputMergeLogCollect'
        )
        self.assertEqual(task.taskType(), "LogCollect")
        step = task.getStep("logCollect1")
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         testArguments['Step1']["CMSSWVersion"])
        self.assertEqual(step.data.application.setup.scramArch,
                         testArguments['Step1']["ScramArch"])

        task = testWorkload.getTaskByPath(
            '/TestWorkload/GENSIM/RECOMergeAODSIMoutput/RECOAODSIMoutputMergeLogCollect'
        )
        self.assertEqual(task.taskType(), "LogCollect")
        step = task.getStep("logCollect1")
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         testArguments['Step3']["CMSSWVersion"])
        self.assertEqual(step.data.application.setup.scramArch,
                         testArguments['Step3']["ScramArch"])

        return
예제 #13
0
    def testStepChainSingleStep(self):
        """
        Build a StepChain with a single step and no input dataset
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        testArguments.update(deepcopy(REQUEST))
        testArguments.pop("Step2")
        testArguments.pop("Step3")
        testArguments['StepChain'] = 1
        testArguments['Step1']['ConfigCacheID'] = injectStepChainConfigSingle(
            self.configDatabase)
        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "GENSIM",
                                    "GravWhatever",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "production")
        self.assertEqual(testWorkload.getCampaign(), "TaskForceUnitTest")
        self.assertEqual(testWorkload.getAcquisitionEra(),
                         "AcquisitionEra_StepChain")
        self.assertEqual(testWorkload.getProcessingString(),
                         "ProcessingString_StepChain")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "PREP-StepChain")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
                         ['CMSSW_7_1_25_patch2'])
        self.assertEqual(testWorkload.data.policies.start.policyName,
                         "MonteCarlo")
        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 4)
        self.assertTrue('GENSIMMergeMINIAODSIMoutput' in tasks)

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.taskType(), "Production", "Wrong task type")
        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventBased",
                         "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 200)
        self.assertEqual(splitParams['events_per_lumi'], 100)
        self.assertTrue(splitParams['performance']['timePerEvent'] >= 144)
        self.assertTrue(splitParams['performance']['sizePerEvent'] >= 512)
        self.assertTrue(
            splitParams['performance']['memoryRequirement'] == 3500)
        # test workload step stuff
        self.assertEqual(sorted(task.listAllStepNames()),
                         ['cmsRun1', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()[0]
        self.assertEqual(outModsAndDsets['outputModule'], 'MINIAODSIMoutput')
        self.assertEqual(
            outModsAndDsets['outputDataset'],
            '/PrimaryDataset-StepChain/AcquisitionEra_StepChain-ProcessingString_StepChain-v1/MINIAODSIM'
        )
        self.assertEqual(task.getSwVersion(), 'CMSSW_7_1_25_patch2')
        self.assertItemsEqual(task.getScramArch(), ['slc6_amd64_gcc481'])
        step = task.getStep("cmsRun1")
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            'GT-Step1')

        return
예제 #14
0
    def test3StepsMemCoresSettingsB(self):
        """
        _test3StepsMemCoresSettingsB_

        Mix Multicore settings at both step and request level and make sure they
        are properly propagated to each step. Three steps in the task.
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        configDocs = injectStepChainConfigMC(self.configDatabase)
        testArguments.update(getThreeStepsOverride())
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]
        testArguments['Step2']['KeepOutput'] = False

        # Test Multicore/Memory settings at step level **only**
        testArguments['Step1']["Multicore"] = 2
        testArguments['Step3']["Multicore"] = 4
        testArguments['Step1']["Memory"] = 2200.0
        testArguments['Step3']["Memory"] = 4400.0
        testArguments['Step1']["EventStreams"] = 2
        testArguments['Step3']["EventStreams"] = 4
        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        prodTask = testWorkload.getTask('StepOne')
        for step in ('cmsRun2', 'stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1,
                             "%s should have 1 core" % step)
        self.assertEqual(
            prodTask.getStepHelper('cmsRun1').getNumberOfCores(),
            testArguments['Step1']["Multicore"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun3').getNumberOfCores(),
            testArguments['Step3']["Multicore"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun1').getNumberOfStreams(),
            testArguments['Step1']["EventStreams"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun3').getNumberOfStreams(),
            testArguments['Step3']["EventStreams"])

        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'],
                         testArguments["Memory"])

        # Test mix of Multicore/Memory settings at both top and step level
        testArguments["Multicore"] = 3
        testArguments['Step1']["Multicore"] = 2
        testArguments['Step3']["Multicore"] = 4
        testArguments["Memory"] = 3300.0
        testArguments['Step1']["Memory"] = 2200.0
        testArguments['Step3']["Memory"] = 4400.0
        testArguments["EventStreams"] = 6
        testArguments['Step1']["EventStreams"] = 4
        testArguments['Step3']["EventStreams"] = 8
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload2", testArguments)
        prodTask = testWorkload.getTask('StepOne')
        for step in ('stageOut1', 'logArch1'):
            stepHelper = prodTask.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1,
                             "%s should have 1 core" % step)
            self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        self.assertEqual(
            prodTask.getStepHelper('cmsRun1').getNumberOfCores(),
            testArguments['Step1']["Multicore"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun2').getNumberOfCores(),
            testArguments["Multicore"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun3').getNumberOfCores(),
            testArguments['Step3']["Multicore"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun1').getNumberOfStreams(),
            testArguments['Step1']["EventStreams"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun2').getNumberOfStreams(),
            testArguments["EventStreams"])
        self.assertEqual(
            prodTask.getStepHelper('cmsRun3').getNumberOfStreams(),
            testArguments['Step3']["EventStreams"])

        perfParams = prodTask.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'],
                         testArguments["Memory"])

        return
예제 #15
0
    def testStepChainMC(self):
        """
        Build a StepChain workload starting from scratch
        """
        testArguments = StepChainWorkloadFactory.getTestArguments()
        # Read in the request
        request = json.load(open(self.jsonTemplate))
        testArguments.update(request['createRequest'])
        testArguments.update({
            "CouchURL": os.environ["COUCHURL"],
            "ConfigCacheUrl": os.environ["COUCHURL"],
            "CouchDBName": "stepchain_t"
        })
        configDocs = injectStepChainConfigMC(self.configDatabase)
        for s in ['Step1', 'Step2', 'Step3']:
            testArguments[s]['ConfigCacheID'] = configDocs[s]

        factory = StepChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", testArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "ProdMinBias",
                                    "MCFakeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        # test workload properties
        self.assertEqual(testWorkload.getDashboardActivity(), "production")
        self.assertEqual(testWorkload.getCampaign(), "Campaign-OVERRIDE-ME")
        self.assertEqual(testWorkload.getAcquisitionEra(), "CMSSW_7_0_0_pre11")
        self.assertEqual(testWorkload.getProcessingString(), "START70_V4")
        self.assertEqual(testWorkload.getProcessingVersion(), 1)
        self.assertEqual(testWorkload.getPrepID(), "Step-00")
        self.assertEqual(sorted(testWorkload.getCMSSWVersions()),
                         ['CMSSW_7_0_0_pre11', 'CMSSW_7_0_0_pre12'])

        # test workload attributes
        self.assertEqual(testWorkload.processingString, "START70_V4")
        self.assertEqual(testWorkload.acquisitionEra, "CMSSW_7_0_0_pre11")
        self.assertEqual(testWorkload.processingVersion, 1)
        self.assertFalse(testWorkload.lumiList, "Wrong lumiList")
        self.assertEqual(testWorkload.data.policies.start.policyName,
                         "MonteCarlo")

        # test workload tasks and steps
        tasks = testWorkload.listAllTaskNames()
        self.assertEqual(len(tasks), 8)
        for t in [
                'ProdMinBias', 'ProdMinBiasMergeAODSIMoutput',
                'ProdMinBiasMergeRECOSIMoutput'
        ]:
            self.assertTrue(t in tasks, "Wrong task name")
        self.assertFalse('ProdMinBiasMergeRAWSIMoutput' in tasks,
                         "Wrong task name")

        task = testWorkload.getTask(tasks[0])
        self.assertEqual(task.name(), "ProdMinBias")
        self.assertEqual(task.getPathName(), "/TestWorkload/ProdMinBias")
        self.assertEqual(task.taskType(), "Production", "Wrong task type")

        splitParams = task.jobSplittingParameters()
        self.assertEqual(splitParams['algorithm'], "EventBased",
                         "Wrong job splitting algo")
        self.assertEqual(splitParams['events_per_job'], 150)
        self.assertEqual(splitParams['events_per_lumi'], 50)
        self.assertFalse(splitParams['lheInputFiles'], "Wrong LHE flag")
        self.assertTrue(splitParams['performance']['timePerEvent'] > 4.75)
        self.assertTrue(splitParams['performance']['sizePerEvent'] > 1233)
        self.assertTrue(
            splitParams['performance']['memoryRequirement'] == 2400)

        self.assertFalse(task.getTrustSitelists(), "Wrong input location flag")
        self.assertFalse(task.inputRunWhitelist(), "Wrong run white list")

        # test workload step stuff
        self.assertEqual(
            sorted(task.listAllStepNames()),
            ['cmsRun1', 'cmsRun2', 'cmsRun3', 'logArch1', 'stageOut1'])
        self.assertEqual(task.getTopStepName(), 'cmsRun1')
        self.assertEqual(task.getStep("cmsRun1").stepType(), "CMSSW")
        self.assertFalse(task.getInputStep(), "Wrong input step")
        outModsAndDsets = task.listOutputDatasetsAndModules()
        outMods = sorted([elem['outputModule'] for elem in outModsAndDsets])
        outDsets = sorted([elem['outputDataset'] for elem in outModsAndDsets])
        self.assertEqual(outMods, ['AODSIMoutput', 'RECOSIMoutput'],
                         "Wrong output modules")
        self.assertTrue(
            '/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterD-START70_V4-v1/AODSIM'
            in outDsets)
        self.assertTrue(
            '/RelValProdMinBias/CMSSW_7_0_0_pre11-FilterC-START70_V4-v1/GEN-SIM-RECO'
            in outDsets)
        self.assertEqual(task.getSwVersion(), 'CMSSW_7_0_0_pre12')
        self.assertEqual(task.getScramArch(), 'slc5_amd64_gcc481')

        step = task.getStep("cmsRun1")
        self.assertFalse(step.data.tree.parent)
        self.assertFalse(getattr(step.data.input, 'inputStepName', None))
        self.assertFalse(getattr(step.data.input, 'inputOutputModule', None))
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
                         'FilterA')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
                         'GEN-SIM')
        self.assertFalse(step.data.output.keep)
        self.assertEqual(sorted(step.data.tree.childNames),
                         ['cmsRun2', 'logArch1', 'stageOut1'])
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         'CMSSW_7_0_0_pre12')
        self.assertEqual(step.data.application.setup.scramArch,
                         'slc5_amd64_gcc481')
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            'START70_V4::All')

        step = task.getStep("cmsRun2")
        self.assertEqual(step.data.tree.parent, "cmsRun1")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun1')
        self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.filterName,
                         'FilterB')
        self.assertEqual(step.data.output.modules.RAWSIMoutput.dataTier,
                         'GEN-SIM-RAW')
        self.assertFalse(step.data.output.keep)
        self.assertEqual(step.data.tree.childNames, ["cmsRun3"])
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         'CMSSW_7_0_0_pre11')
        self.assertEqual(step.data.application.setup.scramArch,
                         'slc5_amd64_gcc481')
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            'START70_V4::All')

        step = task.getStep("cmsRun3")
        self.assertEqual(step.data.tree.parent, "cmsRun2")
        self.assertEqual(step.data.input.inputStepName, 'cmsRun2')
        self.assertEqual(step.data.input.inputOutputModule, 'RAWSIMoutput')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.filterName,
                         'FilterC')
        self.assertEqual(step.data.output.modules.AODSIMoutput.filterName,
                         'FilterD')
        self.assertEqual(step.data.output.modules.RECOSIMoutput.dataTier,
                         'GEN-SIM-RECO')
        self.assertEqual(step.data.output.modules.AODSIMoutput.dataTier,
                         'AODSIM')
        self.assertTrue(step.data.output.keep)
        self.assertFalse(step.data.tree.childNames)
        self.assertEqual(step.data.application.setup.cmsswVersion,
                         'CMSSW_7_0_0_pre11')
        self.assertEqual(step.data.application.setup.scramArch,
                         'slc5_amd64_gcc481')
        self.assertEqual(
            step.data.application.configuration.arguments.globalTag,
            'START70_V4::All')

        return