Пример #1
0
    def testMCWithLHE(self):
        """
        _testMCWithLHE_

        Create a MonteCarlo workflow with a variation on the type of work
        done, this refers to the previous LHEStepZero where the input
        can be .lhe files and there is more than one lumi per job.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
        defaultArguments["LheInputFiles"] = "True"
        defaultArguments["EventsPerJob"] = 200
        defaultArguments["EventsPerLumi"] = 50

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()

        productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
        splitting = productionTask.jobSplittingParameters()
        self.assertEqual(splitting["events_per_job"], 200)
        self.assertEqual(splitting["events_per_lumi"], 50)
        self.assertEqual(splitting["lheInputFiles"], True)
        self.assertFalse(splitting["deterministicPileup"])

        return
Пример #2
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
        pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
        pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)

        # now that the workload was created and args validated, we can add this PileupConfig
        pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"])

        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
Пример #3
0
    def testInvalidKeys_initialize_clone(self):
        """
        Test some undefined/assignment user arguments in initialize_clone
        """
        # test with unknown key
        requestArgs = {'Multicore': 1, 'WRONG': 'Alan'}
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # test with assignment key
        requestArgs = {'Multicore': 1, 'TrustSitelists': False}
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
        requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}}
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # make sure unsupported keys are remove from both top and inner dicts
        requestArgs = {}
        originalArgs = deepcopy(taskChainOriginalArgs)
        originalArgs['BAD'] = 'KEY'
        originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here']
        originalArgs['Task1']['Alan'] = False
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        self.assertFalse('BAD' in cloneArgs)
        self.assertFalse('SiteWhitelist' in cloneArgs['Task1'])
        self.assertFalse('Alan' in cloneArgs['Task1'])
Пример #4
0
    def testMonteCarlo(self):
        """
        _testMonteCarlo_

        Create a Monte Carlo workflow and verify that it is injected correctly
        into WMBS and invoke its detailed test.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "Production",
                                    "SomeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()

        return
Пример #5
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
        pileupMcArgs["PileupConfig"] = {"cosmics": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"],
                                        "minbias": ["/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"]}
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)

        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
Пример #6
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
        pileupMcArgs["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
        pileupMcArgs["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", pileupMcArgs)

        # now that the workload was created and args validated, we can add this PileupConfig
        pileupMcArgs["PileupConfig"] = parsePileupConfig(
            pileupMcArgs["MCPileup"], pileupMcArgs["DataPileup"])

        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
Пример #7
0
    def testMCWithPileup(self):
        """
        _testMCWithPileup_

        Create a Monte Carlo workflow and verify that it is injected correctly
        into WMBS and invoke its detailed test.
        The input configuration includes pileup input files.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        # Add pileup inputs
        defaultArguments["MCPileup"] = COSMICS_PU
        defaultArguments["DataPileup"] = DATA_PU
        defaultArguments["DeterministicPileup"] = True

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()
        productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
        cmsRunStep = productionTask.getStep("cmsRun1").getTypeHelper()
        pileupData = cmsRunStep.getPileup()
        self.assertEqual(pileupData.data.dataset, [DATA_PU])
        self.assertEqual(pileupData.mc.dataset, [COSMICS_PU])

        splitting = productionTask.jobSplittingParameters()
        self.assertTrue(splitting["deterministicPileup"])
        return
Пример #8
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = MonteCarloWorkloadFactory.getTestArguments()
        pileupMcArgs["PileupConfig"] = {
            "cosmics": [
                "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
            ],
            "minbias": [
                "/Mu/PenguinsPenguinsEverywhere-SingleMu-HorriblyJaundicedYellowEyedPenginsSearchingForCarrots-v31/RECO"
            ]
        }
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["ConfigCacheID"] = self.injectGenerationConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", pileupMcArgs)

        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
Пример #9
0
    def validateSchema(self, schema):
        """
        _validateSchema_

        Check for required fields, and some skim facts
        """
        MonteCarloWorkloadFactory.validateSchema(self, schema)
        #Now let's validate new stuff
        self.validateEventsPerLumi(schema)
Пример #10
0
    def testMonteCarloExtension(self):
        """
        _testMonteCarloExtension_

        Create a Monte Carlo workflow and verify that it is injected correctly
        into WMBS and invoke its detailed test. This uses a non-zero first
        lumi. Check that the splitting arguments are correctly
        set for the lfn counter.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
        defaultArguments["FirstLumi"] = 10001
        defaultArguments["EventsPerJob"] = 100
        defaultArguments["FirstEvent"] = 10001
        #defaultArguments["FirstEvent"] = 10001

        initial_lfn_counter = 100  # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload,
                                    "Production",
                                    "SomeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()

        productionTask = testWorkload.getTaskByPath('/TestWorkload/Production')
        productionSplitting = productionTask.jobSplittingParameters()
        self.assertTrue("initial_lfn_counter" in productionSplitting,
                        "No initial lfn counter was stored")
        self.assertEqual(productionSplitting["initial_lfn_counter"],
                         initial_lfn_counter, "Wrong initial LFN counter")

        for outputMod in ["OutputA", "OutputB"]:
            mergeTask = testWorkload.getTaskByPath(
                '/TestWorkload/Production/ProductionMerge%s' % outputMod)
            mergeSplitting = mergeTask.jobSplittingParameters()
            self.assertTrue("initial_lfn_counter" in mergeSplitting,
                            "No initial lfn counter was stored")
            self.assertEqual(mergeSplitting["initial_lfn_counter"],
                             initial_lfn_counter, "Wrong initial LFN counter")

        return
Пример #11
0
    def testMemCoresSettings(self):
        """
        _testMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all tasks and steps.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", defaultArguments)

        # test default values
        taskObj = testWorkload.getTask('Production')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = taskObj.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
            self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = taskObj.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        # now test case where args are provided
        defaultArguments["Multicore"] = 6
        defaultArguments["Memory"] = 4600.0
        defaultArguments["EventStreams"] = 3
        testWorkload = factory.factoryWorkloadConstruction(
            "TestWorkload", defaultArguments)
        taskObj = testWorkload.getTask('Production')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = taskObj.getStepHelper(step)
            if step == 'cmsRun1':
                self.assertEqual(stepHelper.getNumberOfCores(),
                                 defaultArguments["Multicore"])
                self.assertEqual(stepHelper.getNumberOfStreams(),
                                 defaultArguments["EventStreams"])
            else:
                self.assertEqual(stepHelper.getNumberOfCores(), 1)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = taskObj.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'],
                         defaultArguments["Memory"])

        return
Пример #12
0
    def __call__(self, workloadName, arguments):
        """
        Create a workload instance for a LHE Step0 request
        Just pass it down to MonteCarlo
        """
        #Override splitting arguments
        # Splitting arguments
        timePerEvent     = int(arguments.get('TimePerEvent', 60))
        filterEfficiency = float(arguments.get('FilterEfficiency', 1.0))
        totalTime        = int(arguments.get('TotalTime', 9 * 3600))
        self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency)
        if arguments.get("LheInputFiles", False) == True \
             or arguments.get("LheInputFiles", False) == "True":
            self.lheInputFiles = True

        # These are mostly place holders because the job splitting algo and
        # parameters will be updated after the workflow has been created.
        eventsPerJob = int(totalTime/timePerEvent/filterEfficiency)
        self.prodJobSplitAlgo  = arguments.setdefault("ProdJobSplitAlgo", 'EventBased')
        self.prodJobSplitArgs  = arguments.setdefault("ProdJobSplitArgs",
                                               {"events_per_job": eventsPerJob,
                                                "events_per_lumi": arguments['EventsPerLumi']})
        self.prodJobSplitArgs.setdefault("lheInputFiles", self.lheInputFiles)
        mcWorkload = MonteCarloWorkloadFactory.__call__(self, workloadName, arguments)
        mcWorkload.setBlockCloseSettings(mcWorkload.getBlockCloseMaxWaitTime(), 5,
                                         250000000, mcWorkload.getBlockCloseMaxSize())

        return mcWorkload
Пример #13
0
    def testMemCoresSettings(self):
        """
        _testMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all tasks and steps.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)

        # test default values
        taskObj = testWorkload.getTask('Production')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = taskObj.getStepHelper(step)
            self.assertEqual(stepHelper.getNumberOfCores(), 1)
            self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = taskObj.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], 2300.0)

        # now test case where args are provided
        defaultArguments["Multicore"] = 6
        defaultArguments["Memory"] = 4600.0
        defaultArguments["EventStreams"] = 3
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)
        taskObj = testWorkload.getTask('Production')
        for step in ('cmsRun1', 'stageOut1', 'logArch1'):
            stepHelper = taskObj.getStepHelper(step)
            if step == 'cmsRun1':
                self.assertEqual(stepHelper.getNumberOfCores(), defaultArguments["Multicore"])
                self.assertEqual(stepHelper.getNumberOfStreams(), defaultArguments["EventStreams"])
            else:
                self.assertEqual(stepHelper.getNumberOfCores(), 1)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)
        # then test Memory requirements
        perfParams = taskObj.jobSplittingParameters()['performance']
        self.assertEqual(perfParams['memoryRequirement'], defaultArguments["Memory"])

        return
def getMCArgs():
    mcArgs = MonteCarloWorkloadFactory.getTestArguments()
    mcArgs.update({
                   "CouchURL": None,
                   "CouchDBName": None,
                   "ConfigCacheDoc" : None
                   })
    mcArgs.pop('ConfigCacheDoc')

    return mcArgs
Пример #15
0
def getMCArgs():
    mcArgs = MonteCarloWorkloadFactory.getTestArguments()
    mcArgs.update({
        "CouchURL": None,
        "CouchDBName": None,
        "ConfigCacheDoc": None
    })
    mcArgs.pop('ConfigCacheDoc')

    return mcArgs
Пример #16
0
    def testMonteCarloExtension(self):
        """
        _testMonteCarloExtension_

        Create a Monte Carlo workflow and verify that it is injected correctly
        into WMBS and invoke its detailed test. This uses a non-zero first
        lumi. Check that the splitting arguments are correctly
        set for the lfn counter.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["CouchDBName"] = TEST_DB_NAME
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()
        defaultArguments["FirstLumi"] = 10001
        defaultArguments["EventsPerJob"] = 100
        defaultArguments["FirstEvent"] = 10001
        # defaultArguments["FirstEvent"] = 10001

        initial_lfn_counter = (
            100
        )  # EventsPerJob == EventsPerLumi, then the number of previous jobs is equal to the number of the initial lumi

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()

        productionTask = testWorkload.getTaskByPath("/TestWorkload/Production")
        productionSplitting = productionTask.jobSplittingParameters()
        self.assertTrue("initial_lfn_counter" in productionSplitting, "No initial lfn counter was stored")
        self.assertEqual(productionSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")

        for outputMod in ["OutputA", "OutputB"]:
            mergeTask = testWorkload.getTaskByPath("/TestWorkload/Production/ProductionMerge%s" % outputMod)
            mergeSplitting = mergeTask.jobSplittingParameters()
            self.assertTrue("initial_lfn_counter" in mergeSplitting, "No initial lfn counter was stored")
            self.assertEqual(mergeSplitting["initial_lfn_counter"], initial_lfn_counter, "Wrong initial LFN counter")

        return
Пример #17
0
    def testMonteCarlo(self):
        """
        _testMonteCarlo_

        Create a Monte Carlo workflow and verify that it is injected correctly
        into WMBS and invoke its detailed test.
        """
        defaultArguments = MonteCarloWorkloadFactory.getTestArguments()
        defaultArguments["CouchURL"] = os.environ["COUCHURL"]
        defaultArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", defaultArguments)

        testWMBSHelper = WMBSHelper(testWorkload, "Production", "SomeBlock", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._commonMonteCarloTest()

        return
Пример #18
0
    def testInvalidKeys_initialize_clone(self):
        """
        Test some undefined/assignment user arguments in initialize_clone
        """
        # test with unknown key
        requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False}
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)

        # test with assignment key
        requestArgs = {
            'Multicore': 1,
            'TrustSitelists': False,
            'ReqMgr2Only': False
        }
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)

        # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
        requestArgs = {
            'SubRequestType': "RelVal",
            "Step2": {
                "LumisPerJob": 5,
                "Campaign": "Step2Camp"
            }
        }
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS,
                                     TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)
Пример #19
0
    def __call__(self, workloadName, arguments):
        """
        Create a workload instance for a LHE Step0 request
        Just pass it down to MonteCarlo
        """
        #Override splitting arguments
        # Splitting arguments
        timePerEvent     = int(arguments.get('TimePerEvent', 60))
        filterEfficiency = float(arguments.get('FilterEfficiency', 1.0))
        totalTime        = int(arguments.get('TotalTime', 9 * 3600))
        self.totalEvents = int(int(arguments['RequestNumEvents']) / filterEfficiency)

        # These are mostly place holders because the job splitting algo and
        # parameters will be updated after the workflow has been created.
        eventsPerJob = int(totalTime/timePerEvent/filterEfficiency)
        self.prodJobSplitAlgo  = arguments.setdefault("ProdJobSplitAlgo", 'EventBased')
        self.prodJobSplitArgs  = arguments.setdefault("ProdJobSplitArgs",
                                               {"events_per_job": eventsPerJob,
                                                "events_per_lumi": arguments['EventsPerLumi']})
        return MonteCarloWorkloadFactory.__call__(self, workloadName, arguments)
Пример #20
0
 def __call__(self, workflowName, args):
     workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args)
     delattr(workload.taskIterator().next().steps().data.application.configuration,
             'configCacheUrl')
     return workload
Пример #21
0
if len(sys.argv) != 3:
    print("Usage:")
    print("./injectMonteCarloWorkflow.py PROCESSING_VERSION NUM_EVENTS")
    sys.exit(1)
else:
    arguments["ProcessingVersion"] = sys.argv[1]
    numEvents = int(sys.argv[2])

connectToDB()

workloadName = "MonteCarlo-%s" % arguments["ProcessingVersion"]
workloadFile = "MonteCarlo-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)

factory = MonteCarloWorkloadFactory()
workload = factory(workloadName, arguments)
workloadPath = os.path.join(workloadName, workloadFile)
workload.setOwner("*****@*****.**")
workload.setSpecUrl(workloadPath)

# Build a sandbox using TaskMaker
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()

workload.save(workloadPath)

myThread = threading.currentThread()
myThread.transaction.begin()
for workloadTask in workload.taskIterator():
Пример #22
0
 def __init__(self):
     MonteCarloWorkloadFactory.__init__(self)
Пример #23
0
 def makeWorkload(self, schema):
     factory = MonteCarloWorkloadFactory()
     return factory(schema['RequestName'], schema).data
Пример #24
0
Unittests for IteratorTools functions
"""

from __future__ import division, print_function
from pprint import pprint
import unittest
from copy import deepcopy
from WMCore.ReqMgr.DataStructs.Request import initialize_clone
from WMCore.WMSpec.StdSpecs.MonteCarlo import MonteCarloWorkloadFactory
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException

### Spec arguments definition with only key and its default value
MC_ARGS = MonteCarloWorkloadFactory.getWorkloadCreateArgs()
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()

### Some original request dicts (ones to be cloned from)
mcOriginalArgs = {
    'Memory': 1234,
    'TimePerEvent': 1.2,
    'RequestType': 'MonteCarlo',
    "LheInputFiles": True,
    "ScramArch": ["slc6_amd64_gcc481"],
    "RequestName": "test_mc"
 def __call__(self, workflowName, args):
     workload = MonteCarloWorkloadFactory.__call__(self, workflowName, args)
     #delattr(workload.taskIterator().next().steps().data.application.configuration,
     #        'configCacheUrl')
     return workload
Пример #26
0
    def testFilesets(self):
        """
        Test workflow tasks, filesets and subscriptions creation
        """
        # expected tasks, filesets, subscriptions, etc
        expOutTasks = ['/TestWorkload/Production',
                       '/TestWorkload/Production/ProductionMergeOutputB',
                       '/TestWorkload/Production/ProductionMergeOutputA']
        expWfTasks = ['/TestWorkload/Production',
                      '/TestWorkload/Production/LogCollect',
                      '/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
                      '/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
                      '/TestWorkload/Production/ProductionMergeOutputA',
                      '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
                      '/TestWorkload/Production/ProductionMergeOutputB',
                      '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect']
        expFsets = ['FILESET_DEFINED_DURING_RUNTIME',
                    '/TestWorkload/Production/unmerged-OutputBUSER',
                    '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
                    '/TestWorkload/Production/ProductionMergeOutputA/merged-MergedRECO',
                    '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
                    '/TestWorkload/Production/ProductionMergeOutputB/merged-MergedUSER',
                    '/TestWorkload/Production/unmerged-logArchive',
                    '/TestWorkload/Production/unmerged-OutputARECO']
        subMaps = ['FILESET_DEFINED_DURING_RUNTIME',
                   (6,
                    '/TestWorkload/Production/ProductionMergeOutputA/merged-logArchive',
                    '/TestWorkload/Production/ProductionMergeOutputA/ProductionOutputAMergeLogCollect',
                    'MinFileBased',
                    'LogCollect'),
                   (3,
                    '/TestWorkload/Production/ProductionMergeOutputB/merged-logArchive',
                    '/TestWorkload/Production/ProductionMergeOutputB/ProductionOutputBMergeLogCollect',
                    'MinFileBased',
                    'LogCollect'),
                   (8,
                    '/TestWorkload/Production/unmerged-logArchive',
                    '/TestWorkload/Production/LogCollect',
                    'MinFileBased',
                    'LogCollect'),
                   (7,
                    '/TestWorkload/Production/unmerged-OutputARECO',
                    '/TestWorkload/Production/ProductionCleanupUnmergedOutputA',
                    'SiblingProcessingBased',
                    'Cleanup'),
                   (5,
                    '/TestWorkload/Production/unmerged-OutputARECO',
                    '/TestWorkload/Production/ProductionMergeOutputA',
                    'ParentlessMergeBySize',
                    'Merge'),
                   (4,
                    '/TestWorkload/Production/unmerged-OutputBUSER',
                    '/TestWorkload/Production/ProductionCleanupUnmergedOutputB',
                    'SiblingProcessingBased',
                    'Cleanup'),
                   (2,
                    '/TestWorkload/Production/unmerged-OutputBUSER',
                    '/TestWorkload/Production/ProductionMergeOutputB',
                    'ParentlessMergeBySize',
                    'Merge')]

        testArguments = MonteCarloWorkloadFactory.getTestArguments()
        testArguments["CouchURL"] = os.environ["COUCHURL"]
        testArguments["CouchDBName"] = TEST_DB_NAME
        testArguments["ConfigCacheID"] = self.injectMonteCarloConfig()

        factory = MonteCarloWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", testArguments)

        myMask = Mask(FirstRun=1, FirstLumi=1, FirstEvent=1, LastRun=1, LastLumi=10, LastEvent=1000)
        testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self.assertItemsEqual(testWorkload.listOutputProducingTasks(), expOutTasks)

        workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
        self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)

        # same function as in WMBSHelper, otherwise we cannot know which fileset name is
        maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
        topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
        expFsets[0] = topFilesetName
        # returns a tuple of id, name, open and last_update
        filesets = self.listFilesets.execute()
        self.assertItemsEqual([item[1] for item in filesets], expFsets)

        subMaps[0] = (1, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production')
        subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
        self.assertItemsEqual(subscriptions, subMaps)

        ### create another top level subscription
        myMask = Mask(FirstRun=1, FirstLumi=11, FirstEvent=1001, LastRun=1, LastLumi=20, LastEvent=2000)
        testWMBSHelper = WMBSHelper(testWorkload, "Production", mask=myMask,
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        workflows = self.listTasksByWorkflow.execute(workflow="TestWorkload")
        self.assertItemsEqual([item['task'] for item in workflows], expWfTasks)

        # same function as in WMBSHelper, otherwise we cannot know which fileset name is
        maskString = ",".join(["%s=%s" % (x, myMask[x]) for x in sorted(myMask)])
        topFilesetName = 'TestWorkload-Production-%s' % md5(maskString).hexdigest()
        expFsets.append(topFilesetName)
        # returns a tuple of id, name, open and last_update
        filesets = self.listFilesets.execute()
        self.assertItemsEqual([item[1] for item in filesets], expFsets)

        subMaps.append((9, topFilesetName, '/TestWorkload/Production', 'EventBased', 'Production'))
        subscriptions = self.listSubsMapping.execute(workflow="TestWorkload", returnTuple=True)
        self.assertItemsEqual(subscriptions, subMaps)
Пример #27
0
 def __init__(self):
     MonteCarloWorkloadFactory.__init__(self)
     self.lheInputFiles = False