Ejemplo n.º 1
0
    def testInvalidKeys_initialize_clone(self):
        """
        Test some undefined/assignment user arguments in initialize_clone
        """
        # test with unknown key
        requestArgs = {'Multicore': 1, 'WRONG': 'Alan'}
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # test with assignment key
        requestArgs = {'Multicore': 1, 'TrustSitelists': False}
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
        requestArgs = {'SubRequestType': "RelVal", "Step2": {"LumisPerJob": 5, "Campaign": "Step2Camp"}}
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(cloneArgs["RequestName"], cloneArgs)

        # make sure unsupported keys are remove from both top and inner dicts
        requestArgs = {}
        originalArgs = deepcopy(taskChainOriginalArgs)
        originalArgs['BAD'] = 'KEY'
        originalArgs['Task1']['SiteWhitelist'] = ['This_key_should_not_be_here']
        originalArgs['Task1']['Alan'] = False
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS, TASK_ARGS)
        self.assertFalse('BAD' in cloneArgs)
        self.assertFalse('SiteWhitelist' in cloneArgs['Task1'])
        self.assertFalse('Alan' in cloneArgs['Task1'])
Ejemplo n.º 2
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = TaskChainWorkloadFactory.getTestArguments()
        pileupMcArgs['Task1']["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
        pileupMcArgs['Task1']["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
        pileupMcArgs['Task1']["ConfigCacheID"] = self.injectGenerationConfig()
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)

        # now that the workload was created and args validated, we can add this PileupConfig
        pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs['Task1']["MCPileup"],
                                                         pileupMcArgs['Task1']["DataPileup"])
        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs["PileupConfig"], task, taskPath)
Ejemplo n.º 3
0
    def testMultithreadedTaskChain(self):
        """
        Test for multithreaded task chains where all steps run with the same
        number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        arguments['Task1']['Multicore'] = 4
        arguments['Task2']['Multicore'] = 4
        arguments['Task3']['Multicore'] = 4
        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction("MultiChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        hlt = self.workload.getTaskByPath('/MultiChain/HLTD')
        reco = self.workload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = self.workload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        hltStep = hlt.getStepHelper("cmsRun1")
        recoStep = reco.getStepHelper("cmsRun1")
        miniAODStep = miniAOD.getStepHelper("cmsRun1")

        self.assertEqual(hltStep.getNumberOfCores(), 4)
        self.assertEqual(recoStep.getNumberOfCores(), 4)
        self.assertEqual(miniAODStep.getNumberOfCores(), 4)

        return
Ejemplo n.º 4
0
    def testPileupFetcherOnMC(self):
        pileupMcArgs = TaskChainWorkloadFactory.getTestArguments()
        pileupMcArgs['Task1']["MCPileup"] = "/Cosmics/ComissioningHI-PromptReco-v1/RECO"
        pileupMcArgs['Task1']["DataPileup"] = "/HighPileUp/Run2011A-v1/RAW"
        pileupMcArgs['Task1']["ConfigCacheID"] = self.injectGenerationConfig()
        pileupMcArgs["CouchDBName"] = "pileupfetcher_t"
        pileupMcArgs["CouchURL"] = os.environ["COUCHURL"]

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestWorkload", pileupMcArgs)

        # now that the workload was created and args validated, we can add this PileupConfig
        pileupMcArgs["PileupConfig"] = parsePileupConfig(pileupMcArgs['Task1']["MCPileup"],
                                                         pileupMcArgs['Task1']["DataPileup"])

        # Since this is test of the fetcher - The loading from WMBS isn't
        # really necessary because the fetching happens before the workflow
        # is inserted into WMBS: feed the workload instance directly into fetcher:
        fetcher = PileupFetcher()
        creator = SandboxCreator()
        pathBase = "%s/%s" % (self.testDir, testWorkload.name())
        for topLevelTask in testWorkload.taskIterator():
            for taskNode in topLevelTask.nodeIterator():
                # this is how the call to PileupFetcher is happening
                # from the SandboxCreator test
                task = WMTask.WMTaskHelper(taskNode)
                taskPath = "%s/WMSandbox/%s" % (pathBase, task.name())
                fetcher.setWorkingDirectory(taskPath)
                # create Sandbox for the fetcher ...
                creator._makePathonPackage(taskPath)
                fetcher(task)
                self._queryPileUpConfigFile(pileupMcArgs, task, taskPath)
Ejemplo n.º 5
0
    def testMultithreadedTaskChain(self):
        """
        Test for multithreaded task chains where all steps run with the same
        number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        arguments['Task1']['Multicore'] = 4
        arguments['Task2']['Multicore'] = 4
        arguments['Task3']['Multicore'] = 4
        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "MultiChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        hlt = self.workload.getTaskByPath('/MultiChain/HLTD')
        reco = self.workload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = self.workload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
        )

        hltStep = hlt.getStepHelper("cmsRun1")
        recoStep = reco.getStepHelper("cmsRun1")
        miniAODStep = miniAOD.getStepHelper("cmsRun1")

        self.assertEqual(hltStep.getNumberOfCores(), 4)
        self.assertEqual(recoStep.getNumberOfCores(), 4)
        self.assertEqual(miniAODStep.getNumberOfCores(), 4)

        return
Ejemplo n.º 6
0
    def testMultithreadedTaskChain(self):
        """
        Test multi-task TaskChain with default and multicore settings
        """
        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        arguments.pop('Multicore', None)
        arguments.pop('Memory', None)
        arguments.pop('EventStreams', None)
        arguments['Task1'].pop('Multicore', None)
        arguments['Task1'].pop('Memory', None)
        arguments['Task2'].pop('Multicore', None)
        arguments['Task2'].pop('Memory', None)
        arguments['Task2'].pop('EventStreams', None)
        arguments['Task3'].pop('Multicore', None)
        arguments['Task3'].pop('Memory', None)
        arguments['Task3'].pop('EventStreams', None)

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), 0)

        # now all with 16 cores, 16 event streams and 8GB of memory inherited from the top level
        arguments['Multicore'] = 16
        arguments['Memory'] = 8000
        arguments['EventStreams'] = 16
        testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])

        return
Ejemplo n.º 7
0
    def testMultithreadedTaskChain(self):
        """
        Test multi-task TaskChain with default and multicore settings
        """
        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        arguments.pop('Multicore', None)
        arguments.pop('Memory', None)
        arguments['Task1'].pop('Multicore', None)
        arguments['Task1'].pop('Memory', None)
        arguments['Task2'].pop('Multicore', None)
        arguments['Task2'].pop('Memory', None)
        arguments['Task3'].pop('Multicore', None)
        arguments['Task3'].pop('Memory', None)

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], 2300.0)
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), 0)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), 0)

        # now all with 16 cores, 16 event streams and 8GB of memory inherited from the top level
        arguments['Multicore'] = 16
        arguments['Memory'] = 8000
        arguments['EventStreams'] = 16
        testWorkload = factory.factoryWorkloadConstruction("MultiChain", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(hlt.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(reco.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(miniAOD.jobSplittingParameters()['performance']['memoryRequirement'], arguments['Memory'])
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])

        return
Ejemplo n.º 8
0
    def testMultithreadedTasksTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
        recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
        aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task2']['Multicore'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])

        self.assertEqual(hltMemory, arguments['Memory'])
        self.assertEqual(recoMemory, arguments['Task2']['Memory'])
        self.assertEqual(aodMemory, arguments['Task3']['Memory'])

        # and test another mix of cores and memory
        arguments['Task1']['Multicore'] = arguments.pop('Multicore', None)
        arguments['Task1']['Memory'] = arguments.pop('Memory', None)
        arguments['Task2'].pop('Multicore', None)
        arguments['Task2'].pop('Memory', None)
        arguments['Task3']['Multicore'] = 2
        testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
        recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
        aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task1']['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])

        self.assertEqual(hltMemory, arguments['Task1']['Memory'])
        self.assertEqual(recoMemory, 2300.0)
        self.assertEqual(aodMemory, arguments['Task3']['Memory'])

        return
Ejemplo n.º 9
0
    def createMCWMSpec(self, name='MonteCarloWorkload'):
        mcArgs = TaskChainWorkloadFactory.getTestArguments()
        mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
        mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])

        wmspec = taskChainWorkload(name, mcArgs)
        wmspec.setSpecUrl("/path/to/workload")
        getFirstTask(wmspec).addProduction(totalevents=10000)
        return wmspec
Ejemplo n.º 10
0
    def createMCWMSpec(self, name='MonteCarloWorkload'):
        mcArgs = TaskChainWorkloadFactory.getTestArguments()
        mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"]
        mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"])

        wmspec = taskChainWorkload(name, mcArgs)
        wmspec.setSpecUrl("/path/to/workload")
        getFirstTask(wmspec).addProduction(totalevents=10000)
        return wmspec
Ejemplo n.º 11
0
    def testTrustFlags(self):
        """
        _testTrustFlags_

        Given a taskChain with 4 tasks, test whether TrustSitelists is set for
        the top level tasks and TrustPUSitelists is properly set to all tasks.
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        testArguments.update(createMultiGTArgs())
        testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
        testArguments["CouchDBName"] = self.testInit.couchDbName
        testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
        testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
        testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
        testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)

        for task in workload.getAllTasks():
            flags = task.getTrustSitelists().values()
            self.assertEqual(flags, [False, False])

        # set both flags to true now
        workload.setTrustLocationFlag(True, True)
        for task in workload.getAllTasks():
            flags = task.getTrustSitelists()
            if task.isTopOfTree():
                self.assertEqual(flags.values(), [True, True])
            elif task.taskType() in ["Cleanup", "LogCollect"]:
                self.assertEqual(flags.values(), [False, False])
            else:
                self.assertFalse(flags['trustlists'])
                self.assertTrue(flags['trustPUlists'])

        # set both to false now
        workload.setTrustLocationFlag(False, False)
        for task in workload.getAllTasks(cpuOnly=True):
            flags = task.getTrustSitelists().values()
            self.assertEqual(flags, [False, False])
        return
Ejemplo n.º 12
0
def getProdArgs():
    mcArgs = TaskChainWorkloadFactory.getTestArguments()
    mcArgs.update({
                   "CouchURL": None,
                   "CouchDBName": None,
                   "ConfigCacheDoc" : None
                   })
    mcArgs.pop('ConfigCacheDoc')

    return mcArgs
Ejemplo n.º 13
0
def getProdArgs():
    mcArgs = TaskChainWorkloadFactory.getTestArguments()
    mcArgs.update({
        "CouchURL": None,
        "CouchDBName": None,
        "ConfigCacheDoc": None
    })
    mcArgs.pop('ConfigCacheDoc')

    return mcArgs
Ejemplo n.º 14
0
    def testMultithreadedTasksTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "MultiChain2", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        hlt = self.workload.getTaskByPath('/MultiChain2/HLTD')
        reco = self.workload.getTaskByPath(
            '/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = self.workload.getTaskByPath(
            '/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT'
        )

        hltMemory = hlt.jobSplittingParameters(
        )['performance']['memoryRequirement']
        recoMemory = reco.jobSplittingParameters(
        )['performance']['memoryRequirement']
        aodMemory = miniAOD.jobSplittingParameters(
        )['performance']['memoryRequirement']

        hltStep = hlt.getStepHelper("cmsRun1")
        recoStep = reco.getStepHelper("cmsRun1")
        miniAODStep = miniAOD.getStepHelper("cmsRun1")

        self.assertEqual(hltStep.getNumberOfCores(), 4)
        self.assertEqual(recoStep.getNumberOfCores(), 8)
        self.assertEqual(miniAODStep.getNumberOfCores(), 1)

        self.assertEqual(recoMemory, 8000.0)
        self.assertEqual(aodMemory, 2000.0)
        self.assertEqual(hltMemory, 4000.0)

        return
Ejemplo n.º 15
0
    def testTrustFlags(self):
        """
        _testTrustFlags_

        Given a taskChain with 4 tasks, test whether TrustSitelists is set for
        the top level tasks and TrustPUSitelists is properly set to all tasks.
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        testArguments.update(createMultiGTArgs())
        testArguments["CouchURL"] = self.testInit.couchUrl
        testArguments["CouchDBName"] = self.testInit.couchDbName
        testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
        testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
        testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
        testArguments["Task4"]["ConfigCacheID"] =  processorDocs['Skims']
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)

        for task in workload.getAllTasks():
            flags = task.getTrustSitelists().values()
            self.assertEqual(flags, [False, False])

        # set both flags to true now
        workload.setTrustLocationFlag(True, True)
        for task in workload.getAllTasks():
            flags = task.getTrustSitelists()
            if task.isTopOfTree():
                self.assertEqual(flags.values(), [True, True])
            elif task.taskType() in ["Merge", "Harvesting", "Cleanup", "LogCollect"]:
                self.assertEqual(flags.values(), [False, False])
            else:
                self.assertFalse(flags['trustlists'])
                self.assertTrue(flags['trustPUlists'])

        # set both to false now
        workload.setTrustLocationFlag(False, False)
        for task in workload.getAllTasks(cpuOnly=True):
            flags = task.getTrustSitelists().values()
            self.assertEqual(flags, [False, False])
        return
Ejemplo n.º 16
0
    def testMultithreadedTasksTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction("MultiChain2", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        hlt = self.workload.getTaskByPath("/MultiChain2/HLTD")
        reco = self.workload.getTaskByPath("/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT")
        miniAOD = self.workload.getTaskByPath(
            "/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT"
        )

        hltMemory = hlt.jobSplittingParameters()["performance"]["memoryRequirement"]
        recoMemory = reco.jobSplittingParameters()["performance"]["memoryRequirement"]
        aodMemory = miniAOD.jobSplittingParameters()["performance"]["memoryRequirement"]

        hltStep = hlt.getStepHelper("cmsRun1")
        recoStep = reco.getStepHelper("cmsRun1")
        miniAODStep = miniAOD.getStepHelper("cmsRun1")

        self.assertEqual(hltStep.getNumberOfCores(), 4)
        self.assertEqual(recoStep.getNumberOfCores(), 8)
        self.assertEqual(miniAODStep.getNumberOfCores(), 1)

        self.assertEqual(recoMemory, 3200.0)
        self.assertEqual(aodMemory, 2000.0)
        self.assertEqual(hltMemory, 2400.0)

        return
Ejemplo n.º 17
0
    def buildMultithreadedTaskChain(self, filename):
        """    d
        Build a TaskChain from several sources and customization
        """

        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()

        # Read in the request
        request = json.load(open(filename))

        # Construct args from the pieces starting with test args ...
        arguments = testArguments

        # ... continuing with the request
        for key in [
            "CMSSWVersion",
            "ScramArch",
            "GlobalTag",
            "ProcessingVersion",
            "Multicore",
            "Memory",
            "TaskChain",
            "Task1",
            "Task2",
            "Task3",
        ]:
            arguments.update({key: request["createRequest"][key]})

        for key in ["SiteBlacklist"]:
            arguments.update({key: request["assignRequest"][key]})

        # ... then some local overrides

        del arguments["ConfigCacheID"]
        del arguments["ConfigCacheUrl"]
        arguments.update({"CouchURL": self.testInit.couchUrl, "CouchDBName": self.testInit.couchDbName})

        # ... now fill in the ConfigCache documents created and override the inputs to link them up

        arguments["Task1"]["ConfigCacheID"] = processorDocs["DigiHLT"]

        arguments["Task2"]["ConfigCacheID"] = processorDocs["Reco"]
        arguments["Task2"]["InputFromOutputModule"] = "writeRAWDIGI"

        arguments["Task3"]["ConfigCacheID"] = processorDocs["ALCAReco"]
        arguments["Task3"]["InputFromOutputModule"] = "writeALCA"

        return arguments
Ejemplo n.º 18
0
    def buildMultithreadedTaskChain(self, filename):
        """    d
        Build a TaskChain from several sources and customization
        """

        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()

        # Read in the request
        request = json.load(open(filename))

        # Construct args from the pieces starting with test args ...
        arguments = testArguments

        # ... continuing with the request
        for key in [
                'CMSSWVersion', 'ScramArch', 'GlobalTag', 'ProcessingVersion',
                'Multicore', 'Memory', 'TaskChain', 'Task1', 'Task2', 'Task3'
        ]:
            arguments.update({key: request['createRequest'][key]})

        for key in ['SiteBlacklist']:
            arguments.update({key: request['assignRequest'][key]})

        # ... then some local overrides

        del arguments['ConfigCacheID']
        del arguments['ConfigCacheUrl']
        arguments.update({
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
        })

        # ... now fill in the ConfigCache documents created and override the inputs to link them up

        arguments['Task1']['ConfigCacheID'] = processorDocs['DigiHLT']

        arguments['Task2']['ConfigCacheID'] = processorDocs['Reco']
        arguments['Task2']['InputFromOutputModule'] = 'writeRAWDIGI'

        arguments['Task3']['ConfigCacheID'] = processorDocs['ALCAReco']
        arguments['Task3']['InputFromOutputModule'] = 'writeALCA'

        return arguments
Ejemplo n.º 19
0
    def buildMultithreadedTaskChain(self, filename):
        """    d
        Build a TaskChain from several sources and customization
        """

        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()

        # Read in the request
        request = json.load(open(filename))

        # Construct args from the pieces starting with test args ...
        arguments = testArguments

        # ... continuing with the request
        for key in ['CMSSWVersion', 'ScramArch', 'GlobalTag', 'ProcessingVersion',
                    'Multicore', 'Memory',
                    'TaskChain', 'Task1', 'Task2', 'Task3']:
            arguments.update({key : request['createRequest'][key]})

        for key in ['SiteBlacklist']:
            arguments.update({key : request['assignRequest'][key]})

        # ... then some local overrides

        del arguments['ConfigCacheID']
        del arguments['ConfigCacheUrl']
        arguments.update({
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
        })

        # ... now fill in the ConfigCache documents created and override the inputs to link them up

        arguments['Task1']['ConfigCacheID'] = processorDocs['DigiHLT']

        arguments['Task2']['ConfigCacheID'] = processorDocs['Reco']
        arguments['Task2']['InputFromOutputModule'] = 'writeRAWDIGI'

        arguments['Task3']['ConfigCacheID'] = processorDocs['ALCAReco']
        arguments['Task3']['InputFromOutputModule'] = 'writeALCA'

        return arguments
Ejemplo n.º 20
0
    def testInvalidKeys_initialize_clone(self):
        """
        Test some undefined/assignment user arguments in initialize_clone
        """
        # test with unknown key
        requestArgs = {'Multicore': 1, 'WRONG': 'Alan', 'ReqMgr2Only': False}
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)

        # test with assignment key
        requestArgs = {
            'Multicore': 1,
            'TrustSitelists': False,
            'ReqMgr2Only': False
        }
        originalArgs = deepcopy(mcOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, MC_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            MonteCarloWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)

        # test mix of StepChain and TaskChain args (it passes the initial filter but raises factory exception)
        requestArgs = {
            'SubRequestType': "RelVal",
            "Step2": {
                "LumisPerJob": 5,
                "Campaign": "Step2Camp"
            }
        }
        originalArgs = deepcopy(taskChainOriginalArgs)
        cloneArgs = initialize_clone(requestArgs, originalArgs, TASKCHAIN_ARGS,
                                     TASK_ARGS)
        with self.assertRaises(WMSpecFactoryException):
            TaskChainWorkloadFactory().factoryWorkloadConstruction(
                cloneArgs["RequestName"], cloneArgs)
Ejemplo n.º 21
0
    def testGeneratorWorkflow(self):
        """
        _testGeneratorWorkflow_
        Test creating a request with an initial generator task
        it mocks a request where there are 2 similar paths starting
        from the generator, each one with a different PrimaryDataset, CMSSW configuration
        and processed dataset. Dropping the RAW output as well.
        Also include an ignored output module to keep things interesting...
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        processorDocs = makeProcessingConfigs(self.configDatabase)

        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 6,
            "IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
            "Task1": {
                "TaskName": "GenSim",
                "ConfigCacheID": generatorDoc,
                "SplittingAlgo": "EventBased",
                "RequestNumEvents": 10000,
                "Seeding": "AutomaticSeeding",
                "PrimaryDataset": "RelValTTBar",
            },
            "Task2": {
                "TaskName": "DigiHLT_new",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgo": "LumiBased",
                "CMSSWVersion": "CMSSW_5_2_6",
                "GlobalTag": "GR_39_P_V5:All",
                "PrimaryDataset": "PURelValTTBar",
                "KeepOutput": False
            },
            "Task3": {
                "TaskName": "DigiHLT_ref",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgo": "EventBased",
                "CMSSWVersion": "CMSSW_5_2_7",
                "GlobalTag": "GR_40_P_V5:All",
                "AcquisitionEra": "ReleaseValidationNewConditions",
                "ProcessingVersion": 3,
                "ProcessingString": "Test",
                "KeepOutput": False
            },
            "Task4": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT_new",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "SplittingAlgo": "FileBased",
                "TransientOutputModules": ["writeRECO"]
            },
            "Task5": {
                "TaskName": "ALCAReco",
                "InputTask": "DigiHLT_ref",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['ALCAReco'],
                "SplittingAlgo": "LumiBased",
            },
            "Task6": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeRECO",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgo": "LumiBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments
        print arguments
        factory = TaskChainWorkloadFactory()

        # Test a malformed task chain definition
        arguments['Task4']['TransientOutputModules'].append('writeAOD')
        self.assertRaises(WMSpecFactoryException, factory.validateSchema,
                          arguments)

        arguments['Task4']['TransientOutputModules'].remove('writeAOD')
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "PullingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            import traceback
            traceback.print_exc()
            self.fail(msg)
Ejemplo n.º 22
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
        lumiDict2 = {"1":[[2,4], [8,40]], "2":[[100,150], [210,210]]}
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "DefaultGlobalTag",
            "LumiList": lumiDict,
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain" : 4,
            "Task1" :{
                "TaskName" : "DigiHLT",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "InputDataset" : "/MinimumBias/Commissioning10-v4/GEN-SIM",
                "SplittingAlgo"  : "FileBased",
            },
            "Task2" : {
                "TaskName" : "Reco",
                "InputTask" : "DigiHLT",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['Reco'],
                "SplittingAlgo" : "FileBased",
                "GlobalTag" : "GlobalTagForReco",
                "CMSSWVersion" : "CMSSW_3_1_2",
                "ScramArch" : "CompatibleRECOArch",
                "PrimaryDataset" : "ZeroBias",
                "LumiList": lumiDict2,
            },
            "Task3" : {
                "TaskName" : "ALCAReco",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeALCA",
                "ConfigCacheID" : processorDocs['ALCAReco'],
                "SplittingAlgo" : "FileBased",
                "GlobalTag" : "GlobalTagForALCAReco",
                "CMSSWVersion" : "CMSSW_3_1_3",
                "ScramArch" : "CompatibleALCAArch",

            },
            "Task4" : {
                "TaskName" : "Skims",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeRECO",
                "ConfigCacheID" : processorDocs['Skims'],
                "SplittingAlgo" : "FileBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)
Ejemplo n.º 23
0
import unittest
from copy import deepcopy

from Utils.PythonVersion import PY3

from WMCore.ReqMgr.DataStructs.Request import initialize_clone, RequestInfo
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException

### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()

### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {
    'Memory': 234,
    'SkimName1': 'skim_2017',
    'SkimInput1': 'RECOoutput',
    'Skim1ConfigCacheID': 'abcdef',
    'IncludeParents': True,
    'TimePerEvent': 1.2,
    'RequestType': 'ReReco',
    'RequestName': 'test_rereco'
}
Ejemplo n.º 24
0
    def testPileupTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)

        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain" : 2,
            "Task1" :{
                "InputDataset" : "/cosmics/whatever-input-v1/GEN-SIM",
                "TaskName" : "DIGI",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "SplittingAlgo" : "LumiBased",
                "LumisPerJob": 4,
                "MCPileup": "/some/cosmics-mc-v1/GEN-SIM",
                "DeterministicPileup": True,
                "CMSSWVersion" : "CMSSW_5_2_6",
                "GlobalTag" : "GR_39_P_V5:All",
                "PrimaryDataset" : "PURelValTTBar",
                "AcquisitionEra": "CMSSW_5_2_6",
                "ProcessingString": "ProcStr_Task1"
            },
            "Task2" : {
                "TaskName" : "RECO",
                "InputTask" : "DIGI",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['Reco'],
                "DataPileup": "/some/minbias-data-v1/GEN-SIM",
                "SplittingAlgo" : "LumiBased",
                "LumisPerJob": 2,
                "GlobalTag": "GR_R_62_V3::All",
                "AcquisitionEra": "CMSSW_5_2_7",
                "ProcessingString": "ProcStr_Task2"
            },
        }
        
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        self.workload = factory.factoryWorkloadConstruction("PullingTheChain", arguments)

        firstTask = self.workload.getTaskByPath("/PullingTheChain/DIGI")
        cmsRunStep = firstTask.getStep("cmsRun1").getTypeHelper()
        pileupData = cmsRunStep.getPileup()
        self.assertFalse(hasattr(pileupData, "data"))
        self.assertEqual(pileupData.mc.dataset, ["/some/cosmics-mc-v1/GEN-SIM"])
        splitting = firstTask.jobSplittingParameters()
        self.assertTrue(splitting["deterministicPileup"])

        secondTask = self.workload.getTaskByPath("/PullingTheChain/DIGI/DIGIMergewriteRAWDIGI/RECO")
        cmsRunStep = secondTask.getStep("cmsRun1").getTypeHelper()
        pileupData = cmsRunStep.getPileup()
        self.assertFalse(hasattr(pileupData, "mc"))
        self.assertEqual(pileupData.data.dataset, ["/some/minbias-data-v1/GEN-SIM"])
        splitting = secondTask.jobSplittingParameters()
        self.assertFalse(splitting["deterministicPileup"])
Ejemplo n.º 25
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
        lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "DefaultGlobalTag",
            "LumiList": lumiDict,
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 4,
            "Task1": {
                "TaskName": "DigiHLT",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
                "SplittingAlgo": "FileBased",
            },
            "Task2": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "SplittingAlgo": "FileBased",
                "GlobalTag": "GlobalTagForReco",
                "CMSSWVersion": "CMSSW_3_1_2",
                "ScramArch": "CompatibleRECOArch",
                "PrimaryDataset": "ZeroBias",
                "LumiList": lumiDict2,
            },
            "Task3": {
                "TaskName": "ALCAReco",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeALCA",
                "ConfigCacheID": processorDocs['ALCAReco'],
                "SplittingAlgo": "FileBased",
                "GlobalTag": "GlobalTagForALCAReco",
                "CMSSWVersion": "CMSSW_3_1_3",
                "ScramArch": "CompatibleALCAArch",
            },
            "Task4": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeRECO",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgo": "FileBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "YankingTheChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        testWMBSHelper = WMBSHelper(self.workload,
                                    "DigiHLT",
                                    "SomeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._checkTask(
            self.workload.getTaskByPath("/YankingTheChain/DigiHLT"),
            arguments['Task1'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
            arguments['Task2'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
            ), arguments['Task3'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
            ), arguments['Task4'], arguments)

        digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
        self.assertEqual(lumiDict, digi.getLumiMask())
        digiStep = digi.getStepHelper("cmsRun1")
        self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])

        # Make sure this task has a different lumilist than the global one
        reco = self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
        self.assertEqual(lumiDict2, reco.getLumiMask())
        recoStep = reco.getStepHelper("cmsRun1")
        self.assertEqual(recoStep.getGlobalTag(),
                         arguments['Task2']['GlobalTag'])
        self.assertEqual(recoStep.getCMSSWVersion(),
                         arguments['Task2']['CMSSWVersion'])
        self.assertEqual(recoStep.getScramArch(),
                         arguments['Task2']['ScramArch'])

        alca = self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"
        )
        self.assertEqual(lumiDict, alca.getLumiMask())
        alcaStep = alca.getStepHelper("cmsRun1")
        self.assertEqual(alcaStep.getGlobalTag(),
                         arguments['Task3']['GlobalTag'])
        self.assertEqual(alcaStep.getCMSSWVersion(),
                         arguments['Task3']['CMSSWVersion'])
        self.assertEqual(alcaStep.getScramArch(),
                         arguments['Task3']['ScramArch'])

        skim = self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"
        )
        skimStep = skim.getStepHelper("cmsRun1")
        self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 14,
                         "Number of output datasets doesn't match")
        self.assertTrue(
            "/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI"
            in outputDatasets,
            "/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI not in output datasets"
        )
        self.assertTrue(
            "/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI"
            in outputDatasets,
            "/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI not in output datasets"
        )
        self.assertTrue(
            "/ZeroBias/ReleaseValidation-reco-v1/RECO" in outputDatasets,
            "/ZeroBias/ReleaseValidation-reco-v1/RECO not in output datasets")
        self.assertTrue(
            "/ZeroBias/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
            "/ZeroBias/ReleaseValidation-AOD-v1/AOD not in output datasets")
        self.assertTrue(
            "/ZeroBias/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
            "/ZeroBias/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
        )
        for i in range(1, 5):
            self.assertTrue(
                "/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO" % i
                in outputDatasets,
                "/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
                % i)
        for i in range(1, 6):
            self.assertTrue(
                "/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD" % i
                in outputDatasets,
                "/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
                % i)

        return
Ejemplo n.º 26
0
    def testProcessingWithScenarios(self):
        """
        _testProcessingWithScenarios_

        Test creating a processing workload with an input dataset that
        uses scenarios instead of configuration files
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 4,
            "Task1": {
                "TaskName": "DigiHLT",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
                "SplittingAlgorithm": "FileBased",
                "SplittingArguments": {
                    "files_per_job": 1
                },
            },
            "Task2": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT",
                "InputFromOutputModule": "writeRAWDIGI",
                "ProcScenario": "pp",
                "ScenarioMethod": "promptReco",
                "ScenarioArguments": {
                    'outputs': [{
                        'dataTier': "RECO",
                        'moduleLabel': "RECOoutput"
                    }, {
                        'dataTier': "AOD",
                        'moduleLabel': "AODoutput"
                    }, {
                        'dataTier': "ALCARECO",
                        'moduleLabel': "ALCARECOoutput"
                    }]
                },
                "SplittingAlgorithm": "FileBased",
                "SplittingArguments": {
                    "files_per_job": 1
                },
            },
            "Task3": {
                "TaskName": "ALCAReco",
                "InputTask": "Reco",
                "InputFromOutputModule": "ALCARECOoutput",
                "ProcScenario": "pp",
                "ScenarioMethod": "alcaReco",
                "ScenarioArguments": {
                    'outputs': [{
                        'dataTier': "ALCARECO",
                        'moduleLabel': "ALCARECOoutput"
                    }]
                },
                "SplittingAlgorithm": "FileBased",
                "SplittingArguments": {
                    "files_per_job": 1
                },
            },
            "Task4": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "RECOoutput",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgorithm": "FileBased",
                "SplittingArguments": {
                    "files_per_job": 10
                },
            }
        }

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory("YankingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)
Ejemplo n.º 27
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        testArguments.update(createMultiGTArgs())
        lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
        testArguments["ConfigCacheUrl"] = self.testInit.couchUrl
        testArguments["CouchDBName"] = self.testInit.couchDbName
        testArguments["Task1"]["LumiList"] = lumiDict
        testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
        testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
        testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
        testArguments["Task4"]["ConfigCacheID"] = processorDocs['Skims']
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)

        testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"),
                        arguments['Task2'],
                        arguments)
        self._checkTask(self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
                        arguments['Task3'], arguments)
        self._checkTask(self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
                        arguments['Task4'], arguments)

        digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
        self.assertEqual(lumiDict, digi.getLumiMask().getCompactList())
        digiStep = digi.getStepHelper("cmsRun1")
        self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])

        # Make sure this task has a different lumilist than the global one
        reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
        recoStep = reco.getStepHelper("cmsRun1")
        self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
        self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
        self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])

        alca = self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
        alcaStep = alca.getStepHelper("cmsRun1")
        self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
        self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
        self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])

        skim = self.workload.getTaskByPath(
            "/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
        skimStep = skim.getStepHelper("cmsRun1")
        self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
        self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDigiFilter-FAKE-v1/RAW-DIGI" in outputDatasets)
        self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-RawDebugDigiFilter-FAKE-v1/RAW-DEBUG-DIGI" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-reco-FAKE-v1/RECO" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-AOD-FAKE-v1/AOD" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-alca-FAKE-v1/ALCARECO" in outputDatasets)
        for i in range(1, 5):
            self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-alca%d-FAKE-v1/ALCARECO" % i in outputDatasets)
        for i in range(1, 6):
            self.assertTrue("/BprimeJetToBZ_M800GeV_Tune4C_13TeV-madgraph-tauola/ReleaseValidation-skim%d-FAKE-v1/RECO-AOD" % i in outputDatasets)

        return
Ejemplo n.º 28
0
    def test1TaskMemCoresSettings(self):
        """
        _test1TaskMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Single step in a task.
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "ConfigCacheUrl": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "TaskChain": 1,
            "Task1": {
                "TaskName": "TaskOne",
                "ConfigCacheID": generatorDoc,
                "RequestNumEvents": 10000,
                "PrimaryDataset": "RelValTTBar",
            },
        }

        testArguments.update(arguments)

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        taskPaths = ['/TestTaskChainWorkload/TaskOne',
                     '/TestTaskChainWorkload/TaskOne/LogCollectForTaskOne',
                     '/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM',
                     '/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM/TaskOnewriteGENSIMMergeLogCollect',
                     '/TestTaskChainWorkload/TaskOne/TaskOneCleanupUnmergedwriteGENSIM']

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    self.assertEqual(stepHelper.getNumberOfCores(), 1)
                    self.assertEqual(stepHelper.getNumberOfStreams(), 0)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() == 'LogCollect':
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)

        # now play with cores at top level
        testArguments['Multicore'] = 2
        testArguments['EventStreams'] = 2
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    if step == 'cmsRun1':
                        self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Multicore'])
                        self.assertEqual(stepHelper.getNumberOfStreams(), testArguments["EventStreams"])
                    else:
                        self.assertEqual(stepHelper.getNumberOfCores(), 1)
                        self.assertEqual(stepHelper.getNumberOfStreams(), 0)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() == 'LogCollect':
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)

        # last but not least, play with cores at task level
        testArguments['Task1']['Multicore'] = 2
        testArguments['Task1']['EventStreams'] = 2
        testArguments.pop('Multicore', None)
        testArguments.pop('EventStreams', None)
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    if step == 'cmsRun1':
                        self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Task1']['Multicore'])
                        self.assertEqual(stepHelper.getNumberOfStreams(), testArguments['Task1']['EventStreams'])
                    else:
                        self.assertEqual(stepHelper.getNumberOfCores(), 1)
                        self.assertEqual(stepHelper.getNumberOfStreams(), 0)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() == 'LogCollect':
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)
                self.assertEqual(stepHelper.getNumberOfStreams(), 0)

        return
Ejemplo n.º 29
0
    def test1TaskMemCoresSettings(self):
        """
        _test1StepMemCoresSettings_

        Make sure the multicore and memory setings are properly propagated to
        all steps. Single step in a task.
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "TaskChain": 1,
            "Task1": {
                "TaskName": "TaskOne",
                "ConfigCacheID": generatorDoc,
                "RequestNumEvents": 10000,
                "PrimaryDataset": "RelValTTBar",
            },
        }

        testArguments.update(arguments)

        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        taskPaths = ['/TestTaskChainWorkload/TaskOne', 
            '/TestTaskChainWorkload/TaskOne/LogCollectForTaskOne', 
            '/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM', 
            '/TestTaskChainWorkload/TaskOne/TaskOneMergewriteGENSIM/TaskOnewriteGENSIMMergeLogCollect', 
            '/TestTaskChainWorkload/TaskOne/TaskOneCleanupUnmergedwriteGENSIM']

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    self.assertEqual(stepHelper.getNumberOfCores(), 1)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() in ('LogCollect'):
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)

        # now play with cores at top level
        testArguments['Multicore'] = 2
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    if step == 'cmsRun1':
                        self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Multicore'])
                    else:
                        self.assertEqual(stepHelper.getNumberOfCores(), 1)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() in ('LogCollect'):
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)

        # last but not least, play with cores at task level
        testArguments['Task1']['Multicore'] = 2
        testArguments.pop('Multicore', None)
        testWorkload = factory.factoryWorkloadConstruction("TestTaskChainWorkload", testArguments)

        for task in taskPaths:
            taskObj = testWorkload.getTaskByPath(task)
            if taskObj.taskType() in ('Production', 'Processing'):
                for step in ('cmsRun1', 'stageOut1', 'logArch1'):
                    stepHelper = taskObj.getStepHelper(step)
                    if step == 'cmsRun1':
                        self.assertEqual(stepHelper.getNumberOfCores(), testArguments['Task1']['Multicore'])
                    else:
                        self.assertEqual(stepHelper.getNumberOfCores(), 1)
                perfParams = taskObj.jobSplittingParameters()['performance']
                self.assertEqual(perfParams['memoryRequirement'], 2300.0)
            elif taskObj.taskType() in ('LogCollect'):
                stepHelper = taskObj.getStepHelper('logCollect1')
                self.assertEqual(stepHelper.getNumberOfCores(), 1)

        return
Ejemplo n.º 30
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        testArguments.update(createMultiGTArgs())
        lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
        testArguments["CouchURL"] = self.testInit.couchUrl
        testArguments["CouchDBName"] = self.testInit.couchDbName
        testArguments["Task1"]["LumiList"] = lumiDict
        testArguments["Task1"]["ConfigCacheID"] = processorDocs['DigiHLT']
        testArguments["Task2"]["ConfigCacheID"] = processorDocs['Reco']
        testArguments["Task3"]["ConfigCacheID"] = processorDocs['ALCAReco']
        testArguments["Task4"]["ConfigCacheID"] =  processorDocs['Skims']
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)

        testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)


        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"), arguments['Task2'],
                        arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
                        arguments['Task3'], arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
                        arguments['Task4'], arguments)

        digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
        self.assertEqual(lumiDict, digi.getLumiMask().getCompactList())
        digiStep = digi.getStepHelper("cmsRun1")
        self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])

        # Make sure this task has a different lumilist than the global one
        reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
        recoStep = reco.getStepHelper("cmsRun1")
        self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
        self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
        self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])

        alca = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
        alcaStep = alca.getStepHelper("cmsRun1")
        self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
        self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
        self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])

        skim = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
        skimStep = skim.getStepHelper("cmsRun1")
        self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
        self.assertTrue("/MinimumBias/ReleaseValidation-RawDigiFilter-FAKE-v1/RAW-DIGI" in outputDatasets)
        self.assertTrue("/MinimumBias/ReleaseValidation-RawDebugDigiFilter-FAKE-v1/RAW-DEBUG-DIGI" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-reco-FAKE-v1/RECO" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-AOD-FAKE-v1/AOD" in outputDatasets)
        self.assertTrue("/ZeroBias/ReleaseValidation-alca-FAKE-v1/ALCARECO" in outputDatasets)
        for i in range(1, 5):
            self.assertTrue("/MinimumBias/ReleaseValidation-alca%d-FAKE-v1/ALCARECO" % i in outputDatasets)
        for i in range(1, 6):
            self.assertTrue("/MinimumBias/ReleaseValidation-skim%d-FAKE-v1/RECO-AOD" % i in outputDatasets)

        return
Ejemplo n.º 31
0
    def testMultithreadedTasksTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """

        arguments = self.buildMultithreadedTaskChain(self.differentNCores)
        factory = TaskChainWorkloadFactory()
        testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
        recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
        aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task2']['Multicore'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])

        self.assertEqual(hltMemory, arguments['Memory'])
        self.assertEqual(recoMemory, arguments['Task2']['Memory'])
        self.assertEqual(aodMemory, arguments['Task3']['Memory'])

        # and test another mix of cores, event streams and memory
        arguments['Task1']['Multicore'] = arguments.pop('Multicore', None)
        arguments['Task1']['Memory'] = arguments.pop('Memory', None)
        arguments['Task2'].pop('Multicore', None)
        arguments['Task2'].pop('Memory', None)
        arguments['Task3']['Multicore'] = 2
        arguments['EventStreams'] = 32
        arguments['Task2']['EventStreams'] = 8
        arguments['Task3']['EventStreams'] = 0
        testWorkload = factory.factoryWorkloadConstruction("MultiChain2", arguments)

        hlt = testWorkload.getTaskByPath('/MultiChain2/HLTD')
        reco = testWorkload.getTaskByPath('/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT')
        miniAOD = testWorkload.getTaskByPath(
            '/MultiChain2/HLTD/HLTDMergewriteRAWDIGI/RECODreHLT/RECODreHLTMergewriteALCA/MINIAODDreHLT')

        hltMemory = hlt.jobSplittingParameters()['performance']['memoryRequirement']
        recoMemory = reco.jobSplittingParameters()['performance']['memoryRequirement']
        aodMemory = miniAOD.jobSplittingParameters()['performance']['memoryRequirement']

        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task1']['Multicore'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfCores(), 1)
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfCores(), arguments['Task3']['Multicore'])
        self.assertEqual(hlt.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['EventStreams'])
        self.assertEqual(reco.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task2']['EventStreams'])
        self.assertEqual(miniAOD.getStepHelper("cmsRun1").getNumberOfStreams(), arguments['Task3']['EventStreams'])

        self.assertEqual(hltMemory, arguments['Task1']['Memory'])
        self.assertEqual(recoMemory, 2300.0)
        self.assertEqual(aodMemory, arguments['Task3']['Memory'])

        return
Ejemplo n.º 32
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        lumiDict = {"1": [[2, 4], [8, 50]], "2": [[100, 200], [210, 210]]}
        lumiDict2 = {"1": [[2, 4], [8, 40]], "2": [[100, 150], [210, 210]]}
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "DefaultGlobalTag",
            "LumiList": lumiDict,
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 4,
            "Task1": {
                "TaskName": "DigiHLT",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "InputDataset": "/MinimumBias/Commissioning10-v4/GEN-SIM",
                "SplittingAlgo": "FileBased",
            },
            "Task2": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "SplittingAlgo": "FileBased",
                "GlobalTag": "GlobalTagForReco",
                "CMSSWVersion": "CMSSW_3_1_2",
                "ScramArch": "CompatibleRECOArch",
                "PrimaryDataset": "ZeroBias",
                "LumiList": lumiDict2,
            },
            "Task3": {
                "TaskName": "ALCAReco",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeALCA",
                "ConfigCacheID": processorDocs['ALCAReco'],
                "SplittingAlgo": "FileBased",
                "GlobalTag": "GlobalTagForALCAReco",
                "CMSSWVersion": "CMSSW_3_1_3",
                "ScramArch": "CompatibleALCAArch",
            },
            "Task4": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeRECO",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgo": "FileBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "YankingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)
Ejemplo n.º 33
0
    def testMultipleGlobalTags(self):
        """
        _testMultipleGlobalTags_

        Test creating a workload that starts in a processing task
        with an input dataset, and has different globalTags
        and CMSSW versions (with corresponding scramArch) in
        each task
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)
        testArguments = TaskChainWorkloadFactory.getTestArguments()
        lumiDict = {"1":[[2,4], [8,50]], "2":[[100,200], [210,210]]}
        lumiDict2 = {"1":[[2,4], [8,40]], "2":[[100,150], [210,210]]}
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "DefaultGlobalTag",
            "LumiList": lumiDict,
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain" : 4,
            "Task1" :{
                "TaskName" : "DigiHLT",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "InputDataset" : "/MinimumBias/Commissioning10-v4/GEN-SIM",
                "SplittingAlgo"  : "FileBased",
            },
            "Task2" : {
                "TaskName" : "Reco",
                "InputTask" : "DigiHLT",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['Reco'],
                "SplittingAlgo" : "FileBased",
                "GlobalTag" : "GlobalTagForReco",
                "CMSSWVersion" : "CMSSW_3_1_2",
                "ScramArch" : "CompatibleRECOArch",
                "PrimaryDataset" : "ZeroBias",
                "LumiList": lumiDict2,
            },
            "Task3" : {
                "TaskName" : "ALCAReco",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeALCA",
                "ConfigCacheID" : processorDocs['ALCAReco'],
                "SplittingAlgo" : "FileBased",
                "GlobalTag" : "GlobalTagForALCAReco",
                "CMSSWVersion" : "CMSSW_3_1_3",
                "ScramArch" : "CompatibleALCAArch",

            },
            "Task4" : {
                "TaskName" : "Skims",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeRECO",
                "ConfigCacheID" : processorDocs['Skims'],
                "SplittingAlgo" : "FileBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        try:
            self.workload = factory.factoryWorkloadConstruction("YankingTheChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)

        testWMBSHelper = WMBSHelper(self.workload, "DigiHLT", "SomeBlock", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)


        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT"), arguments['Task1'], arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco"), arguments['Task2'],
                        arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco"),
                        arguments['Task3'], arguments)
        self._checkTask(self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims"),
                        arguments['Task4'], arguments)

        digi = self.workload.getTaskByPath("/YankingTheChain/DigiHLT")
        self.assertEqual(lumiDict, digi.getLumiMask())
        digiStep = digi.getStepHelper("cmsRun1")
        self.assertEqual(digiStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(digiStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(digiStep.getScramArch(), arguments['ScramArch'])

        # Make sure this task has a different lumilist than the global one
        reco = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco")
        self.assertEqual(lumiDict2, reco.getLumiMask())
        recoStep = reco.getStepHelper("cmsRun1")
        self.assertEqual(recoStep.getGlobalTag(), arguments['Task2']['GlobalTag'])
        self.assertEqual(recoStep.getCMSSWVersion(), arguments['Task2']['CMSSWVersion'])
        self.assertEqual(recoStep.getScramArch(), arguments['Task2']['ScramArch'])

        alca = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteALCA/ALCAReco")
        self.assertEqual(lumiDict, alca.getLumiMask())
        alcaStep = alca.getStepHelper("cmsRun1")
        self.assertEqual(alcaStep.getGlobalTag(), arguments['Task3']['GlobalTag'])
        self.assertEqual(alcaStep.getCMSSWVersion(), arguments['Task3']['CMSSWVersion'])
        self.assertEqual(alcaStep.getScramArch(), arguments['Task3']['ScramArch'])

        skim = self.workload.getTaskByPath("/YankingTheChain/DigiHLT/DigiHLTMergewriteRAWDIGI/Reco/RecoMergewriteRECO/Skims")
        skimStep = skim.getStepHelper("cmsRun1")
        self.assertEqual(skimStep.getGlobalTag(), arguments['GlobalTag'])
        self.assertEqual(skimStep.getCMSSWVersion(), arguments['CMSSWVersion'])
        self.assertEqual(skimStep.getScramArch(), arguments['ScramArch'])

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 14, "Number of output datasets doesn't match")
        self.assertTrue("/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI" in outputDatasets,
                        "/MinimumBias/ReleaseValidation-RawDigiFilter-v1/RAW-DIGI not in output datasets")
        self.assertTrue("/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI" in outputDatasets,
                        "/MinimumBias/ReleaseValidation-RawDebugDigiFilter-v1/RAW-DEBUG-DIGI not in output datasets")
        self.assertTrue("/ZeroBias/ReleaseValidation-reco-v1/RECO" in outputDatasets,
                        "/ZeroBias/ReleaseValidation-reco-v1/RECO not in output datasets")
        self.assertTrue("/ZeroBias/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
                        "/ZeroBias/ReleaseValidation-AOD-v1/AOD not in output datasets")
        self.assertTrue("/ZeroBias/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
                        "/ZeroBias/ReleaseValidation-alca-v1/ALCARECO not in output datasets")
        for i in range(1, 5):
            self.assertTrue("/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO" % i in outputDatasets,
                            "/MinimumBias/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets" % i)
        for i in range(1, 6):
            self.assertTrue("/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD" % i in outputDatasets,
                            "/MinimumBias/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets" % i)

        return
Ejemplo n.º 34
0
    def testGeneratorWorkflow(self):
        """
        _testGeneratorWorkflow_
        Test creating a request with an initial generator task
        it mocks a request where there are 2 similar paths starting
        from the generator, each one with a different PrimaryDataset, CMSSW configuration
        and processed dataset. Dropping the RAW output as well.
        Also include an ignored output module to keep things interesting...
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        processorDocs = makeProcessingConfigs(self.configDatabase)

        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 6,
            "IgnoredOutputModules": ["writeSkim2", "writeRAWDEBUGDIGI"],
            "Task1": {
                "TaskName": "GenSim",
                "ConfigCacheID": generatorDoc,
                "SplittingAlgo": "EventBased",
                "RequestNumEvents": 10000,
                "Seeding": "AutomaticSeeding",
                "PrimaryDataset": "RelValTTBar",
            },
            "Task2": {
                "TaskName": "DigiHLT_new",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgo": "LumiBased",
                "CMSSWVersion": "CMSSW_5_2_6",
                "GlobalTag": "GR_39_P_V5:All",
                "PrimaryDataset": "PURelValTTBar",
                "KeepOutput": False
            },
            "Task3": {
                "TaskName": "DigiHLT_ref",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgo": "EventBased",
                "CMSSWVersion": "CMSSW_5_2_7",
                "GlobalTag": "GR_40_P_V5:All",
                "AcquisitionEra": "ReleaseValidationNewConditions",
                "ProcessingVersion": 3,
                "ProcessingString": "Test",
                "KeepOutput": False
            },
            "Task4": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT_new",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "SplittingAlgo": "FileBased",
                "TransientOutputModules": ["writeRECO"]
            },
            "Task5": {
                "TaskName": "ALCAReco",
                "InputTask": "DigiHLT_ref",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['ALCAReco'],
                "SplittingAlgo": "LumiBased",
            },
            "Task6": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeRECO",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgo": "LumiBased",
            }
        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()

        # Test a malformed task chain definition
        arguments['Task4']['TransientOutputModules'].append('writeAOD')
        self.assertRaises(WMSpecFactoryException, factory.validateSchema,
                          arguments)

        arguments['Task4']['TransientOutputModules'].remove('writeAOD')
        try:
            self.workload = factory.factoryWorkloadConstruction(
                "PullingTheChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            import traceback
            traceback.print_exc()
            self.fail(msg)

        testWMBSHelper = WMBSHelper(self.workload,
                                    "GenSim",
                                    "SomeBlock",
                                    cachepath=self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(
            testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim")

        self._checkTask(firstTask, arguments['Task1'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"),
            arguments['Task2'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"),
            arguments['Task3'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"
            ), arguments['Task4'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"
            ), arguments['Task5'], arguments)
        self._checkTask(
            self.workload.getTaskByPath(
                "/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"
            ), arguments['Task6'], arguments)

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 11,
                         "Number of output datasets doesn't match")
        self.assertTrue(
            "/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM"
            in outputDatasets,
            "/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets"
        )
        self.assertFalse(
            "/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets,
            "/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets")
        self.assertTrue(
            "/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
            "/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets")
        self.assertTrue(
            "/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO"
            in outputDatasets,
            "/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets"
        )
        for i in range(1, 5):
            self.assertTrue(
                "/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i
                in outputDatasets,
                "/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets"
                % i)
        for i in range(1, 6):
            if i == 2:
                continue
            self.assertTrue(
                "/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i
                in outputDatasets,
                "/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets"
                % i)

        return
Ejemplo n.º 35
0
 def __call__(self, workflowName, args):
     workload = TaskChainWorkloadFactory.__call__(self, workflowName, args)
     #delattr(workload.taskIterator().next().steps().data.application.configuration,
     #        'configCacheUrl')
     return workload
Ejemplo n.º 36
0
    def testGeneratorWorkflow(self):
        """
        _testGeneratorWorkflow_
        Test creating a request with an initial generator task
        it mocks a request where there are 2 similar paths starting
        from the generator, each one with a different PrimaryDataset, CMSSW configuration
        and processed dataset. Dropping the RAW output as well.
        Also include an ignored output module to keep things interesting...
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        processorDocs = makeProcessingConfigs(self.configDatabase)

        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain" : 6,
            "IgnoredOutputModules" : ["writeSkim2", "writeRAWDEBUGDIGI"],
            "Task1" :{
                "TaskName" : "GenSim",
                "ConfigCacheID" : generatorDoc,
                "SplittingAlgo"  : "EventBased",
                "RequestNumEvents" : 10000,
                "Seeding" : "AutomaticSeeding",
                "PrimaryDataset" : "RelValTTBar",
            },
            "Task2" : {
                "TaskName" : "DigiHLT_new",
                "InputTask" : "GenSim",
                "InputFromOutputModule" : "writeGENSIM",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "SplittingAlgo" : "LumiBased",
                "CMSSWVersion" : "CMSSW_5_2_6",
                "GlobalTag" : "GR_39_P_V5:All",
                "PrimaryDataset" : "PURelValTTBar",
                "KeepOutput" : False
            },
            "Task3" : {
                "TaskName" : "DigiHLT_ref",
                "InputTask" : "GenSim",
                "InputFromOutputModule" : "writeGENSIM",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "SplittingAlgo" : "EventBased",
                "CMSSWVersion" : "CMSSW_5_2_7",
                "GlobalTag" : "GR_40_P_V5:All",
                "AcquisitionEra" : "ReleaseValidationNewConditions",
                "ProcessingVersion" : 3,
                "ProcessingString" : "Test",
                "KeepOutput" : False
            },
            "Task4" : {
                "TaskName" : "Reco",
                "InputTask" : "DigiHLT_new",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['Reco'],
                "SplittingAlgo" : "FileBased",
                "TransientOutputModules" : ["writeRECO"]
            },
            "Task5" : {
                "TaskName" : "ALCAReco",
                "InputTask" : "DigiHLT_ref",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['ALCAReco'],
                "SplittingAlgo" : "LumiBased",

            },
            "Task6" : {
                "TaskName" : "Skims",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeRECO",
                "ConfigCacheID" : processorDocs['Skims'],
                "SplittingAlgo" : "LumiBased",

            }

        }
        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()

        # Test a malformed task chain definition
        arguments['Task4']['TransientOutputModules'].append('writeAOD')
        self.assertRaises(WMSpecFactoryException, factory.validateSchema, arguments)

        arguments['Task4']['TransientOutputModules'].remove('writeAOD')
        try:
            self.workload = factory.factoryWorkloadConstruction("PullingTheChain", arguments)
        except Exception as ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            import traceback
            traceback.print_exc()
            self.fail(msg)

        testWMBSHelper = WMBSHelper(self.workload, "GenSim", "SomeBlock", cachepath = self.testInit.testDir)
        testWMBSHelper.createTopLevelFileset()
        testWMBSHelper._createSubscriptionsInWMBS(testWMBSHelper.topLevelTask, testWMBSHelper.topLevelFileset)

        firstTask = self.workload.getTaskByPath("/PullingTheChain/GenSim")

        self._checkTask(firstTask, arguments['Task1'], arguments)
        self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new"), arguments['Task2'], arguments)
        self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref"), arguments['Task3'], arguments)
        self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco"),
                        arguments['Task4'], arguments)
        self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_ref/ALCAReco"),
                        arguments['Task5'], arguments)
        self._checkTask(self.workload.getTaskByPath("/PullingTheChain/GenSim/GenSimMergewriteGENSIM/DigiHLT_new/Reco/Skims"),
                        arguments['Task6'], arguments)

        # Verify the output datasets
        outputDatasets = self.workload.listOutputDatasets()
        self.assertEqual(len(outputDatasets), 11, "Number of output datasets doesn't match")
        self.assertTrue("/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM" in outputDatasets,
                        "/RelValTTBar/ReleaseValidation-GenSimFilter-v1/GEN-SIM not in output datasets")
        self.assertFalse("/RelValTTBar/ReleaseValidation-reco-v1/RECO" in outputDatasets,
                        "/RelValTTBar/ReleaseValidation-reco-v1/RECO in output datasets")
        self.assertTrue("/RelValTTBar/ReleaseValidation-AOD-v1/AOD" in outputDatasets,
                        "/RelValTTBar/ReleaseValidation-AOD-v1/AOD not in output datasets")
        self.assertTrue("/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO" in outputDatasets,
                        "/RelValTTBar/ReleaseValidation-alca-v1/ALCARECO not in output datasets")
        for i in range(1, 5):
            self.assertTrue("/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO" % i in outputDatasets,
                            "/RelValTTBar/ReleaseValidation-alca%d-v1/ALCARECO not in output datasets" % i)
        for i in range(1, 6):
            if i == 2:
                continue
            self.assertTrue("/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD" % i in outputDatasets,
                            "/RelValTTBar/ReleaseValidation-skim%d-v1/RECO-AOD not in output datasets" % i)

        return
Ejemplo n.º 37
0
    def testGeneratorWorkflow(self):
        """
        _testGeneratorWorkflow_
        Test creating a request with an initial generator task
        it mocks a request where there are 2 similar paths starting
        from the generator, each one with a different PrimaryDataset, CMSSW configuration
        and processed dataset. Dropping the RAW output as well.
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        processorDocs = makeProcessingConfigs(self.configDatabase)

        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 6,
            "Task1": {
                "TaskName": "GenSim",
                "ConfigCacheID": generatorDoc,
                "SplittingAlgorithm": "EventBased",
                "SplittingArguments": {
                    "events_per_job": 250
                },
                "RequestNumEvents": 10000,
                "Seeding": "Automatic",
                "PrimaryDataset": "RelValTTBar",
            },
            "Task2": {
                "TaskName": "DigiHLT_new",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgorithm": "LumiBased",
                "SplittingArguments": {
                    "lumis_per_job": 2
                },
                "CMSSWVersion": "CMSSW_5_2_6",
                "GlobalTag": "GR_39_P_V5:All",
                "PrimaryDataset": "PURelValTTBar",
                "KeepOutput": False
            },
            "Task3": {
                "TaskName": "DigiHLT_ref",
                "InputTask": "GenSim",
                "InputFromOutputModule": "writeGENSIM",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgorithm": "EventBased",
                "SplittingArguments": {
                    "events_per_job": 100
                },
                "CMSSWVersion": "CMSSW_5_2_7",
                "GlobalTag": "GR_40_P_V5:All",
                "AcquisitionEra": "ReleaseValidationNewConditions",
                "ProcessingVersion": 3,
                "ProcessingString": "Test",
                "KeepOutput": False
            },
            "Task4": {
                "TaskName": "Reco",
                "InputTask": "DigiHLT_new",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "SplittingAlgorithm": "FileBased",
                "SplittingArguments": {
                    "files_per_job": 1
                },
            },
            "Task5": {
                "TaskName": "ALCAReco",
                "InputTask": "DigiHLT_ref",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['ALCAReco'],
                "SplittingAlgorithm": "LumiBased",
                "SplittingArguments": {
                    "lumis_per_job": 8
                },
            },
            "Task6": {
                "TaskName": "Skims",
                "InputTask": "Reco",
                "InputFromOutputModule": "writeRECO",
                "ConfigCacheID": processorDocs['Skims'],
                "SplittingAlgorithm": "LumiBased",
                "SplittingArguments": {
                    "lumis_per_job": 10
                },
            }
        }

        factory = TaskChainWorkloadFactory()

        try:
            self.workload = factory("PullingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)
Ejemplo n.º 38
0
    def testPileupTaskChain(self):
        """
        Test for multithreaded task chains where each step
        may run with a different number of cores
        """
        processorDocs = makeProcessingConfigs(self.configDatabase)

        testArguments = TaskChainWorkloadFactory.getTestArguments()
        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist": ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain": 2,
            "Task1": {
                "InputDataset": "/cosmics/whatever-input-v1/GEN-SIM",
                "TaskName": "DIGI",
                "ConfigCacheID": processorDocs['DigiHLT'],
                "SplittingAlgo": "LumiBased",
                "LumisPerJob": 4,
                "MCPileup": "/some/cosmics-mc-v1/GEN-SIM",
                "DeterministicPileup": True,
                "CMSSWVersion": "CMSSW_5_2_6",
                "GlobalTag": "GR_39_P_V5:All",
                "PrimaryDataset": "PURelValTTBar",
                "AcquisitionEra": "CMSSW_5_2_6",
                "ProcessingString": "ProcStr_Task1"
            },
            "Task2": {
                "TaskName": "RECO",
                "InputTask": "DIGI",
                "InputFromOutputModule": "writeRAWDIGI",
                "ConfigCacheID": processorDocs['Reco'],
                "DataPileup": "/some/minbias-data-v1/GEN-SIM",
                "SplittingAlgo": "LumiBased",
                "LumisPerJob": 2,
                "GlobalTag": "GR_R_62_V3::All",
                "AcquisitionEra": "CMSSW_5_2_7",
                "ProcessingString": "ProcStr_Task2"
            },
        }

        testArguments.update(arguments)
        arguments = testArguments

        factory = TaskChainWorkloadFactory()
        self.workload = factory.factoryWorkloadConstruction(
            "PullingTheChain", arguments)

        firstTask = self.workload.getTaskByPath("/PullingTheChain/DIGI")
        cmsRunStep = firstTask.getStep("cmsRun1").getTypeHelper()
        pileupData = cmsRunStep.getPileup()
        self.assertFalse(hasattr(pileupData, "data"))
        self.assertEqual(pileupData.mc.dataset,
                         ["/some/cosmics-mc-v1/GEN-SIM"])
        splitting = firstTask.jobSplittingParameters()
        self.assertTrue(splitting["deterministicPileup"])

        secondTask = self.workload.getTaskByPath(
            "/PullingTheChain/DIGI/DIGIMergewriteRAWDIGI/RECO")
        cmsRunStep = secondTask.getStep("cmsRun1").getTypeHelper()
        pileupData = cmsRunStep.getPileup()
        self.assertFalse(hasattr(pileupData, "mc"))
        self.assertEqual(pileupData.data.dataset,
                         ["/some/minbias-data-v1/GEN-SIM"])
        splitting = secondTask.jobSplittingParameters()
        self.assertFalse(splitting["deterministicPileup"])
Ejemplo n.º 39
0
 def __call__(self, workflowName, args):
     workload = TaskChainWorkloadFactory.__call__(self, workflowName, args)
     #delattr(workload.taskIterator().next().steps().data.application.configuration,
     #        'configCacheUrl')
     return workload
Ejemplo n.º 40
0
    def testGeneratorWorkflow(self):
        """
        _testGeneratorWorkflow_
        Test creating a request with an initial generator task
        it mocks a request where there are 2 similar paths starting
        from the generator, each one with a different PrimaryDataset, CMSSW configuration
        and processed dataset. Dropping the RAW output as well.
        Also include an ignored output module to keep things interesting...
        """
        generatorDoc = makeGeneratorConfig(self.configDatabase)
        processorDocs = makeProcessingConfigs(self.configDatabase)


        arguments = {
            "AcquisitionEra": "ReleaseValidation",
            "Requestor": "*****@*****.**",
            "CMSSWVersion": "CMSSW_3_5_8",
            "ScramArch": "slc5_ia32_gcc434",
            "ProcessingVersion": 1,
            "GlobalTag": "GR10_P_v4::All",
            "CouchURL": self.testInit.couchUrl,
            "CouchDBName": self.testInit.couchDbName,
            "SiteWhitelist" : ["T1_CH_CERN", "T1_US_FNAL"],
            "DashboardHost": "127.0.0.1",
            "DashboardPort": 8884,
            "TaskChain" : 6,
            "IgnoredOutputModules" : ["writeSkim2", "writeRAWDEBUGDIGI"],
            "Task1" :{
                "TaskName" : "GenSim",
                "ConfigCacheID" : generatorDoc,
                "SplittingAlgorithm"  : "EventBased",
                "SplittingArguments" : {"events_per_job" : 250},
                "RequestNumEvents" : 10000,
                "Seeding" : "Automatic",
                "PrimaryDataset" : "RelValTTBar",
            },
            "Task2" : {
                "TaskName" : "DigiHLT_new",
                "InputTask" : "GenSim",
                "InputFromOutputModule" : "writeGENSIM",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "SplittingAlgorithm" : "LumiBased",
                "SplittingArguments" : {"lumis_per_job" : 2 },
                "CMSSWVersion" : "CMSSW_5_2_6",
                "GlobalTag" : "GR_39_P_V5:All",
                "PrimaryDataset" : "PURelValTTBar",
                "KeepOutput" : False
            },
            "Task3" : {
                "TaskName" : "DigiHLT_ref",
                "InputTask" : "GenSim",
                "InputFromOutputModule" : "writeGENSIM",
                "ConfigCacheID" : processorDocs['DigiHLT'],
                "SplittingAlgorithm" : "EventBased",
                "SplittingArguments" : {"events_per_job" : 100 },
                "CMSSWVersion" : "CMSSW_5_2_7",
                "GlobalTag" : "GR_40_P_V5:All",
                "AcquisitionEra" : "ReleaseValidationNewConditions",
                "ProcessingVersion" : 3,
                "ProcessingString" : "Test",
                "KeepOutput" : False
            },
            "Task4" : {
                "TaskName" : "Reco",
                "InputTask" : "DigiHLT_new",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['Reco'],
                "SplittingAlgorithm" : "FileBased",
                "SplittingArguments" : {"files_per_job" : 1 },
                "TransientOutputModules" : ["writeRECO"]
            },
            "Task5" : {
                "TaskName" : "ALCAReco",
                "InputTask" : "DigiHLT_ref",
                "InputFromOutputModule" : "writeRAWDIGI",
                "ConfigCacheID" : processorDocs['ALCAReco'],
                "SplittingAlgorithm" : "LumiBased",
                "SplittingArguments" : {"lumis_per_job" : 8 },

            },
            "Task6" : {
                "TaskName" : "Skims",
                "InputTask" : "Reco",
                "InputFromOutputModule" : "writeRECO",
                "ConfigCacheID" : processorDocs['Skims'],
                "SplittingAlgorithm" : "LumiBased",
                "SplittingArguments" : {"lumis_per_job" : 10 },

            }

        }

        factory = TaskChainWorkloadFactory()

        # Test a malformed task chain definition
        arguments['Task4']['TransientOutputModules'].append('writeAOD')
        self.assertRaises(WMSpecFactoryException, factory.validateSchema, arguments)

        arguments['Task4']['TransientOutputModules'].remove('writeAOD')
        try:
            factory.validateSchema(arguments)
            self.workload = factory("PullingTheChain", arguments)
        except Exception, ex:
            msg = "Error invoking TaskChainWorkloadFactory:\n%s" % str(ex)
            self.fail(msg)
Ejemplo n.º 41
0
from __future__ import division, print_function

import unittest
from copy import deepcopy

from WMCore.ReqMgr.DataStructs.Request import initialize_clone
from WMCore.WMSpec.StdSpecs.ReReco import ReRecoWorkloadFactory
from WMCore.WMSpec.StdSpecs.StepChain import StepChainWorkloadFactory
from WMCore.WMSpec.StdSpecs.TaskChain import TaskChainWorkloadFactory
from WMCore.WMSpec.WMSpecErrors import WMSpecFactoryException

### Spec arguments definition with only key and its default value
RERECO_ARGS = ReRecoWorkloadFactory.getWorkloadCreateArgs()
STEPCHAIN_ARGS = StepChainWorkloadFactory.getWorkloadCreateArgs()
TASKCHAIN_ARGS = TaskChainWorkloadFactory.getWorkloadCreateArgs()
# inner Task/Step definition
STEP_ARGS = StepChainWorkloadFactory.getChainCreateArgs()
TASK_ARGS = TaskChainWorkloadFactory.getChainCreateArgs()

### Some original request dicts (ones to be cloned from)
rerecoOriginalArgs = {'Memory': 234, 'SkimName1': 'skim_2017', 'SkimInput1': 'RECOoutput',
                      'Skim1ConfigCacheID': 'abcdef',
                      'TimePerEvent': 1.2, 'RequestType': 'ReReco', 'RequestName': 'test_rereco'}
stepChainOriginalArgs = {'Memory': 1234, 'TimePerEvent': 1.2, 'RequestType': 'StepChain',
                         "ScramArch": ["slc6_amd64_gcc481"], "RequestName": "test_stepchain",
                         "Step1": {"ConfigCacheID": "blah", "GlobalTag": "PHYS18", "BlockWhitelist": ["A", "B"]},
                         "Step2": {"AcquisitionEra": "ACQERA", "ProcessingVersion": 3, "LumisPerJob": 10},
                         "StepChain": 2, "ConfigCacheID": None}
taskChainOriginalArgs = {'PrepID': None, 'Campaign': "MainTask", 'RequestType': 'TaskChain',
                         "ScramArch": ["slc6_amd64_gcc481", "slc7_amd64_gcc630"], "RequestName": "test_taskchain",