示例#1
0
文件: TaskChain.py 项目: dmwm/WMCore
    def modifyTaskConfiguration(self, taskConf,
                                firstTask=False, generator=False):
        """
        _modifyTaskConfiguration_

        Modify the TaskConfiguration according to the specifications
        in getWorkloadCreateArgs and getChainCreateArgs.
        It does type casting and assigns default values if key is not
        present, unless default value is None.
        """
        taskArguments = self.getChainCreateArgs(firstTask, generator)
        for argument in taskArguments:
            if argument not in taskConf and taskArguments[argument]["default"] is not None:
                taskConf[argument] = taskArguments[argument]["default"]
            elif argument in taskConf:
                taskConf[argument] = taskArguments[argument]["type"](taskConf[argument])

        if generator:
            taskConf["SplittingAlgo"] = "EventBased"
            # Adjust totalEvents according to the filter efficiency
            taskConf["RequestNumEvents"] = int(taskConf.get("RequestNumEvents", 0) / \
                                               taskConf.get("FilterEfficiency"))
            taskConf["SizePerEvent"] = taskConf.get("SizePerEvent", self.sizePerEvent) * \
                                       taskConf.get("FilterEfficiency")

        taskConf["SplittingArguments"] = {}
        if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
            taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
                                                                                             taskConf.get("EventsPerLumi"),
                                                                                             taskConf.get("TimePerEvent",
                                                                                                          self.timePerEvent),
                                                                                             taskConf.get("RequestNumEvents"))
            if firstTask:
                self.eventsPerJob = taskConf["EventsPerJob"]
                self.eventsPerLumi = taskConf["EventsPerLumi"]
            taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
            if taskConf["SplittingAlgo"] == "EventBased":
                taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
            else:
                taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600  # 2 days
            taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
        elif taskConf["SplittingAlgo"] == "LumiBased":
            taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
        elif taskConf["SplittingAlgo"] == "FileBased":
            taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]

        taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])

        taskConf["PileupConfig"] = parsePileupConfig(taskConf.get("MCPileup"),
                                                     taskConf.get("DataPileup"))
        # Adjust the pileup splitting
        taskConf["SplittingArguments"].setdefault("deterministicPileup", taskConf['DeterministicPileup'])

        return
示例#2
0
    def modifyTaskConfiguration(self, taskConf,
                                firstTask=False, generator=False):
        """
        _modifyTaskConfiguration_

        Modify the TaskConfiguration according to the specifications
        in getWorkloadCreateArgs and getChainCreateArgs.
        It does type casting and assigns default values if key is not
        present, unless default value is None.
        """
        taskArguments = self.getChainCreateArgs(firstTask, generator)
        for argument in taskArguments:
            if argument not in taskConf and taskArguments[argument]["default"] is not None:
                taskConf[argument] = taskArguments[argument]["default"]
            elif argument in taskConf:
                taskConf[argument] = taskArguments[argument]["type"](taskConf[argument])

        if generator:
            taskConf["SplittingAlgo"] = "EventBased"
            # Adjust totalEvents according to the filter efficiency
            taskConf["RequestNumEvents"] = int(taskConf.get("RequestNumEvents", 0) / \
                                               taskConf.get("FilterEfficiency"))
            taskConf["SizePerEvent"] = taskConf.get("SizePerEvent", self.sizePerEvent) * \
                                       taskConf.get("FilterEfficiency")

        taskConf["SplittingArguments"] = {}
        if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
            taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
                                                                                             taskConf.get("EventsPerLumi"),
                                                                                             taskConf.get("TimePerEvent",
                                                                                                          self.timePerEvent))
            if firstTask:
                self.eventsPerJob = taskConf["EventsPerJob"]
                self.eventsPerLumi = taskConf["EventsPerLumi"]
            taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
            if taskConf["SplittingAlgo"] == "EventBased":
                taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
            else:
                taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600  # 2 days
            taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
        elif taskConf["SplittingAlgo"] == "LumiBased":
            taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
        elif taskConf["SplittingAlgo"] == "FileBased":
            taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]

        taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])

        taskConf["PileupConfig"] = parsePileupConfig(taskConf.get("MCPileup"),
                                                     taskConf.get("DataPileup"))
        # Adjust the pileup splitting
        taskConf["SplittingArguments"].setdefault("deterministicPileup", taskConf['DeterministicPileup'])

        return
示例#3
0
    def testCalcEvtsPerJobLumi(self):
        """
        _testCalcEvtsPerJobLumi_

        Check that EventsPerJob and EventsPerLumi are properly calculated
        for EventBased job splitting.
        """
        self.assertEqual((123, 123), StdBase.calcEvtsPerJobLumi(123, 345, 1))
        self.assertEqual((123, 123), StdBase.calcEvtsPerJobLumi(123, None, 1))

        self.assertEqual((28800, 100), StdBase.calcEvtsPerJobLumi(None, 100, 1))
        self.assertEqual((600, 100), StdBase.calcEvtsPerJobLumi(None, 100, 50.5))
        self.assertEqual((570, 570), StdBase.calcEvtsPerJobLumi(None, 1000, 50.5))

        self.assertEqual((23040, 23040), StdBase.calcEvtsPerJobLumi(None, None, 1.25))
        self.assertEqual((229, 229), StdBase.calcEvtsPerJobLumi(None, None, 125.5))

        self.assertEqual((23528, 11764), StdBase.calcEvtsPerJobLumi(24000, 11764, 10.157120496967591))
        self.assertEqual((2835, 2835), StdBase.calcEvtsPerJobLumi(None, 11764, 10.157120496967591))

        return
示例#4
0
    def modifyJobSplitting(self, taskConf, generator):
        """
        _modifyJobSplitting_

        Adapt job splitting according to the first step configuration
        or lack of some of them.
        """
        if generator:
            requestNumEvts = int(taskConf.get("RequestNumEvents", 0))
            filterEff = taskConf.get("FilterEfficiency")
            # Adjust totalEvents according to the filter efficiency
            taskConf["SplittingAlgo"] = "EventBased"
            taskConf["RequestNumEvents"] = int(requestNumEvts / filterEff)
            taskConf["SizePerEvent"] = self.sizePerEvent * filterEff

        taskConf["SplittingArguments"] = {}
        if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
            taskConf["EventsPerJob"], taskConf[
                "EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(
                    taskConf.get("EventsPerJob"),
                    taskConf.get("EventsPerLumi"), self.timePerEvent,
                    taskConf.get("RequestNumEvents"))
            self.eventsPerJob = taskConf["EventsPerJob"]
            self.eventsPerLumi = taskConf["EventsPerLumi"]
            taskConf["SplittingArguments"]["events_per_job"] = taskConf[
                "EventsPerJob"]
            if taskConf["SplittingAlgo"] == "EventBased":
                taskConf["SplittingArguments"]["events_per_lumi"] = taskConf[
                    "EventsPerLumi"]
            else:
                taskConf["SplittingArguments"][
                    "job_time_limit"] = 48 * 3600  # 2 days
            taskConf["SplittingArguments"]["lheInputFiles"] = taskConf[
                "LheInputFiles"]
        elif taskConf["SplittingAlgo"] == "LumiBased":
            taskConf["SplittingArguments"]["lumis_per_job"] = taskConf[
                "LumisPerJob"]
        elif taskConf["SplittingAlgo"] == "FileBased":
            taskConf["SplittingArguments"]["files_per_job"] = taskConf[
                "FilesPerJob"]

        taskConf["SplittingArguments"].setdefault("include_parents",
                                                  taskConf['IncludeParents'])
        taskConf["SplittingArguments"].setdefault("deterministicPileup",
                                                  self.deterministicPileup)

        return
示例#5
0
    def __call__(self, workloadName, arguments):
        """
        Store the arguments in attributes with the proper
        formatting.
        """
        StdBase.__call__(self, workloadName, arguments)

        # Adjust the events by the filter efficiency
        self.totalEvents = int(self.requestNumEvents / self.filterEfficiency)

        # We don't write out every event in MC,
        # adjust the size per event accordingly
        self.sizePerEvent = self.sizePerEvent * self.filterEfficiency

        # Tune the splitting, only EventBased is allowed for MonteCarlo
        # 8h jobs are CMS standard, set the default with that in mind
        self.prodJobSplitAlgo = "EventBased"
        self.eventsPerJob, self.eventsPerLumi = StdBase.calcEvtsPerJobLumi(
            self.eventsPerJob, self.eventsPerLumi, self.timePerEvent)

        self.prodJobSplitArgs = {
            "events_per_job": self.eventsPerJob,
            "events_per_lumi": self.eventsPerLumi,
            "lheInputFiles": self.lheInputFiles
        }

        # Transform the pileup as required by the CMSSW step
        self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
        # Adjust the pileup splitting
        self.prodJobSplitArgs.setdefault("deterministicPileup",
                                         self.deterministicPileup)

        # Production can be extending statistics,
        # need to move the initial lfn counter
        self.previousJobCount = 0
        if self.firstLumi > 1:
            self.previousJobCount = int(
                math.ceil((self.firstEvent - 1) / self.eventsPerJob))
            self.prodJobSplitArgs[
                "initial_lfn_counter"] = self.previousJobCount

        # Feed values back to save in couch
        arguments['EventsPerJob'] = self.eventsPerJob

        return self.buildWorkload()
示例#6
0
    def __call__(self, workloadName, arguments):
        """
        Store the arguments in attributes with the proper
        formatting.
        """
        StdBase.__call__(self, workloadName, arguments)

        # Adjust the events by the filter efficiency
        self.totalEvents = int(self.requestNumEvents / self.filterEfficiency)

        # We don't write out every event in MC,
        # adjust the size per event accordingly
        self.sizePerEvent = self.sizePerEvent * self.filterEfficiency

        # Tune the splitting, only EventBased is allowed for MonteCarlo
        # 8h jobs are CMS standard, set the default with that in mind
        self.prodJobSplitAlgo = "EventBased"
        self.eventsPerJob, self.eventsPerLumi = StdBase.calcEvtsPerJobLumi(self.eventsPerJob,
                                                                           self.eventsPerLumi,
                                                                           self.timePerEvent)

        self.prodJobSplitArgs = {"events_per_job": self.eventsPerJob,
                                 "events_per_lumi": self.eventsPerLumi,
                                 "lheInputFiles": self.lheInputFiles}

        # Transform the pileup as required by the CMSSW step
        self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
        # Adjust the pileup splitting
        self.prodJobSplitArgs.setdefault("deterministicPileup", self.deterministicPileup)

        # Production can be extending statistics,
        # need to move the initial lfn counter
        self.previousJobCount = 0
        if self.firstLumi > 1:
            self.previousJobCount = int(math.ceil((self.firstEvent - 1) / self.eventsPerJob))
            self.prodJobSplitArgs["initial_lfn_counter"] = self.previousJobCount

        # Feed values back to save in couch
        arguments['EventsPerJob'] = self.eventsPerJob

        return self.buildWorkload()
示例#7
0
文件: StepChain.py 项目: dmwm/WMCore
    def modifyJobSplitting(self, taskConf, generator):
        """
        _modifyJobSplitting_

        Adapt job splitting according to the first step configuration
        or lack of some of them.
        """
        if generator:
            requestNumEvts = int(taskConf.get("RequestNumEvents", 0))
            filterEff = taskConf.get("FilterEfficiency")
            # Adjust totalEvents according to the filter efficiency
            taskConf["SplittingAlgo"] = "EventBased"
            taskConf["RequestNumEvents"] = int(requestNumEvts / filterEff)
            taskConf["SizePerEvent"] = self.sizePerEvent * filterEff

        taskConf["SplittingArguments"] = {}
        if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
            taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
                                                                                             taskConf.get("EventsPerLumi"),
                                                                                             self.timePerEvent,
                                                                                             taskConf.get("RequestNumEvents"))
            self.eventsPerJob = taskConf["EventsPerJob"]
            self.eventsPerLumi = taskConf["EventsPerLumi"]
            taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
            if taskConf["SplittingAlgo"] == "EventBased":
                taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
            else:
                taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600  # 2 days
            taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
        elif taskConf["SplittingAlgo"] == "LumiBased":
            taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
        elif taskConf["SplittingAlgo"] == "FileBased":
            taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]

        taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])
        taskConf["SplittingArguments"].setdefault("deterministicPileup", self.deterministicPileup)

        return