def modifyTaskConfiguration(self, taskConf,
firstTask=False, generator=False):
"""
_modifyTaskConfiguration_
Modify the TaskConfiguration according to the specifications
in getWorkloadCreateArgs and getChainCreateArgs.
It does type casting and assigns default values if key is not
present, unless default value is None.
"""
taskArguments = self.getChainCreateArgs(firstTask, generator)
for argument in taskArguments:
if argument not in taskConf and taskArguments[argument]["default"] is not None:
taskConf[argument] = taskArguments[argument]["default"]
elif argument in taskConf:
taskConf[argument] = taskArguments[argument]["type"](taskConf[argument])
if generator:
taskConf["SplittingAlgo"] = "EventBased"
# Adjust totalEvents according to the filter efficiency
taskConf["RequestNumEvents"] = int(taskConf.get("RequestNumEvents", 0) / \
taskConf.get("FilterEfficiency"))
taskConf["SizePerEvent"] = taskConf.get("SizePerEvent", self.sizePerEvent) * \
taskConf.get("FilterEfficiency")
taskConf["SplittingArguments"] = {}
if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
taskConf.get("EventsPerLumi"),
taskConf.get("TimePerEvent",
self.timePerEvent),
taskConf.get("RequestNumEvents"))
if firstTask:
self.eventsPerJob = taskConf["EventsPerJob"]
self.eventsPerLumi = taskConf["EventsPerLumi"]
taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
if taskConf["SplittingAlgo"] == "EventBased":
taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
else:
taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600 # 2 days
taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
elif taskConf["SplittingAlgo"] == "LumiBased":
taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
elif taskConf["SplittingAlgo"] == "FileBased":
taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]
taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])
taskConf["PileupConfig"] = parsePileupConfig(taskConf.get("MCPileup"),
taskConf.get("DataPileup"))
# Adjust the pileup splitting
taskConf["SplittingArguments"].setdefault("deterministicPileup", taskConf['DeterministicPileup'])
return
def modifyTaskConfiguration(self, taskConf,
firstTask=False, generator=False):
"""
_modifyTaskConfiguration_
Modify the TaskConfiguration according to the specifications
in getWorkloadCreateArgs and getChainCreateArgs.
It does type casting and assigns default values if key is not
present, unless default value is None.
"""
taskArguments = self.getChainCreateArgs(firstTask, generator)
for argument in taskArguments:
if argument not in taskConf and taskArguments[argument]["default"] is not None:
taskConf[argument] = taskArguments[argument]["default"]
elif argument in taskConf:
taskConf[argument] = taskArguments[argument]["type"](taskConf[argument])
if generator:
taskConf["SplittingAlgo"] = "EventBased"
# Adjust totalEvents according to the filter efficiency
taskConf["RequestNumEvents"] = int(taskConf.get("RequestNumEvents", 0) / \
taskConf.get("FilterEfficiency"))
taskConf["SizePerEvent"] = taskConf.get("SizePerEvent", self.sizePerEvent) * \
taskConf.get("FilterEfficiency")
taskConf["SplittingArguments"] = {}
if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
taskConf.get("EventsPerLumi"),
taskConf.get("TimePerEvent",
self.timePerEvent))
if firstTask:
self.eventsPerJob = taskConf["EventsPerJob"]
self.eventsPerLumi = taskConf["EventsPerLumi"]
taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
if taskConf["SplittingAlgo"] == "EventBased":
taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
else:
taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600 # 2 days
taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
elif taskConf["SplittingAlgo"] == "LumiBased":
taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
elif taskConf["SplittingAlgo"] == "FileBased":
taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]
taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])
taskConf["PileupConfig"] = parsePileupConfig(taskConf.get("MCPileup"),
taskConf.get("DataPileup"))
# Adjust the pileup splitting
taskConf["SplittingArguments"].setdefault("deterministicPileup", taskConf['DeterministicPileup'])
return
def testCalcEvtsPerJobLumi(self):
"""
_testCalcEvtsPerJobLumi_
Check that EventsPerJob and EventsPerLumi are properly calculated
for EventBased job splitting.
"""
self.assertEqual((123, 123), StdBase.calcEvtsPerJobLumi(123, 345, 1))
self.assertEqual((123, 123), StdBase.calcEvtsPerJobLumi(123, None, 1))
self.assertEqual((28800, 100), StdBase.calcEvtsPerJobLumi(None, 100, 1))
self.assertEqual((600, 100), StdBase.calcEvtsPerJobLumi(None, 100, 50.5))
self.assertEqual((570, 570), StdBase.calcEvtsPerJobLumi(None, 1000, 50.5))
self.assertEqual((23040, 23040), StdBase.calcEvtsPerJobLumi(None, None, 1.25))
self.assertEqual((229, 229), StdBase.calcEvtsPerJobLumi(None, None, 125.5))
self.assertEqual((23528, 11764), StdBase.calcEvtsPerJobLumi(24000, 11764, 10.157120496967591))
self.assertEqual((2835, 2835), StdBase.calcEvtsPerJobLumi(None, 11764, 10.157120496967591))
return
def modifyJobSplitting(self, taskConf, generator):
"""
_modifyJobSplitting_
Adapt job splitting according to the first step configuration
or lack of some of them.
"""
if generator:
requestNumEvts = int(taskConf.get("RequestNumEvents", 0))
filterEff = taskConf.get("FilterEfficiency")
# Adjust totalEvents according to the filter efficiency
taskConf["SplittingAlgo"] = "EventBased"
taskConf["RequestNumEvents"] = int(requestNumEvts / filterEff)
taskConf["SizePerEvent"] = self.sizePerEvent * filterEff
taskConf["SplittingArguments"] = {}
if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
taskConf["EventsPerJob"], taskConf[
"EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(
taskConf.get("EventsPerJob"),
taskConf.get("EventsPerLumi"), self.timePerEvent,
taskConf.get("RequestNumEvents"))
self.eventsPerJob = taskConf["EventsPerJob"]
self.eventsPerLumi = taskConf["EventsPerLumi"]
taskConf["SplittingArguments"]["events_per_job"] = taskConf[
"EventsPerJob"]
if taskConf["SplittingAlgo"] == "EventBased":
taskConf["SplittingArguments"]["events_per_lumi"] = taskConf[
"EventsPerLumi"]
else:
taskConf["SplittingArguments"][
"job_time_limit"] = 48 * 3600 # 2 days
taskConf["SplittingArguments"]["lheInputFiles"] = taskConf[
"LheInputFiles"]
elif taskConf["SplittingAlgo"] == "LumiBased":
taskConf["SplittingArguments"]["lumis_per_job"] = taskConf[
"LumisPerJob"]
elif taskConf["SplittingAlgo"] == "FileBased":
taskConf["SplittingArguments"]["files_per_job"] = taskConf[
"FilesPerJob"]
taskConf["SplittingArguments"].setdefault("include_parents",
taskConf['IncludeParents'])
taskConf["SplittingArguments"].setdefault("deterministicPileup",
self.deterministicPileup)
return
def __call__(self, workloadName, arguments):
"""
Store the arguments in attributes with the proper
formatting.
"""
StdBase.__call__(self, workloadName, arguments)
# Adjust the events by the filter efficiency
self.totalEvents = int(self.requestNumEvents / self.filterEfficiency)
# We don't write out every event in MC,
# adjust the size per event accordingly
self.sizePerEvent = self.sizePerEvent * self.filterEfficiency
# Tune the splitting, only EventBased is allowed for MonteCarlo
# 8h jobs are CMS standard, set the default with that in mind
self.prodJobSplitAlgo = "EventBased"
self.eventsPerJob, self.eventsPerLumi = StdBase.calcEvtsPerJobLumi(
self.eventsPerJob, self.eventsPerLumi, self.timePerEvent)
self.prodJobSplitArgs = {
"events_per_job": self.eventsPerJob,
"events_per_lumi": self.eventsPerLumi,
"lheInputFiles": self.lheInputFiles
}
# Transform the pileup as required by the CMSSW step
self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
# Adjust the pileup splitting
self.prodJobSplitArgs.setdefault("deterministicPileup",
self.deterministicPileup)
# Production can be extending statistics,
# need to move the initial lfn counter
self.previousJobCount = 0
if self.firstLumi > 1:
self.previousJobCount = int(
math.ceil((self.firstEvent - 1) / self.eventsPerJob))
self.prodJobSplitArgs[
"initial_lfn_counter"] = self.previousJobCount
# Feed values back to save in couch
arguments['EventsPerJob'] = self.eventsPerJob
return self.buildWorkload()
def __call__(self, workloadName, arguments):
"""
Store the arguments in attributes with the proper
formatting.
"""
StdBase.__call__(self, workloadName, arguments)
# Adjust the events by the filter efficiency
self.totalEvents = int(self.requestNumEvents / self.filterEfficiency)
# We don't write out every event in MC,
# adjust the size per event accordingly
self.sizePerEvent = self.sizePerEvent * self.filterEfficiency
# Tune the splitting, only EventBased is allowed for MonteCarlo
# 8h jobs are CMS standard, set the default with that in mind
self.prodJobSplitAlgo = "EventBased"
self.eventsPerJob, self.eventsPerLumi = StdBase.calcEvtsPerJobLumi(self.eventsPerJob,
self.eventsPerLumi,
self.timePerEvent)
self.prodJobSplitArgs = {"events_per_job": self.eventsPerJob,
"events_per_lumi": self.eventsPerLumi,
"lheInputFiles": self.lheInputFiles}
# Transform the pileup as required by the CMSSW step
self.pileupConfig = parsePileupConfig(self.mcPileup, self.dataPileup)
# Adjust the pileup splitting
self.prodJobSplitArgs.setdefault("deterministicPileup", self.deterministicPileup)
# Production can be extending statistics,
# need to move the initial lfn counter
self.previousJobCount = 0
if self.firstLumi > 1:
self.previousJobCount = int(math.ceil((self.firstEvent - 1) / self.eventsPerJob))
self.prodJobSplitArgs["initial_lfn_counter"] = self.previousJobCount
# Feed values back to save in couch
arguments['EventsPerJob'] = self.eventsPerJob
return self.buildWorkload()
def modifyJobSplitting(self, taskConf, generator):
"""
_modifyJobSplitting_
Adapt job splitting according to the first step configuration
or lack of some of them.
"""
if generator:
requestNumEvts = int(taskConf.get("RequestNumEvents", 0))
filterEff = taskConf.get("FilterEfficiency")
# Adjust totalEvents according to the filter efficiency
taskConf["SplittingAlgo"] = "EventBased"
taskConf["RequestNumEvents"] = int(requestNumEvts / filterEff)
taskConf["SizePerEvent"] = self.sizePerEvent * filterEff
taskConf["SplittingArguments"] = {}
if taskConf["SplittingAlgo"] in ["EventBased", "EventAwareLumiBased"]:
taskConf["EventsPerJob"], taskConf["EventsPerLumi"] = StdBase.calcEvtsPerJobLumi(taskConf.get("EventsPerJob"),
taskConf.get("EventsPerLumi"),
self.timePerEvent,
taskConf.get("RequestNumEvents"))
self.eventsPerJob = taskConf["EventsPerJob"]
self.eventsPerLumi = taskConf["EventsPerLumi"]
taskConf["SplittingArguments"]["events_per_job"] = taskConf["EventsPerJob"]
if taskConf["SplittingAlgo"] == "EventBased":
taskConf["SplittingArguments"]["events_per_lumi"] = taskConf["EventsPerLumi"]
else:
taskConf["SplittingArguments"]["job_time_limit"] = 48 * 3600 # 2 days
taskConf["SplittingArguments"]["lheInputFiles"] = taskConf["LheInputFiles"]
elif taskConf["SplittingAlgo"] == "LumiBased":
taskConf["SplittingArguments"]["lumis_per_job"] = taskConf["LumisPerJob"]
elif taskConf["SplittingAlgo"] == "FileBased":
taskConf["SplittingArguments"]["files_per_job"] = taskConf["FilesPerJob"]
taskConf["SplittingArguments"].setdefault("include_parents", taskConf['IncludeParents'])
taskConf["SplittingArguments"].setdefault("deterministicPileup", self.deterministicPileup)
return