def createScriptLines(opts, pklIn):
"""
prepares a bash script fragment which tweaks the PSet params according to opts
returns a string containing the script lines separated by '\n'
"""
runAndLumis = {}
if opts.runAndLumis:
runAndLumis = readFileFromTarball(opts.runAndLumis,
'run_and_lumis.tar.gz')
inputFiles = {}
if opts.inputFile:
inputFiles = readFileFromTarball(opts.inputFile, 'input_files.tar.gz')
# build a tweak object with the needed changes to be applied to PSet
tweak = PSetTweak()
# add tweaks
# inputFile will always be present
# inputFile can have three formats depending on wether secondary input files are used:
# 1. a single LFN as a string : "/store/.....root"
# 2. a list of LFNs : ["/store/.....root", "/store/....root", ...]
# 3. a list of dictionaries (one per file) with keys: 'lfn' and 'parents'
# value for 'lfn' is a string, value for 'parents' is a list of {'lfn':lfn} dictionaries
# [{'lfn':inputlfn, 'parents':[{'lfn':parentlfn1},{'lfn':parentlfn2}], ....]},...]
# to properly prepare the tweak we reuse code fom WMTweak.py:
# https://github.com/dmwm/WMCore/blob/bb573b442a53717057c169b05ae4fae98f31063b/src/python/PSetTweaks/WMTweak.py#L415-L441
primaryFiles = []
secondaryFiles = []
for inputFile in inputFiles:
# make sure input is always in format 3.
if not isinstance(inputFile, dict):
inputFile = {'lfn': inputFile, 'parents': []}
if inputFile["lfn"].startswith("MCFakeFile"):
# for MC which uses "EmptySource" there must be no inputFile
continue
primaryFiles.append(inputFile["lfn"])
for secondaryFile in inputFile["parents"]:
secondaryFiles.append(secondaryFile["lfn"])
print("Adding %d files to 'fileNames' attr" % len(primaryFiles))
print("Adding %d files to 'secondaryFileNames' attr" % len(secondaryFiles))
if len(primaryFiles) > 0:
tweak.addParameter(
"process.source.fileNames",
"customTypeCms.untracked.vstring(%s)" % primaryFiles)
if len(secondaryFiles) > 0:
tweak.addParameter(
"process.source.secondaryFileNames",
"customTypeCms.untracked.vstring(%s)" % secondaryFiles)
# for rearranging runsAndLumis into the structure needed by CMSSW, reuse code taken from
# https://github.com/dmwm/WMCore/blob/bb573b442a53717057c169b05ae4fae98f31063b/src/python/PSetTweaks/WMTweak.py#L482
if runAndLumis:
lumisToProcess = []
for run in runAndLumis.keys():
lumiPairs = runAndLumis[run]
for lumiPair in lumiPairs:
if len(lumiPair) != 2:
# Do nothing
continue
lumisToProcess.append("%s:%s-%s:%s" %
(run, lumiPair[0], run, lumiPair[1]))
tweak.addParameter(
"process.source.lumisToProcess",
"customTypeCms.untracked.VLuminosityBlockRange(%s)" %
lumisToProcess)
# how many events to process
if opts.firstEvent:
tweak.addParameter(
"process.source.firstEvent",
"customTypeCms.untracked.uint32(%s)" % opts.firstEvent)
if opts.firstEvent is None or opts.lastEvent is None:
# what to process is define in runAndLumis, we do no split by events here
maxEvents = -1
else:
# for MC CRAB passes 1st/last event, but cmsRun wants 1st ev + MaxEvents
maxEvents = int(opts.lastEvent) - int(opts.firstEvent) + 1
opts.lastEvent = None # for MC there has to be no lastEvent
tweak.addParameter("process.maxEvents.input",
"customTypeCms.untracked.int32(%s)" % maxEvents)
if opts.lastEvent:
tweak.addParameter(
"process.source.lastEvent",
"customTypeCms.untracked.uint32(%s)" % opts.lastEvent)
# firstLumi, firstRun and eventsPerLumi are used for MC
if opts.firstLumi:
tweak.addParameter(
"process.source.firstLuminosityBlock",
"customTypeCms.untracked.uint32(%s)" % opts.firstLumi)
if opts.firstRun:
tweak.addParameter(
"process.source.firstRun",
"customTypeCms.untracked.uint32(%s)" % opts.firstRun)
if opts.eventsPerLumi:
numberEventsInLuminosityBlock = "customTypeCms.untracked.uint32(%s)" % opts.eventsPerLumi
tweak.addParameter("process.source.numberEventsInLuminosityBlock",
numberEventsInLuminosityBlock)
# time-limited running is used by automatic splitting probe jobs
if opts.maxRuntime:
maxSecondsUntilRampdown = "customTypeCms.untracked.int32(%s)" % opts.maxRuntime
tweak.addParameter("process.maxSecondsUntilRampdown.input",
maxSecondsUntilRampdown)
# event limiter for testing
if opts.oneEventMode in ["1", "True", True]:
tweak.addParameter("process.maxEvents.input",
"customTypeCms.untracked.int32(1)")
# make sure that FJR contains useful statistics, reuse code from
# https://github.com/dmwm/WMCore/blob/c2fa70af3b4c5285d50e6a8bf48636232f738340/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py#L289-L307
tweak.addParameter("process.CPU", "customTypeCms.Service('CPU')")
tweak.addParameter(
"process.Timing",
"customTypeCms.Service('Timing', summaryOnly=cms.untracked.bool(True))"
)
tweak.addParameter(
"process.SimpleMemoryCheck",
"customTypeCms.Service('SimpleMemoryCheck', jobReportOutputOnly=cms.untracked.bool(True))"
)
# tweak !
psetTweakJson = "PSetTweak.json"
tweak.persist(psetTweakJson, formatting='simplejson')
procScript = "edm_pset_tweak.py"
pklOut = pklIn + '-tweaked'
# we always create untracked psets in our tweaks
cmd = "%s --input_pkl %s --output_pkl %s --json %s --create_untracked_psets" % (
procScript, pklIn, pklOut, psetTweakJson)
commandLines = createTweakingCommandLines(cmd, pklIn, pklOut)
# there a few more things to do which require running different EDM/CMSSW commands
#1. enable LazyDownload of LHE files (if needed)
if opts.lheInputFiles:
pklOut = pklIn + '-lazy'
procScript = "cmssw_enable_lazy_download.py"
cmd = "%s --input_pkl %s --output_pkl %s" % (procScript, pklIn, pklOut)
moreLines = createTweakingCommandLines(cmd, pklIn, pklOut)
commandLines += moreLines
#2. make sure random seeds are initialized
pklOut = pklIn + '-seeds'
procScript = "cmssw_handle_random_seeds.py"
cmd = "%s --input_pkl %s --output_pkl %s --seeding dummy" % (procScript,
pklIn, pklOut)
moreLines += createTweakingCommandLines(cmd, pklIn, pklOut)
commandLines += moreLines
#3. make sure that process.maxEvents.input is propagated to Producers, see:
# https://github.com/dmwm/WMCore/blob/85d6d423f0a85fdedf78b65ca8b7b81af9263789/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py#L448-L465
pklOut = pklIn + '-nEvents'
procScript = 'cmssw_handle_nEvents.py'
cmd = "%s --input_pkl %s --output_pkl %s" % (procScript, pklIn, pklOut)
moreLines = createTweakingCommandLines(cmd, pklIn, pklOut)
commandLines += moreLines
return commandLines
