def createScriptLines(opts, pklIn): """ prepares a bash script fragment which tweaks the PSet params according to opts returns a string containing the script lines separated by '\n' """ runAndLumis = {} if opts.runAndLumis: runAndLumis = readFileFromTarball(opts.runAndLumis, 'run_and_lumis.tar.gz') inputFiles = {} if opts.inputFile: inputFiles = readFileFromTarball(opts.inputFile, 'input_files.tar.gz') # build a tweak object with the needed changes to be applied to PSet tweak = PSetTweak() # add tweaks # inputFile will always be present # inputFile can have three formats depending on wether secondary input files are used: # 1. a single LFN as a string : "/store/.....root" # 2. a list of LFNs : ["/store/.....root", "/store/....root", ...] # 3. a list of dictionaries (one per file) with keys: 'lfn' and 'parents' # value for 'lfn' is a string, value for 'parents' is a list of {'lfn':lfn} dictionaries # [{'lfn':inputlfn, 'parents':[{'lfn':parentlfn1},{'lfn':parentlfn2}], ....]},...] # to properly prepare the tweak we reuse code fom WMTweak.py: # https://github.com/dmwm/WMCore/blob/bb573b442a53717057c169b05ae4fae98f31063b/src/python/PSetTweaks/WMTweak.py#L415-L441 primaryFiles = [] secondaryFiles = [] for inputFile in inputFiles: # make sure input is always in format 3. if not isinstance(inputFile, dict): inputFile = {'lfn': inputFile, 'parents': []} if inputFile["lfn"].startswith("MCFakeFile"): # for MC which uses "EmptySource" there must be no inputFile continue primaryFiles.append(inputFile["lfn"]) for secondaryFile in inputFile["parents"]: secondaryFiles.append(secondaryFile["lfn"]) print("Adding %d files to 'fileNames' attr" % len(primaryFiles)) print("Adding %d files to 'secondaryFileNames' attr" % len(secondaryFiles)) if len(primaryFiles) > 0: tweak.addParameter( "process.source.fileNames", "customTypeCms.untracked.vstring(%s)" % primaryFiles) if len(secondaryFiles) > 0: tweak.addParameter( "process.source.secondaryFileNames", "customTypeCms.untracked.vstring(%s)" % secondaryFiles) # for rearranging runsAndLumis into the structure needed by CMSSW, reuse code taken from # https://github.com/dmwm/WMCore/blob/bb573b442a53717057c169b05ae4fae98f31063b/src/python/PSetTweaks/WMTweak.py#L482 if runAndLumis: lumisToProcess = [] for run in runAndLumis.keys(): lumiPairs = runAndLumis[run] for lumiPair in lumiPairs: if len(lumiPair) != 2: # Do nothing continue lumisToProcess.append("%s:%s-%s:%s" % (run, lumiPair[0], run, lumiPair[1])) tweak.addParameter( "process.source.lumisToProcess", "customTypeCms.untracked.VLuminosityBlockRange(%s)" % lumisToProcess) # how many events to process if opts.firstEvent: tweak.addParameter( "process.source.firstEvent", "customTypeCms.untracked.uint32(%s)" % opts.firstEvent) if opts.firstEvent is None or opts.lastEvent is None: # what to process is define in runAndLumis, we do no split by events here maxEvents = -1 else: # for MC CRAB passes 1st/last event, but cmsRun wants 1st ev + MaxEvents maxEvents = int(opts.lastEvent) - int(opts.firstEvent) + 1 opts.lastEvent = None # for MC there has to be no lastEvent tweak.addParameter("process.maxEvents.input", "customTypeCms.untracked.int32(%s)" % maxEvents) if opts.lastEvent: tweak.addParameter( "process.source.lastEvent", "customTypeCms.untracked.uint32(%s)" % opts.lastEvent) # firstLumi, firstRun and eventsPerLumi are used for MC if opts.firstLumi: tweak.addParameter( "process.source.firstLuminosityBlock", "customTypeCms.untracked.uint32(%s)" % opts.firstLumi) if opts.firstRun: tweak.addParameter( "process.source.firstRun", "customTypeCms.untracked.uint32(%s)" % opts.firstRun) if opts.eventsPerLumi: numberEventsInLuminosityBlock = "customTypeCms.untracked.uint32(%s)" % opts.eventsPerLumi tweak.addParameter("process.source.numberEventsInLuminosityBlock", numberEventsInLuminosityBlock) # time-limited running is used by automatic splitting probe jobs if opts.maxRuntime: maxSecondsUntilRampdown = "customTypeCms.untracked.int32(%s)" % opts.maxRuntime tweak.addParameter("process.maxSecondsUntilRampdown.input", maxSecondsUntilRampdown) # event limiter for testing if opts.oneEventMode in ["1", "True", True]: tweak.addParameter("process.maxEvents.input", "customTypeCms.untracked.int32(1)") # make sure that FJR contains useful statistics, reuse code from # https://github.com/dmwm/WMCore/blob/c2fa70af3b4c5285d50e6a8bf48636232f738340/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py#L289-L307 tweak.addParameter("process.CPU", "customTypeCms.Service('CPU')") tweak.addParameter( "process.Timing", "customTypeCms.Service('Timing', summaryOnly=cms.untracked.bool(True))" ) tweak.addParameter( "process.SimpleMemoryCheck", "customTypeCms.Service('SimpleMemoryCheck', jobReportOutputOnly=cms.untracked.bool(True))" ) # tweak ! psetTweakJson = "PSetTweak.json" tweak.persist(psetTweakJson, formatting='simplejson') procScript = "edm_pset_tweak.py" pklOut = pklIn + '-tweaked' # we always create untracked psets in our tweaks cmd = "%s --input_pkl %s --output_pkl %s --json %s --create_untracked_psets" % ( procScript, pklIn, pklOut, psetTweakJson) commandLines = createTweakingCommandLines(cmd, pklIn, pklOut) # there a few more things to do which require running different EDM/CMSSW commands #1. enable LazyDownload of LHE files (if needed) if opts.lheInputFiles: pklOut = pklIn + '-lazy' procScript = "cmssw_enable_lazy_download.py" cmd = "%s --input_pkl %s --output_pkl %s" % (procScript, pklIn, pklOut) moreLines = createTweakingCommandLines(cmd, pklIn, pklOut) commandLines += moreLines #2. make sure random seeds are initialized pklOut = pklIn + '-seeds' procScript = "cmssw_handle_random_seeds.py" cmd = "%s --input_pkl %s --output_pkl %s --seeding dummy" % (procScript, pklIn, pklOut) moreLines += createTweakingCommandLines(cmd, pklIn, pklOut) commandLines += moreLines #3. make sure that process.maxEvents.input is propagated to Producers, see: # https://github.com/dmwm/WMCore/blob/85d6d423f0a85fdedf78b65ca8b7b81af9263789/src/python/WMCore/WMRuntime/Scripts/SetupCMSSWPset.py#L448-L465 pklOut = pklIn + '-nEvents' procScript = 'cmssw_handle_nEvents.py' cmd = "%s --input_pkl %s --output_pkl %s" % (procScript, pklIn, pklOut) moreLines = createTweakingCommandLines(cmd, pklIn, pklOut) commandLines += moreLines return commandLines