def run(self,component,wd,firstEvent,nEvents): if firstEvent != 0: raise RuntimeError("The preprocessor can't skip events at the moment") fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1,1)) if fineSplitFactor > 1: if len(component.files) != 1: raise RuntimeError("Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files)) evtsInFile = edmFileLs(component.files[0])['events'] if nEvents in (None, -1) or nEvents > evtsInFile: nEvents = evtsInFile nEvents = int(ceil(nEvents/float(fineSplitFactor))) firstEvent = fineSplitIndex * nEvents # Now we will run on these events, and the output will contain only those # Thus, we switch off fine-split in the component component.fineSplit = (1,1) if nEvents is None: nEvents = -1 self.maybePrefetchFiles(component) cmsswConfig = imp.load_source("cmsRunProcess",os.path.expandvars(self.configFile)) inputfiles= [] for fn in component.files : if not re.match("file:.*",fn) and not re.match("root:.*",fn) : fn="file:"+fn inputfiles.append(fn) # Four cases: # - no options, cmsswConfig with initialize function # run initialize with default parameters # - filled options, cmsswConfig with initialize function # pass on options to initialize # - no options, classic cmsswConfig # legacy mode # - filled options, classic cmsswConfig # legacy mode but warn that options are not passed on if hasattr(cmsswConfig, "initialize"): if len(self.options) == 0: cmsswConfig.process = cmsswConfig.initialize() else: cmsswConfig.process = cmsswConfig.initialize(**self.options) else: if len(self.options) == 0: pass else: print("WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig") cmsswConfig.process.source.fileNames = inputfiles # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream. # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well). cmsswConfig.process.maxEvents.input = 1 if (fineSplitFactor>1 and nEvents==0) else nEvents cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(0 if (fineSplitFactor>1 and nEvents==0) else firstEvent) #fixme: implement skipEvent / firstevent outfilename=wd+"/cmsswPreProcessing.root" # for outName in cmsswConfig.process.endpath.moduleNames(): for module in cmsswConfig.process.endpaths.viewvalues(): for outName in module.moduleNames(): out = getattr(cmsswConfig.process,outName) if not hasattr(out,"fileName"): continue out.fileName = outfilename if not hasattr(component,"options"): component.options = CFG(name="postCmsrunOptions") #use original as primary and new as secondary #component.options.inputFiles= component.files #component.options.secondaryInputFiles=[outfilename] #use new as primary and original as secondary if self.addOrigAsSecondary: component.options.secondaryInputFiles= component.files component.options.inputFiles=[outfilename] component.files=[outfilename] configfile=wd+"/cmsRun_config.py" f = open(configfile, 'w') f.write(cmsswConfig.process.dumpPython()) f.close() runstring="%s %s >& %s/cmsRun.log" % (self.command,configfile,wd) print("Running pre-processor: %s " %runstring) ret=os.system(runstring) if ret != 0: print("CMSRUN failed") exit(ret) return component
def run(self, component, wd, firstEvent, nEvents): if firstEvent != 0: raise RuntimeError( "The preprocessor can't skip events at the moment") fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1, 1)) if fineSplitFactor > 1: if len(component.files) != 1: raise RuntimeError( "Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (component.name, component.files)) evtsInFile = edmFileLs(component.files[0])['events'] if nEvents in (None, -1) or nEvents > evtsInFile: nEvents = evtsInFile nEvents = int(ceil(nEvents / float(fineSplitFactor))) firstEvent = fineSplitIndex * nEvents # Now we will run on these events, and the output will contain only those # Thus, we switch off fine-split in the component component.fineSplit = (1, 1) if nEvents is None: nEvents = -1 self.maybePrefetchFiles(component) cmsswConfig = imp.load_source("cmsRunProcess", os.path.expandvars(self.configFile)) inputfiles = [] for fn in component.files: # if not re.match("file:.*",fn) and not re.match("root:.*",fn) : # fn="file:"+fn inputfiles.append(fn) secondaryfiles = [] for fn in component.secondaryfiles: # if not re.match("file:.*",fn) and not re.match("root:.*",fn) : # fn="file:"+fn secondaryfiles.append(fn) # Four cases: # - no options, cmsswConfig with initialize function # run initialize with default parameters # - filled options, cmsswConfig with initialize function # pass on options to initialize # - no options, classic cmsswConfig # legacy mode # - filled options, classic cmsswConfig # legacy mode but warn that options are not passed on if hasattr(cmsswConfig, "initialize"): if len(self.options) == 0: cmsswConfig.process = cmsswConfig.initialize() else: cmsswConfig.process = cmsswConfig.initialize(**self.options) else: if len(self.options) == 0: pass else: print "WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig" cmsswConfig.process.source.fileNames = inputfiles if not hasattr(cmsswConfig.process.source, "secondaryFileNames"): cmsswConfig.process.source.secondaryFileNames = cmsswConfig.cms.untracked.vstring( ) cmsswConfig.process.source.secondaryFileNames = secondaryfiles # cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream. # Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well). cmsswConfig.process.maxEvents.input = 1 if ( fineSplitFactor > 1 and nEvents == 0) else nEvents cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32( 0 if (fineSplitFactor > 1 and nEvents == 0) else firstEvent) #fixme: implement skipEvent / firstevent outfilename = wd + "/cmsswPreProcessing.root" # for outName in cmsswConfig.process.endpath.moduleNames(): for module in cmsswConfig.process.endpaths.viewvalues(): for outName in module.moduleNames(): out = getattr(cmsswConfig.process, outName) if not hasattr(out, "fileName"): continue out.fileName = outfilename if not hasattr(component, "options"): component.options = CFG(name="postCmsrunOptions") #use original as primary and new as secondary #component.options.inputFiles= component.files #component.options.secondaryInputFiles=[outfilename] #use new as primary and original as secondary if self.addOrigAsSecondary: component.options.secondaryInputFiles = component.files component.options.inputFiles = [outfilename] component.files = [outfilename] configfile = wd + "/cmsRun_config.py" f = open(configfile, 'w') f.write(cmsswConfig.process.dumpPython()) f.close() print "sed -i s/inf\)/float\( \\'inf\\'\)\)/g " + configfile print "sed -i s/inf,/float\(\\'inf\\'\),/g " + configfile os.system("sed -i s/inf\)/float\(\\'inf\\'\)\)/g " + configfile) os.system("sed -i s/inf,/float\(\\'inf\\'\),/g " + configfile) runstring = "%s %s >& %s/cmsRun.log" % (self.command, configfile, wd) print "Running pre-processor: %s " % runstring ret = os.system(runstring) if ret != 0: print "CMSRUN failed" exit(ret) return component
def run(self, component, wd, firstEvent, nEvents): if firstEvent != 0: raise RuntimeError, "The preprocessor can't skip events at the moment" fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit', (1, 1)) if fineSplitFactor > 1: if len(component.files) != 1: raise RuntimeError, "Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % ( component.name, component.files) evtsInFile = edmFileLs(component.files[0])['events'] if nEvents in (None, -1) or nEvents > evtsInFile: nEvents = evtsInFile nEvents = int(ceil(nEvents / float(fineSplitFactor))) firstEvent = fineSplitIndex * nEvents # Now we will run on these events, and the output will contain only those # Thus, we switch off fine-split in the component component.fineSplit = (1, 1) if nEvents is None: nEvents = -1 self.maybePrefetchFiles(component) cmsswConfig = imp.load_source("cmsRunProcess", os.path.expandvars(self.configFile)) inputfiles = [] for fn in component.files: if not re.match("file:.*", fn) and not re.match("root:.*", fn): fn = "file:" + fn inputfiles.append(fn) cmsswConfig.process.source.fileNames = inputfiles cmsswConfig.process.maxEvents.input = nEvents cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32( firstEvent) #fixme: implement skipEvent / firstevent outfilename = wd + "/cmsswPreProcessing.root" # for outName in cmsswConfig.process.endpath.moduleNames(): for module in cmsswConfig.process.endpaths.viewvalues(): for outName in module.moduleNames(): out = getattr(cmsswConfig.process, outName) out.fileName = outfilename if not hasattr(component, "options"): component.options = CFG(name="postCmsrunOptions") #use original as primary and new as secondary #component.options.inputFiles= component.files #component.options.secondaryInputFiles=[outfilename] #use new as primary and original as secondary if self.addOrigAsSecondary: component.options.secondaryInputFiles = component.files component.options.inputFiles = [outfilename] component.files = [outfilename] configfile = wd + "/cmsRun_config.py" f = open(configfile, 'w') f.write(cmsswConfig.process.dumpPython()) f.close() runstring = "%s %s >& %s/cmsRun.log" % (self.command, configfile, wd) print "Running pre-processor: %s " % runstring ret = os.system(runstring) if ret != 0: print "CMSRUN failed" exit(ret) return component