def run(self, component, wd, firstEvent, nEvents):
print wd, firstEvent, nEvents
if nEvents is None:
nEvents = -1
cmsswConfig = imp.load_source("cmsRunProcess", self.configFile)
inputfiles = []
for fn in component.files:
if not re.match("file:.*", fn) and not re.match("root:.*", fn):
fn = "file:" + fn
inputfiles.append(fn)
cmsswConfig.process.source.fileNames = inputfiles
cmsswConfig.process.maxEvents.input = nEvents
#fixme: implement skipEvent / firstevent
outfilename = wd + "/cmsswPreProcessing.root"
for outName in cmsswConfig.process.endpath.moduleNames():
out = getattr(cmsswConfig.process, outName)
out.fileName = outfilename
if not hasattr(component, "options"):
component.options = CFG(name="postCmsrunOptions")
#use original as primary and new as secondary
#component.options.inputFiles= component.files
#component.options.secondaryInputFiles=[outfilename]
#use new as primary and original as secondary
component.options.secondaryInputFiles = component.files
component.options.inputFiles = [outfilename]
component.files = [outfilename]
configfile = wd + "/cmsRun_config.py"
f = open(configfile, 'w')
f.write(cmsswConfig.process.dumpPython())
f.close()
runstring = "%s %s >& %s/cmsRun.log" % (self.command, configfile, wd)
print "Running pre-processor: %s " % runstring
ret = os.system(runstring)
if ret != 0:
print "CMSRUN failed"
exit(ret)
return component
def run(self, component, wd, firstEvent, nEvents):
if firstEvent != 0:
raise RuntimeError(
"The preprocessor can't skip events at the moment")
fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit',
(1, 1))
if fineSplitFactor > 1:
if len(component.files) != 1:
raise RuntimeError(
"Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s"
% (component.name, component.files))
evtsInFile = edmFileLs(component.files[0])['events']
if nEvents in (None, -1) or nEvents > evtsInFile:
nEvents = evtsInFile
nEvents = int(ceil(nEvents / float(fineSplitFactor)))
firstEvent = fineSplitIndex * nEvents
# Now we will run on these events, and the output will contain only those
# Thus, we switch off fine-split in the component
component.fineSplit = (1, 1)
if nEvents is None:
nEvents = -1
self.maybePrefetchFiles(component)
cmsswConfig = imp.load_source("cmsRunProcess",
os.path.expandvars(self.configFile))
inputfiles = []
for fn in component.files:
# if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
# fn="file:"+fn
inputfiles.append(fn)
secondaryfiles = []
for fn in component.secondaryfiles:
# if not re.match("file:.*",fn) and not re.match("root:.*",fn) :
# fn="file:"+fn
secondaryfiles.append(fn)
# Four cases:
# - no options, cmsswConfig with initialize function
# run initialize with default parameters
# - filled options, cmsswConfig with initialize function
# pass on options to initialize
# - no options, classic cmsswConfig
# legacy mode
# - filled options, classic cmsswConfig
# legacy mode but warn that options are not passed on
if hasattr(cmsswConfig, "initialize"):
if len(self.options) == 0:
cmsswConfig.process = cmsswConfig.initialize()
else:
cmsswConfig.process = cmsswConfig.initialize(**self.options)
else:
if len(self.options) == 0:
pass
else:
print "WARNING: cmsswPreprocessor received options but can't pass on to cmsswConfig"
cmsswConfig.process.source.fileNames = inputfiles
if not hasattr(cmsswConfig.process.source, "secondaryFileNames"):
cmsswConfig.process.source.secondaryFileNames = cmsswConfig.cms.untracked.vstring(
)
cmsswConfig.process.source.secondaryFileNames = secondaryfiles
# cmsRun will not create the output file if maxEvents==0, leading to crash of the analysis downstream.
# Thus, we set nEvents = 1 if the input file is empty (the output file will be empty as well).
cmsswConfig.process.maxEvents.input = 1 if (
fineSplitFactor > 1 and nEvents == 0) else nEvents
cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(
0 if (fineSplitFactor > 1 and nEvents == 0) else firstEvent)
#fixme: implement skipEvent / firstevent
outfilename = wd + "/cmsswPreProcessing.root"
# for outName in cmsswConfig.process.endpath.moduleNames():
for module in cmsswConfig.process.endpaths.viewvalues():
for outName in module.moduleNames():
out = getattr(cmsswConfig.process, outName)
if not hasattr(out, "fileName"): continue
out.fileName = outfilename
if not hasattr(component, "options"):
component.options = CFG(name="postCmsrunOptions")
#use original as primary and new as secondary
#component.options.inputFiles= component.files
#component.options.secondaryInputFiles=[outfilename]
#use new as primary and original as secondary
if self.addOrigAsSecondary:
component.options.secondaryInputFiles = component.files
component.options.inputFiles = [outfilename]
component.files = [outfilename]
configfile = wd + "/cmsRun_config.py"
f = open(configfile, 'w')
f.write(cmsswConfig.process.dumpPython())
f.close()
print "sed -i s/inf\)/float\( \\'inf\\'\)\)/g " + configfile
print "sed -i s/inf,/float\(\\'inf\\'\),/g " + configfile
os.system("sed -i s/inf\)/float\(\\'inf\\'\)\)/g " + configfile)
os.system("sed -i s/inf,/float\(\\'inf\\'\),/g " + configfile)
runstring = "%s %s >& %s/cmsRun.log" % (self.command, configfile, wd)
print "Running pre-processor: %s " % runstring
ret = os.system(runstring)
if ret != 0:
print "CMSRUN failed"
exit(ret)
return component
def run(self, component, wd, firstEvent, nEvents):
if firstEvent != 0:
raise RuntimeError, "The preprocessor can't skip events at the moment"
fineSplitIndex, fineSplitFactor = getattr(component, 'fineSplit',
(1, 1))
if fineSplitFactor > 1:
if len(component.files) != 1:
raise RuntimeError, "Any component with fineSplit > 1 is supposed to have just a single file, while %s has %s" % (
component.name, component.files)
evtsInFile = edmFileLs(component.files[0])['events']
if nEvents in (None, -1) or nEvents > evtsInFile:
nEvents = evtsInFile
nEvents = int(ceil(nEvents / float(fineSplitFactor)))
firstEvent = fineSplitIndex * nEvents
# Now we will run on these events, and the output will contain only those
# Thus, we switch off fine-split in the component
component.fineSplit = (1, 1)
if nEvents is None:
nEvents = -1
self.maybePrefetchFiles(component)
cmsswConfig = imp.load_source("cmsRunProcess",
os.path.expandvars(self.configFile))
inputfiles = []
for fn in component.files:
if not re.match("file:.*", fn) and not re.match("root:.*", fn):
fn = "file:" + fn
inputfiles.append(fn)
cmsswConfig.process.source.fileNames = inputfiles
cmsswConfig.process.maxEvents.input = nEvents
cmsswConfig.process.source.skipEvents = cmsswConfig.cms.untracked.uint32(
firstEvent)
#fixme: implement skipEvent / firstevent
outfilename = wd + "/cmsswPreProcessing.root"
# for outName in cmsswConfig.process.endpath.moduleNames():
for module in cmsswConfig.process.endpaths.viewvalues():
for outName in module.moduleNames():
out = getattr(cmsswConfig.process, outName)
out.fileName = outfilename
if not hasattr(component, "options"):
component.options = CFG(name="postCmsrunOptions")
#use original as primary and new as secondary
#component.options.inputFiles= component.files
#component.options.secondaryInputFiles=[outfilename]
#use new as primary and original as secondary
if self.addOrigAsSecondary:
component.options.secondaryInputFiles = component.files
component.options.inputFiles = [outfilename]
component.files = [outfilename]
configfile = wd + "/cmsRun_config.py"
f = open(configfile, 'w')
f.write(cmsswConfig.process.dumpPython())
f.close()
runstring = "%s %s >& %s/cmsRun.log" % (self.command, configfile, wd)
print "Running pre-processor: %s " % runstring
ret = os.system(runstring)
if ret != 0:
print "CMSRUN failed"
exit(ret)
return component