def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False, histFileName=None, histDirName=None, outputbranchsel=None, maxEntries=None, firstEntry=0, prefetch=False, longTermCache=False, outtreeName="Friends"): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName self.histFile = None self.histDirName = None if self.jobReport and not self.haddFileName: print("Because you requested a FJR we assume you want the final " \ "hadd. No name specified for the output file, will use tree.root") self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None if outputbranchsel is not None: self.outputbranchsel = BranchSelection(outputbranchsel) elif outputbranchsel is None and branchsel is not None: # Use the same branches in the output as in input self.outputbranchsel = BranchSelection(branchsel) else: self.outputbranchsel = None self.histFileName = histFileName self.histDirName = histDirName # 2^63 - 1, largest int64 self.maxEntries = maxEntries if maxEntries else 9223372036854775807 self.firstEntry = firstEntry self.prefetch = prefetch # prefetch files to TMPDIR using xrdcp # keep cached files across runs (it's then up to you to clean up the temp) self.longTermCache = longTermCache self.outtreeName = outtreeName
def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName if self.jobReport and not self.haddFileName: print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None
def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False, histFileName=None, histDirName=None, outputbranchsel=None, SMSMasses=None, doISR=None): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName self.histFile = None self.histDirName = None self.hcount = ROOT.TH1F("Count", "Count", 1, 0, 1) self.hsumofweights = ROOT.TH1F("SumWeights", "SumWeights", 1, 0, 1) self.SMSMasses = SMSMasses self.doISR = doISR if SMSMasses != None: self.hsmscount = ROOT.TH2F("SMSCount", "SMSCount", 2000, -0.5, 1999.5, 2000, -0.5, 1999.5) if doISR != None: self.isrconstant = 1. if self.jobReport and not self.haddFileName: print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None self.outputbranchsel = BranchSelection( outputbranchsel) if outputbranchsel else None self.histFileName = histFileName self.histDirName = histDirName
def __init__(self,outputDir,inputFiles,cut=None,branchsel=None,modules=[],compression="LZMA:9",friend=False,postfix=None, jsonInput=None,noOut=False,justcount=False,provenance=False,haddFileName=None,fwkJobReport=False,histFileName=None,histDirName=None, outputbranchsel=None,maxEntries=None,firstEntry=0, prefetch=False,longTermCache=False, reportEvery=10000, postProcOut = sys.stdout, postProcErr = sys.stdout, eventLoopOut = sys.stdout): self.outputDir=outputDir self.inputFiles=inputFiles self.cut=cut self.modules=modules self.compression=compression self.postfix=postfix self.json=jsonInput self.noOut=noOut self.friend=friend self.justcount=justcount self.provenance=provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName=haddFileName self.histFile = None self.histDirName = None if self.jobReport and not self.haddFileName : print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName="tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None self.outputbranchsel = BranchSelection(outputbranchsel) if outputbranchsel else None self.histFileName=histFileName self.histDirName=histDirName self.maxEntries = maxEntries if maxEntries else 9223372036854775807L # 2^63 - 1, largest int64 self.firstEntry = firstEntry self.prefetch = prefetch # prefetch files to TMPDIR using xrdcp self.longTermCache = longTermCache # keep cached files across runs (it's then up to you to clean up the temp) self.reportEvery=reportEvery self.postProcOut = postProcOut self.postProcErr = postProcErr self.eventLoopOut = eventLoopOut if self.postProcOut != sys.stdout: sys.stdout = self.postProcOut if self.postProcErr != sys.stdout: sys.stderr = self.postProcErr
def __init__(self,outputDir,inputFiles,cut=None,branchsel=None,modules=[],compression="LZMA:9",friend=False,postfix=None, jsonInput=None,noOut=False,justcount=False,provenance=False,haddFileName=None,fwkJobReport=False): self.outputDir=outputDir self.inputFiles=inputFiles self.cut=cut self.modules=modules self.compression=compression self.postfix=postfix self.json=jsonInput self.noOut=noOut self.friend=friend self.justcount=justcount self.provenance=provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName=haddFileName if self.jobReport and not self.haddFileName : print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName="tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None
class PostProcessor: def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False, histFileName=None, histDirName=None, outputbranchsel=None, maxEntries=None, firstEntry=0, prefetch=False, longTermCache=False): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName self.histFile = None self.histDirName = None if self.jobReport and not self.haddFileName: print("Because you requested a FJR we assume you want the final " \ "hadd. No name specified for the output file, will use tree.root") self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None if outputbranchsel != None: self.outputbranchsel = BranchSelection(outputbranchsel) elif outputbranchsel == None and branchsel != None: # Use the same branches in the output as in input self.outputbranchsel = BranchSelection(branchsel) else: self.outputbranchsel = None self.histFileName = histFileName self.histDirName = histDirName # 2^63 - 1, largest int64 self.maxEntries = maxEntries if maxEntries else 9223372036854775807 self.firstEntry = firstEntry self.prefetch = prefetch # prefetch files to TMPDIR using xrdcp # keep cached files across runs (it's then up to you to clean up the temp) self.longTermCache = longTermCache def prefetchFile(self, fname, verbose=True): tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp" if not fname.startswith("root://"): return fname, False rndchars = "".join([hex(ord(i))[2:] for i in os.urandom( 8)]) if not self.longTermCache else "long_cache-id%d-%s" \ % (os.getuid(), hashlib.sha1(fname).hexdigest()) localfile = "%s/%s-%s.root" \ % (tmpdir, os.path.basename(fname).replace(".root", ""), rndchars) if self.longTermCache and os.path.exists(localfile): if verbose: print("Filename %s is already available in local path %s " \ % (fname, localfile)) return localfile, False try: if verbose: print("Filename %s is remote, will do a copy to local path %s"\ % (fname, localfile)) start = time.time() subprocess.check_output(["xrdcp", "-f", "-N", fname, localfile]) if verbose: print("Time used for transferring the file locally: %.2f s"\ % (time.time() - start)) return localfile, (not self.longTermCache) except: if verbose: print( "Error: could not save file locally, will run from remote") if os.path.exists(localfile): if verbose: print("Deleting partially transferred file %s" % localfile) try: os.unlink(localfile) except: pass return fname, False def run(self): outpostfix = self.postfix if self.postfix != None else ( "_Friend" if self.friend else "_Skim") if not self.noOut: if self.compression != "none": ROOT.gInterpreter.ProcessLine("#include <Compression.h>") (algo, level) = self.compression.split(":") compressionLevel = int(level) if algo == "LZMA": compressionAlgo = ROOT.ROOT.kLZMA elif algo == "ZLIB": compressionAlgo = ROOT.ROOT.kZLIB elif algo == "LZ4": compressionAlgo = ROOT.ROOT.kLZ4 else: raise RuntimeError("Unsupported compression %s" % algo) else: compressionLevel = 0 print("Will write selected trees to " + self.outputDir) if not self.justcount: if not os.path.exists(self.outputDir): os.system("mkdir -p " + self.outputDir) else: compressionLevel = 0 if self.noOut: if len(self.modules) == 0: raise RuntimeError( "Running with --noout and no modules does nothing!") # Open histogram file, if desired if (self.histFileName != None and self.histDirName == None) or ( self.histFileName == None and self.histDirName != None): raise RuntimeError( "Must specify both histogram file and histogram directory!") elif self.histFileName != None and self.histDirName != None: self.histFile = ROOT.TFile.Open(self.histFileName, "RECREATE") else: self.histFile = None for m in self.modules: if hasattr(m, 'writeHistFile') and m.writeHistFile: m.beginJob(histFile=self.histFile, histDirName=self.histDirName) else: m.beginJob() fullClone = (len(self.modules) == 0) outFileNames = [] t0 = time.time() totEntriesRead = 0 for fname in self.inputFiles: ffnames = [] if "," in fname: fnames = fname.split(',') fname, ffnames = fnames[0], fnames[1:] # open input file if self.prefetch: ftoread, toBeDeleted = self.prefetchFile(fname) inFile = ROOT.TFile.Open(ftoread) else: inFile = ROOT.TFile.Open(fname) # get input tree inTree = inFile.Get("Events") if inTree == None: inTree = inFile.Get("Friends") nEntries = min(inTree.GetEntries() - self.firstEntry, self.maxEntries) totEntriesRead += nEntries # pre-skimming elist, jsonFilter = preSkim(inTree, self.json, self.cut, maxEntries=self.maxEntries, firstEntry=self.firstEntry) if self.justcount: print('Would select %d / %d entries from %s (%.2f%%)' % (elist.GetN() if elist else nEntries, nEntries, fname, (elist.GetN() if elist else nEntries) / (0.01 * nEntries) if nEntries else 0)) if self.prefetch: if toBeDeleted: os.unlink(ftoread) continue else: print('Pre-select %d entries out of %s (%.2f%%)' % (elist.GetN() if elist else nEntries, nEntries, (elist.GetN() if elist else nEntries) / (0.01 * nEntries) if nEntries else 0)) inAddFiles = [] inAddTrees = [] for ffname in ffnames: inAddFiles.append(ROOT.TFile.Open(ffname)) inAddTree = inAddFiles[-1].Get("Events") if inAddTree == None: inAddTree = inAddFiles[-1].Get("Friends") inAddTrees.append(inAddTree) inTree.AddFriend(inAddTree) if fullClone: # no need of a reader (no event loop), but set up the elist if available if elist: inTree.SetEntryList(elist) else: # initialize reader inTree = InputTree(inTree, elist) # prepare output file if not self.noOut: outFileName = os.path.join( self.outputDir, os.path.basename(fname).replace(".root", outpostfix + ".root")) outFile = ROOT.TFile.Open(outFileName, "RECREATE", "", compressionLevel) outFileNames.append(outFileName) if compressionLevel: outFile.SetCompressionAlgorithm(compressionAlgo) # prepare output tree if self.friend: outTree = FriendOutput(inFile, inTree, outFile) else: outTree = FullOutput( inFile, inTree, outFile, branchSelection=self.branchsel, outputbranchSelection=self.outputbranchsel, fullClone=fullClone, maxEntries=self.maxEntries, firstEntry=self.firstEntry, jsonFilter=jsonFilter, provenance=self.provenance) else: outFile = None outTree = None if self.branchsel: self.branchsel.selectBranches(inTree) # process events, if needed if not fullClone: eventRange = range( self.firstEntry, self.firstEntry + nEntries) if nEntries > 0 and not elist else None (nall, npass, timeLoop) = eventLoop(self.modules, inFile, outFile, inTree, outTree, eventRange=eventRange, maxEvents=self.maxEntries) print( 'Processed %d preselected entries from %s (%s entries). Finally selected %d entries' % (nall, fname, nEntries, npass)) else: nall = nEntries print('Selected %d / %d entries from %s (%.2f%%)' % (outTree.tree().GetEntries(), nall, fname, outTree.tree().GetEntries() / (0.01 * nall) if nall else 0)) # now write the output if not self.noOut: outTree.write() outFile.Close() print("Done %s" % outFileName) if self.jobReport: self.jobReport.addInputFile(fname, nall) if self.prefetch: if toBeDeleted: os.unlink(ftoread) for m in self.modules: m.endJob() print("Total time %.1f sec. to process %i events. Rate = %.1f Hz." % ((time.time() - t0), totEntriesRead, totEntriesRead / (time.time() - t0))) if self.haddFileName: haddnano = "./haddnano.py" if os.path.isfile( "./haddnano.py") else "haddnano.py" os.system("%s %s %s" % (haddnano, self.haddFileName, " ".join(outFileNames))) if self.jobReport: self.jobReport.addOutputFile(self.haddFileName) self.jobReport.save()
class PostProcessor: def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName if self.jobReport and not self.haddFileName: print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None def run(self): if not self.noOut: outpostfix = self.postfix if self.postfix != None else ( "_Friend" if self.friend else "_Skim") if self.compression != "none": ROOT.gInterpreter.ProcessLine("#include <Compression.h>") (algo, level) = self.compression.split(":") compressionLevel = int(level) if algo == "LZMA": compressionAlgo = ROOT.ROOT.kLZMA elif algo == "ZLIB": compressionAlgo = ROOT.ROOT.kZLIB else: raise RuntimeError("Unsupported compression %s" % algo) else: compressionLevel = 0 print "Will write selected trees to " + self.outputDir if not self.justcount: if not os.path.exists(self.outputDir): os.system("mkdir -p " + self.outputDir) if self.noOut: if len(self.modules) == 0: raise RuntimeError( "Running with --noout and no modules does nothing!") for m in self.modules: m.beginJob() fullClone = (len(self.modules) == 0) outFileNames = [] t0 = time.clock() totEntriesRead = 0 for fname in self.inputFiles: # open input file inFile = ROOT.TFile.Open(fname) #get input tree inTree = inFile.Get("Events") totEntriesRead += inTree.GetEntries() # pre-skimming elist, jsonFilter = preSkim(inTree, self.json, self.cut) if self.justcount: print 'Would select %d entries from %s' % ( elist.GetN() if elist else inTree.GetEntries(), fname) continue else: print 'Pre-select %d entries out of %s ' % ( elist.GetN() if elist else inTree.GetEntries(), inTree.GetEntries()) if fullClone: # no need of a reader (no event loop), but set up the elist if available if elist: inTree.SetEntryList(elist) else: # initialize reader inTree = InputTree(inTree, elist) # prepare output file outFileName = os.path.join( self.outputDir, os.path.basename(fname).replace(".root", outpostfix + ".root")) outFile = ROOT.TFile.Open(outFileName, "RECREATE", "", compressionLevel) outFileNames.append(outFileName) if compressionLevel: outFile.SetCompressionAlgorithm(compressionAlgo) # prepare output tree if self.friend: outTree = FriendOutput(inFile, inTree, outFile) else: outTree = FullOutput(inFile, inTree, outFile, branchSelection=self.branchsel, fullClone=fullClone, jsonFilter=jsonFilter, provenance=self.provenance) # process events, if needed if not fullClone: (nall, npass, timeLoop) = eventLoop(self.modules, inFile, outFile, inTree, outTree) print 'Processed %d preselected entries from %s (%s entries). Finally selected %d entries' % ( nall, fname, inTree.GetEntries(), npass) else: print 'Selected %d entries from %s' % ( outTree.tree().GetEntries(), fname) # now write the output outTree.write() outFile.Close() print "Done %s" % outFileName if self.jobReport: self.jobReport.addInputFile(fname, nall) for m in self.modules: m.endJob() print totEntriesRead / (time.clock() - t0), "Hz" if self.haddFileName: os.system( "./haddnano.py %s %s" % (self.haddFileName, " ".join(outFileNames)) ) #FIXME: remove "./" once haddnano.py is distributed with cms releases if self.jobReport: self.jobReport.addOutputFile(self.haddFileName) self.jobReport.save()
class PostProcessor: def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False, histFileName=None, histDirName=None, outputbranchsel=None, SMSMasses=None, doISR=None): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName self.histFile = None self.histDirName = None self.hcount = ROOT.TH1F("Count", "Count", 1, 0, 1) self.hsumofweights = ROOT.TH1F("SumWeights", "SumWeights", 1, 0, 1) self.SMSMasses = SMSMasses self.doISR = doISR if SMSMasses != None: self.hsmscount = ROOT.TH2F("SMSCount", "SMSCount", 2000, -0.5, 1999.5, 2000, -0.5, 1999.5) if doISR != None: self.isrconstant = 1. if self.jobReport and not self.haddFileName: print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None self.outputbranchsel = BranchSelection( outputbranchsel) if outputbranchsel else None self.histFileName = histFileName self.histDirName = histDirName def run(self): outpostfix = self.postfix if self.postfix != None else ( "_Friend" if self.friend else "_Skim") if not self.noOut: if self.compression != "none": ROOT.gInterpreter.ProcessLine("#include <Compression.h>") (algo, level) = self.compression.split(":") compressionLevel = int(level) if algo == "LZMA": compressionAlgo = ROOT.ROOT.kLZMA elif algo == "ZLIB": compressionAlgo = ROOT.ROOT.kZLIB else: raise RuntimeError("Unsupported compression %s" % algo) else: compressionLevel = 0 print "Will write selected trees to " + self.outputDir if not self.justcount: if not os.path.exists(self.outputDir): os.system("mkdir -p " + self.outputDir) else: compressionLevel = 0 if self.noOut: if len(self.modules) == 0: raise RuntimeError( "Running with --noout and no modules does nothing!") # Open histogram file, if desired if (self.histFileName != None and self.histDirName == None) or ( self.histFileName == None and self.histDirName != None): raise RuntimeError( "Must specify both histogram file and histogram directory!") elif self.histFileName != None and self.histDirName != None: self.histFile = ROOT.TFile.Open(self.histFileName, "RECREATE") else: self.histFile = None for m in self.modules: if hasattr(m, 'writeHistFile') and m.writeHistFile: m.beginJob(histFile=self.histFile, histDirName=self.histDirName) else: m.beginJob() fullClone = (len(self.modules) == 0) outFileNames = [] t0 = time.clock() totEntriesRead = 0 for fname in self.inputFiles: # open input file inFile = ROOT.TFile.Open(fname) #get input tree inTree = inFile.Get("Events") totEntriesRead += inTree.GetEntries() self.hcount.SetBinContent(1, inTree.GetEntries()) ROOT.gROOT.SetBatch(True) if self.SMSMasses != None: inTree.Draw( "MaxIf$(GenPart_mass, abs(GenPart_pdgId) == %i):MaxIf$(GenPart_mass, abs(GenPart_pdgId) == %i) >> hSMS(2000, -0.5, 1999.5, 2000, -0.5, 1999.5)" % (self.SMSMasses[0], self.SMSMasses[1])) self.hsmscount = ROOT.gDirectory.Get('hSMS') if self.doISR != None: #Dirty ISR recipe for EWKinos #Need to correct for each mass point #Can't correct per sample (wrong normalization), need to save whole unskimmed histogram per point an then postprocess pt1 = "MaxIf$(GenPart_pt, abs(GenPart_pdgId) == %i && GenPart_status == 22)" % self.doISR[ 0] pt2 = "MaxIf$(GenPart_pt, abs(GenPart_pdgId) == %i && GenPart_status == 22)" % self.doISR[ 1] phi1 = "MaxIf$(GenPart_pt, abs(GenPart_pdgId) == %i && GenPart_status == 22)" % self.doISR[ 0] phi2 = "MaxIf$(GenPart_pt, abs(GenPart_pdgId) == %i && GenPart_status == 22)" % self.doISR[ 1] pt_ISR = "hypot(%s + %s * cos(%s-%s), %s*sin(%s - %s))" % ( pt1, pt2, phi2, phi1, pt2, phi2, phi1) inTree.Draw( " %s : MaxIf$(GenPart_mass, abs(GenPart_pdgId) == %i) : MaxIf$(GenPart_mass, abs(GenPart_pdgId) == %i) >> hISR(1000, -0.5, 1999.5, 1000, -0.5, 1999.5, 20, 0, 1000)" % (pt_ISR, self.SMSMasses[0], self.SMSMasses[1])) self.hISR = ROOT.gDirectory.Get("hISR") if inTree.GetBranchStatus("genWeight"): inTree.Project("SumWeightsTemp", "1.0", "genWeight") sow = ROOT.gROOT.FindObject("SumWeightsTemp").Integral() self.hsumofweights.SetBinContent(1, sow) # pre-skimming elist, jsonFilter = preSkim(inTree, self.json, self.cut) if self.justcount: print 'Would select %d entries from %s' % ( elist.GetN() if elist else inTree.GetEntries(), fname) continue else: print 'Pre-select %d entries out of %s ' % ( elist.GetN() if elist else inTree.GetEntries(), inTree.GetEntries()) if fullClone: # no need of a reader (no event loop), but set up the elist if available if elist: inTree.SetEntryList(elist) else: # initialize reader inTree = InputTree(inTree, elist) # prepare output file if not self.noOut: outFileName = os.path.join( self.outputDir, os.path.basename(fname).replace(".root", outpostfix + ".root")) outFile = ROOT.TFile.Open(outFileName, "RECREATE", "", compressionLevel) outFileNames.append(outFileName) if compressionLevel: outFile.SetCompressionAlgorithm(compressionAlgo) # prepare output tree if self.friend: outTree = FriendOutput(inFile, inTree, outFile) else: outTree = FullOutput( inFile, inTree, outFile, branchSelection=self.branchsel, outputbranchSelection=self.outputbranchsel, fullClone=fullClone, jsonFilter=jsonFilter, provenance=self.provenance) else: outFile = None outTree = None # process events, if needed if not fullClone: (nall, npass, timeLoop) = eventLoop(self.modules, inFile, outFile, inTree, outTree) print 'Processed %d preselected entries from %s (%s entries). Finally selected %d entries' % ( nall, fname, inTree.GetEntries(), npass) else: nall = inTree.GetEntries() print 'Selected %d entries from %s' % ( outTree.tree().GetEntries(), fname) # now write the output if not self.noOut: print "Start writing" self.hcount.Write() print "Start writing" if self.SMSMasses != None: self.hsmscount.Write() print "Start writing" if self.doISR != None: self.hISR.Write() print "Start writing" self.hsumofweights.Write() outTree.write() outFile.Close() print "Done %s" % outFileName if self.jobReport: self.jobReport.addInputFile(fname, nall) for m in self.modules: m.endJob() print totEntriesRead / (time.clock() - t0), "Hz" if self.haddFileName: os.system( "./haddnano.py %s %s" % (self.haddFileName, " ".join(outFileNames)) ) #FIXME: remove "./" once haddnano.py is distributed with cms releases if self.jobReport: self.jobReport.addOutputFile(self.haddFileName) self.jobReport.save()
class PostProcessor: def __init__(self, outputDir, inputFiles, cut=None, branchsel=None, modules=[], compression="LZMA:9", friend=False, postfix=None, jsonInput=None, noOut=False, justcount=False, provenance=False, haddFileName=None, fwkJobReport=False, histFileName=None, histDirName=None, outputbranchsel=None, maxEntries=None, firstEntry=0, prefetch=False, longTermCache=False, saveHistoGenWeights=False, allowNoPostfix=False): self.outputDir = outputDir self.inputFiles = inputFiles self.cut = cut self.modules = modules self.compression = compression self.postfix = postfix self.allowNoPostfix = allowNoPostfix self.json = jsonInput self.noOut = noOut self.friend = friend self.justcount = justcount self.provenance = provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName = haddFileName self.saveHistoGenWeights = saveHistoGenWeights self.histFile = None self.histDirName = None if self.jobReport and not self.haddFileName: print("Because you requested a FJR we assume you want the final " \ "hadd. No name specified for the output file, will use tree.root") self.haddFileName = "tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None if outputbranchsel is not None: self.outputbranchsel = BranchSelection(outputbranchsel) elif outputbranchsel is None and branchsel is not None: # Use the same branches in the output as in input self.outputbranchsel = BranchSelection(branchsel) else: self.outputbranchsel = None self.histFileName = histFileName self.histDirName = histDirName # 2^63 - 1, largest int64 self.maxEntries = maxEntries if maxEntries else 9223372036854775807 self.firstEntry = firstEntry self.prefetch = prefetch # prefetch files to TMPDIR using xrdcp # keep cached files across runs (it's then up to you to clean up the temp) self.longTermCache = longTermCache def prefetchFile(self, fname, verbose=True): tmpdir = os.environ['TMPDIR'] if 'TMPDIR' in os.environ else "/tmp" if not fname.startswith("root://"): return fname, False rndchars = "".join([hex(ord(i))[2:] for i in os.urandom( 8)]) if not self.longTermCache else "long_cache-id%d-%s" \ % (os.getuid(), hashlib.sha1(fname).hexdigest()) localfile = "%s/%s-%s.root" \ % (tmpdir, os.path.basename(fname).replace(".root", ""), rndchars) if self.longTermCache and os.path.exists(localfile): if verbose: print("Filename %s is already available in local path %s " \ % (fname, localfile)) return localfile, False try: if verbose: print("Filename %s is remote, will do a copy to local path %s"\ % (fname, localfile)) start = time.time() subprocess.check_output(["xrdcp", "-f", "-N", fname, localfile]) if verbose: print("Time used for transferring the file locally: %.2f s"\ % (time.time() - start)) return localfile, (not self.longTermCache) except: if verbose: print( "Error: could not save file locally, will run from remote") if os.path.exists(localfile): if verbose: print("Deleting partially transferred file %s" % localfile) try: os.unlink(localfile) except: pass return fname, False def run(self): outpostfix = self.postfix if self.postfix is not None else ( "_Friend" if self.friend else "_Skim") if self.allowNoPostfix and self.postfix is None: outpostfix = "" if not self.noOut: if self.compression != "none": ROOT.gInterpreter.ProcessLine("#include <Compression.h>") (algo, level) = self.compression.split(":") compressionLevel = int(level) if algo == "LZMA": compressionAlgo = ROOT.ROOT.kLZMA elif algo == "ZLIB": compressionAlgo = ROOT.ROOT.kZLIB elif algo == "LZ4": compressionAlgo = ROOT.ROOT.kLZ4 else: raise RuntimeError("Unsupported compression %s" % algo) else: compressionLevel = 0 print("Will write selected trees to " + self.outputDir) if not self.justcount: if not os.path.exists(self.outputDir): os.system("mkdir -p " + self.outputDir) else: compressionLevel = 0 if self.noOut: if len(self.modules) == 0: raise RuntimeError( "Running with --noout and no modules does nothing!") # Open histogram file, if desired if (self.histFileName is not None and self.histDirName is None) or ( self.histFileName is None and self.histDirName is not None): raise RuntimeError( "Must specify both histogram file and histogram directory!") elif self.histFileName is not None and self.histDirName is None: self.histFile = ROOT.TFile.Open(self.histFileName, "RECREATE") else: self.histFile = None for m in self.modules: if hasattr(m, 'writeHistFile') and m.writeHistFile: m.beginJob(histFile=self.histFile, histDirName=self.histDirName) else: m.beginJob() fullClone = (len(self.modules) == 0) outFileNames = [] t0 = time.time() totEntriesRead = 0 for fname in self.inputFiles: ffnames = [] if "," in fname: fnames = fname.split(',') fname, ffnames = fnames[0], fnames[1:] # open input file if self.prefetch: ftoread, toBeDeleted = self.prefetchFile(fname) inFile = ROOT.TFile.Open(ftoread) else: inFile = ROOT.TFile.Open(fname) if not inFile: print 'ERROR: file does not exist, check!' print ' filename:', fname exit(0) # get input tree inTree = inFile.Get("Events") if inTree is None: inTree = inFile.Get("Friends") nEntries = min(inTree.GetEntries() - self.firstEntry, self.maxEntries) # first check that the histogram with weights is not already in the file hasWeightHistograms = False if inFile.GetListOfKeys().Contains( "hGenWeights") and inFile.GetListOfKeys().Contains( "hNumWeights"): hasWeightHistograms = True print "Histogram hGenWeights already exists, I will just copy it without recreating it" if self.saveHistoGenWeights and inTree.GetName( ) == "Events" and not hasWeightHistograms: print "Histogram hGenWeights does not exist yet, I will create it" # check that the tree contains all the original events, otherwise the sum of gen weights will miss some tmpTreeRuns = inFile.Get("Runs") for ievt, event in enumerate(tmpTreeRuns): if ievt: break # only need first event (but there should be only 1 here) nGenEvents = event.genEventCount if nGenEvents != inTree.GetEntries(): raise RuntimeError( "I am creating the histogram with genWeight, but tree Events has less entries than genEventCount in tree Runs (%s instead of %s). The sum of weights will thus be wrong, please check" % (str(inTree.GetEntries()), str(nGenEvents))) # saving distribution of genWeight for offline usage # idea is to fill the distribution of Log10(genWeight) with the sign, so to have a histogram from about -10 to 10 # with about 10k bins (genWeights can take valus spanning several orders of magnitude, especially for fancy weights) # then one can compute the sum of genWeight in a given range using its integral (using Log10(threshold) ). # This somehow relies on having always |genWeight|>1, should it be < 1 the Log would change the sign. # So for the purpose of choosing the bin to be filled, we use |value| or 1.001, whatever is larger (this will not affect the integral) # then, need a second histogram to keep the integer number of events in each bin, so to allow for clipping of large weights hGenWeights = ROOT.TH1D("hGenWeights", "distribution of Log10(genWeight)", 4800, -12.0, 12.0) hNumWeights = ROOT.TH1D( "hNumWeights", "distribution of Log10(genWeight) (unweighted)", 4800, -12.0, 12.0) drawResult = inTree.Draw( "TMath::Sign(1.0,genWeight)*TMath::Log10(max(1.001,abs(genWeight)))>>hGenWeights", "genWeight", "goff", nEntries, self.firstEntry) drawResult = inTree.Draw( "TMath::Sign(1.0,genWeight)*TMath::Log10(max(1.001,abs(genWeight)))>>hNumWeights", "1", "goff", nEntries, self.firstEntry) totEntriesRead += nEntries # pre-skimming elist, jsonFilter = preSkim(inTree, self.json, self.cut, maxEntries=self.maxEntries, firstEntry=self.firstEntry) if self.justcount: print('Would select %d / %d entries from %s (%.2f%%)' % (elist.GetN() if elist else nEntries, nEntries, fname, (elist.GetN() if elist else nEntries) / (0.01 * nEntries) if nEntries else 0)) if self.prefetch: if toBeDeleted: os.unlink(ftoread) continue else: print('Pre-select %d entries out of %s (%.2f%%)' % (elist.GetN() if elist else nEntries, nEntries, (elist.GetN() if elist else nEntries) / (0.01 * nEntries) if nEntries else 0)) inAddFiles = [] inAddTrees = [] for ffname in ffnames: inAddFiles.append(ROOT.TFile.Open(ffname)) inAddTree = inAddFiles[-1].Get("Events") if inAddTree is None: inAddTree = inAddFiles[-1].Get("Friends") inAddTrees.append(inAddTree) inTree.AddFriend(inAddTree) if fullClone: # no need of a reader (no event loop), but set up the elist if available if elist: inTree.SetEntryList(elist) else: # initialize reader inTree = InputTree(inTree, elist) # prepare output file if not self.noOut: outFileName = os.path.join( self.outputDir, os.path.basename(fname).replace(".root", outpostfix + ".root")) outFile = ROOT.TFile.Open(outFileName, "RECREATE", "", compressionLevel) outFileNames.append(outFileName) if compressionLevel: outFile.SetCompressionAlgorithm(compressionAlgo) # prepare output tree if self.friend: outTree = FriendOutput(inFile, inTree, outFile) else: outTree = FullOutput( inFile, inTree, outFile, branchSelection=self.branchsel, outputbranchSelection=self.outputbranchsel, fullClone=fullClone, maxEntries=self.maxEntries, firstEntry=self.firstEntry, jsonFilter=jsonFilter, provenance=self.provenance) else: outFile = None outTree = None if self.branchsel: self.branchsel.selectBranches(inTree) # process events, if needed if not fullClone: eventRange = range( self.firstEntry, self.firstEntry + nEntries) if nEntries > 0 and not elist else None (nall, npass, timeLoop) = eventLoop(self.modules, inFile, outFile, inTree, outTree, eventRange=eventRange, maxEvents=self.maxEntries) print( 'Processed %d preselected entries from %s (%s entries). Finally selected %d entries' % (nall, fname, nEntries, npass)) else: nall = nEntries print('Selected %d / %d entries from %s (%.2f%%)' % (outTree.tree().GetEntries(), nall, fname, outTree.tree().GetEntries() / (0.01 * nall) if nall else 0)) # now write the output if not self.noOut: outTree.write() if not hasWeightHistograms: if self.saveHistoGenWeights: hGenWeights.Write(hGenWeights.GetName()) hNumWeights.Write(hNumWeights.GetName()) outFile.Close() print("Done %s" % outFileName) if self.jobReport: self.jobReport.addInputFile(fname, nall) if self.prefetch: if toBeDeleted: os.unlink(ftoread) for m in self.modules: m.endJob() print("Total time %.1f sec. to process %i events. Rate = %.1f Hz." % ((time.time() - t0), totEntriesRead, totEntriesRead / (time.time() - t0))) if self.haddFileName: haddnano = "./haddnano.py" if os.path.isfile( "./haddnano.py") else "haddnano.py" os.system("%s %s %s" % (haddnano, self.haddFileName, " ".join(outFileNames))) if self.jobReport: self.jobReport.addOutputFile(self.haddFileName) self.jobReport.save()
import subprocess import shutil from PhysicsTools.NanoAODTools.postprocessing.framework.jobreport import JobReport # from PhysicsTools.NanoAODTools.postprocessing.framework.postprocessor import * #this takes care of converting the input files from CRAB from PhysicsTools.NanoAODTools.postprocessing.framework.crabhelper import inputFiles#,runsAndLumis # from PhysicsTools.NanoAODTools.postprocessing.examples.exampleModule import * #p=PostProcessor(".",inputFiles(),"Jet_pt>200",modules=[exampleModuleConstr()],provenance=True,fwkJobReport=True,jsonInput=runsAndLumis()) # p.run() # cmssw_base=os.getenv("CMSSW_BASE") # cmssw_base="./src/" # inputFiles="dcap://dcap01.jinr-t1.ru:22125/pnfs/jinr-t1.ru/data/cms/store/mc/RunIIFall17NanoAOD/DYJetsToLL_1J_TuneCP5_13TeV-amcatnloFXFX-pythia8/NANOAODSIM/PU2017_12Apr2018_94X_mc2017_realistic_v14-v1/00000/00580AEF-EB91-E811-9049-0CC47A4C8F0C.root" job_report=JobReport() bin_path="%s/src/BSM3GAna/BSM3GAna/data/Analyzer/"%(os.environ['CMSSW_BASE']) output_file="tree.root" output_log="output_log.log" if os.path.exists(output_log): os.remove(output_log) if os.path.exists(output_file): os.remove(output_file) cwd=os.getcwd() os.chdir(bin_path) command="./Analyzer -in "+" ".join(inputFiles())+" -out "+output_file+" -C PartDet"
#!/usr/bin/env python import os from PhysicsTools.NanoAODTools.postprocessing.framework.crabhelper import inputFiles, runsAndLumis from PhysicsTools.NanoAODTools.postprocessing.framework.jobreport import JobReport from Sphaleron_Ana.NanoAODReader.NanoReader import * nFiles = 0 outFilesList = [] jobReport = JobReport() for f in inputFiles(): nFiles += 1 outStr = "out" + str(nFiles) outFilesList.append(outStr) nEvents = NanoReader(inputFileName=f, outputFileName=outStr, nJobs=1, jobNum=1, json=runsAndLumis()) print("events are %i \n" % nEvents) jobReport.addInputFile(f, nEvents) #hadd outputs hadd_cmnd = "./haddnano.py tree.root " for f in outFilesList: hadd_cmnd += f + " " print("executing : %s" % hadd_cmnd) os.system(hadd_cmnd) print("ls") os.system("ls")
class PostProcessor : def __init__(self,outputDir,inputFiles,cut=None,branchsel=None,modules=[],compression="LZMA:9",friend=False,postfix=None, jsonInput=None,noOut=False,justcount=False,provenance=False,haddFileName=None,fwkJobReport=False): self.outputDir=outputDir self.inputFiles=inputFiles self.cut=cut self.modules=modules self.compression=compression self.postfix=postfix self.json=jsonInput self.noOut=noOut self.friend=friend self.justcount=justcount self.provenance=provenance self.jobReport = JobReport() if fwkJobReport else None self.haddFileName=haddFileName if self.jobReport and not self.haddFileName : print "Because you requested a FJR we assume you want the final hadd. No name specified for the output file, will use tree.root" self.haddFileName="tree.root" self.branchsel = BranchSelection(branchsel) if branchsel else None def run(self) : if not self.noOut: outpostfix = self.postfix if self.postfix != None else ("_Friend" if self.friend else "_Skim") if self.compression != "none": ROOT.gInterpreter.ProcessLine("#include <Compression.h>") (algo, level) = self.compression.split(":") compressionLevel = int(level) if algo == "LZMA": compressionAlgo = ROOT.ROOT.kLZMA elif algo == "ZLIB": compressionAlgo = ROOT.ROOT.kZLIB else: raise RuntimeError("Unsupported compression %s" % algo) else: compressionLevel = 0 print "Will write selected trees to "+self.outputDir if not self.justcount: if not os.path.exists(self.outputDir): os.system("mkdir -p "+self.outputDir) if self.noOut: if len(self.modules) == 0: raise RuntimeError("Running with --noout and no modules does nothing!") for m in self.modules: m.beginJob() fullClone = (len(self.modules) == 0) outFileNames=[] for fname in self.inputFiles: # open input file inFile = ROOT.TFile.Open(fname) #get input tree inTree = inFile.Get("Events") # pre-skimming elist,jsonFilter = preSkim(inTree, self.json, self.cut) if self.justcount: print 'Would select %d entries from %s'%(elist.GetN() if elist else inTree.GetEntries(), fname) continue if fullClone: # no need of a reader (no event loop), but set up the elist if available if elist: inTree.SetEntryList(elist) else: # initialize reader inTree = InputTree(inTree, elist) # prepare output file outFileName = os.path.join(self.outputDir, os.path.basename(fname).replace(".root",outpostfix+".root")) outFile = ROOT.TFile.Open(outFileName, "RECREATE", "", compressionLevel) outFileNames.append(outFileName) if compressionLevel: outFile.SetCompressionAlgorithm(compressionAlgo) # prepare output tree if self.friend: outTree = FriendOutput(inFile, inTree, outFile) else: outTree = FullOutput(inFile, inTree, outFile, branchSelection = self.branchsel, fullClone = fullClone, jsonFilter = jsonFilter,provenance=self.provenance) # process events, if needed if not fullClone: (nall, npass, time) = eventLoop(self.modules, inFile, outFile, inTree, outTree) print 'Processed %d entries from %s, selected %d entries' % (nall, fname, npass) else: print 'Selected %d entries from %s' % (outTree.tree().GetEntries(), fname) # now write the output outTree.write() outFile.Close() print "Done %s" % outFileName if self.jobReport: self.jobReport.addInputFile(fname,nall) for m in self.modules: m.endJob() if self.haddFileName : os.system("./haddnano.py %s %s" %(self.haddFileName," ".join(outFileNames))) #FIXME: remove "./" once haddnano.py is distributed with cms releases if self.jobReport : self.jobReport.addOutputFile(self.haddFileName) self.jobReport.save()