default='', help= "for eff. scale factor computation: restrict to this list of comma separated bins (Starting at bin 1)" ) (opts, args) = parser.parse_args(argv) if opts.config == "": opts.config = ["config"] # Import after configure to get help message from myutils import BetterConfigParser, mvainfo, ParseInfo config = BetterConfigParser() config.read(opts.config) if len(opts.fitType) < 1: if config.has_option('Fit', 'FitType'): opts.fitType = config.get('Fit', 'FitType') else: opts.fitType = "shapes_prefit" # run plotter if len(opts.regions) < 1 or opts.regions.strip() == 'None': regions = eval(config.get('Fit', 'regions')).keys() else: regions = opts.regions.strip().split(',') for i in range(len(regions)): if '+' in regions[i]: regions[i] = regions[i].split('+') print("REGIONS:", regions) scaleFactorTable = []
print "filelist[0]:",filelist[0]; else: print '' vhbbPlotDef=opts.config[0].split('/')[0]+'/vhbbPlotDef.ini' opts.config.append(vhbbPlotDef)#adds it to the config list config = BetterConfigParser() config.read(opts.config) #path = opts.path region = opts.region # additional blinding cut: addBlindingCut = None if config.has_option('Plot_general','addBlindingCut'):#contained in plots, cut on the event number addBlindingCut = config.get('Plot_general','addBlindingCut') print 'adding add. blinding cut' print "Compile external macros" print "=======================\n" #get locations: Wdir=config.get('Directories','Wdir')# working direcoty containing the ouput samplesinfo=config.get('Directories','samplesinfo')# samples_nosplit.cfg path = config.get('Directories','plottingSamples')# from which samples to plot section='Plot:%s'%region info = ParseInfo(samplesinfo,path) #creates a list of Samples by reading the info in samples_nosplit.cfg and the conentent of the path.
else: print '' vhbbPlotDef = opts.config[0].split('/')[0] + '/vhbbPlotDef.ini' opts.config.append(vhbbPlotDef) #adds it to the config list config = BetterConfigParser() config.read(opts.config) #path = opts.path region = opts.region # additional blinding cut: addBlindingCut = None if config.has_option( 'Plot_general', 'addBlindingCut'): #contained in plots, cut on the event number addBlindingCut = config.get('Plot_general', 'addBlindingCut') print 'adding add. blinding cut' print "Compile external macros" print "=======================\n" #get locations: Wdir = config.get('Directories', 'Wdir') # working direcoty containing the ouput samplesinfo = config.get('Directories', 'samplesinfo') # samples_nosplit.cfg path = config.get('Directories', 'plottingSamples') # from which samples to plot
class XbbRun: def __init__(self, opts): # get file list self.filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None print "len(filelist)",len(self.filelist), if len(self.filelist) > 0: print "filelist[0]:", self.filelist[0] else: print '' # config self.debug = 'XBBDEBUG' in os.environ self.verifyCopy = True self.opts = opts self.config = BetterConfigParser() self.config.read(opts.config) self.channel = self.config.get('Configuration', 'channel') # load namespace, TODO VHbbNameSpace = self.config.get('VHbbNameSpace', 'library') ROOT.gSystem.Load(VHbbNameSpace) # directories self.pathIN = self.config.get('Directories', opts.inputDir) self.pathOUT = self.config.get('Directories', opts.outputDir) self.tmpDir = self.config.get('Directories', 'scratch') print 'INput samples:\t%s'%self.pathIN print 'OUTput samples:\t%s'%self.pathOUT self.fileLocator = FileLocator(config=self.config) # check if given sample identifier uniquely matches a samples from config matchingSamples = ParseInfo(samples_path=self.pathIN, config=self.config).find(identifier=opts.sampleIdentifier) if len(matchingSamples) != 1: print "ERROR: need exactly 1 sample identifier as input with -S !!" print matchingSamples exit(1) self.sample = matchingSamples[0] # collections self.collections = [x.strip() for x in opts.addCollections.split(',') if len(x.strip()) > 0] if len(opts.addCollections.strip())>0 else [] if len(self.collections) < 1: print "\x1b[31mWARNING: no collections added! Specify the collections to add with the --addCollections option!\x1b[0m" print 'collections to add:', self.collections self.collections = self.parseCollectionList(self.collections) print 'after parsing:', self.collections # temorary folder to save the files of this job on the scratch temporaryName = self.sample.identifier + '/' + uuid.uuid4().hex # input files self.subJobs = [] if opts.join: print("INFO: join input files! This is an experimental feature!") # translate naming convention of .txt file to imported files after the prep step inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(x) for x in self.filelist] self.subJobs.append({ 'inputFileNames': self.filelist, 'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep], 'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]), 'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]), }) else: # create separate subjob for all files (default!) for inputFileName in self.filelist: inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(inputFileName)] self.subJobs.append({ 'inputFileNames': [inputFileName], 'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep], 'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]), 'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]), }) # lists of single modules can be given instead of a module, "--addCollections Sys.all" # [Sys] # all = ['Sys.Vtype', 'Sys.Leptons', ...] # TODO: make it fully recursive def parseCollectionList(self, collections): collectionsListsReplaced = [] for collection in collections: if '.' in collection: section = collection.split('.')[0] key = collection.split('.')[1] listExpression = self.config.get(section, key).strip() if listExpression.startswith('[') and listExpression.endswith(']'): listParsed = eval(listExpression) for i in listParsed: collectionsListsReplaced.append(i) else: collectionsListsReplaced.append(collection) else: collectionsListsReplaced.append(collection) return collectionsListsReplaced # run all subjobs def run(self): nFilesProcessed = 0 nFilesFailed = 0 for subJob in self.subJobs: # only process if output is non-existing/broken or --force was used if self.opts.force or not self.fileLocator.isValidRootFile(subJob['outputFileName']): # create directories outputFolder = '/'.join(subJob['outputFileName'].split('/')[:-1]) tmpFolder = '/'.join(subJob['tmpFileName'].split('/')[:-1]) self.fileLocator.makedirs(outputFolder) self.fileLocator.makedirs(tmpFolder) # load sample tree sampleTree = SampleTree(subJob['localInputFileNames'], config=self.config) if not sampleTree.tree: print "trying fallback...", len(subJob['inputFileNames']) if len(subJob['inputFileNames']) == 1: # try original naming scheme if reading directly from Heppy/Nano ntuples (without prep) fileNameOriginal = self.pathIN + '/' + subJob['inputFileNames'][0] print "FO:", fileNameOriginal xrootdRedirector = self.fileLocator.getRedirector(fileNameOriginal) sampleTree = SampleTree([fileNameOriginal], config=self.config, xrootdRedirector=xrootdRedirector) if not sampleTree.tree: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED!\x1b[0m" nFilesFailed += 1 continue else: print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED! (old naming scheme not supported for joining multipel files)\x1b[0m" nFilesFailed += 1 continue # to use this syntax, use "--addCollections Sys.Vtype" for a config file entry like this: # [Sys] # Vtype = VtypeCorrector.VtypeCorrector(channel='Zll') # (instead of passing the tree in the constructor, the setTree method can be used) pyModules = [] versionTable = [] for collection in self.collections: if '.' in collection: section = collection.split('.')[0] key = collection.split('.')[1] if self.config.has_section(section) and self.config.has_option(section, key): pyCode = self.config.get(section, key) elif '(' in collection and collection.endswith(')'): print "WARNING: config option", collection, " not found, interpreting it as Python code!" pyCode = collection else: print "\x1b[31mERROR: config option not found:", collection, ". To specify Python code directly, pass a complete constructor, e.g. --addCollections 'Module.Class()'. Module has to be placed in python/myutils/ folder.\x1b[0m" raise Exception("ConfigError") # import module from myutils moduleName = pyCode.split('(')[0].split('.')[0].strip() if self.debug: print "DEBUG: import module:", moduleName print("\x1b[33mDEBUG: " + collection + ": run PYTHON code:\n"+pyCode+"\x1b[0m") globals()[moduleName] = importlib.import_module(".{module}".format(module=moduleName), package="myutils") # get object wObject = eval(pyCode) # pass the tree and other variables if needed to finalize initialization if hasattr(wObject, "customInit") and callable(getattr(wObject, "customInit")): wObject.customInit({'config': self.config, 'sampleTree': sampleTree, 'tree': sampleTree.tree, 'sample': self.sample, 'channel': self.channel, 'pathIN': self.pathIN, 'pathOUT': self.pathOUT, }) # add callbacks if the objects provides any if hasattr(wObject, "processEvent") and callable(getattr(wObject, "processEvent")): sampleTree.addCallback('event', wObject.processEvent) for cb in ["finish", "prepareOutput"]: if hasattr(wObject, cb) and callable(getattr(wObject, cb)): sampleTree.addCallback(cb, getattr(wObject, cb)) # add branches if hasattr(wObject, "getBranches") and callable(getattr(wObject, "getBranches")): sampleTree.addOutputBranches(wObject.getBranches()) pyModules.append(wObject) versionTable.append([moduleName, wObject.getVersion() if hasattr(wObject, "getVersion") else 0]) else: print "\x1b[31mERROR: config option not found:", collection, " the format should be: [Section].[Option]\x1b[0m" raise Exception("ConfigError") for moduleName, moduleVersion in versionTable: print " > {m}:{v}".format(m=moduleName, v=moduleVersion) # DEPRECATED, do not use anymore ---> use BranchTools.TreeFormulas() if 'addbranches' in self.collections: writeNewVariables = eval(self.config.get("Regression", "writeNewVariablesDict")) sampleTree.addOutputBranches(writeNewVariables) # DEPRECATED, do not use anymore ---> use BranchTools.Drop() if 'removebranches' in self.collections: bl_branch = eval(config.get('Branches', 'useless_branch')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) bl_branch = eval(config.get('Branches', 'useless_after_sys')) for br in bl_branch: sampleTree.addBranchToBlacklist(br) # define output file sampleTree.addOutputTree(subJob['tmpFileName'], cut='1', branches='*', friend=self.opts.friend) # run processing for pyModule in pyModules: if hasattr(pyModule, "beforeProcessing"): getattr(pyModule, "beforeProcessing")() sampleTree.process() for pyModule in pyModules: if hasattr(pyModule, "afterProcessing"): getattr(pyModule, "afterProcessing")() # if output trees have been produced: copy temporary file to output folder if sampleTree.getNumberOfOutputTrees() > 0: try: self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'copy ', subJob['tmpFileName'], subJob['outputFileName'] if self.verifyCopy: if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print 'INFO: output at final destination broken, try to copy again from scratch disk to final destination...' self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True) print 'INFO: second attempt copy done!' if not self.fileLocator.isValidRootFile(subJob['outputFileName']): print '\x1b[31mERROR: output still broken!\x1b[0m' nFilesFailed += 1 raise Exception("FileCopyError") else: print 'INFO: file is good after second attempt!' except Exception as e: print e print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m" # delete temporary file try: self.fileLocator.rm(subJob['tmpFileName']) except Exception as e: print e print "WARNING: could not delete file on scratch!" # clean up if hasattr(wObject, "cleanUp") and callable(getattr(wObject, "cleanUp")): getattr(wObject, "cleanUp")() else: print 'SKIP:', subJob['inputFileNames'] if nFilesFailed > 0: raise Exception("ProcessingIncomplete")
btagDown = BTagShape("../data/csvdiscr.root") btagDown.computeFunctions(-1., 0.) btagFUp = BTagShape("../data/csvdiscr.root") btagFUp.computeFunctions(0., +1.) btagFDown = BTagShape("../data/csvdiscr.root") btagFDown.computeFunctions(0., -1.) else: ROOT.gSystem.Load(btagLibrary) from ROOT import BTagShapeInterface btagNom = BTagShapeInterface("../data/csvdiscr.root", 0, 0) btagUp = BTagShapeInterface("../data/csvdiscr.root", +1, 0) btagDown = BTagShapeInterface("../data/csvdiscr.root", -1, 0) btagFUp = BTagShapeInterface("../data/csvdiscr.root", 0, +1.) btagFDown = BTagShapeInterface("../data/csvdiscr.root", 0, -1.) lhe_weight_map = False if not config.has_option( 'LHEWeights', 'weights_per_bin') else eval( config.get('LHEWeights', 'weights_per_bin')) print '\t - %s' % (job.name) print('opening ' + pathIN + '/' + job.prefix + job.identifier + '.root') input = ROOT.TFile.Open( pathIN + '/' + job.prefix + job.identifier + '.root', 'read') output = ROOT.TFile.Open( tmpDir + '/' + job.prefix + job.identifier + '.root', 'recreate') input.cd() if lhe_weight_map and 'DY' in job.name: inclusiveJob = info.get_sample('DY') print inclusiveJob.name inclusive = ROOT.TFile.Open(pathIN + inclusiveJob.get_path, 'read') inclusive.cd()
btagDown = BTagShape("../data/csvdiscr.root") btagDown.computeFunctions(-1.,0.) btagFUp = BTagShape("../data/csvdiscr.root") btagFUp.computeFunctions(0.,+1.) btagFDown = BTagShape("../data/csvdiscr.root") btagFDown.computeFunctions(0.,-1.) else: ROOT.gSystem.Load(btagLibrary) from ROOT import BTagShapeInterface btagNom = BTagShapeInterface("../data/csvdiscr.root",0,0) btagUp = BTagShapeInterface("../data/csvdiscr.root",+1,0) btagDown = BTagShapeInterface("../data/csvdiscr.root",-1,0) btagFUp = BTagShapeInterface("../data/csvdiscr.root",0,+1.) btagFDown = BTagShapeInterface("../data/csvdiscr.root",0,-1.) lhe_weight_map = False if not config.has_option('LHEWeights', 'weights_per_bin') else eval(config.get('LHEWeights', 'weights_per_bin')) print '\t - %s' %(job.name) print('opening '+pathIN+'/'+job.prefix+job.identifier+'.root') input = ROOT.TFile.Open(pathIN+'/'+job.prefix+job.identifier+'.root','read') output = ROOT.TFile.Open(tmpDir+'/'+job.prefix+job.identifier+'.root','recreate') input.cd() if lhe_weight_map and 'DY' in job.name: inclusiveJob = info.get_sample('DY') print inclusiveJob.name inclusive = ROOT.TFile.Open(pathIN+inclusiveJob.get_path,'read') inclusive.cd() obj = ROOT.TObject for key in ROOT.gDirectory.GetListOfKeys():
btagFUp.computeFunctions(0.0, +1.0) btagFDown = BTagShape("../data/csvdiscr.root") btagFDown.computeFunctions(0.0, -1.0) else: ROOT.gSystem.Load(btagLibrary) from ROOT import BTagShapeInterface btagNom = BTagShapeInterface("../data/csvdiscr.root", 0, 0) btagUp = BTagShapeInterface("../data/csvdiscr.root", +1, 0) btagDown = BTagShapeInterface("../data/csvdiscr.root", -1, 0) btagFUp = BTagShapeInterface("../data/csvdiscr.root", 0, +1.0) btagFDown = BTagShapeInterface("../data/csvdiscr.root", 0, -1.0) lhe_weight_map = ( False if not config.has_option("LHEWeights", "weights_per_bin") else eval(config.get("LHEWeights", "weights_per_bin")) ) print "\t - %s" % (job.name) print ("opening " + pathIN + "/" + job.prefix + job.identifier + ".root") input = ROOT.TFile.Open(pathIN + "/" + job.prefix + job.identifier + ".root", "read") output = ROOT.TFile.Open(tmpDir + "/" + job.prefix + job.identifier + ".root", "recreate") input.cd() if lhe_weight_map and "DY" in job.name: inclusiveJob = info.get_sample("DY") print inclusiveJob.name inclusive = ROOT.TFile.Open(pathIN + inclusiveJob.get_path, "read") inclusive.cd() obj = ROOT.TObject
#print config.get('dc:%s'%var,'range') nBins = int(config.get('dc:%s'%var,'range').split(',')[0]) xMin = float(config.get('dc:%s'%var,'range').split(',')[1]) xMax = float(config.get('dc:%s'%var,'range').split(',')[2]) ROOToutname = config.get('dc:%s'%var,'dcName') RCut = config.get('dc:%s'%var,'cut') signals = config.get('dc:%s'%var,'signal').split(' ') datas = config.get('dc:%s'%var,'data') Datacardbin = config.get('dc:%s'%var,'dcBin') anType = config.get('dc:%s'%var,'type') setup = eval(config.get('LimitGeneral','setup')) doSYS = config.get('dc:%s'%var,'doSYS') doBin = config.get('dc:%s'%var,'doBin') #Systematics: if config.has_option('LimitGeneral','addSample_sys'): addSample_sys = eval(config.get('LimitGeneral','addSample_sys')) additionals = [addSample_sys[key] for key in addSample_sys] else: addSample_sys = None additionals = [] #find out if BDT or MJJ: bdt = False mjj = False cr = False if str(anType) == 'BDT': bdt = True systematics = eval(config.get('LimitGeneral','sys_BDT')) elif str(anType) == 'Mjj': mjj = True
# parse histogram config: treevar = config.get('dc:%s'%var,'var') name = config.get('dc:%s'%var,'wsVarName') title = name nBins = int(config.get('dc:%s'%var,'range').split(',')[0]) xMin = float(config.get('dc:%s'%var,'range').split(',')[1]) xMax = float(config.get('dc:%s'%var,'range').split(',')[2]) ROOToutname = config.get('dc:%s'%var,'dcName') RCut = config.get('dc:%s'%var,'cut') signals = config.get('dc:%s'%var,'signal').split(' ') datas = config.get('dc:%s'%var,'dcBin') Datacardbin=config.get('dc:%s'%var,'dcBin') anType = config.get('dc:%s'%var,'type') setup=eval(config.get('LimitGeneral','setup')) #Systematics: if config.has_option('LimitGeneral','addSample_sys'): addSample_sys = eval(config.get('LimitGeneral','addSample_sys')) additionals = [addSample_sys[key] for key in addSample_sys] else: addSample_sys = None additionals = [] #find out if BDT or MJJ: bdt = False mjj = False cr = False if str(anType) == 'BDT': bdt = True systematics = eval(config.get('LimitGeneral','sys_BDT')) elif str(anType) == 'Mjj': mjj = True systematics = eval(config.get('LimitGeneral','sys_Mjj'))
print "Using", ('dc:%s' % var, 'var') print name print title print nBins print xMin print xMax print ROOToutname print RCut print signals print datas print Datacardbin print anType print setup #Systematics: if config.has_option('LimitGeneral', 'addSample_sys'): addSample_sys = eval(config.get('LimitGeneral', 'addSample_sys')) additionals = [addSample_sys[key] for key in addSample_sys] else: addSample_sys = None additionals = [] #find out if BDT or MJJ: bdt = False mjj = False cr = False lhe = [] if str(anType) == 'BDT': bdt = True systematics = eval(config.get('LimitGeneral', 'sys_BDT')) if config.has_option('LimitGeneral', 'sys_lhe_BDT'): lhe = eval(config.get('LimitGeneral', 'sys_lhe_BDT'))
parser.add_option("-r","--regions", dest="regions", default='', help="cut region identifiers, separated by comma") parser.add_option("-t","--type", dest="fitType", default='', help="shapes_prefit, shapes_fit_b, shapes_fit_s") (opts, args) = parser.parse_args(argv) if opts.config == "": opts.config = ["config"] # Import after configure to get help message from myutils import BetterConfigParser, mvainfo, ParseInfo config = BetterConfigParser() config.read(opts.config) if len(opts.fitType) < 1: if config.has_option('Fit','FitType'): opts.fitType = config.get('Fit','FitType') else: opts.fitType = "shapes_prefit" # run plotter if len(opts.regions) < 1: regions = eval(config.get('Fit', 'regions')).keys() else: regions = opts.regions.split(',') for region in regions: plotter = PostfitPlotter(config=config, region=region, directory=opts.fitType) plotter.prepare() plotter.run()
from myutils import BetterConfigParser, printc, ParseInfo, mvainfo, StackMaker, HistoMaker #opts.config.append('13TeVconfig/vhbbPlotDef.ini') #print 'Plot Config List: ',opts.config config = BetterConfigParser() config.read(opts.config) #path = opts.path region = opts.region batch = opts.batch print batch # additional blinding cut: addBlindingCut = None if config.has_option('Plot_general', 'addBlindingCut'): addBlindingCut = config.get('Plot_general', 'addBlindingCut') print 'adding add. blinding cut' #get locations: Wdir = config.get('Directories', 'Wdir') # the samples_nosplit.cfg file in 13TeV samplesinfo = config.get('Directories', 'samplesinfo') # which directory to take the samples from(sys_out currently) path = config.get('Directories', 'plottingSamples') # Add external macros # For batch jobs lock the compliation #lock = Lock()
if opts.config =="": opts.config = "config" from myutils import BetterConfigParser, printc, ParseInfo, mvainfo, StackMaker, HistoMaker print opts.config opts.config.append('heppy13TeVconfig/vhbbPlotDef.ini') config = BetterConfigParser() config.read(opts.config) #path = opts.path region = opts.region # additional blinding cut: addBlindingCut = None if config.has_option('Plot_general','addBlindingCut'): addBlindingCut = config.get('Plot_general','addBlindingCut') print 'adding add. blinding cut' #get locations: Wdir=config.get('Directories','Wdir') samplesinfo=config.get('Directories','samplesinfo') path = config.get('Directories','plottingSamples') section='Plot:%s'%region info = ParseInfo(samplesinfo,path) #----------Histo from trees------------ def doPlot():
# opts.config.append('13TeVconfig/vhbbPlotDef.ini') # print 'Plot Config List: ',opts.config config = BetterConfigParser() config.read(opts.config) # path = opts.path region = opts.region batch = opts.batch print batch # additional blinding cut: addBlindingCut = None if config.has_option("Plot_general", "addBlindingCut"): addBlindingCut = config.get("Plot_general", "addBlindingCut") print "adding add. blinding cut" # get locations: Wdir = config.get("Directories", "Wdir") # the samples_nosplit.cfg file in 13TeV samplesinfo = config.get("Directories", "samplesinfo") # which directory to take the samples from(sys_out currently) path = config.get("Directories", "plottingSamples") # Add external macros # For batch jobs lock the compliation
print "Using", ("dc:%s" % var, "var") print name print title print nBins print xMin print xMax print ROOToutname print RCut print signals print datas print Datacardbin print anType print setup # Systematics: if config.has_option("LimitGeneral", "addSample_sys"): addSample_sys = eval(config.get("LimitGeneral", "addSample_sys")) additionals = [addSample_sys[key] for key in addSample_sys] else: addSample_sys = None additionals = [] # find out if BDT or MJJ: bdt = False mjj = False cr = False if str(anType) == "BDT": bdt = True systematics = eval(config.get("LimitGeneral", "sys_BDT")) elif str(anType) == "Mjj": mjj = True systematics = eval(config.get("LimitGeneral", "sys_Mjj"))