def diffConfigFile(tag1, tag2, fileName):
print("file:", fileName)
config1 = BetterConfigParser()
config2 = BetterConfigParser()
config1.read(tag1 + 'config/' + fileName)
config2.read(tag2 + 'config/' + fileName)
sections = list(set(config1.sections() + config2.sections()))
commonSections = [
x for x in sections
if x in config1.sections() and x in config2.sections()
]
print("common sections:", commonSections)
for section in commonSections:
firstItemInSection = True
items1d = {x[0]: x[1] for x in config1.items(section, raw=True)}
items2d = {x[0]: x[1] for x in config2.items(section, raw=True)}
items1 = [x[0] for x in config1.items(section, raw=True)]
items2 = [x[0] for x in config2.items(section, raw=True)]
items = list(set(items1 + items2))
commonItems = list(
set([x for x in items if x in items1 and x in items2]))
for item in commonItems:
if item not in os.environ:
try:
item1 = config1.get(section, item)
item2 = config2.get(section, item)
item1 = items1d[item]
item2 = items2d[item]
if item1 != item2:
if firstItemInSection:
firstItemInSection = False
print('-' * 80)
print(' \x1b[34m' + fileName + '\x1b[0m: ' +
section)
print('-' * 80)
print(section + ':' + item)
d = difflib.Differ()
result = list(d.compare([item1], [item2]))
for line in result:
if line.startswith('+'):
color = '\x1b[31m'
elif line.startswith('-'):
color = '\x1b[32m'
elif line.startswith('?'):
color = '\x1b[34m'
print(' ' + color + line + '\x1b[0m')
except Exception as e:
pass
def diffConfigFile(tag1, tag2, fileName):
print("file:", fileName)
config1 = BetterConfigParser()
config2 = BetterConfigParser()
config1.read(tag1 + 'config/' + fileName)
config2.read(tag2 + 'config/' + fileName)
sections = list(set(config1.sections() + config2.sections()))
commonSections = [x for x in sections if x in config1.sections() and x in config2.sections()]
print("common sections:", commonSections)
for section in commonSections:
firstItemInSection = True
items1d = {x[0]:x[1] for x in config1.items(section, raw=True)}
items2d = {x[0]:x[1] for x in config2.items(section, raw=True)}
items1 = [x[0] for x in config1.items(section, raw=True)]
items2 = [x[0] for x in config2.items(section, raw=True)]
items = list(set(items1 + items2))
commonItems = list(set([x for x in items if x in items1 and x in items2]))
for item in commonItems:
if item not in os.environ:
try:
item1 = config1.get(section, item)
item2 = config2.get(section, item)
item1 = items1d[item]
item2 = items2d[item]
if item1 != item2:
if firstItemInSection:
firstItemInSection = False
print('-'*80)
print(' \x1b[34m' + fileName + '\x1b[0m: ' + section)
print('-'*80)
print(section + ':' + item)
d = difflib.Differ()
result = list(d.compare([item1], [item2]))
for line in result:
if line.startswith('+'):
color = '\x1b[31m'
elif line.startswith('-'):
color = '\x1b[32m'
elif line.startswith('?'):
color = '\x1b[34m'
print(' '+color+line+'\x1b[0m')
except Exception as e:
pass
parser.add_option("-V",
"--validate",
dest="validate",
default=True,
action="store_false",
help="Make validation plot the weights")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
from myutils import BetterConfigParser
print opts.config
config = BetterConfigParser()
config.read(opts.config)
prefix = config.get('LHEWeights', 'prefix')
newpostfix = config.get('LHEWeights', 'newpostfix')
inclusive = config.get('LHEWeights', 'inclusive')
files = eval(config.get('LHEWeights', 'fileList'))
lheBin = eval(config.get('LHEWeights', 'lheBin'))
fileList = []
for file in files:
new_entry = [prefix + file + '.root', files[file], []]
if file == inclusive:
fileList.insert(0, new_entry)
else:
fileList.append(new_entry)
print fileList
print lheBin
else:
print ''
vhbbPlotDef=opts.config[0].split('/')[0]+'/vhbbPlotDef.ini'
opts.config.append(vhbbPlotDef)#adds it to the config list
config = BetterConfigParser()
config.read(opts.config)
#path = opts.path
region = opts.region
# additional blinding cut:
addBlindingCut = None
if config.has_option('Plot_general','addBlindingCut'):#contained in plots, cut on the event number
addBlindingCut = config.get('Plot_general','addBlindingCut')
print 'adding add. blinding cut'
print "Compile external macros"
print "=======================\n"
#get locations:
Wdir=config.get('Directories','Wdir')# working direcoty containing the ouput
samplesinfo=config.get('Directories','samplesinfo')# samples_nosplit.cfg
path = config.get('Directories','plottingSamples')# from which samples to plot
section='Plot:%s'%region
info = ParseInfo(samplesinfo,path) #creates a list of Samples by reading the info in samples_nosplit.cfg and the conentent of the path.
dest="config",
default=[],
action="append",
help="directory config")
#parser.add_option("-S", "--samples", dest="names", default="",
# help="samples you want to run on")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
#namelist=opts.names.split(',')
#print "namelist:",namelist
pathIN = config.get('Directories', 'PREPin')
pathOUT = config.get('Directories', 'PREPout')
samplesinfo = config.get('Directories', 'samplesinfo')
sampleconf = BetterConfigParser()
sampleconf.read(samplesinfo)
prefix = sampleconf.get('General', 'prefix')
info = ParseInfo(samples_path=pathIN, config=config)
print "samplesinfo:", samplesinfo
cross_sections = {}
samples = []
for job in info:
if not job.identifier in samples:
if type(job.xsec) is list: job.xsec = job.xsec[0]
cross_sections[job.identifier] = job.xsec
class XbbRun:
def __init__(self, opts):
# get file list
self.filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None
print "len(filelist)",len(self.filelist),
if len(self.filelist) > 0:
print "filelist[0]:", self.filelist[0]
else:
print ''
# config
self.debug = 'XBBDEBUG' in os.environ
self.verifyCopy = True
self.opts = opts
self.config = BetterConfigParser()
self.config.read(opts.config)
self.channel = self.config.get('Configuration', 'channel')
# load namespace, TODO
VHbbNameSpace = self.config.get('VHbbNameSpace', 'library')
ROOT.gSystem.Load(VHbbNameSpace)
# directories
self.pathIN = self.config.get('Directories', opts.inputDir)
self.pathOUT = self.config.get('Directories', opts.outputDir)
self.tmpDir = self.config.get('Directories', 'scratch')
print 'INput samples:\t%s'%self.pathIN
print 'OUTput samples:\t%s'%self.pathOUT
self.fileLocator = FileLocator(config=self.config)
# check if given sample identifier uniquely matches a samples from config
matchingSamples = ParseInfo(samples_path=self.pathIN, config=self.config).find(identifier=opts.sampleIdentifier)
if len(matchingSamples) != 1:
print "ERROR: need exactly 1 sample identifier as input with -S !!"
print matchingSamples
exit(1)
self.sample = matchingSamples[0]
# collections
self.collections = [x.strip() for x in opts.addCollections.split(',') if len(x.strip()) > 0] if len(opts.addCollections.strip())>0 else []
if len(self.collections) < 1:
print "\x1b[31mWARNING: no collections added! Specify the collections to add with the --addCollections option!\x1b[0m"
print 'collections to add:', self.collections
self.collections = self.parseCollectionList(self.collections)
print 'after parsing:', self.collections
# temorary folder to save the files of this job on the scratch
temporaryName = self.sample.identifier + '/' + uuid.uuid4().hex
# input files
self.subJobs = []
if opts.join:
print("INFO: join input files! This is an experimental feature!")
# translate naming convention of .txt file to imported files after the prep step
inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(x) for x in self.filelist]
self.subJobs.append({
'inputFileNames': self.filelist,
'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep],
'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]),
'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]),
})
else:
# create separate subjob for all files (default!)
for inputFileName in self.filelist:
inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(inputFileName)]
self.subJobs.append({
'inputFileNames': [inputFileName],
'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep],
'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]),
'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]),
})
# lists of single modules can be given instead of a module, "--addCollections Sys.all"
# [Sys]
# all = ['Sys.Vtype', 'Sys.Leptons', ...]
# TODO: make it fully recursive
def parseCollectionList(self, collections):
collectionsListsReplaced = []
for collection in collections:
if '.' in collection:
section = collection.split('.')[0]
key = collection.split('.')[1]
listExpression = self.config.get(section, key).strip()
if listExpression.startswith('[') and listExpression.endswith(']'):
listParsed = eval(listExpression)
for i in listParsed:
collectionsListsReplaced.append(i)
else:
collectionsListsReplaced.append(collection)
else:
collectionsListsReplaced.append(collection)
return collectionsListsReplaced
# run all subjobs
def run(self):
nFilesProcessed = 0
nFilesFailed = 0
for subJob in self.subJobs:
# only process if output is non-existing/broken or --force was used
if self.opts.force or not self.fileLocator.isValidRootFile(subJob['outputFileName']):
# create directories
outputFolder = '/'.join(subJob['outputFileName'].split('/')[:-1])
tmpFolder = '/'.join(subJob['tmpFileName'].split('/')[:-1])
self.fileLocator.makedirs(outputFolder)
self.fileLocator.makedirs(tmpFolder)
# load sample tree
sampleTree = SampleTree(subJob['localInputFileNames'], config=self.config)
if not sampleTree.tree:
print "trying fallback...", len(subJob['inputFileNames'])
if len(subJob['inputFileNames']) == 1:
# try original naming scheme if reading directly from Heppy/Nano ntuples (without prep)
fileNameOriginal = self.pathIN + '/' + subJob['inputFileNames'][0]
print "FO:", fileNameOriginal
xrootdRedirector = self.fileLocator.getRedirector(fileNameOriginal)
sampleTree = SampleTree([fileNameOriginal], config=self.config, xrootdRedirector=xrootdRedirector)
if not sampleTree.tree:
print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED!\x1b[0m"
nFilesFailed += 1
continue
else:
print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED! (old naming scheme not supported for joining multipel files)\x1b[0m"
nFilesFailed += 1
continue
# to use this syntax, use "--addCollections Sys.Vtype" for a config file entry like this:
# [Sys]
# Vtype = VtypeCorrector.VtypeCorrector(channel='Zll')
# (instead of passing the tree in the constructor, the setTree method can be used)
pyModules = []
versionTable = []
for collection in self.collections:
if '.' in collection:
section = collection.split('.')[0]
key = collection.split('.')[1]
if self.config.has_section(section) and self.config.has_option(section, key):
pyCode = self.config.get(section, key)
elif '(' in collection and collection.endswith(')'):
print "WARNING: config option", collection, " not found, interpreting it as Python code!"
pyCode = collection
else:
print "\x1b[31mERROR: config option not found:", collection, ". To specify Python code directly, pass a complete constructor, e.g. --addCollections 'Module.Class()'. Module has to be placed in python/myutils/ folder.\x1b[0m"
raise Exception("ConfigError")
# import module from myutils
moduleName = pyCode.split('(')[0].split('.')[0].strip()
if self.debug:
print "DEBUG: import module:", moduleName
print("\x1b[33mDEBUG: " + collection + ": run PYTHON code:\n"+pyCode+"\x1b[0m")
globals()[moduleName] = importlib.import_module(".{module}".format(module=moduleName), package="myutils")
# get object
wObject = eval(pyCode)
# pass the tree and other variables if needed to finalize initialization
if hasattr(wObject, "customInit") and callable(getattr(wObject, "customInit")):
wObject.customInit({'config': self.config,
'sampleTree': sampleTree,
'tree': sampleTree.tree,
'sample': self.sample,
'channel': self.channel,
'pathIN': self.pathIN,
'pathOUT': self.pathOUT,
})
# add callbacks if the objects provides any
if hasattr(wObject, "processEvent") and callable(getattr(wObject, "processEvent")):
sampleTree.addCallback('event', wObject.processEvent)
for cb in ["finish", "prepareOutput"]:
if hasattr(wObject, cb) and callable(getattr(wObject, cb)):
sampleTree.addCallback(cb, getattr(wObject, cb))
# add branches
if hasattr(wObject, "getBranches") and callable(getattr(wObject, "getBranches")):
sampleTree.addOutputBranches(wObject.getBranches())
pyModules.append(wObject)
versionTable.append([moduleName, wObject.getVersion() if hasattr(wObject, "getVersion") else 0])
else:
print "\x1b[31mERROR: config option not found:", collection, " the format should be: [Section].[Option]\x1b[0m"
raise Exception("ConfigError")
for moduleName, moduleVersion in versionTable:
print " > {m}:{v}".format(m=moduleName, v=moduleVersion)
# DEPRECATED, do not use anymore ---> use BranchTools.TreeFormulas()
if 'addbranches' in self.collections:
writeNewVariables = eval(self.config.get("Regression", "writeNewVariablesDict"))
sampleTree.addOutputBranches(writeNewVariables)
# DEPRECATED, do not use anymore ---> use BranchTools.Drop()
if 'removebranches' in self.collections:
bl_branch = eval(config.get('Branches', 'useless_branch'))
for br in bl_branch:
sampleTree.addBranchToBlacklist(br)
bl_branch = eval(config.get('Branches', 'useless_after_sys'))
for br in bl_branch:
sampleTree.addBranchToBlacklist(br)
# define output file
sampleTree.addOutputTree(subJob['tmpFileName'], cut='1', branches='*', friend=self.opts.friend)
# run processing
for pyModule in pyModules:
if hasattr(pyModule, "beforeProcessing"):
getattr(pyModule, "beforeProcessing")()
sampleTree.process()
for pyModule in pyModules:
if hasattr(pyModule, "afterProcessing"):
getattr(pyModule, "afterProcessing")()
# if output trees have been produced: copy temporary file to output folder
if sampleTree.getNumberOfOutputTrees() > 0:
try:
self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True)
print 'copy ', subJob['tmpFileName'], subJob['outputFileName']
if self.verifyCopy:
if not self.fileLocator.isValidRootFile(subJob['outputFileName']):
print 'INFO: output at final destination broken, try to copy again from scratch disk to final destination...'
self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True)
print 'INFO: second attempt copy done!'
if not self.fileLocator.isValidRootFile(subJob['outputFileName']):
print '\x1b[31mERROR: output still broken!\x1b[0m'
nFilesFailed += 1
raise Exception("FileCopyError")
else:
print 'INFO: file is good after second attempt!'
except Exception as e:
print e
print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m"
# delete temporary file
try:
self.fileLocator.rm(subJob['tmpFileName'])
except Exception as e:
print e
print "WARNING: could not delete file on scratch!"
# clean up
if hasattr(wObject, "cleanUp") and callable(getattr(wObject, "cleanUp")):
getattr(wObject, "cleanUp")()
else:
print 'SKIP:', subJob['inputFileNames']
if nFilesFailed > 0:
raise Exception("ProcessingIncomplete")
sys.exit(123)
en = opts.tag
#create the list with the samples to run over
samplesList = opts.samples.split(",")
# timestamp = time.asctime().replace(' ','_').replace(':','-')
timestamp = time.strftime("%Y_%m_%d-%H_%M_%S")
if (debugPrintOUts): print 'samplesList', samplesList
if (debugPrintOUts): print 'timestamp', timestamp
# the list of the config is taken from the path config
pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths.ini' % (en))
_configs = pathconfig.get('Configuration', 'List').split(" ")
configs = ['%sconfig/' % (en) + c for c in _configs]
if (debugPrintOUts): print 'configs', configs
if (debugPrintOUts): print 'opts.ftag', opts.ftag
if not opts.ftag == '':
tagDir = pathconfig.get('Directories', 'tagDir')
if (debugPrintOUts): print 'tagDir', tagDir
DirStruct = {
'tagDir': tagDir,
'ftagdir': '%s/%s/' % (tagDir, opts.ftag),
'logpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'Logs'),
'plotpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'Plots'),
'limitpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'Limits'),
'confpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'config')
print ('opts.filelist="'+opts.filelist+'"')
filelist=filter(None,opts.filelist.replace(' ', '').split(';'))
print (filelist)
print ("len(filelist)",len(filelist),)
if len(filelist)>0:
print ("filelist[0]:",filelist[0])
else:
print ('')
#Import after configure to get help message
from myutils import BetterConfigParser, progbar, printc, ParseInfo, MvaEvaluator
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
#get locations:
Wdir=config.get('Directories','Wdir')
samplesinfo=config.get('Directories','samplesinfo')
#read shape systematics
systematics=config.get('systematics','systematics')
#systematics
INpath = config.get('Directories','MVAin')
OUTpath = config.get('Directories','MVAout')
info = ParseInfo(samplesinfo,INpath)
arglist = ''
#parser.add_option("-P", "--path", dest="path", default="",
# help="path to samples")
parser.add_option("-S", "--samples", dest="names", default="",
help="samples you want to run on")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
TrainFlag = eval(config.get('Analysis','TrainFlag'))
btagLibrary = config.get('BTagReshaping','library')
samplesinfo=config.get('Directories','samplesinfo')
VHbbNameSpace=config.get('VHbbNameSpace','library')
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs=eval(config.get('AngularLike','backgrounds'))
ang_yield=eval(config.get('AngularLike','yields'))
#path=opts.path
pathIN = config.get('Directories','SYSin')
pathOUT = config.get('Directories','SYSout')
tmpDir = os.environ["TMPDIR"]
print 'INput samples:\t%s'%pathIN
parser = OptionParser()
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="directory config")
parser.add_option("-S", "--sampleIdentifier", dest="sampleIdentifier", default="",
help="samples you want to run on")
parser.add_option("-f", "--fileList", dest="fileList", default="",
help="list of files you want to run on")
parser.add_option("-l", "--limit", dest="limit", default=None,
help="max number of files to process")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
fileList = FileList.decompress(opts.fileList) if len(opts.fileList)>0 else None
pathOUT = config.get('Directories','PREPout')
samplesinfo=config.get('Directories','samplesinfo')
samplefiles = config.get('Directories','samplefiles')
sampleconf = BetterConfigParser()
sampleconf.read(samplesinfo)
whereToLaunch = config.get('Configuration','whereToLaunch')
info = ParseInfo(samplesinfo, None)
samples = [x for x in info if not x.subsample and (len(opts.sampleIdentifier) == 0 or x.identifier in opts.sampleIdentifier.split(','))]
treeCopier = copytreePSI.CopyTreePSI(config=config)
if opts.limit and len(samples) > int(opts.limit):
samples= samples[:int(opts.limit)]
for sample in samples:
treeCopier.copytreePSI(pathIN=samplefiles, pathOUT=pathOUT, folderName=sample.identifier, skimmingCut=sample.addtreecut, fileList=fileList)
"--config",
dest="config",
default=[],
action="append",
help="configuration file")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
#Import after configure to get help message
from myutils import BetterConfigParser, progbar, printc, ParseInfo, MvaEvaluator
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
#get locations:
Wdir = config.get('Directories', 'Wdir')
samplesinfo = config.get('Directories', 'samplesinfo')
#systematics
INpath = config.get('Directories', 'MVAin')
OUTpath = config.get('Directories', 'MVAout')
#read shape systematics
systematics = config.get('systematics', 'systematics')
info = ParseInfo(samplesinfo, INpath)
arglist = opts.discr #RTight_blavla,bsbsb
sampleName = section
sampleNameShort = '_'.join(sampleName.split('_')[0:7]).strip()
if '_ext' in sampleName:
sampleNameShort = sampleNameShort.split('_ext')[0]
elif '_backup' in sampleName:
sampleNameShort = sampleNameShort.split('_backup')[0]
if sampleNameShort in sampleDict:
sampleDict[sampleNameShort].append(sampleName)
else:
sampleDict[sampleNameShort] = [sampleName]
if verify:
extweight = 0
try:
extweight = float(config.get(section, 'extweight'))
except:
pass
sampleWeights[sampleNameShort] = float(
sampleWeights[sampleNameShort]
) + extweight if sampleNameShort in sampleWeights else extweight
if verify:
for sampleNameShort, totalWeight in sampleWeights.iteritems():
if sampleNameShort in sampleDict and len(
sampleDict[sampleNameShort]) > 1:
print sampleNameShort, ":", totalWeight
else:
for sample, extParts in sampleDict.iteritems():
if len(extParts) > 1:
print '-' * 80
dest="fileList",
default="",
help="list of files you want to run on")
parser.add_option("-l",
"--limit",
dest="limit",
default=None,
help="max number of files to process")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
fileList = FileList.decompress(
opts.fileList) if len(opts.fileList) > 0 else None
pathOUT = config.get('Directories', 'PREPout')
samplefiles = config.get('Directories', 'samplefiles')
sampleconf = config
whereToLaunch = config.get('Configuration', 'whereToLaunch')
info = ParseInfo(samples_path=None, config=config)
samples = [
x for x in info
if not x.subsample and (len(opts.sampleIdentifier) == 0 or x.identifier in
opts.sampleIdentifier.split(','))
]
treeCopier = copytreePSI.CopyTreePSI(config=config)
if opts.limit and len(samples) > int(opts.limit):
samples = samples[:int(opts.limit)]
for sample in samples:
class XbbRun:
def __init__(self, opts):
# get file list
self.filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None
print "len(filelist)",len(self.filelist),
if len(self.filelist) > 0:
print "filelist[0]:", self.filelist[0]
else:
print ''
# config
self.debug = 'XBBDEBUG' in os.environ
self.verifyCopy = True
self.opts = opts
self.config = BetterConfigParser()
self.config.read(opts.config)
samplesinfo = self.config.get('Directories', 'samplesinfo')
self.channel = self.config.get('Configuration', 'channel')
# load namespace, TODO
VHbbNameSpace = self.config.get('VHbbNameSpace', 'library')
ROOT.gSystem.Load(VHbbNameSpace)
# directories
self.pathIN = self.config.get('Directories', opts.inputDir)
self.pathOUT = self.config.get('Directories', opts.outputDir)
self.tmpDir = self.config.get('Directories', 'scratch')
print 'INput samples:\t%s'%self.pathIN
print 'OUTput samples:\t%s'%self.pathOUT
self.fileLocator = FileLocator(config=self.config)
# check if given sample identifier uniquely matches a samples from config
matchingSamples = ParseInfo(samplesinfo, self.pathIN).find(identifier=opts.sampleIdentifier)
if len(matchingSamples) != 1:
print "ERROR: need exactly 1 sample identifier as input with -S !!"
print matchingSamples
exit(1)
self.sample = matchingSamples[0]
# collections
self.collections = [x.strip() for x in opts.addCollections.split(',') if len(x.strip()) > 0] if len(opts.addCollections.strip())>0 else []
if len(self.collections) < 1:
print "\x1b[31mWARNING: no collections added! Specify the collections to add with the --addCollections option!\x1b[0m"
print 'collections to add:', self.collections
self.collections = self.parseCollectionList(self.collections)
print 'after parsing:', self.collections
# temorary folder to save the files of this job on the scratch
temporaryName = self.sample.identifier + '/' + uuid.uuid4().hex
# input files
self.subJobs = []
if opts.join:
print("INFO: join input files! This is an experimental feature!")
# translate naming convention of .txt file to imported files after the prep step
inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(x) for x in self.filelist]
self.subJobs.append({
'inputFileNames': self.filelist,
'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep],
'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]),
'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]),
})
else:
# create separate subjob for all files (default!)
for inputFileName in self.filelist:
inputFileNamesAfterPrep = [self.fileLocator.getFilenameAfterPrep(inputFileName)]
self.subJobs.append({
'inputFileNames': [inputFileName],
'localInputFileNames': ["{path}/{subfolder}/{filename}".format(path=self.pathIN, subfolder=self.sample.identifier, filename=localFileName) for localFileName in inputFileNamesAfterPrep],
'outputFileName': "{path}/{subfolder}/{filename}".format(path=self.pathOUT, subfolder=self.sample.identifier, filename=inputFileNamesAfterPrep[0]),
'tmpFileName': "{path}/{subfolder}/{filename}".format(path=self.tmpDir, subfolder=temporaryName, filename=inputFileNamesAfterPrep[0]),
})
# lists of single modules can be given instead of a module, "--addCollections Sys.all"
# [Sys]
# all = ['Sys.Vtype', 'Sys.Leptons', ...]
# TODO: make it fully recursive
def parseCollectionList(self, collections):
collectionsListsReplaced = []
for collection in collections:
if '.' in collection:
section = collection.split('.')[0]
key = collection.split('.')[1]
listExpression = self.config.get(section, key).strip()
if listExpression.startswith('[') and listExpression.endswith(']'):
listParsed = eval(listExpression)
for i in listParsed:
collectionsListsReplaced.append(i)
else:
collectionsListsReplaced.append(collection)
else:
collectionsListsReplaced.append(collection)
return collectionsListsReplaced
# run all subjobs
def run(self):
nFilesProcessed = 0
nFilesFailed = 0
for subJob in self.subJobs:
# only process if output is non-existing/broken or --force was used
if self.opts.force or not self.fileLocator.isValidRootFile(subJob['outputFileName']):
# create directories
outputFolder = '/'.join(subJob['outputFileName'].split('/')[:-1])
tmpFolder = '/'.join(subJob['tmpFileName'].split('/')[:-1])
self.fileLocator.makedirs(outputFolder)
self.fileLocator.makedirs(tmpFolder)
# load sample tree
sampleTree = SampleTree(subJob['localInputFileNames'], config=self.config)
if not sampleTree.tree:
print "trying fallback...", len(subJob['inputFileNames'])
if len(subJob['inputFileNames']) == 1:
# try original naming scheme if reading directly from Heppy/Nano ntuples (without prep)
fileNameOriginal = self.pathIN + '/' + subJob['inputFileNames'][0]
print "FO:", fileNameOriginal
xrootdRedirector = self.fileLocator.getRedirector(fileNameOriginal)
sampleTree = SampleTree([fileNameOriginal], config=self.config, xrootdRedirector=xrootdRedirector)
if not sampleTree.tree:
print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED!\x1b[0m"
nFilesFailed += 1
continue
else:
print "\x1b[31mERROR: file does not exist or is broken, will be SKIPPED! (old naming scheme not supported for joining multipel files)\x1b[0m"
nFilesFailed += 1
continue
# to use this syntax, use "--addCollections Sys.Vtype" for a config file entry like this:
# [Sys]
# Vtype = VtypeCorrector.VtypeCorrector(channel='Zll')
# (instead of passing the tree in the constructor, the setTree method can be used)
pyModules = []
for collection in self.collections:
if '.' in collection:
section = collection.split('.')[0]
key = collection.split('.')[1]
pyCode = self.config.get(section, key)
# import module from myutils
moduleName = pyCode.split('(')[0].split('.')[0].strip()
if self.debug:
print "DEBUG: import module:", moduleName
print("\x1b[33mDEBUG: " + collection + ": run PYTHON code:\n"+pyCode+"\x1b[0m")
globals()[moduleName] = importlib.import_module(".{module}".format(module=moduleName), package="myutils")
# get object
wObject = eval(pyCode)
# pass the tree and other variables if needed to finalize initialization
if hasattr(wObject, "customInit") and callable(getattr(wObject, "customInit")):
wObject.customInit({'config': self.config,
'sampleTree': sampleTree,
'tree': sampleTree.tree,
'sample': self.sample,
'channel': self.channel,
'pathIN': self.pathIN,
'pathOUT': self.pathOUT,
})
# add callbacks if the objects provides any
if hasattr(wObject, "processEvent") and callable(getattr(wObject, "processEvent")):
sampleTree.addCallback('event', wObject.processEvent)
# add branches
if hasattr(wObject, "getBranches") and callable(getattr(wObject, "getBranches")):
sampleTree.addOutputBranches(wObject.getBranches())
pyModules.append(wObject)
# DEPRECATED, do not use anymore ---> use BranchTools.TreeFormulas()
if 'addbranches' in self.collections:
writeNewVariables = eval(self.config.get("Regression", "writeNewVariablesDict"))
sampleTree.addOutputBranches(writeNewVariables)
# DEPRECATED, do not use anymore ---> use BranchTools.Drop()
if 'removebranches' in self.collections:
bl_branch = eval(config.get('Branches', 'useless_branch'))
for br in bl_branch:
sampleTree.addBranchToBlacklist(br)
bl_branch = eval(config.get('Branches', 'useless_after_sys'))
for br in bl_branch:
sampleTree.addBranchToBlacklist(br)
# define output file
sampleTree.addOutputTree(subJob['tmpFileName'], cut='1', branches='*', friend=self.opts.friend)
# run processing
for pyModule in pyModules:
if hasattr(pyModule, "beforeProcessing"):
getattr(pyModule, "beforeProcessing")()
sampleTree.process()
for pyModule in pyModules:
if hasattr(pyModule, "afterProcessing"):
getattr(pyModule, "afterProcessing")()
# if output trees have been produced: copy temporary file to output folder
if sampleTree.getNumberOfOutputTrees() > 0:
try:
self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True)
print 'copy ', subJob['tmpFileName'], subJob['outputFileName']
if self.verifyCopy:
if not self.fileLocator.isValidRootFile(subJob['outputFileName']):
print 'INFO: output at final destination broken, try to copy again from scratch disk to final destination...'
self.fileLocator.cp(subJob['tmpFileName'], subJob['outputFileName'], force=True)
print 'INFO: second attempt copy done!'
if not self.fileLocator.isValidRootFile(subJob['outputFileName']):
print '\x1b[31mERROR: output still broken!\x1b[0m'
nFilesFailed += 1
raise Exception("FileCopyError")
else:
print 'INFO: file is good after second attempt!'
except Exception as e:
print e
print "\x1b[31mERROR: copy from scratch to final destination failed!!\x1b[0m"
# delete temporary file
try:
self.fileLocator.rm(subJob['tmpFileName'])
except Exception as e:
print e
print "WARNING: could not delete file on scratch!"
# clean up
if hasattr(wObject, "cleanUp") and callable(getattr(wObject, "cleanUp")):
getattr(wObject, "cleanUp")()
else:
print 'SKIP:', subJob['inputFileNames']
if nFilesFailed > 0:
raise Exception("ProcessingIncomplete")
dest='maxEvents',
default=10000,
help="max number of events per file. Default 10000")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
config = BetterConfigParser()
config.read(opts.config)
import ROOT
ROOT.gROOT.SetBatch(True)
namelistIN = opts.samples
namelist = namelistIN.split(',')
samplesinfo = config.get('Directories', 'samplesinfo')
INpath = config.get('Directories', 'samplepath') + '/envW/sys/'
OUTpath = INpath # for the moemnt OUTpath = INpath ->> TO FIX
info = parse_info(samplesinfo, INpath)
for job in info:
if eval(job.active):
if job.name in namelist:
#get trees:
print INpath + '/' + job.prefix + job.identifier + '.root'
sourceFile = ROOT.TFile.Open(
INpath + '/' + job.prefix + job.identifier + '.root', 'read')
Tree = sourceFile.Get('tree')
obj = ROOT.TObject
nentries = Tree.GetEntries()
print nentries
parser.add_option("-M", "--max-events", dest='maxEvents', default=10000,
help="max number of events per file. Default 10000")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
config = BetterConfigParser()
config.read(opts.config)
import ROOT
ROOT.gROOT.SetBatch(True)
namelistIN=opts.samples
namelist=namelistIN.split(',')
samplesinfo=config.get('Directories','samplesinfo')
INpath=config.get('Directories','samplepath')+'/envW/sys/'
OUTpath=INpath # for the moemnt OUTpath = INpath ->> TO FIX
info = parse_info(samplesinfo,INpath)
for job in info:
if eval(job.active):
if job.name in namelist:
#get trees:
print INpath+'/'+job.prefix+job.identifier+'.root'
sourceFile = ROOT.TFile.Open(INpath+'/'+job.prefix+job.identifier+'.root','read')
Tree = sourceFile.Get('tree')
obj = ROOT.TObject
nentries = Tree.GetEntries()
print nentries
# parser.add_option("-P", "--path", dest="path", default="",
# help="path to samples")
parser.add_option("-S", "--samples", dest="names", default="", help="samples you want to run on")
parser.add_option(
"-C", "--config", dest="config", default=[], action="append", help="configuration defining the plots to make"
)
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo, TreeCache
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
TrainFlag = eval(config.get("Analysis", "TrainFlag"))
btagLibrary = config.get("BTagReshaping", "library")
samplesinfo = config.get("Directories", "samplesinfo")
VHbbNameSpace = config.get("VHbbNameSpace", "library")
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs = eval(config.get("AngularLike", "backgrounds"))
ang_yield = eval(config.get("AngularLike", "yields"))
# path=opts.path
pathIN = config.get("Directories", "SYSin")
pathOUT = config.get("Directories", "SYSout")
tmpDir = os.environ["TMPDIR"]
print "INput samples:\t%s" % pathIN
#opts.config.append('13TeVconfig/vhbbPlotDef.ini')
#print 'Plot Config List: ',opts.config
config = BetterConfigParser()
config.read(opts.config)
#path = opts.path
region = opts.region
batch = opts.batch
print batch
# additional blinding cut:
addBlindingCut = None
if config.has_option('Plot_general', 'addBlindingCut'):
addBlindingCut = config.get('Plot_general', 'addBlindingCut')
print 'adding add. blinding cut'
#get locations:
Wdir = config.get('Directories', 'Wdir')
# the samples_nosplit.cfg file in 13TeV
samplesinfo = config.get('Directories', 'samplesinfo')
# which directory to take the samples from(sys_out currently)
path = config.get('Directories', 'plottingSamples')
# Add external macros
# For batch jobs lock the compliation
#lock = Lock()
#sleep(np.random.rand(1,1)*120)
opts.config = "config"
from myutils import BetterConfigParser, printc, ParseInfo, mvainfo, StackMaker, HistoMaker
print opts.config
opts.config.append('heppy13TeVconfig/vhbbPlotDef.ini')
config = BetterConfigParser()
config.read(opts.config)
#path = opts.path
region = opts.region
# additional blinding cut:
addBlindingCut = None
if config.has_option('Plot_general','addBlindingCut'):
addBlindingCut = config.get('Plot_general','addBlindingCut')
print 'adding add. blinding cut'
#get locations:
Wdir=config.get('Directories','Wdir')
samplesinfo=config.get('Directories','samplesinfo')
path = config.get('Directories','plottingSamples')
section='Plot:%s'%region
info = ParseInfo(samplesinfo,path)
#----------Histo from trees------------
def doPlot():
vars = (config.get(section, 'vars')).split(',')
print "Please provide a task.\n-J prep:\tpreparation of Trees\n-J sys:\t\twrite regression and systematics\n-J eval:\tcreate MVA output\n-J plot:\tproduce Plots\n-J dc:\t\twrite workspaces and datacards"
sys.exit(123)
en = opts.tag
#create the list with the samples to run over
samplesList = opts.samples.split(",")
timestamp = time.asctime().replace(' ', '_').replace(':', '-')
if (debugPrintOUts): print 'samplesList', samplesList
if (debugPrintOUts): print 'timestamp', timestamp
# the list of the config is taken from the path config
pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths' % (en))
_configs = pathconfig.get('Configuration', 'List').split(" ")
configs = ['%sconfig/' % (en) + c for c in _configs]
if (debugPrintOUts): print 'configs', configs
if (debugPrintOUts): print 'opts.ftag', opts.ftag
if not opts.ftag == '':
tagDir = pathconfig.get('Directories', 'tagDir')
# DEBUG PURPOSE ONLY
# sys.exit()
if (debugPrintOUts): print 'tagDir', tagDir
DirStruct = {
'tagDir': tagDir,
'ftagdir': '%s/%s/' % (tagDir, opts.ftag),
'logpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'Logs'),
'plotpath': '%s/%s/%s/' % (tagDir, opts.ftag, 'Plots'),
parser.add_option("-A", "--apply", dest="apply", default=True, action="store_true",
help="configuration defining the")
parser.add_option("-W", "--weight", dest="weights", default=True, action="store_false",
help="Calculate the weights")
parser.add_option("-V", "--validate", dest="validate", default=False, action="store_false",
help="Make validation plot the weights")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
from myutils import BetterConfigParser
print opts.config
config = BetterConfigParser()
config.read(opts.config)
prefix = config.get('lheWeights','prefix')
newpostfix = config.get('lheWeights','newpostfix')
inclusive = config.get('lheWeights','inclusive')
files = eval(config.get('lheWeights','fileList'))
lheBin = eval(config.get('lheWeights','lheBin'))
print '\n========= Looping over Drell Yan Samples ==========\n'
fileList = []
for file in files:
new_entry = [prefix+file+'.root',files[file],[]]
print new_entry
if file == inclusive:
fileList.insert(0, new_entry)
else:
fileList.append(new_entry)
parser.add_option("-r","--regions", dest="regions", default='',
help="cut region identifiers, separated by comma")
parser.add_option("-t","--type", dest="fitType", default='',
help="shapes_prefit, shapes_fit_b, shapes_fit_s")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = ["config"]
# Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo
config = BetterConfigParser()
config.read(opts.config)
if len(opts.fitType) < 1:
if config.has_option('Fit','FitType'):
opts.fitType = config.get('Fit','FitType')
else:
opts.fitType = "shapes_prefit"
# run plotter
if len(opts.regions) < 1:
regions = eval(config.get('Fit', 'regions')).keys()
else:
regions = opts.regions.split(',')
for region in regions:
plotter = PostfitPlotter(config=config, region=region, directory=opts.fitType)
plotter.prepare()
plotter.run()
help="configuration tag")
parser.add_option("-t","--trainingRegions", dest="trainingRegions", default='',
help="cut region identifier")
parser.add_option("-l","--load", dest="loadHDF", action="store_const", default=False, const=True,
help="load hdf files")
parser.add_option("--force", dest="force", action="store_const", default=False, const=True,
help="force reloading samples (use it when cuts have changed)")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = ["config"]
# Import after configure to get help message
if len(opts.tag.strip()) > 1:
config = BetterConfigParser()
config.read("{tag}config/paths.ini".format(tag=opts.tag))
configFiles = config.get("Configuration", "List").split(' ')
opts.config = ["{tag}config/{file}".format(tag=opts.tag, file=x.strip()) for x in configFiles]
print("reading config files:", opts.config)
# load config
config = BetterConfigParser()
config.read(opts.config)
converter = SampleTreesToDataFrameConverter(config,config_name=opts.tag)
if not opts.loadHDF:
converter.loadSamples(safe_hdf=True,force=opts.force)
dfs = converter.getDataFrames()
"--local",
dest="local",
default=True,
help="True to run it locally. False to run on batch system using config")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
#Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo, TreeCache
#load config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
run = opts.training
gui = opts.verbose
#GLOABAL rescale from Train/Test Spliiting:
global_rescale = 2.
#get locations:
MVAdir = config.get('Directories', 'vhbbpath') + '/data/'
samplesinfo = config.get('Directories', 'samplesinfo')
#systematics
systematics = config.get('systematics', 'systematics')
systematics = systematics.split(' ')
weightF = config.get('Weights', 'weightF')
default='',
help="cut regions identifier")
parser.add_option("-C",
"--config",
dest="config",
default=[],
action="append",
help="configuration file")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
# Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo
# load config
config = BetterConfigParser()
config.read(opts.config)
# if no region is given in argument, run it for all of them
regionsListString = opts.regions if len(
opts.regions.strip()) > 0 else config.get('LimitGeneral', 'List')
regions = [
x.strip() for x in regionsListString.split(',') if len(x.strip()) > 0
]
for region in regions:
mergeDC = MergeDatacards(config=config, region=region)
isComplete = mergeDC.prepare()
if isComplete:
mergeDC.run()
# Import heppy configurations
argv = sys.argv
parser = OptionParser()
parser.add_option("-S", "--samples", dest="names", default="",
help="samples you want to run on")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
config = BetterConfigParser()
config.read(opts.config)
samplesinfo = config.get('Directories','samplesinfo')
tmpDir = os.environ["TMPDIR"]
pathIN = config.get('Directories','SYSout')
pathOUT = config.get('Directories','SYSout')
name = config.get('TrainRegression', 'name')
namelist = opts.names.split(',')
info = ParseInfo(samplesinfo,pathIN)
# Now load kinFit modules/namspaces
ROOT.gSystem.Load("libPhysicsToolsKinFitter.so")
sampleName = section
sampleNameShort = '_'.join(sampleName.split('_')[0:7]).strip()
if '_ext' in sampleName:
sampleNameShort = sampleNameShort.split('_ext')[0]
elif '_backup' in sampleName:
sampleNameShort = sampleNameShort.split('_backup')[0]
if sampleNameShort in sampleDict:
sampleDict[sampleNameShort].append(sampleName)
else:
sampleDict[sampleNameShort] = [sampleName]
if verify:
extweight = 0
try:
extweight = float(config.get(section, 'extweight'))
except:
pass
sampleWeights[sampleNameShort] = float(sampleWeights[sampleNameShort]) + extweight if sampleNameShort in sampleWeights else extweight
if verify:
for sampleNameShort,totalWeight in sampleWeights.iteritems():
if sampleNameShort in sampleDict and len(sampleDict[sampleNameShort])>1:
print sampleNameShort,":",totalWeight
else:
for sample,extParts in sampleDict.iteritems():
if len(extParts) > 1:
print '-'*80
print sample,":"
getExtWeights(config,extParts)
#path_ = '/mnt/t3nfs01/data01/shome/gaperrin/VHbb/CMSSW_7_4_3/src/Xbb/python/logs_v25/MERGESYSCACHINGDCSPLIT_SB_M/Limits/'
#path_ = '/mnt/t3nfs01/data01/shome/gaperrin/VHbb/CMSSW_7_4_3/src/Xbb/python/logs_v25/MERGESYSCACHINGDCSPLIT_SB_Mjj_08_08_17_v3/Limits/'
#path_ = '/mnt/t3nfs01/data01/shome/gaperrin/VHbb/CMSSW_7_4_3/src/Xbb/python/logs_v25/MSCACHINGSPLIT_eval_v8_sigPS_All_rmvbr_VH_7/Limits/'
path_ = '/mnt/t3nfs01/data01/shome/berger_p2/VHbb/CMSSW_9_4_0_pre3/src/Xbb/python/logs_v25/rundc-all-test12/Limits'
#------------------------------------------------------
#---------- Mjj ---------------------------------------
#mode = 'Mjj'
#xMin=0
#xMax=255
#masses = ['125']
#Abins = ['highPt','lowPt','medPt']
#channels= ['Zee','Zmm']
#------------------------------------------------------
path = config.get('Directories', 'limits')
outpath = config.get('Directories', 'plotpath')
setup = eval(config.get('LimitGeneral', 'setup'))
Dict = eval(config.get('LimitGeneral', 'Dict'))
MCs = [Dict[s] for s in setup]
sys_BDT = ['CUETP8M1'] + eval(config.get('LimitGeneral', 'sys_BDT'))
#systematicsnaming = eval(config.get('LimitGeneral','systematicsnaming'))
#systematicsnaming = eval(config.get('LimitGeneral','systematicsnaming_HighPt'))
#systematicsnaming = eval(config.get('LimitGeneral','systematicsnaming_LowPt'))
systematicsnaming = eval(config.get('LimitGeneral', 'systematicsnaming'))
systs = [systematicsnaming[s] for s in sys_BDT]
sys_weight = eval(config.get('LimitGeneral', 'weightF_sys'))
for sw in sys_weight:
import os
argv = sys.argv
#------------------------------------------------------------------------------
# prepare sample .txt files from ntuple folders
#------------------------------------------------------------------------------
#get files info from config
parser = OptionParser()
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="directory config")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
inputPath = config.get('Directories', 'samplepath')
outputPath = config.get('Directories', 'samplefiles')
print 'INPUT: ', inputPath
print 'OUTPUT:', outputPath
print '-'*80
if '/pnfs/' in inputPath:
inputPath = '/pnfs/' + inputPath.split('/pnfs/')[-1]
sampleNameReplacementRules = {
'RunIISummer16MiniAODv2_': '',
}
sampleFileReplacementRules = {
'/pnfs/psi.ch/cms/trivcat': ''
}
# find all folders which contain ntuples
#!/usr/bin/env python
import os,sys,ROOT
from optparse import OptionParser
ROOT.gROOT.SetBatch(True)
#--CONFIGURE---------------------------------------------------------------------
argv = sys.argv
parser = OptionParser()
parser.add_option("-S", "--section", dest="section", default="Cuts",
help="Config section")
parser.add_option("-V", "--value", dest="var", default="",
help="Config value")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration file")
(opts, args) = parser.parse_args(argv)
from myutils import BetterConfigParser
config = BetterConfigParser()
config.read(opts.config)
print config.get(opts.section,opts.var)
parser.add_option("-F", "--force", dest="force", action="store_true", help="overwrite existing files", default=False)
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None
print "len(filelist)",len(filelist),
if len(filelist) > 0:
print "filelist[0]:", filelist[0]
else:
print ''
debug = 'XBBDEBUG' in os.environ
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
TrainFlag = eval(config.get('Analysis','TrainFlag'))
btagLibrary = config.get('BTagReshaping','library')
samplesinfo=config.get('Directories','samplesinfo')
channel=config.get('Configuration','channel')
VHbbNameSpace=config.get('VHbbNameSpace','library')
ROOT.gSystem.Load(VHbbNameSpace)
pathIN = config.get('Directories','SYSin')
pathOUT = config.get('Directories','SYSout')
tmpDir = config.get('Directories','scratch')
print 'INput samples:\t%s'%pathIN
print 'OUTput samples:\t%s'%pathOUT
fileLocator = FileLocator(config=config)
# samples
class TBmaster(object):
def __init__(self, TB, client, psiSubscription, Logger, color='black', psiVersion='psi46expert'):
self.TB = TB
self.client = client
self.psiSubscription = psiSubscription
self.color = color
self.Logger = Logger
self.TBSubscription = '/TB%s'%self.TB
self.client.subscribe(self.TBSubscription)
self.dir = ''
self.psiVersion = psiVersion
if self.psiVersion.lower().endswith("psi46expert"):
self.version ='psi46expert'
elif self.psiVersion.lower().endswith("pyxar"):
self.version ='pyxar'
elif self.psiVersion.lower().endswith("pxar"):
self.version ='pxar'
else:
self.version = 'unknown'
self.failed = False
self.busy = False
self.testName ='unkown'
self.testNo = -1
self.TestEnd = False
self.DoTest= False
self.ClosePSI= False
self.Abort = False
self.LogFile = ""
self.RootFile = ""
# default value in Vcal units, -1 means use untrimmed parameters
# this setting is overwritten if [Test Trim] is specified in the ini file
# containg testParameters=Vcal=*
self.trimVcal = -1
self.init = BetterConfigParser()
self.init.read("../config/elComandante.ini")
try:
testParameters = self.init.get('Test Trim','testParameters')
pos1 = testParameters.find("=")
if pos1 > 0:
testParametersName = testParameters[0:pos1]
testParametersValue = testParameters[pos1+1:]
if testParametersName.lower() == "vcal":
self.trimVcal = int(testParametersValue)
self.Logger << "TB%s: using option '-T %s' when calling pxar"%(self.TB, self.trimVcal)
except:
self.Logger << "TB%s: no [Test Trim] section found in ini file, using untrimmed parameters"%self.TB
def _spawn(self,executestr):
my_env = os.environ
if my_env.has_key("LD_PRELOAD"):
my_env["LD_PRELOAD"] = "/opt/glibc-2.14/lib/libc.so.6:" + my_env["LD_PRELOAD"]
else:
my_env["LD_PRELOAD"] = "/opt/glibc-2.14/lib/libc.so.6:"
self.proc = subprocess.Popen([executestr,''], shell = True, stdout = subprocess.PIPE, stdin = subprocess.PIPE, env = my_env)
self.busy = True
def _kill(self):
try:
self.proc.kill()
self.Logger.warning("PSI%s KILLED"%self.TB)
except:
self.Logger.warning("nothing to be killed")
def _abort(self):
self.Logger.warning('ABORT!')
self._kill()
self.Abort = False
return True
def _resetVariables(self):
self.busy = False
self.failed = False
self.TestEnd = False
self.DoTest = False
self.ClosePSI = False
self.Abort = False
def _readAllSoFar(self, retVal = ''):
while (select.select([self.proc.stdout],[],[],0)[0]!=[]) and self.proc.poll() is None:
retVal += self.proc.stdout.read(1)
return retVal
@staticmethod
def findError(stat):
return any([Error in stat for Error in ['error','Error','anyOtherString','command not found']])
def _readout(self):
internalFailed = False
self.Logger << '>>> Aquire Testboard %s <<<'%self.TB
# self._answer(self)
while self.proc.poll() is None and self.ClosePSI==False:
if self.Abort:
internalFailed = self._abort()
lines = ['']
lines = self._readAllSoFar(lines[-1]).split('\n')
for a in range(len(lines)-1):
line=lines[a]
hesays=line.rstrip()
self.client.send(self.TBSubscription,'%s\n'%hesays)
self.Logger.printcolor("psi46@TB%s >> %s"%(self.TB,hesays),self.color)
if self.findError(line.rstrip()):
self.Logger << 'The following error triggered the exception:'
self.Logger.warning(line.rstrip())
self.client.send(self.psiSubscription, 'psi46@TB%s - Error >> %s\n'%(self.TB,line.rstrip()))
self.client.send(self.TBSubscription, 'Error >> %s\n'%(line.rstrip()))
internalFailed = True
self.failed = True
self._kill()
if 'command not found' in line.strip():
self.Logger.warning("psi46expert for TB%s not found"%self.TB)
if self.Abort:
internalFailed = self._abort()
self.failed = internalFailed or self.failed
self.Logger << '>>> Release Testboard %s <<<'%self.TB
self.TestEnd = True
self.busy = False
# check if log and root files have been written
try:
with open(self.LogFile) as f:
lines = f.readlines()
if not "welcome to pxar" in lines[0].lower() or not ("this is the end, my friend" in lines[-1].lower() or "pixsetup free fpxarmemory" in lines[-1].lower()):
print "\x1b[46m\x1b[97mWARNING: incomplete logfile: '%s'! \x1b[0m"%self.LogFile
internalFailed = True
self.failed = True
except:
if len(self.LogFile) > 0:
print "\x1b[46m\x1b[97mWARNING: can't open logfile: '%s'! \x1b[0m"%self.LogFile
internalFailed = True
self.failed = True
if len(self.RootFile) > 0 and not os.path.isfile(self.RootFile):
print "\x1b[45m\x1b[97mCRITICAL: .root file does not exist: %s! \x1b[0m"%self.RootFile
internalFailed = True
self.failed = True
return internalFailed
def _answer(self):
name = self.get_directory_name()
if self.failed:
self.client.send(self.psiSubscription,':STAT:TB%s! %s:failed\n'%(self.TB,name))
self.Logger.warning(':Test %s failed in TB%s'%(name,self.TB))
self.client.send(self.psiSubscription,':STAT:TB%s! %s:failed\n'%(self.TB,name))
elif self.busy:
self.client.send(self.psiSubscription,':STAT:TB%s! %s:busy\n'%(self.TB,name))
#self.Logger << ':Test %s busy in TB%s'%(name,self.TB)
else:
self.client.send(self.psiSubscription,':STAT:TB%s! %s:finished\n'%(self.TB,name))
self.Logger << ':Test %s finished in TB%s'%(name,self.TB)
def get_directory_name(self):
dir = self.dir.rstrip('/')
name = dir.split('/')[-1]
return name
def executeTest(self,whichTest,dir,fname):
self._resetVariables()
self.dir = dir
self.Logger << 'executing psi46 %s in TB%s'%(whichTest,self.TB)
if self.version == 'pyxar':
executestr='%s --dir %s --nogui < %s'%(self.psiVersion,dir,whichTest)
elif self.version == 'pxar':
# cat test | ../bin/pXar -d whereever
#executestr = 'cat {testfile} | {psiVersion} -dir {dir} -r {rootfilename}.root -log {logfilename}.log'.format(testfile = whichTest, psiVersion = self.psiVersion, dir = dir, rootfilename = fname, logfilename = fname)
logIDString = 'TB%s'%self.TB
trimParameters = ''
if self.trimVcal >= 0:
trimParameters = '-T %i'%self.trimVcal
executestr = 'cat %(testfile)s | %(psiVersion)s -d %(dir)s -v DEBUG %(trim)s -r %(rootfilename)s.root -L %(logIDString)s'%{'testfile' : whichTest, 'psiVersion' : self.psiVersion, 'dir' : dir, 'rootfilename' : fname, 'trim' : trimParameters, 'logIDString' : logIDString}
else:
executestr='%s -dir %s -f %s -r %s.root -log %s.log'%(self.psiVersion,dir,whichTest,fname,fname)
self.LogFile = "%s/%s.log"%(dir, fname)
self.RootFile = "%s/%s.root"%(dir, fname)
self._spawn(executestr)
self.failed=self._readout()
self._answer()
def openTB(self,dir,fname,poff=False):
self._resetVariables()
self.dir = dir
self.Logger << 'open TB%s'%(self.TB)
if self.version == 'pyxar':
executestr='%s --dir %s --nogui'%(self.psiVersion,dir)
elif self.version == 'pxar':
# cat test | ../bin/pXar -d whereever
trimParameters = ''
if self.trimVcal >= 0:
trimParameters = '-T %i'%self.trimVcal
executestr = '%(psiVersion)s -d %(dir)s %(trim)s -r %(rootfilename)s.root'%{'psiVersion' : self.psiVersion, 'dir' : dir, 'rootfilename' : fname, 'trim' : trimParameters}
else:
executestr='%s -dir %s -r %s.root -log %s.log'%(self.psiVersion,dir,fname,fname)
if poff:
executestr+=' -t poff -i'
self.Logger << 'exec string = %s'%executestr
self._spawn(executestr)
self.failed=self._readout()
self.Logger << 'done, failed = %s'%self.failed
self._answer()
while not self.ClosePSI:
pass
self.Logger << 'CLOSE TB %s HERE'%(self.TB)
self.proc.communicate(input='exit\n')[0]
self.proc.poll()
if (None == self.proc.returncode):
try:
self.proc.send_signal(signal.SIGINT)
except:
self.Logger << 'Process already killed'
self._answer()
def sendTBStatus(self):
self._answer()
parser.add_option("-F", "--force", dest="force", action="store_true", help="overwrite existing files", default=False)
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None
print "len(filelist)",len(filelist),
if len(filelist) > 0:
print "filelist[0]:", filelist[0]
else:
print ''
debug = 'XBBDEBUG' in os.environ
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
TrainFlag = eval(config.get('Analysis','TrainFlag'))
btagLibrary = config.get('BTagReshaping','library')
samplesinfo=config.get('Directories','samplesinfo')
channel=config.get('Configuration','channel')
VHbbNameSpace=config.get('VHbbNameSpace','library')
ROOT.gSystem.Load(VHbbNameSpace)
pathIN = config.get('Directories','SYSin')
pathOUT = config.get('Directories','SYSout')
tmpDir = config.get('Directories','scratch')
print 'INput samples:\t%s'%pathIN
print 'OUTput samples:\t%s'%pathOUT
fileLocator = FileLocator(config=config)
# samples
#parse args and setup logdir
args = parser.parse_args()
Logger = printer()
Logger.set_name("Psi46Log")
Logger.set_prefix('')
Logger.set_logfile('%s'%(args.loggingDir),'psi46Handler.log')
Logger <<'ConfigDir: "%s"'%args.configDir
configDir= args.configDir
numTB = int(args.numTB)
#load config
config = BetterConfigParser()
config.read(configDir+'/elComandante.conf')
#config
clientName = "psi46"
serverZiel=config.get('subsystem','Ziel')
Port = int(config.get('subsystem','Port'))
serverPort = int(config.get('subsystem','Port'))
psiSubscription = config.get('subsystem','psiSubscription')
errorSubscription = "/error"
#construct
client = sClient(serverZiel,serverPort,clientName)
#subscribe
client.subscribe(psiSubscription)
#----------------------------------------------------
#handler
def handler(signum, frame):
Logger << 'Close Connection'
client.closeConnection()
#Logger << 'Signal handler called with signal', signum
#Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo, TreeCache
import os
if os.path.exists("../interface/DrawFunctions_C.so"):
print 'ROOT.gROOT.LoadMacro("../interface/DrawFunctions_C.so")'
ROOT.gROOT.LoadMacro("../interface/DrawFunctions_C.so")
#load config
config = BetterConfigParser()
config.read(opts.config)
run = opts.training
gui = opts.verbose
anaTag = config.get("Analysis", "tag")
#print "Compile external macros"
#print "=======================\n"
## compile external macros to compute variables on the fly
#ROOT.gSystem.CompileMacro("../plugins/PU.C")
#GLOABAL rescale from Train/Test Spliiting:
global_rescale = 2.
#get locations:
MVAdir = config.get('Directories', 'vhbbpath') + '/python/weights/'
samplesinfo = config.get('Directories', 'samplesinfo')
#systematics
# opts.config.append('13TeVconfig/vhbbPlotDef.ini')
# print 'Plot Config List: ',opts.config
config = BetterConfigParser()
config.read(opts.config)
# path = opts.path
region = opts.region
batch = opts.batch
print batch
# additional blinding cut:
addBlindingCut = None
if config.has_option("Plot_general", "addBlindingCut"):
addBlindingCut = config.get("Plot_general", "addBlindingCut")
print "adding add. blinding cut"
# get locations:
Wdir = config.get("Directories", "Wdir")
# the samples_nosplit.cfg file in 13TeV
samplesinfo = config.get("Directories", "samplesinfo")
# which directory to take the samples from(sys_out currently)
path = config.get("Directories", "plottingSamples")
# Add external macros
# For batch jobs lock the compliation
# lock = Lock()
def drawFromDC():
config = BetterConfigParser()
config.read(opts.config)
print "opts.config:", opts.config
dataname = ''
if 'Zmm' in opts.bin: dataname = 'Zmm'
elif 'Zee' in opts.bin: dataname = 'Zee'
elif 'Wmunu' in opts.bin: dataname = 'Wmn'
elif 'Wenu' in opts.bin: dataname = 'Wen'
elif 'Znunu' in opts.bin: dataname = 'Znn'
elif 'Wtn' in opts.bin: dataname = 'Wtn'
print 'Variable printing'
print opts.var
if (opts.var == ''):
var = 'BDT'
if dataname == 'Zmm' or dataname == 'Zee': var = 'BDT_Zll'
elif dataname == 'Wmn' or dataname == 'Wen': var = 'BDT_Wln'
elif dataname == 'Znn':
if 'HighPt' in opts.bin: var = 'BDT_ZnnHighPt'
elif 'LowPt' in opts.bin: var = 'BDT_ZnnLowPt'
elif 'LowCSV' in opts.bin: var = 'BDT_ZnnLowCSV'
else: var = 'BDT_Znn'
if dataname == '' or var == 'BDT':
raise RuntimeError, "Did not recognise mode or var from %s" % opts.bin
else:
var = opts.var
region = 'BDT'
ws_var = config.get('plotDef:%s' % var, 'relPath')
ws_var = ROOT.RooRealVar(ws_var, ws_var, -1., 1.)
blind = eval(config.get('Plot:%s' % region, 'blind'))
Stack = StackMaker(config, var, region, True)
if 'LowPt' in opts.bin or 'ch1_Wenu' == opts.bin or 'ch2_Wmunu' == opts.bin:
print 'Niklas %s' % opts.bin
Stack.addFlag2 = 'Low p_{T}(V)'
elif 'MedPt' in opts.bin or 'ch1_Wenu2' == opts.bin or 'ch2_Wmunu2' == opts.bin:
Stack.addFlag2 = 'Intermediate p_{T}(V)'
elif 'HighPt' in opts.bin or 'ch1_Wenu3' == opts.bin or 'ch2_Wmunu3' == opts.bin:
Stack.addFlag2 = 'High p_{T}(V)'
else:
Stack.addFlag2 = ''
preFit = False
addName = 'PostFit_%s' % (opts.fit)
if not opts.mlfit:
addName = 'PreFit'
preFit = True
Stack.options['pdfName'] = '%s_%s_%s.pdf' % (var, opts.bin, addName)
log = eval(config.get('Plot:%s' % region, 'log'))
setup = config.get('Plot_general', 'setup').split(',')
if dataname == 'Zmm' or dataname == 'Zee':
try:
setup.remove('W1b')
setup.remove('W2b')
setup.remove('Wlight')
setup.remove('WH')
except:
print '@INFO: Wb / Wligh / WH not present in the datacard'
# if not dataname == 'Znn' and 'QCD' in setup:
# setup.remove('QCD')
Stack.setup = setup
Dict = eval(config.get('LimitGeneral', 'Dict'))
lumi = eval(config.get('Plot_general', 'lumi'))
options = copy(opts)
options.dataname = "data_obs"
options.mass = 0
options.format = "%8.3f +/- %6.3f"
options.channel = opts.bin
options.excludeSyst = []
options.norm = False
options.stat = False
options.bin = True # fake that is a binary output, so that we parse shape lines
options.out = "tmp.root"
options.fileName = args[0]
options.cexpr = False
options.fixpars = False
options.libs = []
options.verbose = 0
options.poisson = 0
options.nuisancesToExclude = []
options.noJMax = None
theBinning = ROOT.RooFit.Binning(Stack.nBins, Stack.xMin, Stack.xMax)
file = open(opts.dc, "r")
os.chdir(os.path.dirname(opts.dc))
DC = parseCard(file, options)
if not DC.hasShapes: DC.hasShapes = True
MB = ShapeBuilder(DC, options)
theShapes = {}
theSyst = {}
nuiVar = {}
if opts.mlfit:
nuiVar = readBestFit(opts.mlfit)
if not opts.bin in DC.bins:
raise RuntimeError, "Cannot open find %s in bins %s of %s" % (
opts.bin, DC.bins, opts.dc)
for b in DC.bins:
if options.channel != None and (options.channel != b): continue
exps = {}
expNui = {}
shapeNui = {}
reducedShapeNui = {}
for (p, e) in DC.exp[b].items(
): # so that we get only self.DC.processes contributing to this bin
exps[p] = [e, []]
expNui[p] = [e, []]
for (lsyst, nofloat, pdf, pdfargs, errline) in DC.systs:
if pdf in ('param', 'flatParam'): continue
# begin skip systematics
skipme = False
for xs in options.excludeSyst:
if re.search(xs, lsyst):
skipme = True
if skipme: continue
# end skip systematics
counter = 0
for p in DC.exp[b].keys(
): # so that we get only self.DC.processes contributing to this bin
if errline[b][p] == 0: continue
# if p == 'QCD' and not 'QCD' in setup: continue
if pdf == 'gmN':
exps[p][1].append(1 / sqrt(pdfargs[0] + 1))
elif pdf == 'gmM':
exps[p][1].append(errline[b][p])
elif type(errline[b][p]) == list:
kmax = max(errline[b][p][0], errline[b][p][1],
1.0 / errline[b][p][0], 1.0 / errline[b][p][1])
exps[p][1].append(kmax - 1.)
elif pdf == 'lnN':
lnNVar = max(errline[b][p], 1.0 / errline[b][p]) - 1.
if not nuiVar.has_key('%s_%s' % (opts.fit, lsyst)):
nui = 0.
else:
nui = nuiVar['%s_%s' % (opts.fit, lsyst)][0]
lnNVar = lnNVar * nuiVar['%s_%s' %
(opts.fit, lsyst)][1]
exps[p][1].append(lnNVar)
expNui[p][1].append(abs(1 - errline[b][p]) * nui)
elif ("shape" in pdf):
#print 'shape %s %s: %s'%(pdf,p,lsyst)
s0 = MB.getShape(b, p)
sUp = MB.getShape(b, p, lsyst + "Up")
sDown = MB.getShape(b, p, lsyst + "Down")
if (s0.InheritsFrom("RooDataHist")):
s0 = ROOT.RooAbsData.createHistogram(
s0, p, ws_var, theBinning)
s0.SetName(p)
sUp = ROOT.RooAbsData.createHistogram(
sUp, p + lsyst + 'Up', ws_var, theBinning)
sUp.SetName(p + lsyst + 'Up')
sDown = ROOT.RooAbsData.createHistogram(
sDown, p + lsyst + 'Down', ws_var, theBinning)
sDown.SetName(p + lsyst + 'Down')
theShapes[p] = s0.Clone()
theShapes[p + lsyst + 'Up'] = sUp.Clone()
theShapes[p + lsyst + 'Down'] = sDown.Clone()
if not nuiVar.has_key('%s_%s' % (opts.fit, lsyst)):
nui = 0.
reducedNui = 1.
else:
nui = nuiVar['%s_%s' % (opts.fit, lsyst)][0]
reducedNui = nuiVar['%s_%s' % (opts.fit, lsyst)][1]
shapeNui[p + lsyst] = nui
reducedShapeNui[lsyst] = reducedNui
if not 'CMS_vhbb_stat' in lsyst:
if counter == 0:
theSyst[lsyst] = s0.Clone()
theSyst[lsyst + 'Up'] = sUp.Clone()
theSyst[lsyst + 'Down'] = sDown.Clone()
else:
theSyst[lsyst].Add(s0)
theSyst[lsyst + 'Up'].Add(sUp.Clone())
theSyst[lsyst + 'Down'].Add(sDown.Clone())
counter += 1
procs = DC.exp[b].keys()
procs.sort()
print "Original procs:", procs
# if not 'QCD' in setup and 'QCD' in procs:
# procs.remove('QCD')
if not 'W2b' in setup and 'WjHF' in procs:
procs.remove('WjHF')
if not 'Wlight' in setup and 'WjLF' in procs:
procs.remove('WjLF')
fmt = ("%%-%ds " % max([len(p) for p in procs])) + " " + options.format
#Compute norm uncertainty and best fit
theNormUncert = {}
theBestFit = {}
for p in procs:
relunc = sqrt(sum([x * x for x in exps[p][1]]))
print fmt % (p, exps[p][0], exps[p][0] * relunc)
theNormUncert[p] = relunc
absBestFit = sum([x for x in expNui[p][1]])
theBestFit[p] = 1. + absBestFit
histos = []
typs = []
setup2 = copy(setup)
shapesUp = [[] for _ in range(0, len(setup2))]
shapesDown = [[] for _ in range(0, len(setup2))]
sigCount = 0
signalList = ['ZH', 'WH']
#signalList = ['VVb']
for p in procs:
b = opts.bin
for s in setup:
if not Dict[s] == p: continue
if s in signalList:
if sigCount == 0:
Overlay = copy(theShapes[Dict[s]])
else:
Overlay.Add(theShapes[Dict[s]])
sigCount += 1
else:
histos.append(theShapes[Dict[s]])
typs.append(s)
for (lsyst, nofloat, pdf, pdfargs, errline) in DC.systs:
if errline[b][p] == 0: continue
if ("shape" in pdf) and not 'CMS_vhbb_stat' in lsyst:
print 'syst %s' % lsyst
shapesUp[setup2.index(s)].append(theShapes[Dict[s] +
lsyst + 'Up'])
shapesDown[setup2.index(s)].append(
theShapes[Dict[s] + lsyst + 'Down'])
#-------------
#Compute absolute uncertainty from shapes
counter = 0
for (lsyst, nofloat, pdf, pdfargs, errline) in DC.systs:
sumErr = 0
for p in procs:
sumErr += errline[b][p]
if ("shape"
in pdf) and not 'CMS_vhbb_stat' in lsyst and not sumErr == 0:
theSystUp = theSyst[lsyst + 'Up'].Clone()
theSystUp.Add(theSyst[lsyst].Clone(), -1.)
theSystUp.Multiply(theSystUp)
theSystDown = theSyst[lsyst + 'Down'].Clone()
theSystDown.Add(theSyst[lsyst].Clone(), -1.)
theSystDown.Multiply(theSystDown)
theSystUp.Scale(reducedShapeNui[lsyst])
theSystDown.Scale(reducedShapeNui[lsyst])
if counter == 0:
theAbsSystUp = theSystUp.Clone()
theAbsSystDown = theSystDown.Clone()
else:
theAbsSystUp.Add(theSystUp.Clone())
theAbsSystDown.Add(theSystDown.Clone())
counter += 1
#-------------
#Best fit for shapes
if not preFit:
histos, Overlay, typs = getBestFitShapes(procs, theShapes, shapeNui,
theBestFit, DC, setup, opts,
Dict)
counter = 0
errUp = []
total = []
errDown = []
nBins = histos[0].GetNbinsX()
print 'total bins %s' % nBins
Error = ROOT.TGraphAsymmErrors(histos[0])
theTotalMC = histos[0].Clone()
for h in range(1, len(histos)):
theTotalMC.Add(histos[h])
total = [[]] * nBins
errUp = [[]] * nBins
errDown = [[]] * nBins
for bin in range(1, nBins + 1):
binError = theTotalMC.GetBinError(bin)
if math.isnan(binError):
binError = 0.
total[bin - 1] = theTotalMC.GetBinContent(bin)
#Stat uncertainty of the MC outline
errUp[bin - 1] = [binError]
errDown[bin - 1] = [binError]
#Relative norm uncertainty of the individual MC
for h in range(0, len(histos)):
print "h:", h
print "bin:", bin
print "histos:", histos
print "theNormUncert:", theNormUncert
print "histos[h]:", histos[h]
errUp[bin - 1].append(histos[h].GetBinContent(bin) *
theNormUncert[histos[h].GetName()])
errDown[bin - 1].append(histos[h].GetBinContent(bin) *
theNormUncert[histos[h].GetName()])
#Shape uncertainty of the MC
for bin in range(1, nBins + 1):
#print sqrt(theSystUp.GetBinContent(bin))
errUp[bin - 1].append(sqrt(theAbsSystUp.GetBinContent(bin)))
errDown[bin - 1].append(sqrt(theAbsSystDown.GetBinContent(bin)))
#Add all in quadrature
totErrUp = [sqrt(sum([x**2 for x in bin])) for bin in errUp]
totErrDown = [sqrt(sum([x**2 for x in bin])) for bin in errDown]
#Make TGraph with errors
for bin in range(1, nBins + 1):
if not total[bin - 1] == 0:
point = histos[0].GetXaxis().GetBinCenter(bin)
Error.SetPoint(bin - 1, point, 1)
Error.SetPointEYlow(bin - 1, totErrDown[bin - 1] / total[bin - 1])
print 'down %s' % (totErrDown[bin - 1] / total[bin - 1])
Error.SetPointEYhigh(bin - 1, totErrUp[bin - 1] / total[bin - 1])
print 'up %s' % (totErrUp[bin - 1] / total[bin - 1])
#-----------------------
#Read data
data0 = MB.getShape(opts.bin, 'data_obs')
if (data0.InheritsFrom("RooDataHist")):
data0 = ROOT.RooAbsData.createHistogram(data0, 'data_obs', ws_var,
theBinning)
data0.SetName('data_obs')
datas = [data0]
datatyps = [None]
datanames = [dataname]
print "blind:", blind
print "'BDT' in var:", 'BDT' in var
if blind and 'BDT' in var:
print "I'm blinding..."
for bin in range(datas[0].GetNbinsX() / 2, datas[0].GetNbinsX() + 1):
datas[0].SetBinContent(bin, 0)
histos.append(copy(Overlay))
if 'ZH' in signalList and 'WH' in signalList:
typs.append('VH')
if 'ZH' in Stack.setup: Stack.setup.remove('ZH')
if 'WH' in Stack.setup: Stack.setup.remove('WH')
Stack.setup.insert(0, 'VH')
elif 'ZH' in signalList:
typs.append('ZH')
elif 'WH' in signalList:
typs.append('WH')
elif 'VVb' in signalList:
typs.append('VVb')
print Stack.setup
Stack.histos = histos
Stack.typs = typs
Stack.datas = datas
Stack.datatyps = datatyps
Stack.datanames = datanames
Stack.overlay = [Overlay]
Stack.AddErrors = Error
if dataname == 'Wtn':
lumi = 18300.
Stack.lumi = lumi
Stack.doPlot()
print 'i am done!\n'
var = opts.variable
#-------------------------------------------------------------------------------
#--read variables from config---------------------------------------------------
# 7 or 8TeV Analysis
print "Compile external macros"
print "=======================\n"
# compile external macros to compute variables on the fly
#ROOT.gSystem.CompileMacro("../plugins/PU.C")
print 'Assigne variables from config'
print '==============================\n'
anaTag = config.get("Analysis", "tag")
if not any([anaTag == '7TeV', anaTag == '8TeV', anaTag == '13TeV']):
raise Exception(
"anaTag %s unknown. Specify 8TeV or 7TeV or 13TeV in the general config"
% anaTag)
# Directories:
Wdir = config.get('Directories', 'Wdir')
vhbbpath = config.get('Directories', 'vhbbpath')
samplesinfo = config.get('Directories', 'samplesinfo')
path = config.get('Directories', 'dcSamples')
outpath = config.get('Directories', 'limits')
optimisation = opts.optimisation
optimisation_training = False
UseTrainSample = eval(config.get('Analysis', 'UseTrainSample'))
if UseTrainSample:
print 'Training events will be used'
parser.add_option("-V", "--variable", dest="variable", default="",
help="variable for shape analysis")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration file")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
var=opts.variable
#-------------------------------------------------------------------------------
# Add external macros
#ROOT.gSystem.CompileMacro("../plugins/PU.C")
#--read variables from config---------------------------------------------------
# 7 or 8TeV Analysis
anaTag = config.get("Analysis","tag")
# Directories:
Wdir=config.get('Directories','Wdir')
vhbbpath=config.get('Directories','vhbbpath')
samplesinfo=config.get('Directories','samplesinfo')
path = config.get('Directories','dcSamples')
outpath=config.get('Directories','limits')
try:
os.stat(outpath)
except:
os.mkdir(outpath)
# parse histogram config:
treevar = config.get('dc:%s'%var,'var')
name = config.get('dc:%s'%var,'wsVarName')
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
weight = opts.weight
evaluate_optimisation = False
if weight != '':
evaluate_optimisation = True
#Import after configure to get help message
from myutils import BetterConfigParser, ParseInfo, MvaEvaluator
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
fileLocator = FileLocator(config=config)
print ("OPTS", opts)
if len(opts.fileList) > 0:
filelist = FileList.decompress(opts.fileList) if len(opts.fileList) > 0 else None
print ("len(filelist)", len(filelist))
if len(filelist) > 0:
print ("filelist[0]:", filelist[0])
else:
filelist = SampleTree({'name': opts.sampleIdentifier, 'folder': config.get('Directories', 'MVAin')}, countOnly=True, splitFilesChunkSize=-1, config=config).getSampleFileNameChunks()[0]
print ("INFO: no file list given, use all files!")
print (len(filelist), filelist)
#get locations:
Wdir = config.get('Directories', 'Wdir')
parser.add_option("-C",
"--config",
dest="config",
default=[],
action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
TrainFlag = eval(config.get('Analysis', 'TrainFlag'))
btagLibrary = config.get('BTagReshaping', 'library')
samplesinfo = config.get('Directories', 'samplesinfo')
VHbbNameSpace = config.get('VHbbNameSpace', 'library')
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs = eval(config.get('AngularLike', 'backgrounds'))
ang_yield = eval(config.get('AngularLike', 'yields'))
#path=opts.path
pathIN = config.get('Directories', 'SYSin')
pathOUT = config.get('Directories', 'SYSout')
tmpDir = os.environ["TMPDIR"]
print 'INput samples:\t%s' % pathIN
def drawFromDC():
config = BetterConfigParser()
config.read(opts.config)
region = opts.region
print "\nopts.config:",opts.config
print "opts:", opts
print "var:", opts.var
print "bin:", opts.bin
#Should Read this from the parser
datanames = config.get('dc:%s'%opts.bin,'data').split(' ')
print 'dataname is', datanames
region = opts.bin
var = opts.var
ws_var = config.get('plotDef:%s'%var,'relPath')
nbin = int(config.get('plotDef:%s'%var,'nBins'))
xmin = float(config.get('plotDef:%s'%var,'min'))
xmax = float(config.get('plotDef:%s'%var,'max'))
blind = eval(opts.blind)
postfit = eval(opts.postfit)
print 'config:', config
print 'var: ', var
print 'region: ', region
print 'blind: ', blind
print 'postfit: ', postfit
Group_dc = eval(config.get('Plot_general','Group_dc'))
Stack=StackMaker(config,var,region,True)
# check for pre or post fit options
preFit = False
addName = 'PostFit_%s' %(opts.fit)
if not opts.mlfit:
addName = 'PreFit'
preFit = True
print '\n-----> Fit Type(opts.fit) : ', opts.fit
print ' (opts.mlfit): ', opts.mlfit
print ' preFit : ', preFit
Stack.options['pdfName'] = '%s_%s_%s.pdf' %(var,opts.bin,addName)
#log = eval(config.get('Plot:%s'%region,'log'))
if 'Zee' in opts.bin or 'Zuu' in opts.bin:
#VH
setup = ['ggZHbb', 'qqZHbb','Zbb','Zb','Z_udscg','TT','VV2b','VVlight','ST']
#VV
#setup = ['VV2b','ggZHbb','qqZHbb','Z_udscg','Zb','Zbb','TT','VVlight','ST']
channel = 'ZllHbb'
if 'Zee' in opts.bin: lep_channel = 'Zee'
elif 'Zuu' in opts.bin: lep_channel = 'Zuu'
#region_dic = {'BDT':'SIG','CRZlight':'Zlf','CRZb':'Zhf','CRttbar':'TT'}
region_dic = {
'BDT':'BDT',
'CRZlight':'CRZlight',
'CRZb':'CRZb',
'CRttbar':'CRttbar',
'ZeeMass_lowpt':'ZeeMass_lowpt',
'ZeeMass_highpt':'ZeeMass_highpt',
'ZuuMass_lowpt':'ZuuMass_lowpt',
'ZuuMass_highpt':'ZuuMass_highpt',
'ZeeMassVV_lowpt':'ZeeMassVV_lowpt',
'ZeeMassVV_highpt':'ZeeMassVV_highpt',
'ZuuMassVV_lowpt':'ZuuMassVV_lowpt',
'ZuuMassVV_highpt':'ZuuMassVV_highpt',
'ZuuMass_Vptbin0':'ZuuMass_Vptbin0',
'ZuuMass_Vptbin1':'ZuuMass_Vptbin1',
'ZuuMass_Vptbin2':'ZuuMass_Vptbin2',
'ZeeMass_Vptbin0':'ZeeMass_Vptbin0',
'ZeeMass_Vptbin1':'ZeeMass_Vptbin1',
'ZeeMass_Vptbin2':'ZeeMass_Vptbin2',
}
print 'opts.bin is', opts.bin
region_name = [region_dic[key] for key in region_dic if (key in opts.bin)]
region_name = region_name[0]
print 'region_name is', region_name
pt_region_dic = {'lowpt':'lowpt','highpt':'highpt','bin0':'bin0','bin1':'bin1','bin2':'bin2'}
pt_region_name = [pt_region_dic[key] for key in pt_region_dic if (key in opts.bin)]
pt_region_name = pt_region_name[0]
else:
print '@ERROR: This is not a Zll region. Aborting'
sys.exit()
Stack.setup = setup
Dict = eval(config.get('LimitGeneral','Dict'))
lumi = eval(config.get('General','lumi'))
Stack.nBins = nbin
Stack.xMin = xmin
Stack.xMax = xmax
print '/n----> The Binning:'
print 'nBins:', Stack.nBins
print 'xMin:', Stack.xMin
print 'xMax:', Stack.xMax
theBinning = ROOT.RooFit.Binning(Stack.nBins,Stack.xMin,Stack.xMax)
#################
#We are now skiping a large part of the orginal code, as everything that remains to be done is to read the postfit plot from the mlfit.root
#How should the scale sys (lnN be handled) ?
#
histos = []
typs = []
shapes = {}
shapesUp = [[] for _ in range(0,len(setup))]
shapesDown = [[] for _ in range(0,len(setup))]
#signalList = ['ggZHbb', 'qqZHbb']
#signalList = []
sigCount = 0
#Overlay ={}
Overlay = []
prefit_overlay = []
postfit_from_wc = False
if opts.mlfit.split('/')[-1] != 'mlfit.root':
postfit_from_wc = True
dirname = ''
####
#Open the mlfit.root and retrieve the mc
print 'opts.mlfit is', opts.mlfit
file = ROOT.TFile.Open(opts.mlfit)
#if file == None: raise RuntimeError, "Cannot open file %s" % theFile
#print '\n\n-----> Fit File: ',file
print '====================='
print 'POSTFIT IS', postfit
print '====================='
if postfit:
if not postfit_from_wc:
if not ROOT.gDirectory.cd('shapes_fit_s'):
print '@ERROR: didn\'t find the shapes_fit_s directory. Aborting'
sys.exit()
else:
pass
else:
if not ROOT.gDirectory.cd('shapes_prefit'):
print '@ERROR: didn\'t find the shapes_prefit directory. Aborting'
sys.exit()
folder_found = False
for dir in ROOT.gDirectory.GetListOfKeys():
dirinfo = dir.GetName().split('_')
print 'dir name is', dir.GetName().split('_')
##if not (dirinfo[0] == channel and dirinfo[2] == lep_channel and dirinfo[3] == region_name and dirinfo[4] == pt_region_name):
print 'dirinfo is', dirinfo
print 'lep_channel is', lep_channel
print 'region_name is', region_name
print 'pt_region_name is', pt_region_name
if not (dirinfo[0] == lep_channel and dirinfo[1] == region_name and dirinfo[2] == pt_region_name):
#for VV
if not (dirinfo[2] == region_name.split('_')[0] and dirinfo[3] == pt_region_name):
if not (len(dirinfo) > 3 and dirinfo[3] == region_name.split('_')[0] and dirinfo[4] == 'Vpt'+pt_region_name):
continue
if postfit_from_wc and 'prefit' in dirinfo:
continue
folder_found = True
dirname = dir.GetName()
#signal, use prefit
for s in setup:
if ('ZHbb' in s and postfit) and not postfit_from_wc:
print 'ERROR'
sys.exit()
ROOT.gDirectory.cd('/shapes_prefit')
ROOT.gDirectory.cd(dirname)
found = False
for subdir in ROOT.gDirectory.GetListOfKeys():
#print 'subdir name is', subdir.GetName()
if subdir.GetName() == Dict[s]:
found = True
hist = rebinHist(gDirectory.Get(subdir.GetName()).Clone(), nbin, xmin, xmax)
histos.append(hist)
typs.append(s)
#print 's is', s
#print 'signalList is', signalList
#if s in signalList:
# hist.SetTitle(s)
# Overlay.append(hist)
# print 'the Histogram title is', hist.GetTitle()
else:
#SF_ZJets = [0.95188, 0.94404, 1.0463]
#SF_TTbar = 1.0373
#;Vpt high
#SF_ZJets = [1.1235, 0.91368, 1.2435]
#SF_TTbar = 1.0601
#Start be getting the SF
print 'Gonna apply SF'
scale = 1
#if 'low' in opts.dc:
# if 'TT' in s: scale = 1.01
# if 'Z_udscg' in s: scale = 0.96
# if 'Zb' in s: scale = 0.99
# if 'Zbb' in s : scale = 1.04
#elif 'high' in opts.dc:
# if 'TT' in s: scale = 1.01
# if 'Z_udscg' in s: scale = 1.03
# if 'Zb' in s: scale = 0.96
# if 'Zbb' in s: scale = 1.23
#else:
# pass
# #ROOT.gDirectory.cd('/shapes_fit_s')
#if not postfit_from_wc:
# ROOT.gDirectory.cd('/shapes_prefit')
# print 'ERROR2'
# sys.exit()
#else:
# scale = 1
ROOT.gDirectory.cd(dirname)
found = False
for subdir in ROOT.gDirectory.GetListOfKeys():
print 'subdir name is', subdir.GetName()
#print 'Dict is ', Dict
if subdir.GetName() == Dict[s] or (postfit_from_wc and subdir.GetName() == s):
found = True
hist = rebinHist(gDirectory.Get(subdir.GetName()).Clone(), nbin, xmin, xmax, scale)
histos.append(hist)
typs.append(s)
print 's is', s
#print 'signalList is', signalList
#if s in signalList:
# hist.SetTitle(s)
# Overlay.append(hist)
# print 'the Histogram title is', hist.GetTitle()
#take prefit distr. for signal
if not found:
print '@ERROR: didn\'t find the postfit histogram. Aborting'
sys.exit()
if not postfit_from_wc:
ROOT.gDirectory.cd('/shapes_prefit/'+dirname)
print 'ERROR3'
sys.exit()
if not postfit_from_wc:
total = rebinHist(gDirectory.Get('total').Clone(), nbin, xmin, xmax)
total.SetTitle('prefit')
prefit_overlay.append(total)
break
if not folder_found:
print '@ERROR: Folder was not found.'
print 'lep_channel', lep_channel
print 'region_name', region_name
print 'pt_region_name', pt_region_name
sys.exit()
#retrieve the data
options = copy(opts)
options.dataname = "data_obs"
options.mass = 0
options.format = "%8.3f +/- %6.3f"
options.channel = opts.bin
options.excludeSyst = []
options.norm = False
options.stat = False
options.bin = True # fake that is a binary output, so that we parse shape lines
options.out = "tmp.root"
options.fileName = args[0]
options.cexpr = False
options.fixpars = False
options.libs = []
options.verbose = 0
options.poisson = 0
options.nuisancesToExclude = []
options.noJMax = None
#This needs to be done from the "dc" root file
print 'file is ',opts.dc
dc_file= open(opts.dc, "r")
os.chdir(os.path.dirname(opts.dc))
DC = parseCard(dc_file, options)
if not DC.hasShapes: DC.hasShapes = True
MB = ShapeBuilder(DC, options)
data0 = MB.getShape(opts.bin,'data_obs')
if (data0.InheritsFrom("RooDataHist")):
data0 = ROOT.RooAbsData.createHistogram(data0,'data_obs',ws_var,theBinning)
data0.SetName('data_obs')
datas=[data0]
if blind and 'BDT' in var:
for bin in range(datas[0].GetNbinsX()-3,datas[0].GetNbinsX()+1):
datas[0].SetBinContent(bin,0)
if blind and 'Mass' in var:
for bin in range(datas[0].GetNbinsX()-13,datas[0].GetNbinsX()-7):
datas[0].SetBinContent(bin,0)
datatyps = [None]
#print '\nshapes!!!', shapes
print '\nOVERLAY!!!', Overlay
#Add all the histos and overlay to the stackmaker such that they can be ploted
#print 'before Stack, histos are', histos
#sys.exit()
Stack.histos = histos
Stack.typs = typs
Stack.overlay = Overlay
Stack.prefit_overlay = prefit_overlay
Stack.datas = datas
Stack.datatyps = datatyps
Stack.datanames= datanames
Stack.AddErrors= True
Stack.lumi = lumi
if 'BDT' in var:
Stack.forceLog = True
Stack.doPlot()
print 'i am done!\n'
if __name__ == "__main__":
# read arguments
argv = sys.argv
parser = OptionParser()
parser.add_option("-t", "--regions", dest="regions", default='',
help="cut regions identifier")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration file")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
# Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo
# load config
config = BetterConfigParser()
config.read(opts.config)
# if no region is given in argument, run it for all of them
regionsListString = opts.regions if len(opts.regions.strip())>0 else config.get('LimitGeneral', 'List')
regions = [x.strip() for x in regionsListString.split(',') if len(x.strip()) > 0]
for region in regions:
mergeDC = MergeDatacards(config=config, region=region)
isComplete = mergeDC.prepare()
if isComplete:
mergeDC.run()
else:
raise Exception("InputDcIncomplete")
def drawFromDC():
config = BetterConfigParser()
config.read(opts.config)
region = opts.region
print "\nopts.config:", opts.config
print "opts:", opts
print "var:", opts.var
print "bin:", opts.bin
dataname = 'Zll'
if 'Zuu' in opts.bin: dataname = 'Zuu'
elif 'Zee' in opts.bin: dataname = 'Zee'
elif 'Wmn' in opts.bin: dataname = 'Wmn'
elif 'Wen' in opts.bin: dataname = 'Wen'
elif 'Znn' in opts.bin: dataname = 'Znn'
elif 'Wtn' in opts.bin: dataname = 'Wtn'
if (opts.var == ''):
var = 'BDT'
if dataname == 'Zmm' or dataname == 'Zee': var = 'BDT_Zll'
elif dataname == 'Wmn' or dataname == 'Wen': var = 'BDT_Wln'
elif dataname == 'Znn': var = 'BDT_Znn'
#if 'HighPt' in opts.bin: var = 'BDT_ZnnHighPt'
#if 'LowPt' in opts.bin: var = 'BDT_ZnnLowPt'
#if 'LowCSV' in opts.bin: var = 'BDT_ZnnLowCSV'
if dataname == '' or var == 'BDT':
raise RuntimeError, 'Did not recognise mode or var from ' + opts.bin
else:
var = opts.var
if opts.var == 'BDT':
if 'LowPt' in opts.bin: var = 'gg_plus_ZH125_low_Zpt'
elif 'MedPt' in opts.bin: var = 'gg_plus_ZH125_low_Zpt'
elif 'HighPt' in opts.bin: var = 'gg_plus_ZH125_high_Zpt'
elif 'VV' in opts.bin: var = 'VV_bdt'
else: var = 'gg_plus_ZH125_high_Zpt'
if opts.var == 'CRBDT':
if 'LowPt' in opts.bin: var = 'gg_plus_ZH125_low_Zpt_CR'
elif 'MedPt' in opts.bin: var = 'gg_plus_ZH125_low_Zpt_CR'
elif 'HighPt' in opts.bin: var = 'gg_plus_ZH125_high_Zpt_CR'
else: var = 'gg_plus_ZH125_high_Zpt_CR'
#if 'BDT' in var:
# region = 'BDT'
#else:
region = opts.bin
ws_var = config.get('plotDef:%s' % var, 'relPath')
print 'ws_var:', ws_var
if opts.var == 'BDT':
ws_var = ROOT.RooRealVar(ws_var, ws_var, -1, 1.)
else:
ws_var = ROOT.RooRealVar(ws_var, ws_var, -1, 1.)
print 'config:', config
print 'var: ', var
#if 'Wen' in region:
# region = 'WenHighPt'
#if 'Wmn' in region:
# region = 'WmnHighPt'
if 'Znn' in region:
region = 'Znn_13TeV_Signal'
print 'region: ', region
blind = eval(config.get('Plot:%s' % region, 'blind'))
Stack = StackMaker(config, var, region, True)
if 'low' in opts.bin or 'ch1_Wenu' == opts.bin or 'ch2_Wmunu' == opts.bin:
Stack.addFlag2 = 'Low p_{T}(V)'
elif 'MedPt' in opts.bin or 'ch1_Wenu2' == opts.bin or 'ch2_Wmunu2' == opts.bin:
Stack.addFlag2 = 'Intermediate p_{T}(V)'
elif 'high' in opts.bin or 'ch1_Wenu3' == opts.bin or 'ch2_Wmunu3' == opts.bin:
Stack.addFlag2 = 'High p_{T}(V)'
# check for pre or post fit options
preFit = False
addName = 'PostFit_%s' % (opts.fit)
if not opts.mlfit:
addName = 'PreFit'
preFit = True
print '\n-----> Fit Type(opts.fit) : ', opts.fit
print ' (opts.mlfit): ', opts.mlfit
print ' preFit : ', preFit
print ' opts.bin : ', opts.bin
Stack.options['pdfName'] = '%s_%s_%s.pdf' % (var, opts.bin, addName)
log = eval(config.get('Plot:%s' % region, 'log'))
if 'Zll' in opts.bin or 'Zee' in opts.bin or 'Zuu' in opts.bin or 'Zmm' in opts.bin:
#setup = config.get('Plot_general','setup').split(',')
setup = [
'ZH', 'ggZH', 'DY2b', 'DY1b', 'DYlight', 'TT', 'VVHF', 'VVLF', 'ST'
]
signalList = ['ZH']
channel = 'ZllHbb'
if 'Zee' in opts.bin: lep_channel = 'Zee'
elif 'Zuu' in opts.bin: lep_channel = 'Zmm'
region_dic = {
'BDT': 'SIG',
' Zlf': 'Zlf',
'Zhf': 'Zhf',
'TT': 'TT',
'13TeV': 'SIG'
}
region_name = [
region_dic[key] for key in region_dic if (key in opts.bin)
]
region_name = 'SIG'
if 'Zlf' in opts.bin: region_name = 'Zlf'
if 'Zhf' in opts.bin: region_name = 'Zhf'
if 'ttbar' in opts.bin: region_name = 'TT'
if 'low' in opts.bin or 'Low' in opts.bin: pt_region_name = 'low'
if 'high' in opts.bin or 'High' in opts.bin: pt_region_name = 'high'
if 'med' in opts.bin or 'Med' in opts.bin: pt_region_name = 'med'
# ['ZllHbb', 'ch1', 'Zmm', 'SIG', 'low']
if 'Zmm' in lep_channel and 'low' in pt_region_name: zll_index = 'ch1'
elif 'Zmm' in lep_channel and 'high' in pt_region_name:
zll_index = 'ch3'
elif 'Zee' in lep_channel and 'high' in pt_region_name:
zll_index = 'ch4'
elif 'Zee' in lep_channel and 'low' in pt_region_name:
zll_index = 'ch2'
else:
zll_index = 'ch2'
# binning
nBins = Stack.nBins
xMin = Stack.xMin
xMax = Stack.xMax
# for stat hists
if 'Zee' in lep_channel and 'low' in pt_region_name and 'minCMVA' not in opts.var:
stat_name = opts.bin
elif 'Zee' in lep_channel and 'high' in pt_region_name and 'minCMVA' not in opts.var:
stat_name = opts.bin
elif 'Zmm' in lep_channel and 'low' in pt_region_name and 'minCMVA' not in opts.var:
stat_name = opts.bin
elif 'Zmm' in lep_channel and 'high' in pt_region_name and 'minCMVA' not in opts.var:
stat_name = opts.bin
else:
stat_name = opts.bin
if 'minCMVA' in opts.var:
if 'low' in pt_region_name:
if 'Zmm' in lep_channel:
if 'Zlf' in region_name: stat_name = 'Zlf_low_Zuu'
if 'Zhf' in region_name: stat_name = 'Zhf_low_Zuu'
if 'TT' in region_name: stat_name = 'ttbar_low_Zuu'
else:
if 'Zlf' in region_name: stat_name = 'Zlf_low_Zee'
if 'Zhf' in region_name: stat_name = 'Zhf_low_Zee'
if 'TT' in region_name: stat_name = 'ttbar_low_Zee'
if 'high' in pt_region_name:
if 'Zmm' in lep_channel:
if 'Zlf' in region_name: stat_name = 'Zlf_high_Zuu'
if 'Zhf' in region_name: stat_name = 'Zhf_high_Zuu'
if 'TT' in region_name: stat_name = 'ttbar_high_Zuu'
else:
if 'Zlf' in region_name: stat_name = 'Zlf_high_Zee'
if 'Zhf' in region_name: stat_name = 'Zhf_high_Zee'
if 'TT' in region_name: stat_name = 'ttbar_high_Zee'
if 'Wmn' in opts.bin or 'Wen' in opts.bin:
setup = [
'WH', 'ZH', 'Wj2b', 'Wj1b', 'Wj0b', 'DY2b', 'DY1b', 'DYlight',
'TT', 'VVHF', 'VVLF', 'ST'
]
signalList = ['ZH', 'WH']
channel = 'WlnHbb'
if 'Wen' in opts.bin:
lep_channel = 'Wen'
if 'tt' in region: region_name = 'ttWen'
elif 'whf' in region:
if 'High' in opts.bin:
region_name = 'whfWenHigh'
else:
region_name = 'whfWenLow'
elif 'wlf' in region:
region_name = 'wlfWen'
else:
region_name = opts.bin
if 'Wmn' in opts.bin:
lep_channel = 'Wmn'
if 'tt' in region: region_name = 'ttWmn'
elif 'whf' in region:
if 'High' in opts.bin:
region_name = 'whfWmnHigh'
else:
region_name = 'whfWmnLow'
elif 'wlf' in region:
region_name = 'wlfWmn'
else:
region_name = opts.bin
pt_region_name = 'none'
nBins = 40
xMin = -1
xMax = 1
stat_name = 'BDT_' + opts.bin + '_'
if 'Znn' in opts.bin:
setup = [
'ZH', 'ggZH', 'WH', 'DY2b', 'DY1b', 'DYlight', 'Wj2b', 'Wj1b',
'Wj0b', 'TT', 'VVHF', 'VVLF', 'ST'
]
signalList = ['ZH']
channel = 'Znn'
lep_channel = 'Znn'
if 'nAddJet1' in opts.bin:
pt_region_name = 'nAddJet1'
elif 'nAddJet0' in opts.bin:
pt_region_name = 'nAddJet0'
else:
pt_region_name = 'High'
region_name = 'SR'
nBins = 25
xMin = -1
xMax = 1
if 'QCD' in opts.dc:
region_name = 'QCD'
stat_name = 'Znn_13TeV_QCD'
elif 'TT' in opts.dc:
region_name = 'TT'
stat_name = 'Znn_13TeV_TT'
elif 'Zbb' in opts.dc:
region_name = 'Zbb'
stat_name = 'Znn_13TeV_Zbb'
elif 'Zlight' in opts.dc:
region_name = 'Zlight'
stat_name = 'Znn_13TeV_Zlight'
else:
stat_name = opts.bin
print '############'
print 'Channel is', channel
print 'lepton channel is', lep_channel
print 'region_name is', region_name
print 'pt region_name is', pt_region_name
print '/n----> The Binning:'
print 'nBins:', nBins
print 'xMin:', xMin
print 'xMax:', xMax
if dataname == 'Zmm' or dataname == 'Zee':
try:
setup.remove('W1b')
setup.remove('W2b')
setup.remove('Wlight')
setup.remove('WH')
except:
print '@INFO: Wb / Wligh / WH not present in the datacard'
if not dataname == 'Znn' and 'QCD' in setup:
setup.remove('QCD')
Stack.setup = setup
Dict = eval(config.get('LimitGeneral', 'Dict'))
lumi = eval(config.get('Plot_general', 'lumi'))
options = copy(opts)
options.dataname = "data_obs"
options.mass = 0
options.format = "%8.3f +/- %6.3f"
options.channel = opts.bin
options.excludeSyst = []
options.norm = False
options.stat = False
options.bin = True # fake that is a binary output, so that we parse shape lines
options.out = "tmp.root"
options.fileName = args[0]
options.filename = region
options.cexpr = False
options.fixpars = False
options.libs = []
options.verbose = 0
options.poisson = 0
options.nuisancesToExclude = []
options.noJMax = None
theBinning = ROOT.RooFit.Binning(nBins, xMin, xMax)
# for prefit erros
prefit_error_histos = []
postfit_error_histos = []
histos = []
typs = []
shapes = {}
shapesUp = [[] for _ in range(0, len(setup))]
shapesDown = [[] for _ in range(0, len(setup))]
sigCount = 0
Overlay = []
prefit_overlay = []
dirname = ''
#Open the mlfit.root and retrieve the mc
file = ROOT.TFile.Open(opts.mlfit)
if file == None: raise RuntimeError, "Cannot open file %s" % opts.mlfit
print '\n\n-----> Fit File: ', file
for dir in ROOT.gDirectory.GetListOfKeys():
dirinfo = dir.GetName().split('_')
print 'dirinfo:', dirinfo
if 'Znn' in dirinfo[0] and 'Znn' not in opts.bin: continue
if 'ZllHbb' in dirinfo[0] and ('Zmm' not in lep_channel
and 'Zee' not in lep_channel):
continue
if 'W' in dirinfo[0] and 'W' not in opts.bin: continue
if 'W' in opts.bin:
print 'channel, lepton channel, region_name:', channel, lep_channel, region_name
#if not (dirinfo[0] == channel and dirinfo[1] == lep_channel and dirinfo[2] == region_name):
if not (dirinfo[0] == region_name and dirinfo[1] == 'postfit'):
continue
else:
print '!!! Match !!!'
if 'Znn' in opts.bin:
print 'channel, lepton channel, region_name, pt_region_name:', channel, lep_channel, region_name, pt_region_name
#if not (dirinfo[0] == lep_channel and dirinfo[2] == pt_region_name and dirinfo[3] == 'postfit'):
if not (dirinfo[0] == lep_channel and dirinfo[2] == region_name
and dirinfo[3] == 'postfit'):
continue
else:
print '!!! Match !!!'
if 'Zuu' in opts.bin or 'Zee' in opts.bin:
print 'channel, zll index, lepton channel, region_name, pt_region_name:', channel, zll_index, lep_channel, region_name, pt_region_name
if not (dirinfo[0] == lep_channel and dirinfo[1] == region_name and
dirinfo[2] == pt_region_name and dirinfo[3] == 'postfit'):
continue
else:
print '!!! Match !!!'
print 'Directory:', dir.GetName()
dirname = dir.GetName()
# Pull out the MC stat uncertainties first
#hists_WenHighPt40.root
#vhbb_WenHighPt40_13TeV.txt
if 'W' in opts.bin:
stat_filename = opts.dc.replace('.txt', '.root')
stat_filename = stat_filename.replace('vhbb', 'hists')
stat_filename = stat_filename.replace('_13TeV', '')
else:
stat_filename = opts.dc.replace('.txt', '.root')
stat_filename = stat_filename.replace('DC_', '')
print '\n Opening card for MC stat hists:', stat_filename
print 'Dir name:', stat_name
stat_file = ROOT.TFile.Open(stat_filename)
if 'W' not in opts.bin: ROOT.gDirectory.cd(stat_name)
stat_hists = {}
for s in setup:
for dir in ROOT.gDirectory.GetListOfKeys():
#print 'dir:', dir.GetName()
if 'W' in opts.bin:
wlvname = dir.GetName().replace(stat_name, '')
if wlvname == Dict[s]:
stat_hists[wlvname] = ROOT.gDirectory.Get(
dir.GetName()).Clone()
if dir.GetName() == Dict[s]:
stat_hists[dir.GetName()] = ROOT.gDirectory.Get(
dir.GetName()).Clone()
#stat_file.Close()
print '\nStat_hists:', stat_hists
file = ROOT.TFile.Open(opts.mlfit)
#ROOT.gDirectory.cd('shapes_fit_s')
#ROOT.gDirectory.cd(fit_dir)
ROOT.gDirectory.cd(dirname)
subdir_list = [x for x in ROOT.gDirectory.GetListOfKeys()]
for s in setup:
print '\ns:', s
found = False
#for subdir in ROOT.gDirectory.GetListOfKeys():
for subdir in subdir_list:
print 'subdir name is', subdir.GetName()
print 'Dict Key is', Dict[s]
if subdir.GetName() == Dict[s]:
found = True
# Set Histos postFit shapes and preFit errors
hist = rebinHist(
ROOT.gDirectory.Get(subdir.GetName()).Clone(), nBins,
xMin, xMax, dirname, subdir, prefit_error_histos,
postfit_error_histos, stat_hists)
histos.append(hist)
typs.append(s)
if s in signalList:
hist.SetTitle(s)
Overlay.append(hist)
#print 'the Histogram title is', hist.GetTitle()
break
if not found:
print '@ERROR: didn\'t find the postfit histogram. Aborting'
hist = ROOT.TH1F(Dict[s], Dict[s], nBins, xMin, xMax)
histos.append(hist)
typs.append(s)
#sys.exit()
#ROOT.gDirectory.cd('/shapes_prefit/'+dirname)
#total = rebinHist(ROOT.gDirectory.Get('total').Clone(), Stack.nBins, Stack.xMin, Stack.xMax)
#total.SetTitle('prefit')
#prefit_overlay.append(total)
break
# Get the total pre/post fit error
#print '\n Calculating final pre/post fit errors'
#print prefit_error_histos
for i, iErrorHist in enumerate(prefit_error_histos):
#print i,iErrorHist
if i == 0:
temp_prefit_error = iErrorHist.Clone()
else:
temp_prefit_error.Add(iErrorHist)
for i, iErrorHist in enumerate(postfit_error_histos):
#print i,iErrorHist
if i == 0:
temp_postfit_error = iErrorHist.Clone()
else:
temp_postfit_error.Add(iErrorHist)
final_prefit_error = ROOT.TGraphAsymmErrors(temp_prefit_error)
final_postfit_error = ROOT.TGraphAsymmErrors(temp_postfit_error)
total = [[]] * nBins
errUp = [[]] * nBins
errDown = [[]] * nBins
total_post = [[]] * nBins
errUp_post = [[]] * nBins
errDown_post = [[]] * nBins
# rebin the final errors
for bin in range(1, nBins + 1):
binError = temp_prefit_error.GetBinError(bin)
total[bin - 1] = temp_prefit_error.GetBinContent(bin)
errUp[bin - 1] = [binError]
errDown[bin - 1] = [binError]
binError_post = temp_postfit_error.GetBinError(bin)
total_post[bin - 1] = temp_postfit_error.GetBinContent(bin)
errUp_post[bin - 1] = [binError_post]
errDown_post[bin - 1] = [binError_post]
#Add all in quadrature
totErrUp = [sqrt(sum([x**2 for x in bin])) for bin in errUp]
totErrDown = [sqrt(sum([x**2 for x in bin])) for bin in errDown]
totErrUp_post = [sqrt(sum([x**2 for x in bin])) for bin in errUp_post]
totErrDown_post = [sqrt(sum([x**2 for x in bin])) for bin in errDown_post]
for bin in range(1, nBins + 1):
if not total[bin - 1] == 0:
point = histos[0].GetXaxis().GetBinCenter(bin)
final_prefit_error.SetPoint(bin - 1, point, 1)
final_prefit_error.SetPointEYlow(
bin - 1, totErrDown[bin - 1] / total[bin - 1])
final_prefit_error.SetPointEYhigh(
bin - 1, totErrUp[bin - 1] / total[bin - 1])
if not total_post[bin - 1] == 0:
point = histos[0].GetXaxis().GetBinCenter(bin)
final_postfit_error.SetPoint(bin - 1, point, 1)
final_postfit_error.SetPointEYlow(
bin - 1, totErrDown_post[bin - 1] / total_post[bin - 1])
final_postfit_error.SetPointEYhigh(
bin - 1, totErrUp_post[bin - 1] / total_post[bin - 1])
# =================================================
##### Read data
print '\n#### Datafile is ', opts.dc
dc_file = open(opts.dc, "r")
os.chdir(os.path.dirname(opts.dc))
DC = parseCard(dc_file, options)
if not DC.hasShapes: DC.hasShapes = True
MB = ShapeBuilder(DC, options)
data0 = MB.getShape(opts.bin, 'data_obs')
print data0
if (data0.InheritsFrom("RooDataHist")):
data0 = ROOT.RooAbsData.createHistogram(data0, 'data_obs', ws_var,
theBinning)
data0.SetName('data_obs')
datas = [data0]
datatyps = [None]
datanames = [dataname]
print '\nDATA HIST:', data0
print 'Data name:', dataname
# if opts.var == 'BDT':
# print '!!!! Blinding !!!!'
# if 'Zee' in dataname or 'Zuu' in dataname:
# for bin in range(4,datas[0].GetNbinsX()+1):
# datas[0].SetBinContent(bin,0)
# if 'Znn' in dataname:
# for bin in range(20,datas[0].GetNbinsX()+1):
# datas[0].SetBinContent(bin,0)
# if 'W' in dataname:
# for bin in range(4,datas[0].GetNbinsX()+1):
# #print datas[0].GetBinContent(bin,0)
# datas[0].SetBinContent(bin,0)
#else:
for bin in range(0, datas[0].GetNbinsX() + 1):
print datas[0].GetBinContent(bin, 0)
# =======================================================
#if 'VV' in opts.bin:
if isVV:
signalList = ['VVHF', ' VVHF']
print 'Signal List:', signalList
#histos.append(copy(Overlay))
if 'ZH' in signalList and 'WH' in signalList:
#typs.append('WH')
#if 'ZH' in Stack.setup: Stack.setup.remove('ZH')
#if 'WH' in Stack.setup: Stack.setup.remove('WH')
#Stack.setup.insert(0,'WH')
#print 'Stack.setup:', Stack.setup
typs.append('WH')
typs.append('ZH')
#elif 'ZH' in signalList:
#Stack.setup.remove('WH')
#typs.append('ggZH')
# typs.append('ZH')
if 'VVb' in signalList or 'VVHF' in signalList:
#typs.append('WH')
typs.append('ZH')
typs.append('VVHF')
if 'VVHF' in Stack.setup:
Stack.setup.remove('VVHF')
Stack.setup.insert(0, 'VVHF')
if 'ZH' in Stack.setup:
Stack.setup.remove('ZH')
Stack.setup.insert(-1, 'ZH')
if 'WH' in Stack.setup:
Stack.setup.remove('WH')
Stack.setup.insert(-1, 'WH')
if 'ggZH' in Stack.setup:
Stack.setup.remove('ggZH')
Stack.setup.insert(-1, 'ggZH')
Stack.nBins = nBins
Stack.xMin = xMin
Stack.xMax = xMax
print '\n-----> Stack.setup(double check)...', Stack
print 'Post Histos:', histos
print 'Datas:', datas
print 'typs:', typs
Stack.histos = histos
Stack.typs = typs
Stack.datas = datas
Stack.datatyps = datatyps
Stack.datanames = datanames
Stack.filename = region
#if opts.var is not 'BDT':
#Stack.prefit_overlay = [prefit_overlay]
#if '13TeV' in region:
#Stack.overlay = [Overlay]
print '\n\n\t\t Overlay: ', Stack.overlay
# Add custom postFit errors
#Stack.AddErrors = final_prefit_error
Stack.AddErrors_Postfit = final_postfit_error
if dataname == 'Wtn':
lumi = 18300.
Stack.lumi = lumi
Stack.doPlot()
print 'i am done!\n'
vhbbPlotDef = opts.config[0].split('/')[0] + '/vhbbPlotDef.ini'
opts.config.append(vhbbPlotDef) #adds it to the config list
config = BetterConfigParser()
config.read(opts.config)
#path = opts.path
region = opts.region
# additional blinding cut:
addBlindingCut = None
if config.has_option(
'Plot_general',
'addBlindingCut'): #contained in plots, cut on the event number
addBlindingCut = config.get('Plot_general', 'addBlindingCut')
print 'adding add. blinding cut'
print "Compile external macros"
print "=======================\n"
#get locations:
Wdir = config.get('Directories',
'Wdir') # working direcoty containing the ouput
samplesinfo = config.get('Directories', 'samplesinfo') # samples_nosplit.cfg
path = config.get('Directories',
'plottingSamples') # from which samples to plot
section = 'Plot:%s' % region
xMin = -1
xMax = 1
masses = ["125"]
Abins = ["HighPt", "LowPt"]
channels = ["Zee", "Zmm"]
# ------------------------------------------------------
# ---------- Mjj ---------------------------------------
# mode = 'Mjj'
# xMin=0
# xMax=255
# masses = ['125']
# Abins = ['highPt','lowPt','medPt']
# channels= ['Zee','Zmm']
# ------------------------------------------------------
path = config.get("Directories", "limits")
outpath = config.get("Directories", "plotpath")
setup = eval(config.get("LimitGeneral", "setup"))
Dict = eval(config.get("LimitGeneral", "Dict"))
MCs = [Dict[s] for s in setup]
sys_BDT = eval(config.get("LimitGeneral", "sys_BDT"))
systematicsnaming = eval(config.get("LimitGeneral", "systematicsnaming"))
systs = [systematicsnaming[s] for s in sys_BDT]
if eval(config.get("LimitGeneral", "weightF_sys")):
systs.append("UEPS")
def myText(txt="CMS Preliminary", ndcX=0, ndcY=0, size=0.8):
parser.add_option("-N","--name", dest="set_name", default='',
help="Parameter setting name. Output files will have this name")
parser.add_option("-L","--local",dest="local", default=True,
help="True to run it locally. False to run on batch system using config")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
#Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo, TreeCache
#load config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
run = opts.training
gui = opts.verbose
print '-----> Input Training Sample(s): ', run
#GLOABAL rescale from Train/Test Spliiting:
global_rescale=2.
#get locations:
MVAdir = config.get('Directories','vhbbpath')+'/data/'
samplesinfo = config.get('Directories','samplesinfo')
#systematics
systematics = config.get('systematics','systematics')
systematics = systematics.split(' ')
#parser.add_option("-P", "--path", dest="path", default="",
# help="path to samples")
parser.add_option("-S", "--samples", dest="names", default="",
help="samples you want to run on")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo, TreeCache
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
TrainFlag = eval(config.get('Analysis','TrainFlag'))
btagLibrary = config.get('BTagReshaping','library')
samplesinfo=config.get('Directories','samplesinfo')
VHbbNameSpace=config.get('VHbbNameSpace','library')
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs=eval(config.get('AngularLike','backgrounds'))
ang_yield=eval(config.get('AngularLike','yields'))
#path=opts.path
pathIN = config.get('Directories','SYSin')
pathOUT = config.get('Directories','SYSout')
tmpDir = os.environ["TMPDIR"]
print 'INput samples:\t%s'%pathIN
sys.exit(123)
en = opts.tag
#create the list with the samples to run over
samplesList=opts.samples.split(",")
timestamp = time.asctime().replace(' ','_').replace(':','-')
if(debugPrintOUts): print 'samplesList',samplesList
if(debugPrintOUts): print 'timestamp',timestamp
# the list of the config is taken from the path config
pathconfig = BetterConfigParser()
pathconfig.read('%sconfig/paths'%(en))
_configs = pathconfig.get('Configuration','List').split(" ")
configs = [ '%sconfig/'%(en) + c for c in _configs ]
if(debugPrintOUts): print 'configs',configs
if(debugPrintOUts): print 'opts.ftag',opts.ftag
if not opts.ftag == '':
tagDir = pathconfig.get('Directories','tagDir')
# DEBUG PURPOSE ONLY
# sys.exit()
if(debugPrintOUts): print 'tagDir',tagDir
DirStruct={'tagDir':tagDir,'ftagdir':'%s/%s/'%(tagDir,opts.ftag),'logpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Logs'),'plotpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Plots'),'limitpath':'%s/%s/%s/'%(tagDir,opts.ftag,'Limits'),'confpath':'%s/%s/%s/'%(tagDir,opts.ftag,'config') }
if(debugPrintOUts): print 'DirStruct',DirStruct
parser.add_option("-C",
"--config",
dest="config",
default=[],
action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo, TreeCache
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
TrainFlag = eval(config.get('Analysis', 'TrainFlag'))
btagLibrary = config.get('BTagReshaping', 'library')
samplesinfo = config.get('Directories', 'samplesinfo')
VHbbNameSpace = config.get('VHbbNameSpace', 'library')
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs = eval(config.get('AngularLike', 'backgrounds'))
ang_yield = eval(config.get('AngularLike', 'yields'))
#path=opts.path
pathIN = config.get('Directories', 'SYSin')
pathOUT = config.get('Directories', 'SYSout')
tmpDir = os.environ["TMPDIR"]
print 'INput samples:\t%s' % pathIN
#get files info from config
parser = OptionParser()
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="directory config")
parser.add_option("-S", "--samples", dest="names", default="",
help="samples you want to run on")
(opts, args) = parser.parse_args(argv)
config = BetterConfigParser()
config.read(opts.config)
namelist=opts.names.split(',')
pathIN = config.get('Directories','PREPin')
pathOUT = config.get('Directories','PREPout')
samplesinfo=config.get('Directories','samplesinfo')
sampleconf = BetterConfigParser()
sampleconf.read(samplesinfo)
prefix=sampleconf.get('General','prefix')
info = ParseInfo(samplesinfo,pathIN)
for job in info:
if not job.name in namelist:
continue
if job.subsample:
continue
copytree(pathIN,pathOUT,prefix,job.prefix,job.identifier,'',job.addtreecut)
def getEventCount(config, sampleIdentifier, cut="1", sampleTree=None):
if not sampleTree:
sampleTree = SampleTree({'name': sampleIdentifier, 'folder': config.get('Directories',args.fromFolder).strip()}, config=config)
h1 = ROOT.TH1D("h1","h1",1,0,2)
nEvents = sampleTree.tree.Draw("1>>h1", "(" + cut + ")*genWeight")
nEventsWeighted = h1.GetBinContent(1)
h1.Delete()
return nEventsWeighted
# load config
config = BetterConfigParser()
configFolder = args.tag + 'config/'
print "config folder:", configFolder
config.read(configFolder + '/paths.ini')
configFiles = [x for x in config.get('Configuration','List').strip().split(' ') if len(x.strip()) > 0]
config = BetterConfigParser()
for configFile in configFiles:
config.read(configFolder + configFile)
print "read config ", configFile
pruneThreshold = float(args.prune)
sampleGroups = []
for x in args.samples.split(','):
sampleGroups.append(x.split('+'))
sampleCuts = args.cuts.strip().split(',')
if args.fc != '':
cutGroups = [x.strip(',').split(',') for x in args.fc.strip(';').split(';')]
# cartesian product
help=
"for eff. scale factor computation: restrict to this list of comma separated bins (Starting at bin 1)"
)
(opts, args) = parser.parse_args(argv)
if opts.config == "":
opts.config = ["config"]
# Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo
config = BetterConfigParser()
config.read(opts.config)
if len(opts.fitType) < 1:
if config.has_option('Fit', 'FitType'):
opts.fitType = config.get('Fit', 'FitType')
else:
opts.fitType = "shapes_prefit"
# run plotter
if len(opts.regions) < 1 or opts.regions.strip() == 'None':
regions = eval(config.get('Fit', 'regions')).keys()
else:
regions = opts.regions.strip().split(',')
for i in range(len(regions)):
if '+' in regions[i]:
regions[i] = regions[i].split('+')
print("REGIONS:", regions)
scaleFactorTable = []
plotCommands = []
#parser.add_option("-P", "--path", dest="path", default="",
# help="path to samples")
parser.add_option("-S", "--samples", dest="names", default="",
help="samples you want to run on")
parser.add_option("-C", "--config", dest="config", default=[], action="append",
help="configuration defining the plots to make")
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = "config"
from myutils import BetterConfigParser, ParseInfo, TreeCache
print opts.config
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis","tag")
TrainFlag = eval(config.get('Analysis','TrainFlag'))
btagLibrary = config.get('BTagReshaping','library')
samplesinfo=config.get('Directories','samplesinfo')
VHbbNameSpace=config.get('VHbbNameSpace','library')
ROOT.gSystem.Load(VHbbNameSpace)
AngLikeBkgs=eval(config.get('AngularLike','backgrounds'))
ang_yield=eval(config.get('AngularLike','yields'))
#path=opts.path
pathIN = config.get('Directories','SYSin')
pathOUT = config.get('Directories','SYSout')
tmpDir = os.environ["TMPDIR"]
print 'INput samples:\t%s'%pathIN
#parse args and setup logdir
args = parser.parse_args()
Logger = printer()
Logger.set_name("Psi46Log")
Logger.set_prefix('')
Logger.set_logfile('%s' % (args.loggingDir), 'psi46Handler.log')
Logger << 'ConfigDir: "%s"' % args.configDir
configDir = args.configDir
numTB = int(args.numTB)
#load config
config = BetterConfigParser()
config.read(configDir + '/elComandante.conf')
#config
clientName = "psi46"
serverZiel = config.get('subsystem', 'Ziel')
Port = int(config.get('subsystem', 'Port'))
serverPort = int(config.get('subsystem', 'Port'))
psiSubscription = config.get('subsystem', 'psiSubscription')
errorSubscription = "/error"
#construct
client = sClient(serverZiel, serverPort, clientName)
#subscribe
client.subscribe(psiSubscription)
#----------------------------------------------------
#handler
def handler(signum, frame):
Logger << 'Close Connection'
parser.add_option("-t","--trainingRegions", dest="trainingRegions", default='',
help="cut region identifier")
parser.add_option("-S","--systematics", dest="systematics", default=2,
help="include systematics (0 for none, 1 for bdtVars, 2 for all (with btagWeights)")
parser.add_option("-x", "--test", dest="test", action="store_true", help="for debugging only!!!", default=False)
(opts, args) = parser.parse_args(argv)
if opts.config =="":
opts.config = ["config"]
# Import after configure to get help message
from myutils import BetterConfigParser, mvainfo, ParseInfo
if len(opts.tag.strip()) > 1:
config = BetterConfigParser()
config.read("{tag}config/paths.ini".format(tag=opts.tag))
configFiles = config.get("Configuration", "List").split(' ')
opts.config = ["{tag}config/{file}".format(tag=opts.tag, file=x.strip()) for x in configFiles]
print("reading config files:", opts.config)
sys = False
btagSys = False
if int(opts.systematics) > 0:
sys = True
if int(opts.systematics) > 1:
btagSys = True
# load config
config = BetterConfigParser()
config.read(opts.config)
converter = SampleTreesToNumpyConverter(config, opts.trainingRegions, useSyst=sys, useWeightSyst=btagSys, testRun=opts.test)
success = converter.run()
if not success:
print(filelist)
print(
"len(filelist)",
len(filelist),
)
if len(filelist) > 0:
print("filelist[0]:", filelist[0])
else:
print('')
#Import after configure to get help message
from myutils import BetterConfigParser, progbar, printc, ParseInfo, MvaEvaluator
config = BetterConfigParser()
config.read(opts.config)
anaTag = config.get("Analysis", "tag")
#get locations:
Wdir = config.get('Directories', 'Wdir')
samplesinfo = config.get('Directories', 'samplesinfo')
#read shape systematics
systematics = config.get('systematics', 'systematics')
#systematics
INpath = config.get('Directories', 'MVAin')
OUTpath = config.get('Directories', 'MVAout')
info = ParseInfo(samplesinfo, INpath)
arglist = ''
def drawFromDC():
config = BetterConfigParser()
config.read(opts.config)
region = opts.region
print "\nopts.config:",opts.config
print "opts:", opts
print "var:", opts.var
print "bin:", opts.bin
dataname = 'Zll'
if 'Zmm' in opts.bin: dataname = 'Zmm'
elif 'Zee' in opts.bin: dataname = 'Zee'
elif 'Wmunu' in opts.bin: dataname = 'Wmn'
elif 'Wenu' in opts.bin: dataname = 'Wen'
elif 'Znunu' in opts.bin: dataname = 'Znn'
elif 'Wtn' in opts.bin: dataname = 'Wtn'
if (opts.var == ''):
var = 'BDT'
if dataname == 'Zmm' or dataname == 'Zee': var = 'BDT_Zll'
elif dataname == 'Wmn' or dataname == 'Wen': var = 'BDT_Wln'
elif dataname == 'Znn':
if 'HighPt' in opts.bin: var = 'BDT_ZnnHighPt'
if 'LowPt' in opts.bin: var = 'BDT_ZnnLowPt'
if 'LowCSV' in opts.bin: var = 'BDT_ZnnLowCSV'
if dataname == '' or var == 'BDT': raise RuntimeError, 'Did not recognise mode or var from '+opts.bin
else:
var = opts.var
if 'BDT' in var:
region = 'BDT'
else:
region = opts.bin
ws_var = config.get('plotDef:%s'%var,'relPath')
if region == 'BDT':
ws_var = ROOT.RooRealVar(ws_var,ws_var,-1.,1.)
else:
ws_var = ROOT.RooRealVar(ws_var,ws_var, 0, 1.)
blind = eval(config.get('Plot:%s'%region,'blind'))
print 'config:', config
print 'var: ', var
print 'region: ', region
Stack=StackMaker(config,var,region,True)
if 'LowPt' in opts.bin or 'ch1_Wenu' == opts.bin or 'ch2_Wmunu' == opts.bin:
Stack.addFlag2 = 'Low p_{T}(V)'
elif 'MedPt' in opts.bin or 'ch1_Wenu2' == opts.bin or 'ch2_Wmunu2' == opts.bin:
Stack.addFlag2 = 'Intermediate p_{T}(V)'
elif 'HighPt' in opts.bin or 'ch1_Wenu3' == opts.bin or 'ch2_Wmunu3' == opts.bin:
Stack.addFlag2 = 'High p_{T}(V)'
# check for pre or post fit options
preFit = False
addName = 'PostFit_%s' %(opts.fit)
if not opts.mlfit:
addName = 'PreFit'
preFit = True
print '\n-----> Fit Type(opts.fit) : ', opts.fit
print ' (opts.mlfit): ', opts.mlfit
print ' preFit : ', preFit
Stack.options['pdfName'] = '%s_%s_%s.pdf' %(var,opts.bin,addName)
log = eval(config.get('Plot:%s'%region,'log'))
setup = config.get('Plot_general','setup').split(',')
if dataname == 'Zmm' or dataname == 'Zee':
try:
setup.remove('W1b')
setup.remove('W2b')
setup.remove('Wlight')
setup.remove('WH')
except:
print '@INFO: Wb / Wligh / WH not present in the datacard'
if not dataname == 'Znn' and 'QCD' in setup:
setup.remove('QCD')
Stack.setup = setup
Dict = eval(config.get('LimitGeneral','Dict'))
lumi = eval(config.get('Plot_general','lumi'))
options = copy(opts)
options.dataname = "data_obs"
options.mass = 0
options.format = "%8.3f +/- %6.3f"
options.channel = opts.bin
options.excludeSyst = []
options.norm = False
options.stat = False
options.bin = True # fake that is a binary output, so that we parse shape lines
options.out = "tmp.root"
options.fileName = args[0]
options.cexpr = False
options.fixpars = False
options.libs = []
options.verbose = 0
options.poisson = 0
options.nuisancesToExclude = []
options.noJMax = None
theBinning = ROOT.RooFit.Binning(Stack.nBins,Stack.xMin,Stack.xMax)
file = open(opts.dc, "r")
os.chdir(os.path.dirname(opts.dc))
print '\nDC Path:', os.path.dirname(opts.dc)
DC = parseCard(file, options)
if not DC.hasShapes: DC.hasShapes = True
MB = ShapeBuilder(DC, options)
theShapes = {}
theSyst = {}
nuiVar = {}
print '\n\n ------> Mlfit File: ', opts.mlfit
if opts.mlfit:
nuiVar = readBestFit(opts.mlfit)
if not opts.bin in DC.bins: raise RuntimeError, "Cannot open find %s in bins %s of %s" % (opts.bin,DC.bins,opts.dc)
print '\n-----> Looping over bins in datacard...'
for b in DC.bins:
print ' bin: ', b
if options.channel != None and (options.channel != b): continue
exps = {}
expNui = {}
shapeNui = {}
reducedShapeNui = {}
for (p,e) in DC.exp[b].items(): # so that we get only self.DC.processes contributing to this bin
exps[p] = [ e, [] ]
expNui[p] = [ e, [] ]
print '\n-----> Datacard systematics: ', DC.systs
for (lsyst,nofloat,pdf,pdfargs,errline) in DC.systs:
print '\n-----> Looping over systematics in datacard: ', (lsyst,nofloat,pdf,pdfargs,errline)
if pdf in ('param', 'flatParam'): continue
# begin skip systematics
skipme = False
for xs in options.excludeSyst:
if re.search(xs, lsyst):
skipme = True
if skipme:
print '\n-----> skipping systematics...'
continue
# end skip systematics
counter = 0
print '\n\t-----> Looping over keys in datacard: ', DC.exp[b].keys()
for p in DC.exp[b].keys(): # so that we get only self.DC.processes contributing to this bin
print '\n\t-----> Looping over process in this bin: ', p
if errline[b][p] == 0: continue
if p == 'QCD' and not 'QCD' in setup: continue
if pdf == 'gmN':
exps[p][1].append(1/sqrt(pdfargs[0]+1));
elif pdf == 'gmM':
exps[p][1].append(errline[b][p]);
elif type(errline[b][p]) == list:
kmax = max(errline[b][p][0], errline[b][p][1], 1.0/errline[b][p][0], 1.0/errline[b][p][1]);
exps[p][1].append(kmax-1.);
elif pdf == 'lnN':
lnNVar = max(errline[b][p], 1.0/errline[b][p])-1.
if not nuiVar.has_key('%s_%s'%(opts.fit,lsyst)):
nui = 0.
else:
nui= nuiVar['%s_%s'%(opts.fit,lsyst)][0]
lnNVar = lnNVar*nuiVar['%s_%s'%(opts.fit,lsyst)][1]
exps[p][1].append(lnNVar)
expNui[p][1].append(abs(1-errline[b][p])*nui);
elif 'shape' in pdf:
print '\n\t-----> Filling the Shapes for this process...'
#print 'shape %s %s: %s'%(pdf,p,lsyst)
s0 = MB.getShape(b,p)
sUp = MB.getShape(b,p,lsyst+"Up")
sDown = MB.getShape(b,p,lsyst+"Down")
if (s0.InheritsFrom("RooDataHist")):
s0 = ROOT.RooAbsData.createHistogram(s0,p,ws_var,theBinning)
s0.SetName(p)
sUp = ROOT.RooAbsData.createHistogram(sUp,p+lsyst+'Up',ws_var,theBinning)
sUp.SetName(p+lsyst+'Up')
sDown = ROOT.RooAbsData.createHistogram(sDown,p+lsyst+'Down',ws_var,theBinning)
sDown.SetName(p+lsyst+'Down')
theShapes[p] = s0.Clone()
theShapes[p+lsyst+'Up'] = sUp.Clone()
theShapes[p+lsyst+'Down'] = sDown.Clone()
if not nuiVar.has_key('%s_%s'%(opts.fit,lsyst)):
nui = 0.
reducedNui = 1.
else:
nui= nuiVar['%s_%s'%(opts.fit,lsyst)][0]
reducedNui= nuiVar['%s_%s'%(opts.fit,lsyst)][1]
shapeNui[p+lsyst] = nui
reducedShapeNui[lsyst] = reducedNui
if not 'CMS_vhbb_stat' in lsyst:
if counter == 0:
theSyst[lsyst] = s0.Clone()
theSyst[lsyst+'Up'] = sUp.Clone()
theSyst[lsyst+'Down'] = sDown.Clone()
else:
theSyst[lsyst].Add(s0)
theSyst[lsyst+'Up'].Add(sUp.Clone())
theSyst[lsyst+'Down'].Add(sDown.Clone())
counter += 1
procs = DC.exp[b].keys(); procs.sort()
if not 'QCD' in setup and 'QCD' in procs:
procs.remove('QCD')
if not 'W2b' in setup and 'WjHF' in procs:
procs.remove('WjHF')
if not 'Wlight' in setup and 'WjLF' in procs:
procs.remove('WjLF')
fmt = ("%%-%ds " % max([len(p) for p in procs]))+" "+options.format;
#Compute norm uncertainty and best fit
theNormUncert = {}
theBestFit = {}
print '\n-----> Computing norm uncertaint and best fit...'
for p in procs:
relunc = sqrt(sum([x*x for x in exps[p][1]]))
print fmt % (p, exps[p][0], exps[p][0]*relunc)
theNormUncert[p] = relunc
absBestFit = sum([x for x in expNui[p][1]])
theBestFit[p] = 1.+absBestFit
histos = []
typs = []
setup2=copy(setup)
shapesUp = [[] for _ in range(0,len(setup2))]
shapesDown = [[] for _ in range(0,len(setup2))]
sigCount = 0
signalList = ['ZH','WH']
# for shape analysis?
for p in procs:
b = opts.bin
print 'process: ', p
print 'setup:',setup
print 'Dict:', Dict
print 'theShapes:', theShapes
for s in setup:
print '-----> Fillings the shapes for: ', s
if not Dict[s] == p: continue
if s in signalList:
if sigCount ==0:
Overlay=copy(theShapes[Dict[s]])
else:
Overlay.Add(theShapes[Dict[s]])
sigCount += 1
else:
histos.append(theShapes[Dict[s]])
typs.append(s)
for (lsyst,nofloat,pdf,pdfargs,errline) in DC.systs:
if errline[b][p] == 0: continue
if ("shape" in pdf) and not 'CMS_vhbb_stat' in lsyst:
print 'syst %s'%lsyst
shapesUp[setup2.index(s)].append(theShapes[Dict[s]+lsyst+'Up'])
shapesDown[setup2.index(s)].append(theShapes[Dict[s]+lsyst+'Down'])
#-------------
#Compute absolute uncertainty from shapes
counter = 0
for (lsyst,nofloat,pdf,pdfargs,errline) in DC.systs:
sumErr = 0
for p in procs:
sumErr += errline[b][p]
print '---> PDF:',pdf, lsyst
if ("shape" in pdf) and not 'CMS_vhbb_stat' in lsyst and not sumErr == 0:
theSystUp = theSyst[lsyst+'Up'].Clone()
theSystUp.Add(theSyst[lsyst].Clone(),-1.)
theSystUp.Multiply(theSystUp)
theSystDown = theSyst[lsyst+'Down'].Clone()
theSystDown.Add(theSyst[lsyst].Clone(),-1.)
theSystDown.Multiply(theSystDown)
theSystUp.Scale(reducedShapeNui[lsyst])
theSystDown.Scale(reducedShapeNui[lsyst])
if counter == 0:
theAbsSystUp = theSystUp.Clone()
theAbsSystDown = theSystDown.Clone()
else:
theAbsSystUp.Add(theSystUp.Clone())
theAbsSystDown.Add(theSystDown.Clone())
counter +=1
#-------------
#Best fit for shapes
if not preFit:
# Set the preFit as an overlay
print '\n Making prefit overlay...'
print procs
i = 0
for hist in theShapes:
if hist not in procs: continue
print 'Process:', hist
print 'Shape:', theShapes[hist]
print 'i:', i
if i == 0:
prefit_overlay=copy(theShapes[hist])
#prefit_overlay=theShapes[hist]
print 'First Integral:', theShapes[hist].Integral()
i+=1
else:
#prefit_overlay.Add(theShapes[hist], 1.0)
prefit_overlay.Add(theShapes[hist])
print 'Integral:', theShapes[hist].Integral()
print 'prefit_overlay:', prefit_overlay
print 'Integral:', prefit_overlay.Integral()
print '\n-----> Getting best fit shapes(for postFit)...'
histos, Overlay, typs = getBestFitShapes(procs,theShapes,shapeNui,theBestFit,DC,setup,opts,Dict)
counter = 0
errUp=[]
total=[]
errDown=[]
nBins = histos[0].GetNbinsX()
#print histos
# temp hack to get histo names right
#names = ['ggZH','DY2B', 'DY1B', 'DYlight', 'TT', 'VV']
#for name,i in enumerate(histos):
# i.SetName(names[name])
#Overlay.SetName('ZH')
# end hack
print '\n total bins %s'%nBins
print '\n histos: ',histos
print '\n theNormUncert: ',theNormUncert
print '\n Overlay: ', Overlay
Error = ROOT.TGraphAsymmErrors(histos[0])
theTotalMC = histos[0].Clone()
for h in range(1,len(histos)):
theTotalMC.Add(histos[h])
total = [[]]*nBins
errUp = [[]]*nBins
errDown = [[]]*nBins
print '\n\n\t\t -----> The Histos: ', histos
for bin in range(1,nBins+1):
binError = theTotalMC.GetBinError(bin)
if math.isnan(binError):
binError = 0.
total[bin-1]=theTotalMC.GetBinContent(bin)
#Stat uncertainty of the MC outline
errUp[bin-1] = [binError]
errDown[bin-1] = [binError]
# Temp hack to fix theNormUncert naming
temp_theNormUncert = {}
for i,hist in enumerate(histos):
for x in theNormUncert:
#print '\nx: ', x
if x in histos[i].GetName():
temp_theNormUncert[histos[i].GetName()] = theNormUncert[x]
#print temp_theNormUncert
#Relative norm uncertainty of the individual MC
for h in range(0,len(histos)):
#errUp[bin-1].append(histos[h].GetBinContent(bin)*theNormUncert[histos[h].GetName()])
#errDown[bin-1].append(histos[h].GetBinContent(bin)*theNormUncert[histos[h].GetName()])
errUp[bin-1].append(histos[h].GetBinContent(bin)*temp_theNormUncert[histos[h].GetName()])
errDown[bin-1].append(histos[h].GetBinContent(bin)*temp_theNormUncert[histos[h].GetName()])
#Shape uncertainty of the MC
for bin in range(1,nBins+1):
#print sqrt(theSystUp.GetBinContent(bin))
errUp[bin-1].append(sqrt(theAbsSystUp.GetBinContent(bin)))
errDown[bin-1].append(sqrt(theAbsSystDown.GetBinContent(bin)))
#Add all in quadrature
totErrUp=[sqrt(sum([x**2 for x in bin])) for bin in errUp]
totErrDown=[sqrt(sum([x**2 for x in bin])) for bin in errDown]
#Make TGraph with errors
for bin in range(1,nBins+1):
if not total[bin-1] == 0:
point=histos[0].GetXaxis().GetBinCenter(bin)
Error.SetPoint(bin-1,point,1)
Error.SetPointEYlow(bin-1,totErrDown[bin-1]/total[bin-1])
#print 'down %s'%(totErrDown[bin-1]/total[bin-1])
Error.SetPointEYhigh(bin-1,totErrUp[bin-1]/total[bin-1])
#print 'up %s'%(totErrUp[bin-1]/total[bin-1])
#-----------------------
#Read data
data0 = MB.getShape(opts.bin,'data_obs')
if (data0.InheritsFrom("RooDataHist")):
data0 = ROOT.RooAbsData.createHistogram(data0,'data_obs',ws_var,theBinning)
data0.SetName('data_obs')
datas=[data0]
datatyps = [None]
datanames=[dataname]
print '\nDATA HIST:', data0
print 'Data name:', dataname
if blind and 'BDT' in var:
for bin in range(10,datas[0].GetNbinsX()+1):
datas[0].SetBinContent(bin,0)
#for bin in range(0,datas[0].GetNbinsX()+1):
# print 'Data in bin x:', datas[0].GetBinContent(bin)
histos.append(copy(Overlay))
if 'ZH' in signalList and 'WH' in signalList:
typs.append('ZH')
if 'ZH' in Stack.setup: Stack.setup.remove('ZH')
if 'WH' in Stack.setup: Stack.setup.remove('WH')
Stack.setup.insert(0,'ZH')
elif 'ZH' in signalList:
typs.append('ZH')
elif 'WH' in signalList:
typs.append('WH')
elif 'VVb' in signalList:
typs.append('VVb')
print '\n-----> Stack.setup(double check)...'
print 'Histos:', histos
print 'typs:', typs
Stack.histos = histos
Stack.typs = typs
Stack.datas = datas
Stack.datatyps = datatyps
Stack.datanames= datanames
Stack.prefit_overlay = [prefit_overlay]
if region == 'BDT':
Stack.overlay = [Overlay]
print '\n\n\t\t Overlay: ',Stack.overlay
Stack.AddErrors=Error
if dataname == 'Wtn':
lumi = 18300.
Stack.lumi = lumi
Stack.doPlot()
print 'i am done!\n'