def filterAndDropSystematics(self,
include='.*',
exclude=None,
verbose=False):
"include and exclude can be either a regex, a single value, or a list"
nBefore = len(self.systematics)
def is_regex(exp):
return exp and '*' in exp
def is_list(exp):
return type(exp) == list
def is_literal_list(exp):
return (exp and ',' in exp)
def is_single_value(exp):
return exp and len(exp)
def str_to_list(exp):
return eval("['{0}']".format(exp))
print 'type include ', type(include), ' type()==list: ', (
type(include) == list)
toBeIncluded = (
[s for s in self.systematics
if s in include] if is_list(include) else
[s for s in self.systematics
if s in str_to_list(include)] if is_literal_list(include) else
filterWithRegexp(self.systematics, include) if is_regex(include)
else str_to_list(include) if is_single_value(include) else self.
systematics)
toBeExcluded = (
[s for s in self.systematics
if s in exclude] if is_list(exclude) else
[s for s in self.systematics
if s in str_to_list(exclude)] if is_literal_list(exclude) else
filterWithRegexp(self, systematics, exclude) if is_regex(exclude)
else str_to_list(exclude) if is_single_value(exclude) else [])
self.systematics = remove_duplicates(
[s for s in toBeIncluded if s not in toBeExcluded])
nAfter = len(self.systematics)
if verbose:
print "%s : dropped %d systematics, left with %s" % (
self.name, nBefore - nAfter, str(self.systematics))
assert self.systematics.count(
'NOM') == 1 or not nBefore, "%s : 'NOM' required %s" % (
self.name, str(self.systematics))
def parse_group_option(options=None, groups=[]):
"Given a list of Group objects, filter them according to options.group and options.exclude_group"
def is_regex(expr=''):
regex_common_operators = '.^$*+?'
return any(o in expr for o in regex_common_operators)
def is_list(expr=''):
return ',' in expr
if options.group:
group_opt = options.group
if is_regex(group_opt):
groups = utils.filterWithRegexp(groups, group_opt, lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name==group_opt]
if options.exclude_group:
group_opt = options.exclude_group
if is_regex(group_opt):
print 'excludeWithRegexp ',group_opt
groups = utils.excludeWithRegexp(groups, group_opt, lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name not in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name!=group_opt]
if options.verbose:
print "parse_group_option: selected ",list(g.name for g in groups)
return groups
def parse_group_option(options=None, groups=[]):
"Given a list of Group objects, filter them according to options.group and options.exclude_group"
def is_regex(expr=''):
regex_common_operators = '.^$*+?'
return any(o in expr for o in regex_common_operators)
def is_list(expr=''):
return ',' in expr
if options.group:
group_opt = options.group
if is_regex(group_opt):
groups = utils.filterWithRegexp(groups, group_opt,
lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name == group_opt]
if options.exclude_group:
group_opt = options.exclude_group
if is_regex(group_opt):
print 'excludeWithRegexp ', group_opt
groups = utils.excludeWithRegexp(groups, group_opt,
lambda g: g.name)
elif is_list(group_opt):
group_list = group_opt.strip('[]').split(',')
groups = [g for g in groups if g.name not in group_list]
else: # assume this is a single group
groups = [g for g in groups if g.name != group_opt]
if options.verbose:
print "parse_group_option: selected ", list(g.name for g in groups)
return groups
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp(
datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(
datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets:
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose:
print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose: print cmd
if submit:
out = utils.getCommandOutput(cmd)
if verbose: print out['stdout']
if not submit:
print "This was a dry run; use '--submit' to actually submit the jobs"
def get_list_of_syst_to_fill(opts):
systematics = ['NOM']
sysOption = opts.syst
excludedSyst = opts.exclude
anySys = sysOption == None
if sysOption == 'fake' or anySys:
systematics += systUtils.fakeSystVariations()
if sysOption == 'object' or anySys:
systematics += systUtils.mcObjectVariations()
if sysOption == 'weight' or anySys:
systematics += systUtils.mcWeightVariations()
if sysOption and sysOption.count(','):
systematics = [
s for s in systUtils.getAllVariations()
if s in sysOption.split(',')
]
elif sysOption in systUtils.getAllVariations():
systematics = [sysOption]
elif not anySys and len(systematics) == 1 and sysOption != 'NOM':
raise ValueError("Invalid syst %s" % str(sysOption))
if excludedSyst:
systematics = [
s for s in systematics
if s not in filterWithRegexp(systematics, excludedSyst)
]
return systematics
def main():
options = parse_options()
inputdf = options.input
outdir = options.output_dir
regexp = options.sample_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
debug = options.debug
utils.mkdirIfNeeded(outdir)
if debug: dataset.Dataset.verbose_parsing = True
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp(datasets, regexp,
lambda _: _.name) if regexp else datasets
datasets = utils.excludeWithRegexp(
datasets, exclude, lambda _: _.name) if exclude else datasets
counter = {'fail': 0, 'pass': 0}
for d in datasets:
outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir,
verbose) else 'fail'
counter[outcome] += 1
if verbose:
print "created %d filelists (%d failures)" % (counter['pass'],
counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=None):
selected_regions = selection_formulas().keys()
if regions:
selected_regions = [r for r in selected_regions if r in regions.split(',')]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions
return selected_regions
def main(filename1, filename2, outdir, regexp, verbose) :
# there are too many histos; by default compare only the ones vs. pt for few selections
relevantHistograms = ['l_pt_den','l_pt_num']
relevantSelections = ['CRPremT2','CR_SSInc','CR_WHSS','CRPremT2','CR_SSInc','CR_WHSS']
outdir = outdir if outdir else guessOutdirFromInputs(filename1, filename2)
if verbose : print "saving output plots to '%s'"%outdir
file1, file2 = r.TFile.Open(filename1), r.TFile.Open(filename2)
histonames1 = getAllHistoNames(file1, onlyTH1=True)
histonames2 = getAllHistoNames(file2, onlyTH1=True)
if verbose :
print '\n'.join(["%s: %d histograms"%(f, len(hs))
for f, hs in [(filename1, histonames1), (filename2, histonames2)]])
commonHistos = [h for h in histonames1 if h in histonames2]
def diff(a, b) : return list(set(a)-set(b))
if len(commonHistos)!=len(histonames2) :
missFrom1, missFrom2 = diff(histonames2, histonames1), diff(histonames1, histonames2)
print ('histograms not in common:\n'
+"%d missing from %s\n"%(len(missFrom1), filename1)
#+'\n\t'.join(missFrom1)
+"%d missing from %s\n"%(len(missFrom2), filename2)
#+'\n\t'.join(missFrom2)
)
label1, label2 = labelFromFilename(filename1), labelFromFilename(filename2)
commonHistos = (filterWithRegexp(commonHistos, regexp) if regexp is not None
else filter(lambda h:
any(s in h for s in relevantHistograms) and
any(s in h for s in relevantSelections),
commonHistos))
canvas = r.TCanvas('diff_fakeMatrix','diff_fakeMatrix')
for h in commonHistos :
outname = outdir+'/'+h
h1, h2 = file1.Get(h), file2.Get(h)
plotComparison(h1, h2, canvas, outname, label1, label2, verbose)
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp (datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets :
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose : print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose : print cmd
if submit :
out = utils.getCommandOutput(cmd)
if verbose : print out['stdout']
if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
def regions_to_plot(include='.*', exclude=None, regions=None):
selected_regions = selection_formulas().keys()
if regions:
selected_regions = [
r for r in selected_regions if r in regions.split(',')
]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(
selected_regions, exclude) if exclude else selected_regions
return selected_regions
def filterAndDropSystematics(self, include='.*', exclude=None, verbose=False) :
nBefore = len(self.systematics)
anyFilter = include or exclude
toBeExcluded = filter(self,systematics, exclude) if exclude else []
systs = ['NOM'] if 'NOM' in self.systematics else []
if include : systs += filterWithRegexp(self.systematics, include)
if exclude : systs = [s for s in systs if toBeExcluded and s not in toBeExcluded]
self.systematics = systs if anyFilter else self.systematics
self.systematics = remove_duplicates(self.systematics)
nAfter = len(self.systematics)
if verbose : print "%s : dropped %d systematics, left with %s"%(self.name, nBefore-nAfter, str(self.systematics))
assert self.systematics.count('NOM')==1 or not nBefore, "%s : 'NOM' required %s"%(self.name, str(self.systematics))
def filterAndDropSystematics(self, include='.*', exclude=None, verbose=False) :
"include and exclude can be either a regex, a single value, or a list"
nBefore = len(self.systematics)
def is_regex(exp) : return exp and '*' in exp
def is_list(exp) : return type(exp)==list
def is_literal_list(exp) : return (exp and ',' in exp)
def is_single_value(exp) : return exp and len(exp)
def str_to_list(exp) : return eval("['{0}']".format(exp))
print 'type include ',type(include),' type()==list: ',(type(include)==list)
toBeIncluded = ([s for s in self.systematics if s in include] if is_list(include) else
[s for s in self.systematics if s in str_to_list(include)] if is_literal_list(include) else
filterWithRegexp(self.systematics, include) if is_regex(include) else
str_to_list(include) if is_single_value(include) else
self.systematics)
toBeExcluded = ([s for s in self.systematics if s in exclude] if is_list(exclude) else
[s for s in self.systematics if s in str_to_list(exclude)] if is_literal_list(exclude) else
filterWithRegexp(self,systematics, exclude) if is_regex(exclude) else
str_to_list(exclude) if is_single_value(exclude) else
[])
self.systematics = remove_duplicates([s for s in toBeIncluded if s not in toBeExcluded])
nAfter = len(self.systematics)
if verbose : print "%s : dropped %d systematics, left with %s"%(self.name, nBefore-nAfter, str(self.systematics))
assert self.systematics.count('NOM')==1 or not nBefore, "%s : 'NOM' required %s"%(self.name, str(self.systematics))
def runFill(opts) :
batchMode = opts.batch
inputFakeDir = opts.input_fake
inputGenDir = opts.input_gen
outputDir = opts.output_dir
sysOption = opts.syst
excludedSyst = opts.exclude
verbose = opts.verbose
if verbose : print "filling histos"
mkdirIfNeeded(outputDir)
systematics = ['NOM']
anySys = sysOption==None
if sysOption=='fake' or anySys : systematics += systUtils.fakeSystVariations()
if sysOption=='object' or anySys : systematics += systUtils.mcObjectVariations()
if sysOption=='weight' or anySys : systematics += systUtils.mcWeightVariations()
if sysOption and sysOption.count(',') : systematics = [s for s in systUtils.getAllVariations() if s in sysOption.split(',')]
elif sysOption in systUtils.getAllVariations() : systematics = [sysOption]
elif not anySys and len(systematics)==1 and sysOption!='NOM' : raise ValueError("Invalid syst %s"%str(sysOption))
if excludedSyst : systematics = [s for s in systematics if s not in filterWithRegexp(systematics, excludedSyst)]
if verbose : print "about to loop over these systematics:\n %s"%str(systematics)
for syst in systematics :
if batchMode :
newOptions = " --input-gen %s" % opts.input_gen
newOptions += " --input-fake %s" % opts.input_fake
newOptions += " --output-dir %s" % opts.output_dir
newOptions += " --verbose %s" % opts.verbose
newOptions += " --syst %s" % syst
template = 'batch/templates/check_hft_fill.sh.template'
script = "batch/hft_%s.sh"%syst
scriptFile = open(script, 'w')
scriptFile.write(open(template).read()
.replace('%(opt)s', newOptions)
.replace('%(logfile)s', 'log/hft/fill_'+syst+'.log')
.replace('%(jobname)s', 'fill_'+syst))
scriptFile.close()
cmd = "sbatch %s"%script
if verbose : print cmd
out = getCommandOutput(cmd)
if verbose : print out['stdout']
if out['stderr'] : print out['stderr']
continue
if verbose : print '---- filling ',syst
samplesPerGroup = allSamplesAllGroups()
[s.setSyst(syst) for g, samples in samplesPerGroup.iteritems() for s in samples]
counters, histos = countAndFillHistos(samplesPerGroup=samplesPerGroup, syst=syst, verbose=verbose, outdir=outputDir)
printCounters(counters)
saveHistos(samplesPerGroup, histos, outputDir, verbose)
def optimizeSelection() :
inputdir, options = parseOptions()
print 'sigreg ',options.sigreg
tag = pickTag(inputdir, options)
sigFiles, bkgFiles = getInputFilenames(inputdir, tag, options) # todo: filter with regexp
sigFiles = dict([(s, k) for s, k in sigFiles.iteritems() if s in filterWithRegexp(sigFiles.keys(), options.sigreg)])
allSamples = dictSum(sigFiles, bkgFiles)
vars = variablesToPlot()
histos = bookHistos(vars, allSamples.keys(), options.ll, options.nj)
counts = fillHistosAndCount(histos, dictSum(sigFiles, bkgFiles), options.ll, options.nj, options.quicktest)
bkgHistos = dict((s, h) for s, h in histos.iteritems() if s in bkgFiles.keys())
sigHistos = dict((s, h) for s, h in histos.iteritems() if s in sigFiles.keys())
plotHistos(bkgHistos, sigHistos, options.plotdir)
printSummary(counts, options.summary)
def get_list_of_syst_to_fill(opts):
systematics = ['NOM']
sysOption = opts.syst
excludedSyst = opts.exclude
anySys = sysOption==None
if sysOption=='fake' or anySys : systematics += systUtils.fakeSystVariations()
if sysOption=='object' or anySys : systematics += systUtils.mcObjectVariations()
if sysOption=='weight' or anySys : systematics += systUtils.mcWeightVariations()
if sysOption and sysOption.count(','):
systematics = [s for s in systUtils.getAllVariations() if s in sysOption.split(',')]
elif sysOption in systUtils.getAllVariations(): systematics = [sysOption]
elif not anySys and len(systematics)==1 and sysOption!='NOM':
raise ValueError("Invalid syst %s"%str(sysOption))
if excludedSyst:
systematics = [s for s in systematics if s not in filterWithRegexp(systematics, excludedSyst)]
return systematics
def regions_to_plot(include='.*', exclude=None, regions=''):
"include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions"
# return ['vr_emu_mue_ss'] # test to debug fake
# return ['vr_emu_ss_razor']
# return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake
# return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs
# return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg
# used to be the default:
# ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os',
# 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss',
# 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss',
# 'sr_mue_os_low_pt1_15'
# ]
selected_regions = selection_formulas().keys()
if regions:
regions = regions if type(regions)==list else regions.split(',') if ',' in regions else [regions] # if it's a comma-sep string, convert it to list
selected_regions = [r for r in selected_regions if r in regions]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(selected_regions, exclude) if exclude else selected_regions
return selected_regions
def main(filename1, filename2, outdir, regexp, verbose):
# there are too many histos; by default compare only the ones vs. pt for few selections
relevantHistograms = ['l_pt_den', 'l_pt_num']
relevantSelections = [
'CRPremT2', 'CR_SSInc', 'CR_WHSS', 'CRPremT2', 'CR_SSInc', 'CR_WHSS'
]
outdir = outdir if outdir else guessOutdirFromInputs(filename1, filename2)
if verbose: print "saving output plots to '%s'" % outdir
file1, file2 = r.TFile.Open(filename1), r.TFile.Open(filename2)
histonames1 = getAllHistoNames(file1, onlyTH1=True)
histonames2 = getAllHistoNames(file2, onlyTH1=True)
if verbose:
print '\n'.join([
"%s: %d histograms" % (f, len(hs))
for f, hs in [(filename1, histonames1), (filename2, histonames2)]
])
commonHistos = [h for h in histonames1 if h in histonames2]
def diff(a, b):
return list(set(a) - set(b))
if len(commonHistos) != len(histonames2):
missFrom1, missFrom2 = diff(histonames2,
histonames1), diff(histonames1,
histonames2)
print('histograms not in common:\n' + "%d missing from %s\n" %
(len(missFrom1), filename1)
#+'\n\t'.join(missFrom1)
+ "%d missing from %s\n" % (len(missFrom2), filename2)
#+'\n\t'.join(missFrom2)
)
label1, label2 = labelFromFilename(filename1), labelFromFilename(filename2)
commonHistos = (filterWithRegexp(commonHistos, regexp)
if regexp is not None else filter(
lambda h: any(s in h for s in relevantHistograms) and
any(s in h for s in relevantSelections), commonHistos))
canvas = r.TCanvas('diff_fakeMatrix', 'diff_fakeMatrix')
for h in commonHistos:
outname = outdir + '/' + h
h1, h2 = file1.Get(h), file2.Get(h)
plotComparison(h1, h2, canvas, outname, label1, label2, verbose)
def main():
options = parse_options()
inputdf = options.input
outdir = options.output_dir
regexp = options.sample_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
debug = options.debug
utils.mkdirIfNeeded(outdir)
if debug : dataset.Dataset.verbose_parsing = True
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp (datasets, regexp, lambda _: _.name) if regexp else datasets
datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets
counter = {'fail':0, 'pass':0}
for d in datasets:
outcome = 'pass' if d.build_filelist(gpatlas_dir(d, tag), outdir, verbose) else 'fail'
counter[outcome] += 1
if verbose:
print "created %d filelists (%d failures)" % (counter['pass'], counter['fail'])
def regions_to_plot(include='.*', exclude=None, regions=''):
"include and exclude are regexp; regions is a string with either one region or a comma-separated list of regions"
# return ['vr_emu_mue_ss'] # test to debug fake
# return ['vr_emu_ss_razor']
# return [k for k in selection_formulas().keys() if ('vr' in k and 'ss' in k)] # test to debug fake
# return [k for k in selection_formulas().keys() if 'vr' not in k] # tmp until I have vrs
# return [k for k in selection_formulas().keys() if 'sr' in k] # tmp dbg
# used to be the default:
# ['sr_emu_os', 'sr_mue_os', 'vr_emu_os', 'vr_mue_os',
# 'sr_emu_ss', 'sr_mue_ss', 'vr_emu_ss', 'vr_mue_ss',
# 'ext_mumu_ss', 'ext_emu_mue_ss', 'ext_emu_pt0_40_ss', 'ext_mue_pt0_40_ss', 'ext_mumu_pt0_40_ss',
# 'sr_mue_os_low_pt1_15'
# ]
selected_regions = selection_formulas().keys()
if regions:
regions = regions if type(
regions) == list else regions.split(',') if ',' in regions else [
regions
] # if it's a comma-sep string, convert it to list
selected_regions = [r for r in selected_regions if r in regions]
selected_regions = utils.filterWithRegexp(selected_regions, include)
selected_regions = utils.excludeWithRegexp(
selected_regions, exclude) if exclude else selected_regions
return selected_regions
if fakerate : return 'fakerate'
if fakepred : return 'fakepred'
def formAndCreateOutdir(basedir, subdir) :
d = basedir+'/'+subdir
if not os.path.isdir(d) : os.makedirs(d)
return d
outdir = formAndCreateOutdir('out/', subdir())
logdir = formAndCreateOutdir('log/', subdir())
batdir = formAndCreateOutdir(scriptDir, subdir())
inputTemplate = "filelist/%(sample)s.txt"
outScriptTemplate = "%(batdir)s/%(sample)s.sh"
outRootTemplate = "%(outdir)s/%(sample)s_%(tag)s.root"
outLogTemplate = "%(logdir)s/%(sample)s_%(tag)s.log"
sampleNames = [d.name for d in datasets if not d.placeholder or alsoph]
sampleNames = filterWithRegexp(sampleNames, regexp)
excludedNames = [] if exclude is None else filterWithRegexp(sampleNames, exclude)
sampleNames = [s for s in sampleNames if s not in excludedNames]
def listExists(dset='', flistDir='./filelist') : return os.path.exists(flistDir+'/'+dset+'.txt')
def fillInScriptTemplate(sample, input, output, otherOptions, outScript, scriptTemplate) :
options = otherOptions
options += ' --WH-sample' if 'WH' in sample else ''
outFile = open(outScript, 'w')
for line in open(scriptTemplate).readlines() :
line = line.replace('${inp}', input)
line = line.replace('${out}', output)
line = line.replace('${opt}', options)
line = line.replace('${sample}', sample)
outFile.write(line)
outFile.close()
def runPlot(opts) :
inputDir = opts.input_dir
outputDir = opts.output_dir
sysOption = opts.syst
excludedSyst = opts.exclude
verbose = opts.verbose
mkdirIfNeeded(outputDir)
buildTotBkg = systUtils.buildTotBackgroundHisto
buildStat = systUtils.buildStatisticalErrorBand
buildSyst = systUtils.buildSystematicErrorBand
groups = allGroups()
selections = allRegions()
variables = variablesToPlot()
for group in groups :
group.setHistosDir(inputDir)
group.exploreAvailableSystematics(verbose)
group.filterAndDropSystematics(sysOption, excludedSyst, verbose)
mkdirIfNeeded(outputDir)
systematics = ['NOM']
anySys = sysOption==None
if sysOption=='fake' or anySys : systematics += systUtils.fakeSystVariations()
if sysOption=='object' or anySys : systematics += systUtils.mcObjectVariations()
if sysOption=='weight' or anySys : systematics += systUtils.mcWeightVariations()
if sysOption and sysOption.count(',') : systematics = [s for s in systUtils.getAllVariations() if s in sysOption.split(',')]
elif sysOption in systUtils.getAllVariations() : systematics = [sysOption]
if not anySys and len(systematics)==1 and sysOption!='NOM' : raise ValueError("Invalid syst %s"%str(sysOption))
if excludedSyst : systematics = [s for s in systematics if s not in filterWithRegexp(systematics, excludedSyst)]
if verbose : print "using the following systematics : %s"%str(systematics)
fakeSystematics = [s for s in systematics if s in systUtils.fakeSystVariations()]
mcSystematics = [s for s in systematics if s in systUtils.mcObjectVariations() + systUtils.mcWeightVariations()]
simBkgs = [g for g in groups if g.isMcBkg]
data, fake, signal = findByName(groups, 'data'), findByName(groups, 'fake'), findByName(groups, 'signal')
for sel in selections :
if verbose : print '-- plotting ',sel
for var in variables :
if verbose : print '---- plotting ',var
for g in groups : g.setSystNominal()
nominalHistoData = data.getHistogram(variable=var, selection=sel, cacheIt=True)
nominalHistoSign = signal.getHistogram(variable=var, selection=sel, cacheIt=True)
nominalHistoFakeBkg = fake.getHistogram(variable=var, selection=sel, cacheIt=True)
nominalHistosSimBkg = dict([(g.name, g.getHistogram(variable=var, selection=sel, cacheIt=True)) for g in simBkgs])
nominalHistosBkg = dict([('fake', nominalHistoFakeBkg)] + [(g, h) for g, h in nominalHistosSimBkg.iteritems()])
nominalHistoTotBkg = buildTotBkg(histoFakeBkg=nominalHistoFakeBkg, histosSimBkgs=nominalHistosSimBkg)
statErrBand = buildStat(nominalHistoTotBkg)
systErrBand = buildSyst(fake=fake, simBkgs=simBkgs, variable=var, selection=sel,
fakeVariations=fakeSystematics, mcVariations=mcSystematics, verbose=verbose)
plotHistos(histoData=nominalHistoData, histoSignal=nominalHistoSign, histoTotBkg=nominalHistoTotBkg,
histosBkg=nominalHistosBkg,
statErrBand=statErrBand, systErrBand=systErrBand,
canvasName=(sel+'_'+var), outdir=outputDir, verbose=verbose)
for group in groups :
summary = group.variationsSummary()
for selection, summarySel in summary.iteritems() :
colW = str(12)
header = ' '.join([('%'+colW+'s')%colName for colName in ['variation', 'yield', 'delta[%]']])
lineTemplate = '%(sys)'+colW+'s'+'%(counts)'+colW+'s'+'%(delta)'+colW+'s'
print "---- summary of variations for %s ----" % group.name
print "--- %s ---" % selection
print header
print '\n'.join(lineTemplate%{'sys':s,
'counts':(("%.3f"%c) if type(c) is float else (str(c)+str(type(c)))),
'delta' :(("%.3f"%d) if type(d) is float else '--' if d==None else (str(d)+str(type(d)))) }
for s,c,d in summarySel)
def formAndCreateOutdir(basedir, subdir):
d = basedir + '/' + subdir
if not os.path.isdir(d): os.makedirs(d)
return d
outdir = formAndCreateOutdir('out/', subdir())
logdir = formAndCreateOutdir('log/', subdir())
batdir = formAndCreateOutdir(scriptDir, subdir())
inputTemplate = "filelist/%(sample)s.txt"
outScriptTemplate = "%(batdir)s/%(sample)s.sh"
outRootTemplate = "%(outdir)s/%(sample)s_%(tag)s.root"
outLogTemplate = "%(logdir)s/%(sample)s_%(tag)s.log"
sampleNames = [d.name for d in datasets if not d.placeholder or alsoph]
sampleNames = filterWithRegexp(sampleNames, regexp)
excludedNames = [] if exclude is None else filterWithRegexp(
sampleNames, exclude)
sampleNames = [s for s in sampleNames if s not in excludedNames]
def listExists(dset='', flistDir='./filelist'):
return os.path.exists(flistDir + '/' + dset + '.txt')
def fillInScriptTemplate(sample, input, output, otherOptions, outScript,
scriptTemplate):
options = otherOptions
options += ' --WH-sample' if 'WH' in sample else ''
outFile = open(outScript, 'w')
for line in open(scriptTemplate).readlines():
