def __init__(self, opts, directory, massPoints, datacardPatterns,
rootfilePatterns, clsType):
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" %
directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" %
fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory,
rf % mass)
else:
rfname = os.path.join(self.datacardDirectory, rf)
if not os.path.isfile(rfname):
raise Exception(
"ROOT file (for shapes) '%s' does not exist!" %
rfname)
aux.addToDictList(self.rootfiles, mass, rfname)
def __init__(self, opts, directory, massPoints, datacardPatterns, rootfilePatterns, clsType):
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" % directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" % fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory, rf % mass)
else:
rfname = os.path.join(self.datacardDirectory, rf)
if not os.path.isfile(rfname):
raise Exception("ROOT file (for shapes) '%s' does not exist!" % rfname)
aux.addToDictList(self.rootfiles, mass, rfname)
def addConfigInfo(of,
dataset,
addLuminosity=True,
dataVersion=None,
additionalText={}):
d = of.mkdir("configInfo")
d.cd()
# configinfo histogram
configinfo = ROOT.TH1F("configinfo", "configinfo", 3, 0, 3)
axis = configinfo.GetXaxis()
def setValue(bin, name, value):
axis.SetBinLabel(bin, name)
configinfo.SetBinContent(bin, value)
setValue(1, "control", 1)
setValue(2, "energy", float(dataset.getEnergy()))
if dataset.isData():
if addLuminosity:
setValue(3, "luminosity", dataset.getLuminosity())
elif dataset.isMC():
setValue(3, "crossSection", 1.0)
configinfo.Write()
configinfo.Delete()
# dataVersion
ds = dataset
while hasattr(ds, "datasets"):
ds = ds.datasets[0]
if dataVersion is None:
dataVersion = ds.dataVersion
dv = ROOT.TNamed("dataVersion", dataVersion)
dv.Write()
dv.Delete()
# codeVersion
codeVersion = ROOT.TNamed("codeVersion", git.getCommitId())
codeVersion.Write()
codeVersion.Delete()
for name, content in additionalText.iteritems():
txt = ROOT.TNamed(name, content)
txt.Write()
txt.Delete()
of.cd()
def addConfigInfo(of, dataset, addLuminosity=True, dataVersion=None, additionalText={}):
d = of.mkdir("configInfo")
d.cd()
# configinfo histogram
configinfo = ROOT.TH1F("configinfo", "configinfo", 3, 0, 3)
axis = configinfo.GetXaxis()
def setValue(bin, name, value):
axis.SetBinLabel(bin, name)
configinfo.SetBinContent(bin, value)
setValue(1, "control", 1)
setValue(2, "energy", float(dataset.getEnergy()))
if dataset.isData():
if addLuminosity:
setValue(3, "luminosity", dataset.getLuminosity())
elif dataset.isMC():
setValue(3, "crossSection", 1.0)
configinfo.Write()
configinfo.Delete()
# dataVersion
ds = dataset
while hasattr(ds, "datasets"):
ds = ds.datasets[0]
if dataVersion is None:
dataVersion = ds.dataVersion
dv = ROOT.TNamed("dataVersion", dataVersion)
dv.Write()
dv.Delete()
# codeVersion
codeVersion = ROOT.TNamed("codeVersion", git.getCommitId())
codeVersion.Write()
codeVersion.Delete()
for name, content in additionalText.iteritems():
txt = ROOT.TNamed(name, content)
txt.Write()
txt.Delete()
of.cd()
def run(self, proof=False, proofWorkers=None):
outputDir = self._outputPrefix + "_" + time.strftime("%y%m%d_%H%M%S")
if self._outputPostfix != "":
outputDir += "_" + self._outputPostfix
# Create output directory
os.mkdir(outputDir)
multicrabCfg = os.path.join(outputDir, "multicrab.cfg")
f = open(multicrabCfg, "w")
for dset in self._datasets:
f.write("[%s]\n\n" % dset.getName())
f.close()
# Copy/merge lumi files
lumifiles = set([d.getLumiFile() for d in self._datasets])
lumidata = {}
for fname in lumifiles:
if not os.path.exists(fname):
continue
f = open(fname)
data = json.load(f)
f.close()
for k in data.keys():
if k in lumidata:
raise Exception(
"Luminosity JSON file %s has a dataset for which the luminosity has already been loaded; please check the luminosity JSON files\n%s"
% (fname, k, "\n".join(lumifiles)))
lumidata.update(data)
if len(lumidata) > 0:
f = open(os.path.join(outputDir, "lumi.json"), "w")
json.dump(lumidata, f, sort_keys=True, indent=2)
f.close()
# Add run range in a json file, if runMin and runMax in pset
rrdata = {}
for aname, analyzerIE in self._analyzers.iteritems():
ana = analyzerIE.getAnalyzer()
if hasattr(ana, "__call__"):
for dset in self._datasets:
if dset.getDataVersion().isData():
ana = ana(dset.getDataVersion())
if ana.__getattr__("runMax") > 0:
rrdata[aname] = "%s-%s" % (ana.__getattr__(
"runMin"), ana.__getattr__("runMax"))
break
if len(rrdata) > 0:
f = open(os.path.join(outputDir, "runrange.json"), "w")
json.dump(rrdata, f, sort_keys=True, indent=2)
f.close()
# Setup proof if asked
_proof = None
if proof:
opt = ""
if proofWorkers is not None:
opt = "workers=%d" % proofWorkers
_proof = ROOT.TProof.Open(opt)
_proof.Exec("gSystem->Load(\"libHPlusAnalysis.so\");")
# Init timing counters
realTimeTotal = 0
cpuTimeTotal = 0
readMbytesTotal = 0
callsTotal = 0
# Process over datasets
ndset = 0
for dset in self._datasets:
# Get data PU distributions from data
# This is done every time for a dataset since memory management is simpler to handle
# if all the histograms in memory are deleted after reading a dataset is finished
hPUs = self._getDataPUhistos()
# Initialize
ndset += 1
inputList = ROOT.TList()
nanalyzers = 0
anames = []
usePUweights = False
useTopPtCorrection = False
nAllEventsPUWeighted = 0.0
for aname, analyzerIE in self._analyzers.iteritems():
if analyzerIE.runForDataset_(dset.getName()):
nanalyzers += 1
analyzer = analyzerIE.getAnalyzer()
if hasattr(analyzer, "__call__"):
analyzer = analyzer(dset.getDataVersion())
if analyzer is None:
raise Exception(
"Analyzer %s was specified as a function, but returned None"
% aname)
if not isinstance(analyzer, Analyzer):
raise Exception(
"Analyzer %s was specified as a function, but returned object of %s instead of Analyzer"
% (aname, analyzer.__class__.__name__))
inputList.Add(
ROOT.TNamed(
"analyzer_" + aname,
analyzer.className_() + ":" + analyzer.config_()))
# ttbar status for top pt corrections
ttbarStatus = "0"
useTopPtCorrection = analyzer.exists(
"useTopPtWeights") and analyzer.__getattr__(
"useTopPtWeights")
useTopPtCorrection = useTopPtCorrection and dset.getName(
).startswith("TT")
if useTopPtCorrection:
ttbarStatus = "1"
inputList.Add(ROOT.TNamed("isttbar", ttbarStatus))
# Pileup reweighting
(puAllEvents, puStatus) = self._parsePUweighting(
dset, analyzer, aname, hPUs, inputList)
nAllEventsPUWeighted += puAllEvents
usePUweights = puStatus
# Sum skim counters (from ttree)
hSkimCounterSum = self._getSkimCounterSum(
dset.getFileNames())
inputList.Add(hSkimCounterSum)
# Add name
anames.append(aname)
if nanalyzers == 0:
print "Skipping %s, no analyzers" % dset.getName()
continue
print "*** Processing dataset (%d/%d): %s" % (
ndset, len(self._datasets), dset.getName())
if dset.getDataVersion().isData():
lumivalue = "--- not available in lumi.json (or lumi.json not available) ---"
if dset.getName() in lumidata.keys():
lumivalue = lumidata[dset.getName()]
print " Luminosity: %s fb-1" % lumivalue
print " Using pileup weights:", usePUweights
if useTopPtCorrection:
print " Using top pt weights: True"
resDir = os.path.join(outputDir, dset.getName(), "res")
resFileName = os.path.join(resDir,
"histograms-%s.root" % dset.getName())
os.makedirs(resDir)
tchain = ROOT.TChain("Events")
for f in dset.getFileNames():
tchain.Add(f)
tchain.SetCacheLearnEntries(1000)
tchain.SetCacheSize(
10000000
) # Set cache size to 10 MB (somehow it is not automatically set contrary to ROOT docs)
tselector = ROOT.SelectorImpl()
# FIXME: TChain.GetEntries() is needed only to give a time
# estimate for the analysis. If this turns out to be slow,
# we could store the number of events along the file names
# (whatever is the method for that)
inputList.Add(ROOT.TNamed("entries", str(tchain.GetEntries())))
if dset.getDataVersion().isMC():
inputList.Add(ROOT.TNamed("isMC", "1"))
else:
inputList.Add(ROOT.TNamed("isMC", "0"))
inputList.Add(ROOT.TNamed("options", self._options.serialize_()))
inputList.Add(ROOT.TNamed("printStatus", "1"))
if _proof is not None:
tchain.SetProof(True)
inputList.Add(
ROOT.TNamed("PROOF_OUTPUTFILE_LOCATION", resFileName))
else:
inputList.Add(ROOT.TNamed("OUTPUTFILE_LOCATION", resFileName))
tselector.SetInputList(inputList)
readBytesStart = ROOT.TFile.GetFileBytesRead()
readCallsStart = ROOT.TFile.GetFileReadCalls()
timeStart = time.time()
clockStart = time.clock()
if self._maxEvents > 0:
tchain.SetCacheEntryRange(0, self._maxEvents)
tchain.Process(tselector, "", self._maxEvents)
else:
tchain.Process(tselector)
if _debugMemoryConsumption:
print " MEMDBG: TChain cache statistics:"
tchain.PrintCacheStats()
# Obtain Nall events for top pt corrections
NAllEventsTopPt = 0
if useTopPtCorrection:
for inname in dset.getFileNames():
fIN = ROOT.TFile.Open(inname)
h = fIN.Get("configInfo/topPtWeightAllEvents")
if h != None:
binNumber = 2 # nominal
if hasattr(analyzer, "topPtSystematicVariation"):
variation = getattr(analyzer,
"topPtSystematicVariation")
if variation == "minus":
binNumber = 0
# FIXME: The bin is to be added to the ttrees
#elif variation == "plus":
#binNumber = 3
#if not h.GetXaxis().GetBinLabel().endsWith("Plus"):
#raise Exception("This should not happen")
if binNumber > 0:
NAllEventsTopPt += h.GetBinContent(binNumber)
else:
raise Exception(
"Warning: Could not obtain N(AllEvents) for top pt reweighting"
)
ROOT.gROOT.GetListOfFiles().Remove(fIN)
fIN.Close()
# Write configInfo
fIN = ROOT.TFile.Open(dset.getFileNames()[0])
cinfo = fIN.Get("configInfo/configinfo")
tf = ROOT.TFile.Open(resFileName, "UPDATE")
configInfo = tf.Get("configInfo")
if configInfo == None:
configInfo = tf.mkdir("configInfo")
configInfo.cd()
dv = ROOT.TNamed("dataVersion", str(dset.getDataVersion()))
dv.Write()
dv.Delete()
cv = ROOT.TNamed("codeVersionAnalysis", git.getCommitId())
cv.Write()
cv.Delete()
if not cinfo == None:
# Add more information to configInfo
n = cinfo.GetNbinsX()
cinfo.SetBins(n + 3, 0, n + 3)
cinfo.GetXaxis().SetBinLabel(n + 1, "isData")
cinfo.GetXaxis().SetBinLabel(n + 2, "isPileupReweighted")
cinfo.GetXaxis().SetBinLabel(n + 3, "isTopPtReweighted")
# Add "isData" column
if not dset.getDataVersion().isMC():
cinfo.SetBinContent(n + 1, cinfo.GetBinContent(1))
# Add "isPileupReweighted" column
if usePUweights:
cinfo.SetBinContent(n + 2,
nAllEventsPUWeighted / nanalyzers)
# Add "isTopPtReweighted" column
if useTopPtCorrection:
cinfo.SetBinContent(n + 3, NAllEventsTopPt)
# Write
cinfo.Write()
ROOT.gROOT.GetListOfFiles().Remove(fIN)
fIN.Close()
# Memory management
configInfo.Delete()
ROOT.gROOT.GetListOfFiles().Remove(tf)
tf.Close()
for item in inputList:
if isinstance(item, ROOT.TObject):
item.Delete()
inputList = None
if hSkimCounterSum != None:
hSkimCounterSum.Delete()
if _debugMemoryConsumption:
print " MEMDBG: gDirectory", ROOT.gDirectory.GetList(
).GetSize()
print " MEMDBG: list ", ROOT.gROOT.GetList().GetSize()
print " MEMDBG: globals ", ROOT.gROOT.GetListOfGlobals(
).GetSize()
#for item in ROOT.gROOT.GetListOfGlobals():
#print item.GetName()
print " MEMDBG: files", ROOT.gROOT.GetListOfFiles(
).GetSize()
#for item in ROOT.gROOT.GetListOfFiles():
# print " %d items"%item.GetList().GetSize()
print " MEMDBG: specials ", ROOT.gROOT.GetListOfSpecials(
).GetSize()
for item in ROOT.gROOT.GetListOfSpecials():
print " " + item.GetName()
#gDirectory.GetList().Delete();
#gROOT.GetList().Delete();
#gROOT.GetListOfGlobals().Delete();
#TIter next(gROOT.GetList());
#while (TObject* o = dynamic_cast<TObject*>(next())) {
#o.Delete();
#}
# Performance and information
timeStop = time.time()
clockStop = time.clock()
readCallsStop = ROOT.TFile.GetFileReadCalls()
readBytesStop = ROOT.TFile.GetFileBytesRead()
calls = ""
if _proof is not None:
tchain.SetProof(False)
queryResult = _proof.GetQueryResult()
cpuTime = queryResult.GetUsedCPU()
readMbytes = queryResult.GetBytes() / 1024 / 1024
else:
cpuTime = clockStop - clockStart
readMbytes = float(readBytesStop -
readBytesStart) / 1024 / 1024
calls = " (%d calls)" % (readCallsStop - readCallsStart)
realTime = timeStop - timeStart
print " Real time %.2f, CPU time %.2f (%.1f %%), read %.2f MB%s, read speed %.2f MB/s" % (
realTime, cpuTime, cpuTime / realTime * 100, readMbytes, calls,
readMbytes / realTime)
print
realTimeTotal += realTime
cpuTimeTotal += cpuTime
readMbytesTotal += readMbytes
print
if len(self._datasets) > 1:
print " Total: Real time %.2f, CPU time %.2f (%.1f %%), read %.2f MB, read speed %.2f MB/s" % (
realTimeTotal, cpuTimeTotal, cpuTimeTotal / realTimeTotal *
100, readMbytesTotal, readMbytesTotal / realTimeTotal)
print " Results are in", outputDir
return outputDir
def run(self, proof=False, proofWorkers=None):
outputDir = self._outputPrefix+"_"+time.strftime("%y%m%d_%H%M%S")
if self._outputPostfix != "":
outputDir += "_"+self._outputPostfix
# Create output directory
os.mkdir(outputDir)
self.Print("Created output directory %s" % (sh_Note + outputDir + sh_Normal), True)
multicrabCfg = os.path.join(outputDir, "multicrab.cfg")
f = open(multicrabCfg, "w")
# For-loop: All datasets to be ran
for dset in self._datasets:
f.write("[%s]\n\n" % dset.getName())
f.close()
# Copy/merge lumi files
lumifiles = set([d.getLumiFile() for d in self._datasets])
lumidata = {}
for fname in lumifiles:
if not os.path.exists(fname):
continue
f = open(fname)
data = json.load(f)
f.close()
for k in data.keys():
if k in lumidata:
msg = "Luminosity JSON file %s has a dataset (%s) for which the luminosity has already been loaded. " % (fname, k)
msg += "Please check the luminosity JSON files:\n%s" % ("\n".join(lumifiles))
raise Exception(sh_Error + msg + sh_Normal)
lumidata.update(data)
if len(lumidata) > 0:
# Add run range in a json file, if runMin and runMax in pset
rrdata = {}
for aname, analyzerIE in self._analyzers.iteritems():
ana = analyzerIE.getAnalyzer()
if hasattr(ana, "__call__"):
for dset in self._datasets:
if dset.getDataVersion().isData():
ana = ana(dset.getDataVersion())
if ana.__getattr__("runMax") > 0:
rrdata[aname] = "%s-%s"%(ana.__getattr__("runMin"),ana.__getattr__("runMax"))
#lumidata[aname] = ana.__getattr__("lumi")
break
if len(rrdata) > 0:
f = open(os.path.join(outputDir, "runrange.json"), "w")
json.dump(rrdata, f, sort_keys=True, indent=2)
f.close()
# Create the luminosity JSON file
f = open(os.path.join(outputDir, "lumi.json"), "w")
json.dump(lumidata, f, sort_keys=True, indent=2)
self.Verbose("Created luminosity json file %s" % (sh_Note + f.name + sh_Normal), True)
f.close()
# Setup proof if asked
_proof = None
if proof:
opt = ""
if proofWorkers is not None:
opt = "workers=%d"%proofWorkers
_proof = ROOT.TProof.Open(opt)
_proof.Exec("gSystem->Load(\"libHPlusAnalysis.so\");")
# Init timing counters
realTimeTotal = 0
cpuTimeTotal = 0
readMbytesTotal = 0
callsTotal = 0
# Print the datasets that will be run on!
self.Print("Will process %d datasets in total:" % (len(self._datasets) ), True)
for i, d in enumerate(self._datasets, 1):
self.Print("%d) %s" % (i, sh_Note + d.getName() + sh_Normal), i==0)
# Process over datasets
ndset = 0
for i, dset in enumerate(self._datasets, 1):
hPUs = self._getDataPUhistos()
# Initialize
ndset += 1
inputList = ROOT.TList()
nanalyzers = 0
anames = []
usePUweights = False
useTopPtCorrection = False
nAllEventsPUWeighted = 0.0
for aname, analyzerIE in self._analyzers.iteritems():
if analyzerIE.runForDataset_(dset.getName()):
nanalyzers += 1
analyzer = analyzerIE.getAnalyzer()
if hasattr(analyzer, "__call__"):
analyzer = analyzer(dset.getDataVersion())
if analyzer is None:
raise Exception("Analyzer %s was specified as a function, but returned None" % aname)
if not isinstance(analyzer, Analyzer):
raise Exception("Analyzer %s was specified as a function, but returned object of %s instead of Analyzer" % (aname, analyzer.__class__.__name__))
inputList.Add(ROOT.TNamed("analyzer_"+aname, analyzer.className_()+":"+analyzer.config_()))
# ttbar status for top pt corrections
ttbarStatus = "0"
useTopPtCorrection = analyzer.exists("useTopPtWeights") and analyzer.__getattr__("useTopPtWeights")
#useTopPtCorrection = useTopPtCorrection and dset.getName().startswith("TT")
useTopPtCorrection = useTopPtCorrection and self.isTTbarDataset(dset)
if useTopPtCorrection:
ttbarStatus = "1"
inputList.Add(ROOT.TNamed("isttbar", ttbarStatus))
# intermediate H+ status for reweighting the NoNeutral samples
intermediateStatus = "0"
if dset.getName().find("IntermediateMassNoNeutral") > 0:
intermediateStatus = "1"
inputList.Add(ROOT.TNamed("isIntermediateNoNeutral", intermediateStatus))
# Pileup reweighting
self.Verbose("Getting pileup reweighting weights", True)
(puAllEvents, puStatus) = self._parsePUweighting(dset, analyzer, aname, hPUs, inputList)
nAllEventsPUWeighted += puAllEvents
usePUweights = puStatus
# Sum skim counters (from ttree)
hSkimCounterSum = self._getSkimCounterSum(dset.getFileNames())
inputList.Add(hSkimCounterSum)
# Add name
anames.append(aname)
if nanalyzers == 0:
self.Print("Skipping %s, no analyzers" % dset.getName(), True)
continue
self.Print("Processing dataset (%d/%d)" % (ndset, len(self._datasets) ))
align= "{:<23} {:<1} {:<60}"
info = {}
info["Dataset"] = dset.getName()
if dset.getDataVersion().isData():
lumivalue = "--- not available in lumi.json (or lumi.json not available) ---"
if dset.getName() in lumidata.keys():
lumivalue = lumidata[dset.getName()]
info["Luminosity"] = str(lumivalue) + " fb-1"
info["UsePUweights"] = usePUweights
info["UseTopPtCorrection"] = useTopPtCorrection
for key in info:
self.Print(align.format(key, ":", info[key]), False)
# Create dir for dataset ROOTT files
resDir = os.path.join(outputDir, dset.getName(), "res")
resFileName = os.path.join(resDir, "histograms-%s.root"%dset.getName())
os.makedirs(resDir)
tchain = ROOT.TChain("Events")
# For-loop: All file names for dataset
for f in dset.getFileNames():
tchain.Add(f)
tchain.SetCacheLearnEntries(1000);
tchain.SetCacheSize(10000000) # Set cache size to 10 MB (somehow it is not automatically set contrary to ROOT docs)
tselector = ROOT.SelectorImpl()
# FIXME: TChain.GetEntries() is needed only to give a time
# estimate for the analysis. If this turns out to be slow,
# we could store the number of events along the file names
# (whatever is the method for that)
inputList.Add(ROOT.TNamed("entries", str(tchain.GetEntries())))
if dset.getDataVersion().isMC():
inputList.Add(ROOT.TNamed("isMC", "1"))
else:
inputList.Add(ROOT.TNamed("isMC", "0"))
inputList.Add(ROOT.TNamed("options", self._options.serialize_()))
inputList.Add(ROOT.TNamed("printStatus", "1"))
if _proof is not None:
tchain.SetProof(True)
inputList.Add(ROOT.TNamed("PROOF_OUTPUTFILE_LOCATION", resFileName))
else:
inputList.Add(ROOT.TNamed("OUTPUTFILE_LOCATION", resFileName))
tselector.SetInputList(inputList)
readBytesStart = ROOT.TFile.GetFileBytesRead()
readCallsStart = ROOT.TFile.GetFileReadCalls()
timeStart = time.time()
clockStart = time.clock()
# Determine how many events to run on for given dataset
if len(self._maxEvents.keys()) > 0:
key = ""
for k in self._maxEvents.keys():
if k.lower() == "all":
key = k
break
maxEv_re = re.compile(k)
match = maxEv_re.search(dset.getName())
if match:
key = k
break
if key == "":
tchain.Process(tselector)
else:
maxEvts = self._maxEvents[key]
if maxEvts == -1:
tchain.Process(tselector)
else:
tchain.SetCacheEntryRange(0, self._maxEvents[key])
tchain.Process(tselector, "", self._maxEvents[key])
else:
tchain.Process(tselector)
if _debugMemoryConsumption:
print " MEMDBG: TChain cache statistics:"
tchain.PrintCacheStats()
# Obtain Nall events for top pt corrections
NAllEventsTopPt = 0
if useTopPtCorrection:
for inname in dset.getFileNames():
fIN = ROOT.TFile.Open(inname)
h = fIN.Get("configInfo/topPtWeightAllEvents")
if h != None:
binNumber = 2 # nominal
if hasattr(analyzer, "topPtSystematicVariation"):
variation = getattr(analyzer, "topPtSystematicVariation")
if variation == "minus":
binNumber = 0
# FIXME: The bin is to be added to the ttrees
#elif variation == "plus":
#binNumber = 3
#if not h.GetXaxis().GetBinLabel().endsWith("Plus"):
#raise Exception("This should not happen")
if binNumber > 0:
NAllEventsTopPt += h.GetBinContent(binNumber)
else:
raise Exception("Warning: Could not obtain N(AllEvents) for top pt reweighting")
ROOT.gROOT.GetListOfFiles().Remove(fIN)
fIN.Close()
# Write configInfo
fIN = ROOT.TFile.Open(dset.getFileNames()[0])
cinfo = fIN.Get("configInfo/configinfo")
tf = ROOT.TFile.Open(resFileName, "UPDATE")
configInfo = tf.Get("configInfo")
if configInfo == None:
configInfo = tf.mkdir("configInfo")
configInfo.cd()
dv = ROOT.TNamed("dataVersion", str(dset.getDataVersion()))
dv.Write()
dv.Delete()
cv = ROOT.TNamed("codeVersionAnalysis", git.getCommitId())
cv.Write()
cv.Delete()
if not cinfo == None:
# Add more information to configInfo
n = cinfo.GetNbinsX()
cinfo.SetBins(n+3, 0, n+3)
cinfo.GetXaxis().SetBinLabel(n+1, "isData")
cinfo.GetXaxis().SetBinLabel(n+2, "isPileupReweighted")
cinfo.GetXaxis().SetBinLabel(n+3, "isTopPtReweighted")
# Add "isData" column
if not dset.getDataVersion().isMC():
cinfo.SetBinContent(n+1, cinfo.GetBinContent(1))
# Add "isPileupReweighted" column
if usePUweights:
cinfo.SetBinContent(n+2, nAllEventsPUWeighted / nanalyzers)
# Add "isTopPtReweighted" column
if useTopPtCorrection:
cinfo.SetBinContent(n+3, NAllEventsTopPt)
# Write
cinfo.Write()
ROOT.gROOT.GetListOfFiles().Remove(fIN);
fIN.Close()
# Memory management
configInfo.Delete()
ROOT.gROOT.GetListOfFiles().Remove(tf);
tf.Close()
for item in inputList:
if isinstance(item, ROOT.TObject):
item.Delete()
inputList = None
if hSkimCounterSum != None:
hSkimCounterSum.Delete()
if _debugMemoryConsumption:
print " MEMDBG: gDirectory", ROOT.gDirectory.GetList().GetSize()
print " MEMDBG: list ", ROOT.gROOT.GetList().GetSize()
print " MEMDBG: globals ", ROOT.gROOT.GetListOfGlobals().GetSize()
#for item in ROOT.gROOT.GetListOfGlobals():
#print item.GetName()
print " MEMDBG: files", ROOT.gROOT.GetListOfFiles().GetSize()
#for item in ROOT.gROOT.GetListOfFiles():
# print " %d items"%item.GetList().GetSize()
print " MEMDBG: specials ", ROOT.gROOT.GetListOfSpecials().GetSize()
for item in ROOT.gROOT.GetListOfSpecials():
print " "+item.GetName()
#gDirectory.GetList().Delete();
#gROOT.GetList().Delete();
#gROOT.GetListOfGlobals().Delete();
#TIter next(gROOT.GetList());
#while (TObject* o = dynamic_cast<TObject*>(next())) {
#o.Delete();
#}
# Performance and information
timeStop = time.time()
clockStop = time.clock()
readCallsStop = ROOT.TFile.GetFileReadCalls()
readBytesStop = ROOT.TFile.GetFileBytesRead()
calls = ""
if _proof is not None:
tchain.SetProof(False)
queryResult = _proof.GetQueryResult()
cpuTime = queryResult.GetUsedCPU()
readMbytes = queryResult.GetBytes()/1024/1024
else:
cpuTime = clockStop-clockStart
readMbytes = float(readBytesStop-readBytesStart)/1024/1024
calls = " (%d calls)" % (readCallsStop-readCallsStart)
realTime = timeStop-timeStart
# Print usage stats in user-friendly formatting
self.PrintStats(readCallsStop, readCallsStart, cpuTime, realTime, readMbytes)
# Time accumulation
realTimeTotal += realTime
cpuTimeTotal += cpuTime
readMbytesTotal += readMbytes
# Total time stats
self.PrintStatsTotal(readMbytes, cpuTimeTotal, realTimeTotal, readMbytesTotal)
# Inform user of location of results
self.Print("Results are in %s" % (sh_Success + outputDir + sh_Normal), True)
return outputDir
def run(self, proof=False, proofWorkers=None):
outputDir = self._outputPrefix + "_" + time.strftime("%y%m%d_%H%M%S")
if self._outputPostfix != "":
outputDir += "_" + self._outputPostfix
# Create output directory
os.mkdir(outputDir)
self.Print(
"Created output directory %s" % (sh_Note + outputDir + sh_Normal),
True)
multicrabCfg = os.path.join(outputDir, "multicrab.cfg")
f = open(multicrabCfg, "w")
# For-loop: All datasets to be ran
for dset in self._datasets:
f.write("[%s]\n\n" % dset.getName())
f.close()
# Copy/merge lumi files
lumifiles = set([d.getLumiFile() for d in self._datasets])
lumidata = {}
for fname in lumifiles:
if not os.path.exists(fname):
continue
f = open(fname)
data = json.load(f)
f.close()
for k in data.keys():
if k in lumidata:
msg = "Luminosity JSON file %s has a dataset (%s) for which the luminosity has already been loaded. " % (
fname, k)
msg += "Please check the luminosity JSON files:\n%s" % (
"\n".join(lumifiles))
raise Exception(sh_Error + msg + sh_Normal)
lumidata.update(data)
if len(lumidata) > 0:
# Add run range in a json file, if runMin and runMax in pset
rrdata = {}
for aname, analyzerIE in self._analyzers.iteritems():
ana = analyzerIE.getAnalyzer()
if hasattr(ana, "__call__"):
for dset in self._datasets:
if dset.getDataVersion().isData():
ana = ana(dset.getDataVersion())
if ana.__getattr__("runMax") > 0:
rrdata[aname] = "%s-%s" % (ana.__getattr__(
"runMin"), ana.__getattr__("runMax"))
#lumidata[aname] = ana.__getattr__("lumi")
break
if len(rrdata) > 0:
f = open(os.path.join(outputDir, "runrange.json"), "w")
json.dump(rrdata, f, sort_keys=True, indent=2)
f.close()
# Create the luminosity JSON file
f = open(os.path.join(outputDir, "lumi.json"), "w")
json.dump(lumidata, f, sort_keys=True, indent=2)
self.Verbose(
"Created luminosity json file %s" %
(sh_Note + f.name + sh_Normal), True)
f.close()
# Setup proof if asked
_proof = None
if proof:
opt = ""
if proofWorkers is not None:
opt = "workers=%d" % proofWorkers
_proof = ROOT.TProof.Open(opt)
_proof.Exec("gSystem->Load(\"libHPlusAnalysis.so\");")
# Init timing counters
realTimeTotal = 0
cpuTimeTotal = 0
readMbytesTotal = 0
callsTotal = 0
# Print the datasets that will be run on!
self.Print(
"Will process %d datasets in total:" % (len(self._datasets)), True)
for i, d in enumerate(self._datasets, 1):
self.Print("%d) %s" % (i, sh_Note + d.getName() + sh_Normal),
i == 0)
# Process over datasets
ndset = 0
for i, dset in enumerate(self._datasets, 1):
hPUs = self._getDataPUhistos()
# Initialize
ndset += 1
inputList = ROOT.TList()
nanalyzers = 0
anames = []
usePUweights = False
useTopPtCorrection = False
nAllEventsPUWeighted = 0.0
for aname, analyzerIE in self._analyzers.iteritems():
if analyzerIE.runForDataset_(dset.getName()):
nanalyzers += 1
analyzer = analyzerIE.getAnalyzer()
if hasattr(analyzer, "__call__"):
analyzer = analyzer(dset.getDataVersion())
if analyzer is None:
raise Exception(
"Analyzer %s was specified as a function, but returned None"
% aname)
if not isinstance(analyzer, Analyzer):
raise Exception(
"Analyzer %s was specified as a function, but returned object of %s instead of Analyzer"
% (aname, analyzer.__class__.__name__))
inputList.Add(
ROOT.TNamed(
"analyzer_" + aname,
analyzer.className_() + ":" + analyzer.config_()))
# ttbar status for top pt corrections
ttbarStatus = "0"
useTopPtCorrection = analyzer.exists(
"useTopPtWeights") and analyzer.__getattr__(
"useTopPtWeights")
#useTopPtCorrection = useTopPtCorrection and dset.getName().startswith("TT")
useTopPtCorrection = useTopPtCorrection and self.isTTbarDataset(
dset)
if useTopPtCorrection:
ttbarStatus = "1"
inputList.Add(ROOT.TNamed("isttbar", ttbarStatus))
# intermediate H+ status for reweighting the NoNeutral samples
intermediateStatus = "0"
if dset.getName().find("IntermediateMassNoNeutral") > 0:
intermediateStatus = "1"
inputList.Add(
ROOT.TNamed("isIntermediateNoNeutral",
intermediateStatus))
# Pileup reweighting
self.Verbose("Getting pileup reweighting weights", True)
(puAllEvents, puStatus) = self._parsePUweighting(
dset, analyzer, aname, hPUs, inputList)
nAllEventsPUWeighted += puAllEvents
usePUweights = puStatus
# Sum skim counters (from ttree)
hSkimCounterSum = self._getSkimCounterSum(
dset.getFileNames())
inputList.Add(hSkimCounterSum)
# Add name
anames.append(aname)
if nanalyzers == 0:
self.Print("Skipping %s, no analyzers" % dset.getName(), True)
continue
self.Print("Processing dataset (%d/%d)" %
(ndset, len(self._datasets)))
align = "{:<23} {:<1} {:<60}"
info = {}
info["Dataset"] = dset.getName()
if dset.getDataVersion().isData():
lumivalue = "--- not available in lumi.json (or lumi.json not available) ---"
if dset.getName() in lumidata.keys():
lumivalue = lumidata[dset.getName()]
info["Luminosity"] = str(lumivalue) + " fb-1"
info["UsePUweights"] = usePUweights
info["UseTopPtCorrection"] = useTopPtCorrection
for key in info:
self.Print(align.format(key, ":", info[key]), False)
# Create dir for dataset ROOTT files
resDir = os.path.join(outputDir, dset.getName(), "res")
resFileName = os.path.join(resDir,
"histograms-%s.root" % dset.getName())
os.makedirs(resDir)
tchain = ROOT.TChain("Events")
# For-loop: All file names for dataset
for f in dset.getFileNames():
tchain.Add(f)
tchain.SetCacheLearnEntries(1000)
tchain.SetCacheSize(
10000000
) # Set cache size to 10 MB (somehow it is not automatically set contrary to ROOT docs)
tselector = ROOT.SelectorImpl()
# FIXME: TChain.GetEntries() is needed only to give a time
# estimate for the analysis. If this turns out to be slow,
# we could store the number of events along the file names
# (whatever is the method for that)
inputList.Add(ROOT.TNamed("entries", str(tchain.GetEntries())))
if dset.getDataVersion().isMC():
inputList.Add(ROOT.TNamed("isMC", "1"))
else:
inputList.Add(ROOT.TNamed("isMC", "0"))
inputList.Add(ROOT.TNamed("options", self._options.serialize_()))
inputList.Add(ROOT.TNamed("printStatus", "1"))
if _proof is not None:
tchain.SetProof(True)
inputList.Add(
ROOT.TNamed("PROOF_OUTPUTFILE_LOCATION", resFileName))
else:
inputList.Add(ROOT.TNamed("OUTPUTFILE_LOCATION", resFileName))
tselector.SetInputList(inputList)
readBytesStart = ROOT.TFile.GetFileBytesRead()
readCallsStart = ROOT.TFile.GetFileReadCalls()
timeStart = time.time()
clockStart = time.clock()
# Determine how many events to run on for given dataset
if len(self._maxEvents.keys()) > 0:
key = ""
for k in self._maxEvents.keys():
if k.lower() == "all":
key = k
break
maxEv_re = re.compile(k)
match = maxEv_re.search(dset.getName())
if match:
key = k
break
if key == "":
tchain.Process(tselector)
else:
maxEvts = self._maxEvents[key]
if maxEvts == -1:
tchain.Process(tselector)
else:
tchain.SetCacheEntryRange(0, self._maxEvents[key])
tchain.Process(tselector, "", self._maxEvents[key])
else:
tchain.Process(tselector)
if _debugMemoryConsumption:
print " MEMDBG: TChain cache statistics:"
tchain.PrintCacheStats()
# Obtain Nall events for top pt corrections
NAllEventsTopPt = 0
if useTopPtCorrection:
for inname in dset.getFileNames():
fIN = ROOT.TFile.Open(inname)
h = fIN.Get("configInfo/topPtWeightAllEvents")
if h != None:
binNumber = 2 # nominal
if hasattr(analyzer, "topPtSystematicVariation"):
variation = getattr(analyzer,
"topPtSystematicVariation")
if variation == "minus":
binNumber = 0
# FIXME: The bin is to be added to the ttrees
#elif variation == "plus":
#binNumber = 3
#if not h.GetXaxis().GetBinLabel().endsWith("Plus"):
#raise Exception("This should not happen")
if binNumber > 0:
NAllEventsTopPt += h.GetBinContent(binNumber)
else:
raise Exception(
"Warning: Could not obtain N(AllEvents) for top pt reweighting"
)
ROOT.gROOT.GetListOfFiles().Remove(fIN)
fIN.Close()
# Write configInfo
fIN = ROOT.TFile.Open(dset.getFileNames()[0])
cinfo = fIN.Get("configInfo/configinfo")
tf = ROOT.TFile.Open(resFileName, "UPDATE")
configInfo = tf.Get("configInfo")
if configInfo == None:
configInfo = tf.mkdir("configInfo")
configInfo.cd()
dv = ROOT.TNamed("dataVersion", str(dset.getDataVersion()))
dv.Write()
dv.Delete()
cv = ROOT.TNamed("codeVersionAnalysis", git.getCommitId())
cv.Write()
cv.Delete()
if not cinfo == None:
# Add more information to configInfo
n = cinfo.GetNbinsX()
cinfo.SetBins(n + 3, 0, n + 3)
cinfo.GetXaxis().SetBinLabel(n + 1, "isData")
cinfo.GetXaxis().SetBinLabel(n + 2, "isPileupReweighted")
cinfo.GetXaxis().SetBinLabel(n + 3, "isTopPtReweighted")
# Add "isData" column
if not dset.getDataVersion().isMC():
cinfo.SetBinContent(n + 1, cinfo.GetBinContent(1))
# Add "isPileupReweighted" column
if usePUweights:
cinfo.SetBinContent(n + 2,
nAllEventsPUWeighted / nanalyzers)
# Add "isTopPtReweighted" column
if useTopPtCorrection:
cinfo.SetBinContent(n + 3, NAllEventsTopPt)
# Write
cinfo.Write()
ROOT.gROOT.GetListOfFiles().Remove(fIN)
fIN.Close()
# Memory management
configInfo.Delete()
ROOT.gROOT.GetListOfFiles().Remove(tf)
tf.Close()
for item in inputList:
if isinstance(item, ROOT.TObject):
item.Delete()
inputList = None
if hSkimCounterSum != None:
hSkimCounterSum.Delete()
if _debugMemoryConsumption:
print " MEMDBG: gDirectory", ROOT.gDirectory.GetList(
).GetSize()
print " MEMDBG: list ", ROOT.gROOT.GetList().GetSize()
print " MEMDBG: globals ", ROOT.gROOT.GetListOfGlobals(
).GetSize()
#for item in ROOT.gROOT.GetListOfGlobals():
#print item.GetName()
print " MEMDBG: files", ROOT.gROOT.GetListOfFiles(
).GetSize()
#for item in ROOT.gROOT.GetListOfFiles():
# print " %d items"%item.GetList().GetSize()
print " MEMDBG: specials ", ROOT.gROOT.GetListOfSpecials(
).GetSize()
for item in ROOT.gROOT.GetListOfSpecials():
print " " + item.GetName()
#gDirectory.GetList().Delete();
#gROOT.GetList().Delete();
#gROOT.GetListOfGlobals().Delete();
#TIter next(gROOT.GetList());
#while (TObject* o = dynamic_cast<TObject*>(next())) {
#o.Delete();
#}
# Performance and information
timeStop = time.time()
clockStop = time.clock()
readCallsStop = ROOT.TFile.GetFileReadCalls()
readBytesStop = ROOT.TFile.GetFileBytesRead()
calls = ""
if _proof is not None:
tchain.SetProof(False)
queryResult = _proof.GetQueryResult()
cpuTime = queryResult.GetUsedCPU()
readMbytes = queryResult.GetBytes() / 1024 / 1024
else:
cpuTime = clockStop - clockStart
readMbytes = float(readBytesStop -
readBytesStart) / 1024 / 1024
calls = " (%d calls)" % (readCallsStop - readCallsStart)
realTime = timeStop - timeStart
# Print usage stats in user-friendly formatting
self.PrintStats(readCallsStop, readCallsStart, cpuTime, realTime,
readMbytes)
# Time accumulation
realTimeTotal += realTime
cpuTimeTotal += cpuTime
readMbytesTotal += readMbytes
# Total time stats
self.PrintStatsTotal(readMbytes, cpuTimeTotal, realTimeTotal,
readMbytesTotal)
# Inform user of location of results
self.Print("Results are in %s" % (sh_Success + outputDir + sh_Normal),
True)
return outputDir
def __init__(self, opts, directory, massPoints, datacardPatterns, rootfilePatterns, clsType):
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" % directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" % fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory, rf % mass)
aux.addToDictList(self.rootfiles, mass, rfname)
else:
rfname = os.path.join(self.datacardDirectory, rf)
aux.addToDictList(self.rootfiles, mass, rfname)
# if root files are not found, try with 1pr and 3pr extensions in the name
if not os.path.isfile(rfname):
rf_1pr = rf.replace(".root","_1pr.root")
rf_3pr = rf.replace(".root","_3pr.root")
rfname_1pr = os.path.join(self.datacardDirectory, rf_1pr % mass)
rfname_3pr = os.path.join(self.datacardDirectory, rf_3pr % mass)
aux.addToDictList(self.rootfiles, mass, rfname_1pr)
aux.addToDictList(self.rootfiles, mass, rfname_3pr)
if not os.path.isfile(rfname_1pr) or not os.path.isfile(rfname_3pr):
# if still not found, raise exception/warning
# raise Exception("ROOT file (for shapes) '%s' does not exist!" % rfname)
print("\033[91mWarning: ROOT file (for shapes) '%s' does not exist!\033[00m" % rfname)
def __init__(self, opts, directory, massPoints, datacardPatterns,
rootfilePatterns, clsType):
'''
Constructor
\param directory Datacard directory
\param massPoints List of mass points to calculate the limit
\param datacardPatterns List of datacard patterns to include in the limit calculation
\param rootfilePatterns List of shape ROOT file patterns to include in the limit calculation
\param clsType Object defining the CLs flavour (either LEPType or LHCType).
'''
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" %
directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" %
fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory,
rf % mass)
aux.addToDictList(self.rootfiles, mass, rfname)
else:
rfname = os.path.join(self.datacardDirectory, rf)
aux.addToDictList(self.rootfiles, mass, rfname)
# if root files are not found, try with 1pr and 3pr extensions in the name
if not os.path.isfile(rfname):
# 1+2 b-jets
# rf_1pr = rf.replace(".root","_2bjets.root")
# rf_3pr = rf.replace(".root","_1bjet.root")
# 0+1 b-jets
# rf_1pr = rf.replace(".root","_1bjets.root")
# rf_3pr = rf.replace(".root","_0bjets.root")
rf_1pr = rf.replace(".root", "_0bjet.root")
rf_3pr = rf.replace(".root", "_1bjet.root")
rf_extra = rf.replace(".root", "_2bjets.root")
rfname_1pr = os.path.join(self.datacardDirectory,
rf_1pr % mass)
rfname_3pr = os.path.join(self.datacardDirectory,
rf_3pr % mass)
rfname_extra = os.path.join(self.datacardDirectory,
rf_extra % mass)
aux.addToDictList(self.rootfiles, mass, rfname_1pr)
aux.addToDictList(self.rootfiles, mass, rfname_3pr)
aux.addToDictList(self.rootfiles, mass, rfname_extra)
if not os.path.isfile(
rfname_1pr) or not os.path.isfile(
rfname_3pr) or not os.path.isfile(
rfname_extra):
# if still not found, raise exception/warning
# raise Exception("ROOT file (for shapes) '%s' does not exist!" % rfname)
print(
"\033[91mWarning: ROOT file (for shapes) '%s' does not exist!\033[00m"
% rfname)
def __init__(self, opts, directory, massPoints, datacardPatterns, rootfilePatterns, clsType):
'''
Constructor
\param directory Datacard directory
\param massPoints List of mass points to calculate the limit
\param datacardPatterns List of datacard patterns to include in the limit calculation
\param rootfilePatterns List of shape ROOT file patterns to include in the limit calculation
\param clsType Object defining the CLs flavour (either LEPType or LHCType).
'''
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" % directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" % fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory, rf % mass)
aux.addToDictList(self.rootfiles, mass, rfname)
else:
rfname = os.path.join(self.datacardDirectory, rf)
aux.addToDictList(self.rootfiles, mass, rfname)
# if root files are not found, try with category A/B/C extensions in the name
if not os.path.isfile(rfname):
rf_a = rf.replace(".root","_a.root")
rf_b = rf.replace(".root","_b.root")
rf_c = rf.replace(".root","_c.root")
rfname_a = os.path.join(self.datacardDirectory, rf_a % mass)
rfname_b = os.path.join(self.datacardDirectory, rf_b % mass)
rfname_c = os.path.join(self.datacardDirectory, rf_c % mass)
aux.addToDictList(self.rootfiles, mass, rfname_a)
aux.addToDictList(self.rootfiles, mass, rfname_b)
aux.addToDictList(self.rootfiles, mass, rfname_c)
if not os.path.isfile(rfname_a) or not os.path.isfile(rfname_b) or not os.path.isfile(rfname_c):
# if still not found, raise exception/warning
# raise Exception("ROOT file (for shapes) '%s' does not exist!" % rfname)
print("\033[91mWarning: ROOT file (for shapes) '%s' does not exist!\033[00m" % rfname)
def __init__(self, opts, directory, massPoints, datacardPatterns,
rootfilePatterns, clsType):
self.opts = opts
self.datacardDirectory = directory
self.massPoints = massPoints
self.datacardPatterns = datacardPatterns
self.rootfilePatterns = rootfilePatterns
self.clsType = clsType.clone()
self.jobsCreated = False
self.datacards = {}
self.rootfiles = {}
self.scripts = []
self.configuration = {}
if not os.path.isdir(directory):
raise Exception("Datacard directory '%s' does not exist" %
directory)
# this is a dictionary dumped to configuration.json
self.configuration = {
"masspoints": massPoints,
"datacards": datacardPatterns,
"rootfiles": rootfilePatterns,
"codeVersion": git.getCommitId(),
"clsType": self.clsType.name(),
}
clsConfig = self.clsType.getConfiguration(self.configuration)
if clsConfig != None:
self.configuration["clsConfig"] = clsConfig
for mass in self.massPoints:
for dc in datacardPatterns:
fname = None
if "%s" in dc:
fname = os.path.join(self.datacardDirectory, dc % mass)
else:
fname = os.path.join(self.datacardDirectory, dc)
if not os.path.isfile(fname):
raise Exception("Datacard file '%s' does not exist!" %
fname)
aux.addToDictList(self.datacards, mass, fname)
for rf in rootfilePatterns:
if rf != None:
rfname = None
if "%s" in rf:
rfname = os.path.join(self.datacardDirectory,
rf % mass)
aux.addToDictList(self.rootfiles, mass, rfname)
else:
rfname = os.path.join(self.datacardDirectory, rf)
aux.addToDictList(self.rootfiles, mass, rfname)
# if root files are not found, try with 1pr and 3pr extensions in the name
if not os.path.isfile(rfname):
rf_1pr = rf.replace(".root", "_1pr.root")
rf_3pr = rf.replace(".root", "_3pr.root")
rfname_1pr = os.path.join(self.datacardDirectory,
rf_1pr % mass)
rfname_3pr = os.path.join(self.datacardDirectory,
rf_3pr % mass)
aux.addToDictList(self.rootfiles, mass, rfname_1pr)
aux.addToDictList(self.rootfiles, mass, rfname_3pr)
if not os.path.isfile(
rfname_1pr) or not os.path.isfile(rfname_3pr):
# if still not found, raise exception/warning
# raise Exception("ROOT file (for shapes) '%s' does not exist!" % rfname)
print(
"\033[91mWarning: ROOT file (for shapes) '%s' does not exist!\033[00m"
% rfname)