def setUp(self):
"""Create dictionaries needed by combiners."""
dirname = os.path.join(os.path.dirname(__file__), 'fixtures')
orfdata = TFile(
os.path.abspath(os.path.join(dirname, 'dqm_data.root')),
'read'
)
orfref = TFile(
os.path.abspath(os.path.join(dirname, 'dqm_ref.root')),
'read'
)
# valid ROOT files
assert(not orfdata.IsZombie())
assert(not orfref.IsZombie())
self.tdir = tempfile.mkdtemp()
self.rfdata = TFile(os.path.join(self.tdir, 'fdata.root'), 'recreate')
self.rfref = TFile(os.path.join(self.tdir, 'fref.root'), 'recreate')
configfile = os.path.abspath(os.path.join(os.path.dirname(__file__), 'analysis_config_test.py'))
with open(configfile, 'r') as inputFile:
exec(inputFile.read())
config = AnalysisConfigWrapper((analysis_config_branches, analysis_config_leaves))
self.mycombiner = config.getTrunk(orfdata.GetName(), orfref.GetName(), r"pedestal\S*")
self.mycombiner.evaluate()
def setUp(self):
"""Create dictionaries needed by combiners."""
dirname = os.path.join(os.path.dirname(__file__), 'fixtures')
orfdata = TFile(
os.path.abspath(os.path.join(dirname, 'dqm_data.root')), 'read')
orfref = TFile(os.path.abspath(os.path.join(dirname, 'dqm_ref.root')),
'read')
# valid ROOT files
assert (not orfdata.IsZombie())
assert (not orfref.IsZombie())
self.tdir = tempfile.mkdtemp()
self.rfdata = TFile(os.path.join(self.tdir, 'fdata.root'), 'recreate')
self.rfref = TFile(os.path.join(self.tdir, 'fref.root'), 'recreate')
hist_recipes = [
(get_avg_trend, 'Vetra/NoiseMon/ADCCMSuppressed',
('RMSNoise_vs_ChipChannel', 'AvgRMSNoise_trend')),
(get_avg_hist, 'Vetra/NoiseMon/ADCCMSuppressed',
('RMSNoise_vs_ChipChannel', 'AvgRMSNoise_all')),
(get_avg_hist, 'Vetra/NoiseMon/ADCCMSuppressed',
('RMSNoise_vs_ChipChannel', 'AvgRMSNoise_R', 'r')),
(get_avg_hist, 'Vetra/NoiseMon/ADCCMSuppressed',
('RMSNoise_vs_ChipChannel', 'AvgRMSNoise_Phi', 'p')),
# (get_avg_hist, 'Vetra/VeloPedestalSubtractorMoni',
# ('Ped_Sub_ADCs_Profile', 'Ped_Sub_ADCs_all'))
]
# histograms: make, save, and cleanup
for recipe in hist_recipes:
href = recipe[0](orfref.GetDirectory(recipe[1]), *recipe[2])
self.rfref.WriteTObject(href)
del href
hdata = recipe[0](orfdata.GetDirectory(recipe[1]), *recipe[2])
self.rfdata.WriteTObject(hdata)
del hdata
self.rfref.Close()
self.rfdata.Close()
# Write DQ database to temp directory (rather than run directory)
Config().dq_db_file_path = os.path.join(self.tdir, Config().dq_db)
configfile = os.path.join(os.path.dirname(__file__),
'analysis_config_test.py')
with open(configfile, 'r') as inputFile:
exec(inputFile.read())
config = AnalysisConfigWrapper(
(analysis_config_branches, analysis_config_leaves))
self.mycombiner = config.getTrunk(orfdata.GetName(), orfref.GetName())
# Results to compare against
self.results = {"score": Score(70.62594356001006), "lvl": ERROR}
def Execute(filename):
# save stdin
savecout = sys.stdout
savecerr = sys.stderr
# redirect stdin to log file
fcout = open('cout.log', 'w')
fcerr = open('cerr.log', 'w')
sys.stdout = fcout
sys.stderr = fcerr
# Open the input file
input = TFile(filename, "OPEN")
# redirect log file to stdin
fcout.close()
fcerr.close()
sys.stdout = savecout
sys.stderr = savecerr
# Check if the file is opened ?
if (not input.IsOpen()) or input.IsZombie():
print "File called '" + sys.argv[1] + "' is not found"
return
# Get the tree
tree = input.Get("MyModule/Event")
if tree == None:
print "TTree called 'MyModule/Event' is not found"
return
# Printing the tree structure
tree.Print()
class EnergyCorrector(object):
"""Generic energy corrector"""
def __init__(self, fnam, histnam='h_cor'):
"""
fnam is a root file containing a 1D histogram giving
the correction factor as a function of eta.
"""
self.file = TFile(fnam)
if self.file.IsZombie():
raise ValueError(fnam + ' cannot be opened')
self.hist = self.file.Get(histnam)
if self.hist == None:
raise ValueError('{h} cannot be found in {f}'.format(h=histnam,
f=fnam))
def correct_p4(self, p4):
"""
returns the corrected 4-momentum.
The 4 momentum is expected to behave as the one of the Candidate class
"""
eta = p4.eta()
pt = p4.pt()
return pt * self.correction_factor(pt, eta)
def correction_factor(self, pt, eta):
"""
returns the correction factor.
takes also pt as this class could be generalized for a 2D calibration.
"""
etabin = self.hist.FindBin(eta)
shift = self.hist.GetBinContent(etabin) / 100.
return shift
def rewtOneHist(dataset, hwts):
fileName = condor_dir + "/" + dataset + ".root"
if not os.path.exists(fileName):
print "WARNING: didn't find ", fileName
return
print "About to reweight histogram in " + fileName
inFile = TFile(fileName, "UPDATE")
if inFile.IsZombie() or not inFile.GetNkeys():
return
inFile.cd()
h = inFile.Get(str(arguments.histToBeReWeighted)).Clone()
if not h:
print " Could not find hist named " + arguments.histToBeReWeighted + " in " + inFile.GetName(
)
return
h.SetDirectory(0)
newName = h.GetName() + str(arguments.suffixRename)
h.SetName(newName)
dir = arguments.histToBeReWeighted
dir = dir[:dir.rfind("/")]
print "Will write hist to directory " + dir
inFile.cd(dir)
tdir = inFile.GetDirectory(dir)
tdir.Delete(newName + ";*")
for i in range(1, h.GetNbinsX() + 1):
val = h.GetBinContent(i)
err = h.GetBinError(i)
binCtr = h.GetBinCenter(i)
wt = hwts.GetBinContent(hwts.FindBin(binCtr))
h.SetBinContent(i, val * wt)
h.SetBinError(i, err * wt)
h.Write()
inFile.Close()
def GetNumberOfEvents(FilesSet):
NumberOfEvents = {'SkimNumber' : {}, 'TotalNumber' : 0}
for histFile in FilesSet:
ScoutFile = TFile(histFile)
if ScoutFile.IsZombie():
print histFile + " is a bad root file."
FilesSet.remove(histFile)
continue
randomChannelDirectory = ""
TotalNumberTmp = 0
for key in ScoutFile.GetListOfKeys():
if key.GetClassName() != "TDirectoryFile" or "CutFlow" not in key.GetName():
continue
randomChannelDirectory = key.GetName()
channelName = randomChannelDirectory[0:len(randomChannelDirectory)-14]
if not NumberOfEvents['SkimNumber'].has_key(channelName):
NumberOfEvents['SkimNumber'][channelName] = 0
OriginalCounterObj = ScoutFile.Get(randomChannelDirectory + "/eventCounter")
SkimCounterObj = ScoutFile.Get(randomChannelDirectory + "/cutFlow")
TotalNumberTmp = 0
if not OriginalCounterObj:
print "Could not find eventCounter histogram in " + str(histFile) + " !"
continue
elif not SkimCounterObj:
print "Could not find cutFlow histogram in " + str(histFile) + " !"
else:
OriginalCounter = OriginalCounterObj.Clone()
OriginalCounter.SetDirectory(0)
TotalNumberTmp = TotalNumberTmp + OriginalCounter.GetBinContent(1)
SkimCounter = SkimCounterObj.Clone()
SkimCounter.SetDirectory(0)
NumberOfEvents['SkimNumber'][channelName] = NumberOfEvents['SkimNumber'][channelName] + SkimCounter.GetBinContent(SkimCounter.GetXaxis().GetNbins())
NumberOfEvents['TotalNumber'] = NumberOfEvents['TotalNumber'] + TotalNumberTmp
return NumberOfEvents
def load_trees_from_file(file_name):
"""
Load all TTree from a ROOT file
@param file_name Name of ROOT file
@return dict with name of tree as key and the tree as value, ROOT file
"""
if not os.path.isfile(file_name):
raise IOError("File %s does not exist." % file_name)
root_file = TFile(file_name)
if root_file.IsZombie():
raise IOError("Can't open root file %s." % file_name)
keys = root_file.GetListOfKeys()
trees = {}
for key in keys:
tree = root_file.Get(key.GetName())
if isinstance(tree, TTree):
trees[key.GetName()] = tree
return trees, root_file
def getDataProfile(outfilename, JSON, pileup, bins, minbias, local=False):
"""Get pileup profile in data with pileupCalc.py tool."""
print ">>> getDataProfile(%s,%d,%s)" % (outfilename, bins, minbias)
if local:
JSON = copyToLocal(JSON)
pileup = copyToLocal(pileup)
command = "./pileupCalc.py -i %s --inputLumiJSON %s --calcMode true --maxPileupBin %d --numPileupBins %d --minBiasXsec %d %s --verbose" % (
JSON, pileup, bins, bins, minbias * 1000, outfilename)
else:
command = "pileupCalc.py -i %s --inputLumiJSON %s --calcMode true --maxPileupBin %d --numPileupBins %d --minBiasXsec %d %s" % (
JSON, pileup, bins, bins, minbias * 1000, outfilename)
print ">>> executing command (this may take a while):"
print ">>> " + command
os.system(command)
# CHECK
if not os.path.isfile(outfilename):
print ">>> Warning! getDataProfile: Could find output file %s!" % (
outfilename)
return
file = TFile(outfilename, 'READ')
if not file or file.IsZombie():
print ">>> Warning! getDataProfile: Could not open output file %s!" % (
outfilename)
return
hist = file.Get('pileup')
print ">>> pileup profile in data with min. bias %s mb has a mean of %.1f" % (
minbias, hist.GetMean())
file.Close()
def getMCProfile(outfilename, indir, samples, channel, year):
"""Get pileup profile in MC by adding Pileup_nTrueInt histograms from a given list of samples."""
print ">>> getMCProfile(%s)" % (outfilename)
nprofiles = 0
histname = 'pileup'
tothist = None
for subdir, samplename in samples:
filename = "%s/%s/%s_%s.root" % (indir, subdir, samplename, channel)
print ">>> %s" % (filename)
file = TFile(filename, 'READ')
if not file or file.IsZombie():
print ">>> Warning! getMCProfile: Could not open %s" % (filename)
continue
hist = file.Get(histname)
if not hist:
print ">>> Warning! getMCProfile: Could not open histogram in %s" % (
filename)
continue
if tothist == None:
tothist = hist.Clone('pileup')
tothist.SetTitle('pileup')
tothist.SetDirectory(0)
nprofiles += 1
else:
tothist.Add(hist)
nprofiles += 1
file.Close()
print ">>> added %d MC profiles, %d entries, %.1f mean" % (
nprofiles, tothist.GetEntries(), tothist.GetMean())
file = TFile(outfilename, 'RECREATE')
tothist.Write('pileup')
file.Close()
def getweigthinfo(files):
totnum = 0.
totnumtrue = 0.
totnumtrueW = 0.
goodfiles = []
for fname in files:
# p = subprocess.Popen(['ls', '-l', fname], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
# info = p.stderr.read()
# if 'cannot access' in info:
# print 'ERROR', fname
# continue
#fname = fname.replace('/eos/uscms', 'root://cmseos.fnal.gov/')
tf = TFile(fname)
if tf.IsZombie() == True:
print "Zombie file: " + fname
continue
goodfiles.append(fname)
tree = tf.Get('Events')
num = tree.GetEntries()
totnum += num
if 'MC' in files[0]:
hist = tf.Get('PUDistribution')
histW = tf.Get('PUDistribution_w')
numtrue = hist.Integral()
numtrueW = histW.Integral()
totnumtrue += numtrue
totnumtrueW += numtrueW
return totnumtrueW, totnumtrue, totnum, goodfiles
def GetListObjects(self):
afile = TFile(self.Filename)
if afile.IsZombie():
print(" error trying to open file: " + self.Filename)
sys.exit()
if self.XML:
print('''
<cuy>
''')
print(' <validation type=\"' + afile.GetName() + '\" file=\"' +
self.Filename + '\" release=\"x.y.z\">')
self.dir = ROOT.gDirectory
self.Loop()
if self.XML:
print('''
</validation>
</cuy>
''')
def readhist(filename, s, var, cat):
file = TFile(filename, "READ")
if file.IsZombie():
print "WARNING: file", filename, "does not exist"
return None
h = file.Get(cat + '/' + var + "/" + s)
h.SetDirectory(0)
return h
def rootFileValid(path):
from ROOT import TFile
result = True
file = TFile(path)
result &= file.GetSize() > 0
result &= not file.TestBit(TFile.kRecovered)
result &= not file.IsZombie()
return result
def rootFileValid(path):
from ROOT import TFile
result = True
file = TFile(path)
result &= file.GetSize() > 0
result &= not file.TestBit(TFile.kRecovered)
result &= not file.IsZombie()
if not result: print "WARNING: omitting damaged file '%s'" % path
return result
def ExtractDatasetInfo(self, dataset):
from ROOT import TFile
name = InstanceName.Get(dataset.name)
filename = self.rootdir + "/" + name + ".root"
rootfile = TFile(filename)
if rootfile.IsZombie():
logging.error("file called '" + self.rootdir + "/" + name +
".root is not found")
return
# Getting data from ROOT file
xsections = rootfile.Get("general/xsections")
if not bool(xsections):
ErrorMsg_BranchNotFound('general/xsections', filename)
return
xerrors = rootfile.Get("general/xerrors")
if not bool(xerrors):
ErrorMsg_BranchNotFound('general/xerrors', filename)
return
nevents = rootfile.Get("general/nevents")
if not bool(nevents):
ErrorMsg_BranchNotFound('general/nevents', filename)
return
# Checking indices
if xsections.GetNoElements() is 0:
ErrorMsg_BranchEmpty("branch 'general/xsections' is empty")
return
if xerrors.GetNoElements() is 0:
ErrorMsg_BranchEmpty("branch 'general/xerrors' is empty")
return
if nevents.GetNoElements() is 0:
ErrorMsg_BranchEmpty("branch 'general/nevents' is empty")
return
if xsections.GetNoElements()!=xerrors.GetNoElements() or \
xerrors.GetNoElements()!=nevents.GetNoElements() or \
nevents.GetNoElements()!=xsections.GetNoElements():
logging.error("the 'general' branches have different size "\
"in the file '"+filename+"'")
return
if xsections.GetNoElements() is not (len(dataset) + 1):
logging.error(
"number of data files do not correspond in the file '" +
filename + "'")
return
# Extracting data
dataset.measured_xsection = xsections[xsections.GetNoElements() - 1]
dataset.measured_xerror = xerrors[xerrors.GetNoElements() - 1]
dataset.measured_n = int(nevents[nevents.GetNoElements() - 1])
def compareMCProfiles(indir, samples, channel, year, tag=""):
"""Compare MC profiles."""
print ">>> compareMCProfiles()"
histname = 'pileup'
outdir = ensureDirectory("plots")
avehist = None
hists = []
if tag and tag[0] != '_': tag = '_' + tag
# GET histograms
for subdir, samplename in samples:
filename = "%s/%s/%s_%s.root" % (indir, subdir, samplename, channel)
print ">>> %s" % (filename)
file = TFile(filename, 'READ')
if not file or file.IsZombie():
print ">>> Warning! compareMCProfiles: Could not open %s" % (
filename)
continue
hist = file.Get(histname)
hist.SetName(samplename)
hist.SetTitle(samplename)
hist.SetDirectory(0)
if not hist:
print ">>> Warning! compareMCProfiles: Could not open histogram in %s" % (
filename)
continue
if avehist == None:
avehist = hist.Clone('average%s' % tag)
avehist.SetTitle('MC average')
avehist.SetDirectory(0)
avehist.Add(hist)
hist.Scale(1. / hist.Integral())
hists.append(hist)
file.Close()
# PLOT
hists = [avehist] + hists
colors = [kBlack] + linecolors
avehist.Scale(1. / avehist.Integral())
plotname = "%s/pileup_MC_%s%s" % (outdir, year, tag)
drawHistsWithRatio(hists,
plotname,
xtitle="Number of true interactions",
ytitle="A.U.",
textsize=0.032,
rmin=0.45,
rmax=1.55,
colors=colors)
for hist in hists:
if hist == avehist: continue
if hist.GetDirectory():
gDirectory.Delete(hist.GetName())
else:
hist.Delete()
return avehist
def ensureTFile(filename,option='update'):
"""Open TFile, checking if the file in the given path exists."""
if not os.path.isfile(filename):
print '>>> ERROR! ScaleFactorTool.ensureTFile: File in path "%s" does not exist!!'%(filename)
exit(1)
file = TFile(filename,option)
if not file or file.IsZombie():
print '>>> ERROR! ScaleFactorTool.ensureTFile Could not open file by name "%s"'%(filename)
exit(1)
return file
def compareEventsToDAS(filenames, dasname):
"""Compare a number of processed events in an output file to the available number of events in DAS."""
dasname = dasname.replace('__', '/')
if args.verbose:
print "compareEventsToDAS: %s, %s" % (filenames, dasname)
#start = time.time()
if isinstance(filenames, str):
filenames = [filenames]
total_processed = 0
nfiles = len(filenames)
for filename in filenames:
file = TFile(filename, 'READ')
if file.IsZombie():
continue
hist = file.Get('cutflow')
if hist:
events_processed = hist.GetBinContent(1)
if args.verbose:
print "%12d events processed in %s " % (events_processed,
filename)
total_processed += events_processed
#else:
# print bcolors.FAIL + '[NG] compareEventsToDAS: no cutflow found in ' + filename + bcolors.ENDC
file.Close()
instance = 'prod/phys03' if 'USER' in dasname else 'prod/global'
dascmd = 'das_client --limit=0 --query=\"summary dataset=/%s instance=%s\"' % (
dasname, instance)
if args.verbose:
print dascmd
dasargs = shlex.split(dascmd)
output, error = subprocess.Popen(dasargs,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
if not "nevents" in output:
print bcolors.BOLD + bcolors.FAIL + ' [NG] Did not find nevents for "%s" in DAS. Return message:' % (
dasname) + bcolors.ENDC
print bcolors.FAIL + ' ' + output + bcolors.ENDC
return False
total_das = Double(output.split('"nevents":')[1].split(',')[0])
fraction = total_processed / total_das
nfiles = ", %d files" % (nfiles) if nfiles > 1 else ""
if fraction > 1.001:
print bcolors.BOLD + bcolors.FAIL + ' [NG] DAS entries = %d, Processed in tree = %d (frac = %.2f > 1%s)' % (
total_das, total_processed, fraction, nfiles) + bcolors.ENDC
elif fraction > 0.8:
print bcolors.BOLD + bcolors.OKBLUE + ' [OK] DAS entries = %d, Processed in tree = %d (frac = %.2f%s)' % (
total_das, total_processed, fraction, nfiles) + bcolors.ENDC
else:
print bcolors.BOLD + bcolors.FAIL + ' [NG] DAS entries = %d, Processed in tree = %d (frac = %.2f < 0.8%s)' % (
total_das, total_processed, fraction, nfiles) + bcolors.ENDC
return True
def updatestatus(jobstatus, outdir, name):
from ROOT import TFile
print("Updating job status")
print("Total: " + str(len(jobstatus)))
# get the qstat job listing
proccommand = 'qstat | grep dx5412'
proc = subprocess.Popen(proccommand, stdout=subprocess.PIPE, shell=True)
qstat_result = proc.stdout.read()
for i in range(len(jobstatus)):
# if job is completed, we don't need to check again
if jobstatus[i] == 2:
continue
# check if the job is still underway
jobinprocess = qstat_result.find((name + str(i)).encode())
if jobinprocess >= 0:
jobstatus[i] = 1
continue
# if the job is not still underway,
# check to see if the job has completed properly
# if not, mark to resubmit
if outdir.startswith('/'):
filename = outdir + '/' + name + str(i) + '.root'
else:
filename = os.getcwd() + '/' + outdir + '/' + name + str(
i) + '.root'
if os.path.isfile(filename):
outputfile = TFile(filename, "READ")
if outputfile.IsZombie():
print("job " + str(i + 1) + " of " + str(len(jobstatus)) +
" complete: file is zombie, resubmit")
jobstatus[i] = 0
os.remove(filename)
elif outputfile.IsOpen():
print("job " + str(i + 1) + " of " + str(len(jobstatus)) +
" complete: ROOT file healthy")
print(filename)
jobstatus[i] = 2
outputfile.Close()
else:
print("job " + str(i + 1) + " of " + str(len(jobstatus)) +
" undefined file status, resubmit")
jobstatus[i] = 0
else:
print("undefined status: job " + str(i + 1) + " of " +
str(len(jobstatus)) + " marked for submission")
jobstatus[i] = 0
return jobstatus
def get_full_tree_name(folder, path_to_root_file, tree_name):
root_file = TFile(path_to_root_file)
if root_file.IsZombie():
raise FileNotFoundError(
'File {} does not exist, abort'.format(path_to_root_file))
if folder not in root_file.GetListOfKeys():
raise NameError('Folder {} does not exist in {}\n'.format(
folder, path_to_root_file))
root_file.Close()
full_tree_name = '/'.join([folder, tree_name])
return full_tree_name
def file_exits_and_not_zombie(file_path):
"""Checks for existence of ROOT file and ensures it is not a zombie and has its keys"""
if os.path.isfile(file_path):
f = TFile(file_path)
if f.IsZombie() or f.GetSeekKeys() == 0:
f.Close()
return False
else:
f.Close()
return True
else:
return False
def ensureTFile(filename, option='READ'):
"""Open TFile, checking if the file in the given path exists."""
if not os.path.isfile(filename):
print '- ERROR! DYCorrection.ensureTFile: File in path "%s" does not exist!!' % (
filename)
exit(1)
file = TFile(filename, option)
if not file or file.IsZombie():
print '- ERROR! DYCorrection.ensureTFile Could not open file by name "%s"' % (
filename)
exit(1)
return file
def ensureTFile(filename, option='READ', **kwargs):
pre = kwargs.get('pre', "")
stop = kwargs.get('exit', True)
if not os.path.isfile(filename):
LOG.fatal('File in path "%s" does not exist' % (filename), pre=pre)
file = TFile(filename, option)
if not file or file.IsZombie():
if stop:
LOG.fatal('Could not open file by name "%s"' % (filename), pre=pre)
else:
LOG.warning('Could not open file by name "%s"' % (filename),
pre=pre)
return file
def do_analysis(file_name, args):
file = TFile(file_name)
if file == None or file.IsZombie() == True:
print('***Error*** :', file_name, 'is not found')
return
print(file_name, 'is opened')
tree_name = args.tree_name
input_tree = file.Get(tree_name)
if input_tree == None:
print('***Error*** :', tree_name, 'is not found in', file_name)
return
print(tree_name, 'is loaded from', file_name)
base = os.path.basename(file_name)
out_name = args.outdir + '/' + base.replace('.root', '_ana.root')
if out_name == file_name:
print('***Error*** :', 'extension of', file_name, 'is not ".root"')
return
out_file = TFile(out_name, 'recreate')
if out_file == None or out_file.IsZombie() == True:
print('***Error*** :', 'opening', out_name, 'is failed')
return
print(out_name, 'is generated')
analysis = main_analysis(input_tree, args)
entries = numpy.array([range(analysis.nentries)]).T
numpy.apply_along_axis(analysis.event_routine, axis=1, arr=entries)
c = TCanvas('c', 'c', 400, 400)
out_file.cd()
for histo in analysis.histo.dict_of_histos.values():
histo.Write()
for image in analysis.histo.dict_of_images.values():
image.Write()
out_file.Close()
def fill(self, yields):
# Import ROOT only here, to avoid it interfering with parsing
from ROOT import TFile, TH1, Double
for s in yields.samples:
for r in yields.regions:
filename = s + '_' + self.channel + '.root'
filepath = os.path.join(self.histopath, filename)
hname, rng = self.regions[r]
rf = TFile(filepath)
if rf.IsZombie():
raise RuntimeError('Failed to open %s', filepath)
htmp = rf.Get(hname)
# print htmp
if not isinstance(htmp, TH1):
raise RuntimeError('Histogram %s not found in %s' %
(hname, filepath))
h = htmp.Clone()
# Adapt the range, if required
xax = h.GetXaxis()
first, last = xax.GetFirst(), xax.GetLast()
xmin, xmax = xax.GetBinLowEdge(first), xax.GetBinUpEdge(last)
if rng is not None:
umin, umax = rng
umin = umin if umin is not None else xmin
umax = umax if umax is not None else xmax
# print umin,umax
# Apply new range (use ROOT to do the difficult bit :P)
xax.SetRangeUser(umin, umax)
# Update limits
first, last = xax.GetFirst(), xax.GetLast()
xmin, xmax = xax.GetBinLowEdge(first), xax.GetBinUpEdge(
last)
err = Double(0.)
nev = h.IntegralAndError(first, last, err)
rf.Close()
print filename, hname, rng,
print ' | ', '%.2f(%d)' % (xmin,
first), '-', last, '%.2f(%d)' % (
xmax, last), ' -- ', nev, err
del h
yields.values[r][s] = (nev, err)
def checkFiles(filelist, directory):
if args.verbose:
print "checkFiles: %s, %s" % (filelist, directory)
if isinstance(filelist, str):
filelist = [filelist]
badfiles = []
ifound = []
for filename in filelist:
file = TFile(filename, 'READ')
isbad = False
if file.IsZombie():
print bcolors.FAIL + '[NG] file %s is a zombie' % (
filename) + bcolors.ENDC
isbad = True
else:
tree = file.Get('tree')
if not isinstance(tree, TTree):
print bcolors.FAIL + '[NG] no tree found in ' + filename + bcolors.ENDC
isbad = True
elif not isinstance(file.Get('cutflow'), TH1):
print bcolors.FAIL + '[NG] no cutflow found in ' + filename + bcolors.ENDC
isbad = True
elif any(s in filename
for s in ['DYJets', 'WJets'
]) and tree.GetMaximum('LHE_Njets') > 10:
print bcolors.WARNING + '[WN] LHE_Njets = %d > 10 in %s' % (
tree.GetMaximum('LHE_Njets'), filename) + bcolors.ENDC
if isbad:
badfiles.append(filename)
#rmcmd = 'rm %s' %filename
#print rmcmd
#os.system(rmcmd)
file.Close()
match = indexpattern.search(filename)
if match: ifound.append(int(match.group(1)))
if len(badfiles) > 0:
print bcolors.FAIL + "[NG] %s: %d out of %d files have no tree!" % (
directory, len(badfiles), len(filelist)) + bcolors.ENDC
return False
# TODO: check all chunks (those>imax)
imax = max(ifound) + 1
if len(filelist) < imax:
imiss = [i for i in range(0, max(ifound)) if i not in ifound]
chunktext = ('chunks ' if len(imiss) > 1 else 'chunk ') + ', '.join(
str(i) for i in imiss)
print bcolors.BOLD + bcolors.WARNING + "[WN] %s missing %d/%d files (%s) ?" % (
directory, len(imiss), len(filelist), chunktext) + bcolors.ENDC
return True
def trim_filelist(*filelist):
#Remove the first element, which is the python script itself
#mylist is a list of all of the input files
mylist = [ myfile for myfile in filelist[0] if myfile[-3:] != '.py' ]
#Dictionary of run/subrun/evt of entry 0 =>
#list of corresponding filename(s)
filedict = {}
#Loop over the input files, build the dictionary
for myfile in mylist:
#Make sure the file isn't a zombie
f = TFile(myfile,'READ')
if f.IsZombie(): continue
#Make sure the file has larlite_id_tree in it
if not f.GetListOfKeys().FindObject('larlite_id_tree'): continue
#Get info from the 0th entry of larlite_id_tree
dummy = f.larlite_id_tree.GetEntry(0)
irun = f.larlite_id_tree._run_id
isub = f.larlite_id_tree._subrun_id
ievt = f.larlite_id_tree._event_id
print "file %s has (%d,%d,%d)"%(myfile,irun,isub,ievt)
#If this file isn't in the dictionary yet, add it
if (irun,isub,ievt) not in filedict.keys():
filedict[(irun,isub,ievt)] = [myfile]
#If it is, append it
else: filedict[(irun,isub,ievt)].append(myfile)
#Close the file
f.Close()
#Now, loop over the dictionary and find the maximum number of files
#associated with one (run,subrun,evt) combo.
#Keep only these files. That way, if you hand in 5 reco2d files and
#5 mcinfo files, and 4 of the reco2d files are valid, you will only
#return those 4 files and the 4 associated mcinfo files.
n_files_max = max([len(i) for i in filedict.values()])
#Build a list of all input files to keep, and return it
return_list = []
for values in filedict.values():
if len(values) < n_files_max: continue
for filename in values:
return_list.append(filename)
return return_list
def TestROOTFile(self, path):
from ROOT import TFile
_tmp_file = TFile(path)
if _tmp_file and not _tmp_file.IsOpen():
return False
elif _tmp_file and _tmp_file.IsOpen() and _tmp_file.IsZombie():
_tmp_file.Close()
return False
elif _tmp_file and _tmp_file.IsOpen() and _tmp_file.TestBit(
TFile.kRecovered):
_tmp_file.Close()
return False
else:
_tmp_file.Close()
return True
def getProcessedDatasets(condor_dir, datasets):
processed_datasets = []
for dataset in datasets:
fileName = condor_dir + "/" + dataset + ".root"
if not os.path.exists(fileName):
if (arguments.verbose):
print "Couldn't find output file for", dataset, "dataset", fileName, "fileName"
continue
testFile = TFile(fileName)
if not (testFile.IsZombie()):
processed_datasets.append(dataset)
if arguments.verbose:
print "Opening testFile: ", fileName
if len(processed_datasets) is 0:
sys.exit(
"Can't find any output root files for the given list of datasets")
return processed_datasets
def filecheck(rootfile):
f = TFile(rootfile)
if (f.IsZombie()):
#print "File corrupted"
f.Close()
return 0
else:
hist = f.FindObjectAny("reportSummaryContents")
#(skip filecheck for HcalTiming files!!)
if (hist == None and rootfile.rfind('HcalTiming') == -1):
#print "File is incomplete"
f.Close()
return 0
else:
#print "File is OK"
f.Close()
return 1
