def tpMap(raw={}, warn=True, book=None):
forward = {}
backward = {}
skipped = []
for fedId, d in raw.iteritems():
if fedId is None:
continue
delta = matching.pipelineDelta(d["header"]["utca"])
for block in d["htrBlocks"].values():
for key, triggerData in block["triggerData"].iteritems():
coords = (block["Crate"], block["Slot"], block["Top"], key)
nSoi = sum(triggerData["SOI"])
if warn and nSoi != 1:
printer.warning("iEntry %d / %s has !=1 SOIs: %s" % (raw[None]["iEntry"], coords, triggerData["SOI"]))
if not nSoi:
continue
soi = triggerData["SOI"].index(1)
l = [soi, delta, 0, 0]
for i, tp9 in enumerate(triggerData["TP"]):
fg = tp9 >> 8
tp8 = tp9 & 0xff
l.append(tp8)
if i == soi:
nBins = 14
book.fill(tp8, "tp_soi_fg%d_%d" % (fg, fedId), nBins, -0.5, nBins - 0.5, title="FED %d;TP E_{SOI};Trigger Towers / bin" % fedId)
forward[coords] = l
return forward, backward, skipped
def eventMaps(chain, s={}, identityMap=False):
forward = {}
backward = {}
forwardBcn = {}
if s["progress"]:
print "Mapping %s:" % s["label"]
try:
def fillEventMap2(chain, iEntry):
if iEntry == nMapMin:
raw.pruneFeds(chain, s)
return fillEventMap(iEntry, raw.unpackedHeader(s), forward, forwardBcn, backward)
nMapMin = 0 # start from beginning
nMapMax = None # look at all entries
nEv = s["nEventsMax"] + s["nEventsSkip"]
if nEv:
if identityMap:
nMapMin = s["nEventsSkip"]
nMapMax = nEv
else:
nMapMax = nEv * 2 # a guess for how far to look not to miss out-of-order events
chainLoop(chain, nMapMin, nMapMax, fillEventMap2, progress=s["progress"], sparseLoop=s["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in eventMaps()")
return forward, backward, forwardBcn
def eventMaps(chain, s={}, nMapMin=0, nMapMax=None):
forward = {}
backward = {}
forwardBcn = {}
treeName = s["treeName"]
fedId0 = s["fedIds"][0]
if treeName != "Events":
branch0 = s["branch"](fedId0)
else:
branch0 = None
if s["progress"]:
print "Mapping %s:" % s["label"]
kargs = {"headerOnly": True,
"nBytesPer": s["nBytesPer"],
"skipWords64": s["skipWords64"],
}
try:
def fillEventMap2(chain, iEntry):
return fillEventMap(chain, iEntry,
treeName, fedId0, branch0, s, kargs,
forward, forwardBcn, backward)
chainLoop(chain, nMapMin, nMapMax, fillEventMap2, progress=s["progress"], sparseLoop=s["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in eventMaps()")
return forward, backward, forwardBcn
def eventMaps(chain, s={}, identityMap=False):
forward = {}
backward = {}
forwardBcn = {}
if s["progress"]:
print("Mapping %s:" % s["label"])
try:
def fillEventMap2(chain, iEntry):
if iEntry == nMapMin:
raw.pruneFeds(chain, s)
return fillEventMap(iEntry, unpack.unpackedHeader(s), forward, forwardBcn, backward)
nMapMin = 0 # start from beginning
nMapMax = None # look at all entries
nEv = s["nEventsMax"] + s["nEventsSkip"]
if nEv:
if identityMap:
nMapMin = s["nEventsSkip"]
nMapMax = nEv
else:
nMapMax = nEv * 2 # a guess for how far to look not to miss out-of-order events
chainLoop(chain, nMapMin, nMapMax, fillEventMap2, progress=s["progress"], sparseLoop=s["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in eventMaps()")
return forward, backward, forwardBcn
def generate_commands_xmlfile(self):
"""
Steps to generate XML commands file for builder.
"""
components_filelist = []
# Get general data from '.ent' files
data_dict = self.generate_entities_data_dict()
# Remove 'self.save_index_file' to avoid building components multiple times
if os.path.exists(self.save_index_file):
os.remove(self.save_index_file)
# Parse LFS or BLFS book
if self.builder_data_dict["book"] == "lfs":
components_filelist = self.parse_lfs_book(data_dict)
elif self.builder_data_dict["book"] == "blfs":
components_filelist = self.parse_blfs_book(data_dict)
else:
msg = "Book '{b}' no to be parsed".format(
b=self.builder_data_dict["book"])
printer.warning(msg)
# Get destination filename
attribute = "{s}_XML_FILENAME".format(
s=self.builder_data_dict["name"].upper())
destination_filename = getattr(config, attribute)
# Write XML file
self.write_commands_xmlfile(components_filelist, data_dict,
destination_filename)
def loop(chain=None,
chainI=None,
outer={},
inner={},
innerEvent={},
options={}):
if outer["progress"]:
print "Looping:"
kargs = {"book": autoBook.autoBook("book")}
kargs.update(options)
try:
def outerInnerCompare2(chain, iEntry):
return outerInnerCompare(chain, iEntry, outer, inner, innerEvent,
chainI, kargs)
nMin = outer["nEventsSkip"]
nMax = (outer["nEventsSkip"] +
outer["nEventsMax"]) if outer["nEventsMax"] else None
chainLoop(chain,
nMin,
nMax,
outerInnerCompare2,
progress=outer["progress"],
sparseLoop=outer["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in loop()")
return kargs["book"]
def htrData(d={},
l={},
iWord16=None,
word16=None,
utca=None,
fedId=None,
warn=True):
if (word16 >> 15):
flavor = (word16 >> 12) & 0x7
dataKey, channelId, channelHeader = channelInit(
iWord16=iWord16,
word16=word16,
flavor=flavor,
utca=utca,
nPreSamples=l["nPreSamples"])
if warn and dataKey == "otherData":
coords = "FED %4d crate %2d slot %2d" % (fedId, l["Crate"],
l["Slot"])
evn = "(EvN 0x%06x, iWord16 %4d, word16 0x%04x)" % (
l["EvN"], iWord16, word16)
printer.warning("unknown flavor %d: %s %s." %
(flavor, coords, evn))
d["dataKey"] = dataKey
d["channelId"] = channelId
l[d["dataKey"]][d["channelId"]] = channelHeader
elif "channelId" in d:
storeChannelData(
dct=l[d["dataKey"]][d["channelId"]],
iWord16=iWord16,
word16=word16,
)
def check_chroot_user_exists(username):
"""
Check 'username' exists by parsing the '/etc/passwd' file.
"""
etc_passwd = os.path.join(config.BASE_DIRECTORY, "etc", "passwd")
result = False
if os.path.exists(etc_passwd) is True:
# .- read 'etc_passwd' into list
data = read_file(etc_passwd).rstrip("\n").split("\n")
# .- get usernames on first column
usernames = [line.split(":")[0] for line in data]
# .- check 'username' is in 'usernames'
if username in usernames:
result = True
else:
msg = """tools.check_chroot_user_exists(): passwd file '{f}' do not exists. \
We can not check if user '{u}' exists or not."""
msg = msg.format(f=etc_passwd, u=username)
printer.warning(msg)
return result
def tpMap(raw={}, warn=True, book=None):
forward = {}
backward = {}
skipped = []
for fedId, d in raw.iteritems():
if fedId is None:
continue
delta = matching.pipelineDelta(d["header"]["utca"])
for block in d["htrBlocks"].values():
for key, triggerData in block["triggerData"].iteritems():
coords = (block["Crate"], block["Slot"], block["Top"], key)
nSoi = sum(triggerData["SOI"])
if warn and nSoi != 1:
printer.warning("iEntry %d / %s has !=1 SOIs: %s" % (raw[None]["iEntry"], coords, triggerData["SOI"]))
if not nSoi:
continue
soi = triggerData["SOI"].index(1)
l = [soi, delta, 0, 0]
for i, tp9 in enumerate(triggerData["TP"]):
fg = tp9 >> 8
tp8 = tp9 & 0xff
l.append(tp8)
if i == soi:
nBins = 14
book.fill(tp8, "tp_soi_fg%d_%d" % (fg, fedId), nBins, -0.5, nBins - 0.5, title="FED %d;TP E_{SOI};Trigger Towers / bin" % fedId)
forward[coords] = l
return forward, backward, skipped
def opts():
parser = oparser(arg="RUN_NUMBER [RUN_NUMBER2 ...]")
look = optparse.OptionGroup(parser, "Options solely for use with look.py")
look.add_option("--quiet",
dest="quiet",
default=False,
action="store_true",
help="Print less to stdout")
look.add_option("--hhmm",
dest="hhmm",
default=None,
type="int",
help="minimum hhmm")
parser.add_option_group(look)
options, args = parser.parse_args()
runs = []
for arg in args:
try:
runs.append(int(arg))
except ValueError:
printer.warning("Could not convert '%s' to an integer." % arg)
if not runs:
printer.error("Please provide a run number as the argument.")
return options, runs
def go(options, run, nRuns):
override(options, run, nRuns)
files = ["dummy"] if options.noLoop else search(run)
if not files:
printer.warning("Did not find a matching file for run %d." % run)
return 1, None, None
files = pruned(files)
nFiles = len(files)
if 2 <= nFiles:
options.file1 = ",".join(files)
if options.hhmm is None:
options.sparseLoop = max(1, options.nEventsMax / nFiles)
else:
options.sparseLoop = -1
else:
options.file1 = files[0]
sw.set_default_feds1(options)
if not options.quiet:
report(files)
return oneRun.main(options)
def download(self):
"""
'download' command.
"""
# 'download' command
# Parse command line arguments
self.download_parser = self.cli.configure_download_parser()
self.download_args = self.download_parser.parse_args(
self.all_args.command[1:])
# Sanitize book name from input
self.download_args.book_name = self.download_args.book_name.lower()
if self.download_args.book_name != "lfs" and self.download_args.book_name != "blfs":
printer.warning("Unknown book name for download: '{b}'".format(
b=self.download_args.book_name))
# self.parser.print_help()
sys.exit(1)
# Run downloader
d = downloader.Downloader(self.download_args.book_name)
# Run download method
if self.download_args.sources is True:
d.download_source()
elif self.download_args.xml is True:
d.download_xml()
else:
printer.warning(
"You must provide any of the '--xml' or '--sources' options")
# self.parser.print_help()
sys.exit(1)
def loop(chain=None, chainI=None, outer={}, inner={}, innerEvent={}, oMapF={}, options={}):
if outer["progress"]:
print("Looping:")
kargs = {"book": autoBook.autoBook("book"),
"warnQuality": outer["warnQuality"],
"raw1": {},
"raw2": {},
"chain": chain,
"chainI": chainI,
"outer": outer,
"inner": inner}
kargs.update(options)
try:
def outerInnerCompare2(chain, iEntry):
kargs["evn"], kargs["orn"] = oMapF[iEntry]
return outerInnerCompare(iEntry, innerEvent, kargs)
nMin = outer["nEventsSkip"]
nMax = (outer["nEventsSkip"] + outer["nEventsMax"]) if outer["nEventsMax"] else None
chainLoop(chain, nMin, nMax, outerInnerCompare2,
progress=outer["progress"], sparseLoop=outer["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in loop()")
return kargs["book"]
def __init__(self):
self.lfsbuilder_src_directory = os.path.dirname(
os.path.realpath(__file__))
self.temporal_folder = os.path.join(self.lfsbuilder_src_directory,
"tmp")
self.basic_parser = None
self.build_parser = None
self.download_parser = None
self.xml_parser = None
self.cli = cli.Cli()
self.actual_owner_uid = pwd.getpwuid(os.stat(__file__).st_uid).pw_name
self.actual_owner_gid = grp.getgrgid(os.stat(__file__).st_gid).gr_name
# Parse sys.args and use dispatcher pattern to
# invoke method named as command
self.basic_parser = self.cli.configure_basic_parser()
self.all_args = self.basic_parser.parse_args()
self.build_args = None
self.download_args = None
self.xml_args = None
# Set boolean configuration flags
self.set_config_option(self.all_args)
if not hasattr(self, self.all_args.command[0]):
printer.warning(
"Unknown command '{c}'".format(c=self.all_args.command[0]))
self.basic_parser.print_help()
sys.exit(1)
# .- Create 'tmp' directory
tools.create_directory(self.temporal_folder)
# Run command
getattr(self, self.all_args.command[0])()
def htrOverviewBits(d={}, book={}, fedId=None, msg="", warn=True):
letters = ["L", "M", "S", "E", "!P", "!V", "!C"]
abbr = "HTR" if "HTR0" in d else "uHTR"
for iHtr in range(15):
key = "%s%d" % (abbr, iHtr)
if key not in d:
continue
h = d[key]
for i, l in enumerate(letters):
x = h.get(
l[-1]) # .get to handle 'LMS' for VME; last char to avoid '!'
if x is None:
continue
if len(l) == 2: # flip if !
x = not x
if x:
book.fill(i,
"htrOverviewBits_%d" % fedId,
7,
-0.5,
6.5,
title="FED %d;;HTRs / bin" % fedId,
xAxisLabels=letters)
if warn and d["utca"] and l[-1] in "CV":
printer.warning("%s / input %2d has bit %s set." %
(msg, iHtr, l))
def run_post_steps(component_data_dict, parent_function):
if component_data_dict["builder_name"] == "toolchain":
parent_function()
else:
# If we are building 'system' we have to avoid running this. It tries to compile
# component twice
printer.warning(
"Skipped 'post' steps to avoid building component twice")
def histogramAdcs(book, fedId, block, channelData, adcs, nTsMax, errf, eq):
nAdcMax = 256
adc = max(channelData["QIE"])
adcs.add(adc)
mp = channelData.get("M&P", 0)
book.fill(adc,
"channel_peak_adc_mp%d_%d" % (mp, fedId),
nAdcMax,
-0.5,
nAdcMax - 0.5,
title="FED %d;Peak ADC (ErrF == 0);Channels / bin" % fedId)
for (i, adc) in enumerate(channelData["QIE"]):
if nTsMax <= i:
break
book.fill((i, adc),
"ADC_vs_TS_%s_%d" % (errf, fedId), (nTsMax, nAdcMax),
(-0.5, -0.5), (nTsMax - 0.5, nAdcMax - 0.5),
title="FED %d (ErrF %s 0);time slice;ADC;Counts / bin" %
(fedId, eq))
# the 32 fibers of HEP17 carrying SiPM data
if block["Crate"] != 34:
continue
fib = 0
if block["Slot"] == 12:
fib += 12 + channelData["Fiber"] - 1
if 13 <= channelData["Fiber"]:
fib -= 2
elif block["Slot"] == 11 and 12 <= channelData["Fiber"]:
fib += channelData["Fiber"] - 12
else:
continue
if (2 <= channelData["Flavor"]) and (not channelData["ErrF"]):
printer.warning("Crate %d Slot %d Fib %d Channel %d has flavor %d" % \
(block["Crate"], block["Slot"], channelData["Fiber"], channelData["FibCh"], channelData["Flavor"]))
book.fill((i, adc),
"ADC_vs_TS_HEP17_%s_fib%d" % (errf, fib), (nTsMax, nAdcMax),
(-0.5, -0.5), (nTsMax - 0.5, nAdcMax - 0.5),
title="HEP17 Fib %d;time slice;ADC;Counts / bin" % fib)
book.fill(adc,
"HEP17_ADC_TS%d" % i,
nAdcMax,
-0.5,
nAdcMax - 0.5,
title="HEP17 TS%d;ADC;Counts / bin" % i)
def molHeader(d={}, words64=[]):
if len(words64) != 2:
printer.warning("molHeader has %d != 2 words" % len(words64))
w1 = swapped64(words64[0])
w2 = swapped64(words64[1])
iblock = (w1 >> 32) & 0x7ff
d[iblock] = {}
d[iblock]["magic"] = words64[0] & 0xffff
d[iblock]["isFirstBlock"] = w1 & (1L << 31)
d[iblock]["isLastBlock"] = w1 & (1L << 30)
d[iblock]["nWord64"] = w1 & 0x3ff
d[iblock]["FEDid"] = (w2 >> 32) & 0xfff
d[iblock]["Trigger"] = w2 & 0xffffff
def molHeader(d={}, words64=[]):
if len(words64) != 2:
printer.warning("molHeader has %d != 2 words" % len(words64))
w1 = swapped64(words64[0])
w2 = swapped64(words64[1])
iblock = (w1 >> 32) & 0x7ff
d[iblock] = {}
d[iblock]["magic"] = words64[0] & 0xffff
d[iblock]["isFirstBlock"] = w1 & (1 << 31)
d[iblock]["isLastBlock"] = w1 & (1 << 30)
d[iblock]["nWord64"] = w1 & 0x3ff
d[iblock]["FEDid"] = (w2 >> 32) & 0xfff
d[iblock]["Trigger"] = w2 & 0xffffff
def get_uid_gid_chroot_username(username):
"""
Return 'UID' and 'GID' values for 'username' in chroot.
"""
etc_passwd = os.path.join(config.BASE_DIRECTORY, "etc", "passwd")
# Index for the 'UID' value in '/etc/passwd'
uid_index = 2
# Index for the 'GID' value in '/etc/passwd'
gid_index = 3
result = {
"uid": None,
"gid": None,
}
if os.path.exists(etc_passwd) is True:
# .- read 'etc_passwd' into list
data = read_file(etc_passwd).rstrip("\n").split("\n")
# .- get usernames on first column
usernames = [line.split(":")[0] for line in data]
# .- check 'username' is in 'usernames'
if username in usernames:
# .- get username index
index = usernames.index(username)
# .- get 'UID' and 'GID'
result = {
"uid": data[index].split(":")[uid_index],
"gid": data[index].split(":")[gid_index]
}
else:
# Username is not in the '/etc/passwd' file
msg = """tools.get_uid_gid_chroot_username(): username '{u}' do not exists \
in '{f}' file. We can not get 'UID' and 'GID' for user '{u}'."""
msg = msg.format(f=etc_passwd, u=username)
printer.warning(msg)
else:
# '/etc/passwd' file do not exists
msg = """tools.get_uid_gid_chroot_username(): passwd file '{f}' do not exists. \
We can not get 'UID' and 'GID' for user '{u}'."""
msg = msg.format(f=etc_passwd, u=username)
printer.warning(msg)
return result
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i
for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
lines = []
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
lines.append(words)
lines.append("-" * len(words))
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" %
(h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
lines.append(" ".join(s))
if len(lines) > 12:
printer.info(
"suppressed printing of match histogram (more than 10 different occupancies)"
)
else:
for line in lines:
print line
f.Close()
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
lines = []
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
lines.append(words)
lines.append("-" * len(words))
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" % (h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
lines.append(" ".join(s))
if len(lines) > 12:
printer.info("suppressed printing of match histogram (more than 10 different occupancies)")
else:
for line in lines:
print(line)
f.Close()
def example(raw1={}, raw2={}, book=None, warnQuality=True, fewerHistos=False, **_):
# sanity check for incoming raw data
for i, raw in enumerate([raw1, raw2]):
if not raw:
continue
# find the number of time samples in the data
print "the type of 'raw' is:", type(raw)
pprint(raw)
nTsMax = raw[None]["firstNTs"]
for fedId, d in sorted(raw.iteritems()):
if fedId is None:
continue
# get the important chunks of the raw data
blocks=d["htrBlocks"].values()
# sanity checks for these chunks
for block in blocks:
if type(block) is not dict:
printer.warning("FED %d block is not dict" % fedId)
continue
elif "channelData" not in block:
printer.warning("FED %d block has no channelData" % fedId)
continue
for channelData in block["channelData"].values():
if channelData["QIE"]:
# check the error flags
errf = "ErrFNZ" if channelData["ErrF"] else "ErrF0"
# some logic for naming two histograms, one for a clean error flag and another with a problematic error flag
eq = "!=" if channelData["ErrF"] else "=="
nAdcMax = 256
for (i, adc) in enumerate(channelData["QIE"]):
if nTsMax <= i:
break
# fill a 2d histogram where bins on y-axis are ADCs and bins on x-axis are time slice index
book.fill((i, adc), "ADC_vs_TS_%s_%d" % (errf, fedId),
(nTsMax, nAdcMax), (-0.5, -0.5), (nTsMax - 0.5, nAdcMax - 0.5),
title="FED %d (ErrF %s 0);time slice;ADC;Counts / bin" % (fedId, eq))
def collectedRaw(tree=None, specs={}):
raw = {}
kargs = {}
for item in ["patterns", "dump", "unpack", "nBytesPer", "skipWords64"]:
kargs[item] = specs[item]
missingFeds = []
for fedId in specs["fedIds"]:
if "branch" in specs:
branch = specs["branch"](fedId)
if specs["treeName"] == "Events":
rawThisFed = wordsOneFed(tree, fedId, specs["rawCollection"],
specs["product"])
elif specs["treeName"] == "CMSRAW":
rawThisFed = wordsOneChunk(tree, branch, loud=False)
else:
rawThisFed = wordsOneBranch(tree=tree, branch=branch)
if rawThisFed is None:
printer.warning("removing FED %d from spec (no branch %s)." %
(fedId, branch))
missingFeds.append(fedId)
continue
raw[fedId] = unpacked(fedData=rawThisFed,
warn=specs["warnUnpack"],
**kargs)
if not raw[fedId]["nBytesSW"]:
printer.warning("removing FED %d from spec (read zero bytes)." %
fedId)
missingFeds.append(fedId)
continue
for fedId in missingFeds:
specs["fedIds"].remove(fedId)
if fedId in raw:
del raw[fedId]
raw[None] = {"iEntry": tree.GetReadEntry()}
for key in ["label", "patterns", "dump", "crateslots"]:
raw[None][key] = specs[key]
return raw
def checkHtrModules(fedId=None, spigots=[], htrBlocks={}):
crates = []
for iBlock, block in htrBlocks.iteritems():
if block["IsTTP"]:
continue
spigot = spigots[iBlock]
expected = hw.expectedVmeHtr(fedId, spigot)
crates.append(block["Crate"])
bad = [block["Top"] != expected["Top"],
block["Slot"] != expected["Slot"],
]
if any(bad):
fields = (fedId, spigot, block["Crate"],
block["Slot"], block["Top"],
expected["Slot"], expected["Top"])
printer.error("FED %3d spigot %2d has moduleId decode to crate %2d slot %2d%s (expected slot %2d%s)" % fields)
if len(set(crates)) != 1:
printer.warning("FED %s contains modules with crate labels %s." % (str(fedId), str(crates)))
def checkHtrModules(fedId=None, spigots=[], htrBlocks={}):
crates = []
for iBlock, block in htrBlocks.iteritems():
if block["IsTTP"]:
continue
spigot = spigots[iBlock]
expected = hw.expectedVmeHtr(fedId, spigot)
crates.append(block["Crate"])
bad = [block["Top"] != expected["Top"],
block["Slot"] != expected["Slot"],
]
if any(bad):
fields = (fedId, spigot, block["Crate"],
block["Slot"], block["Top"],
expected["Slot"], expected["Top"])
printer.error("FED %3d spigot %2d has moduleId decode to crate %2d slot %2d%s (expected slot %2d%s)" % fields)
if len(set(crates)) != 1:
printer.warning("FED %s contains modules with crate labels %s." % (str(fedId), str(crates)))
def check_non_privileged_user():
# Create the config.NON_PRIVILEGED_USERNAME does not exist
if tools.check_user_exists(config.NON_PRIVILEGED_USERNAME) is False:
msg = "User '{user}' expecified in the 'config.NON_PRIVILEGED_USERNAME' \
variable doesn't exist"
msg = msg.format(user=config.NON_PRIVILEGED_USERNAME)
printer.error(msg)
# Create this user and group
printer.warning("User '{u}' does not exists. Creating it".format(
u=config.NON_PRIVILEGED_USERNAME))
cmd = "groupadd {U}".format(u=config.NON_PRIVILEGED_USERNAME)
tools.run_program_without_output(cmd)
cmd = "useradd -s /bin/bash -g {u} -m -k /dev/null {u}"
cmd = cmd.format(u=config.NON_PRIVILEGED_USERNAME)
tools.run_program_without_output(cmd)
printer.info("User '{u}' created".format(u=config.NON_PRIVILEGED_USERNAME))
def fedList(s=""):
out = []
d = fedMap()
for x in s.split(","):
if not x:
continue
y = d.get(x)
if y is None:
try:
y = int(x)
out.append(y)
except ValueError:
printer.warning("Could not convert '%s' to an integer" % x)
else:
out += y
return out
def fedList(s=""):
out = []
d = fedMap()
for x in s.split(","):
if not x:
continue
y = d.get(x)
if y is None:
try:
y = int(x)
out.append(y)
except ValueError:
printer.warning("Could not convert '%s' to an integer" % x)
else:
out += y
return out
def collectedRaw(tree=None, specs={}):
raw = {}
kargs = {}
for item in ["patterns", "dump", "unpack", "nBytesPer", "skipWords64"]:
kargs[item] = specs[item]
missingFeds = []
for fedId in specs["fedIds"]:
if "branch" in specs:
branch = specs["branch"](fedId)
if specs["treeName"] == "Events":
rawThisFed = wordsOneFed(tree, fedId, specs["rawCollection"], specs["product"])
elif specs["treeName"] == "CMSRAW":
rawThisFed = wordsOneChunk(tree, branch, loud=False)
else:
rawThisFed = wordsOneBranch(tree=tree, branch=branch)
if rawThisFed is None:
printer.warning("removing FED %d from spec (no branch %s)." % (fedId, branch))
missingFeds.append(fedId)
continue
raw[fedId] = unpacked(fedData=rawThisFed,
warn=specs["warnUnpack"],
**kargs)
if not raw[fedId]["nBytesSW"]:
printer.warning("removing FED %d from spec (read zero bytes)." % fedId)
missingFeds.append(fedId)
continue
for fedId in missingFeds:
specs["fedIds"].remove(fedId)
if fedId in raw:
del raw[fedId]
raw[None] = {"iEntry": tree.GetReadEntry()}
for key in ["label", "patterns", "dump", "crateslots"]:
raw[None][key] = specs[key]
return raw
def loop(chain=None,
chainI=None,
outer={},
inner={},
innerEvent={},
oMapF={},
options={}):
if outer["progress"]:
print "Looping:"
kargs = {
"book": autoBook.autoBook("book"),
"warnQuality": outer["warnQuality"],
"raw1": {},
"raw2": {},
"chain": chain,
"chainI": chainI,
"outer": outer,
"inner": inner
}
kargs.update(options)
try:
def outerInnerCompare2(chain, iEntry):
kargs["evn"], kargs["orn"] = oMapF[iEntry]
return outerInnerCompare(iEntry, innerEvent, kargs)
nMin = outer["nEventsSkip"]
nMax = (outer["nEventsSkip"] +
outer["nEventsMax"]) if outer["nEventsMax"] else None
chainLoop(chain,
nMin,
nMax,
outerInnerCompare2,
progress=outer["progress"],
sparseLoop=outer["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in loop()")
return kargs["book"]
def loop(chain=None, chainI=None, outer={}, inner={}, innerEvent={}, options={}):
if outer["progress"]:
print "Looping:"
kargs = {"book": autoBook.autoBook("book")}
kargs.update(options)
try:
def outerInnerCompare2(chain, iEntry):
return outerInnerCompare(chain, iEntry, outer, inner, innerEvent, chainI, kargs)
nMin = outer["nEventsSkip"]
nMax = (outer["nEventsSkip"] + outer["nEventsMax"]) if outer["nEventsMax"] else None
chainLoop(chain, nMin, nMax, outerInnerCompare2,
progress=outer["progress"], sparseLoop=outer["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in loop()")
return kargs["book"]
def printChannelSummary(outputFile):
f = r.TFile(outputFile)
hs = []
for name in ["MatchedFibersCh%d" % i for i in range(3)] + ["MatchedTriggerTowers"]:
h = f.Get(name)
if h:
hs.append(h)
if hs:
words = ["nMatched"]
words += ["FibCh%d" % i for i in range(3)]
words.append(" TPs")
words = " ".join(words)
print words
print "-" * len(words)
else:
return
for iBin in range(0, 2 + hs[0].GetNbinsX()):
xs = [h.GetBinCenter(iBin) for h in hs]
if len(set(xs)) != 1:
printer.error("binnings for nMatched do not match.")
return
for h in hs:
w = h.GetBinWidth(iBin)
if 1.0e-6 < abs(w - 1.0):
printer.warning("Histogram %s bin %d has width %g" % (h.GetName(), iBin, w))
x = xs[0]
cs = [h.GetBinContent(iBin) for h in hs]
if any(cs):
s = [" %4d" % x]
s += ["%4d" % c for c in cs]
print " ".join(s)
f.Close()
def htrOverviewBits(d={}, book={}, fedId=None, msg="", warn=True):
letters = ["L", "M", "S", "E", "!P", "!V", "!C"]
abbr = "HTR" if "HTR0" in d else "uHTR"
for iHtr in range(15):
key = "%s%d" % (abbr, iHtr)
if key not in d:
continue
h = d[key]
for i, l in enumerate(letters):
x = h.get(l[-1]) # .get to handle 'LMS' for VME; last char to avoid '!'
if x is None:
continue
if len(l) == 2: # flip if !
x = not x
if x:
book.fill(i, "htrOverviewBits_%d" % fedId, 7, -0.5, 6.5,
title="FED %d;;HTRs / bin" % fedId,
xAxisLabels=letters)
if warn and d["utca"] and l[-1] in "CV":
printer.warning("%s / input %2d has bit %s set." % (msg, iHtr, l))
def htrData(d={}, l={}, iWord16=None, word16=None,
patterns={}, utca=None, fedId=None, warn=True):
if (word16 >> 15):
flavor = (word16 >> 12) & 0x7
dataKey, channelId, channelHeader = channelInit(iWord16=iWord16,
word16=word16,
flavor=flavor,
utca=utca)
if warn and dataKey == "otherData":
coords = "FED %4d crate %2d slot %2d" % (fedId, l["Crate"], l["Slot"])
evn = "(EvN 0x%06x, iWord16 %4d, word16 0x%04x)" % (l["EvN"], iWord16, word16)
printer.warning("unknown flavor %d: %s %s." % (flavor, coords, evn))
d["dataKey"] = dataKey
d["channelId"] = channelId
l[d["dataKey"]][d["channelId"]] = channelHeader
elif "channelId" in d:
storeChannelData(dct=l[d["dataKey"]][d["channelId"]],
iWord16=iWord16,
word16=word16,
)
def eventMaps(chain, s={}, nMapMin=0, nMapMax=None):
forward = {}
backward = {}
forwardBcn = {}
treeName = s["treeName"]
fedId0 = s["fedIds"][0]
if treeName != "Events":
branch0 = s["branch"](fedId0)
else:
branch0 = None
if s["progress"]:
print "Mapping %s:" % s["label"]
kargs = {
"headerOnly": True,
"nBytesPer": s["nBytesPer"],
"skipWords64": s["skipWords64"],
}
try:
def fillEventMap2(chain, iEntry):
return fillEventMap(chain, iEntry, treeName, fedId0, branch0, s,
kargs, forward, forwardBcn, backward)
chainLoop(chain,
nMapMin,
nMapMax,
fillEventMap2,
progress=s["progress"],
sparseLoop=s["sparseLoop"])
except KeyboardInterrupt:
printer.warning("KeyboardInterrupt in eventMaps()")
return forward, backward, forwardBcn
def other(d={}, words64=[]):
if not words64:
return
if words64[0] & 0xffff == 0x5a47:
molHeader(d, words64)
elif words64[0] == 0xbadc0ffeebadcafe:
d["magic"] = words64[0]
if len(words64) != 2:
printer.warning("badcoffee header has %d != 2 words" % len(words64))
else:
d["nWord64"] = words64[1]
else:
if len(words64) != 1:
printer.warning("other header has %d != 1 words" % len(words64))
else:
w = words64[0]
d["magic"] = w & 0xffffffff
n32 = w >> 32
if n32 & 0x1:
printer.warning("found odd number of 32bit words: %d" % n32)
d["nWord64"] = n32 >> 1
def other(d={}, words64=[]):
if not words64:
return
if words64[0] & 0xffff == 0x5a47:
molHeader(d, words64)
elif words64[0] == 0xbadc0ffeebadcafe:
d["magic"] = words64[0]
if len(words64) != 2:
printer.warning("badcoffee header has %d != 2 words" %
len(words64))
else:
d["nWord64"] = words64[1]
else:
if len(words64) != 1:
printer.warning("other header has %d != 1 words" % len(words64))
else:
w = words64[0]
d["magic"] = w & 0xffffffff
n32 = w >> 32
if n32 & 0x1:
printer.warning("found odd number of 32bit words: %d" % n32)
d["nWord64"] = n32 >> 1
def oneHtr(iBlock=None, p={}, printColumnHeaders=None, dump=None, crateslots=[], utca=None,
nonMatchedQie=[], nonMatchedTp=[]):
try:
zs = p.get("ZS")
except TypeError as e:
print "iBlock='%s':" % str(iBlock), e
return
if "nWord16Qie" in p:
col = "nWord16Qie"
elif utca:
col = "DataLength16"
elif p.get("IsTTP"):
col = " "
else:
coords = "crate %2d slot %2d%1s" % (p.get("Crate", -1), p.get("Slot", -1), p.get("Top", "x"))
printer.warning("unpacking did not succeed enough to print more about %s" % coords)
return
out = []
if printColumnHeaders:
columns = ["iWord16",
" EvN",
" OrN5",
" BcN",
"Cr",
"Sl",
" Fl",
"FrmtV",
"nPre",
#"nWordTP",
col,
"nSamp",
"EvN8",
" CRC" + ("32" if utca else ""),
]
if zs:
columns += [" ", "ZSMask: Thr1, Thr24, ThrTP"]
out.append(" ".join(columns))
strings = [" %05d" % p["0Word16"],
" 0x%07x" % p["EvN"],
"0x%02x" % p["OrN5"],
"%4d" % p["BcN"],
"%2d" % p["Crate"],
"%2d%1s" % (p["Slot"], p["Top"]),
"%2d" % p.get("FWFlavor", -1), # absent in uHTR
" 0x%01x" % p["FormatVer"],
" %2d" % p["nPreSamples"],
#" %3d " % p["nWord16Tp"],
]
if utca:
s = " %4d/%4d" % (p.get(col, -1), p.get(col+"T", -1))
strings.append(s)
else:
strings.append(" %3d " % p.get(col, -1))
strings += [" %2d" % p.get("nSamples", -1), # absent in uHTR
" 0x%02x" % p["EvN8"],
]
if utca:
strings.append("0x%08x" % p["CRC"])
elif "Qie" in col:
strings.append("0x%04x" % p["CRC"])
else:
strings.append(" TTP ")
if zs:
strings += ["",
"0x%04x" % zs["Mask"],
" %3d" % zs["Threshold1"],
" %3d" % zs["Threshold24"],
" %3d" % zs["ThresholdTP"],
]
out.append(" ".join(strings))
printer.green("\n".join(out))
anyHtrDataPrinted = False
if 4 <= dump:
kargs = {"skipFibChs": [0, 2, 3, 4, 5, 6, 7] if (4 <= dump <= 6) else [],
"nonMatched": nonMatchedQie,
"latency": p.get("Latency"),
"zs": p.get("ZS"),
"skipErrF": [3],
}
if dump == 6 or 8 <= dump:
kargs["skipErrF"] = []
if p["IsTTP"]:
cd = ttpData(p["ttpInput"], p["ttpOutput"], p["ttpAlgoDep"])
else:
if crateslots and (100*p["Crate"] + p["Slot"]) not in crateslots:
return
cd = htrChannelData(p["channelData"].values(),
crate=p["Crate"],
slot=p["Slot"],
top=p["Top"],
**kargs)
if len(cd) >= 2:
printer.yellow(cd[0])
printer.msg("\n".join(cd[1:]))
anyHtrDataPrinted = True
if 5 <= dump:
kargs = {"skipZeroTps": dump <= 7,
"crate": p["Crate"],
"slot": p["Slot"],
"top": p["Top"],
"nonMatched": nonMatchedTp,
}
if utca:
td = uhtrTriggerData(p["triggerData"], **kargs)
else:
td = htrTriggerData(p["triggerData"], zs=zs, **kargs)
if len(td) >= 2:
printer.yellow(td[0])
printer.msg("\n".join(td[1:]))
anyHtrDataPrinted = True
return anyHtrDataPrinted
def testexample(raw1={},
raw2={},
book=None,
warnQuality=True,
fewerHistos=False,
**_):
# sanity check for incoming raw data
for i, raw in enumerate([raw1, raw2]):
if not raw:
continue
# find the number of time samples in the data
nTsMax = raw[None]["firstNTs"]
for fedId, d in sorted(raw.iteritems()):
if fedId != INSPECT_FEDID:
continue
# get the important chunks of the raw data
blocks = d["htrBlocks"].values()
header = d["header"]
evt = header["EvN"]
shunt = (evt - 1) // SHUNT_EVENT_UNIT
print "EVENT", evt
# sanity checks for these chunks
for block in blocks:
if type(block) is not dict:
printer.warning("FED %d block is not dict" % fedId)
continue
elif "channelData" not in block:
printer.warning("FED %d block has no channelData" % fedId)
continue
# pick a channel >> L48
if block["Crate"] != CHANNEL["crate"] or block[
"Slot"] != CHANNEL["slot"]:
continue
for channelData in block["channelData"].values(
): # channel loop
# pick a channel >> L43
if channelData["Fiber"] != CHANNEL["fiber"] or channelData[
"FibCh"] != CHANNEL["fibCh"]:
continue
# print block["Crate"], block["Slot"], channelData["Fiber"], channelData["FibCh"]
if channelData["QIE"]:
# check the error flags
errf = "ErrFNZ" if channelData["ErrF"] else "ErrF0"
# some logic for naming two histograms, one for a clean error flag and another with a problematic error flag
eq = "!=" if channelData["ErrF"] else "=="
nAdcMax = 256
for (ts, adc) in enumerate(
channelData["QIE"]
): # TS loop : each channel is consist of 10TS
if nTsMax <= ts:
break
# fill a 2d histogram where bins on y-axis are ADCs and bins on x-axis are time slice index
book.fill(
(ts, adc),
"ADC_vs_TS_%s_%d_%d_%d_%d_%d" %
(errf, fedId, block["Crate"], block["Slot"],
channelData["Fiber"], channelData["FibCh"]),
(nTsMax, nAdcMax), (-0.5, -0.5),
(nTsMax - 0.5, nAdcMax - 0.5),
title=
"FED %d (ErrF %s 0);time slice;ADC;Counts / bin"
% (fedId, eq))
if ts is 3: # use max time slice
# fill a 2d histogram where bins on y-axis are ADCs and bins on x-axis are shunt setting
book.fill(
(shunt, adc),
"ADC_vs_Shunt_%s_%d_%d_%d_%d_%d" %
(errf, fedId, block["Crate"],
block["Slot"], channelData["Fiber"],
channelData["FibCh"]),
(MAX_SHUNT, nAdcMax), (0, -0.5),
(MAX_SHUNT, nAdcMax - 0.5),
title=
"FED %d (ErrF %s 0);shunt setting;ADC;Counts / bin"
% (fedId, eq))
# pedestal subtraction : for each event, the pedestal was estimated by the average of the first two time samples.
charge = 0.0
ADC_pedsub = 0.0
pedCharge = 0.0
pedADC = 0.0
rms_pedADC = 0.0
rms_pedCharge = 0.0
time_slice = {
"ped_ts": [0, 1],
"sum_ts": [3]
} #,4,5,6]} # The total charge collected for each channel per event, q, was q = (TS3+TS4+TS5+TS6)-{(TS0+TS1)/2}
ped_tsNum = len(time_slice["ped_ts"])
sum_tsNum = len(time_slice["sum_ts"])
if shunt in SELECTED_SHUNT:
for (ts, adc) in enumerate(channelData["QIE"]):
if ts in time_slice["ped_ts"]:
pedADC += adc
pedCharge += float(ADCtoCharge[shunt][adc])
if ts is time_slice["ped_ts"][-1]:
pedADC /= ped_tsNum
pedCharge /= ped_tsNum
rms_pedADC = ((pedADC -
PED_ADC_MEAN)**2)**0.5
rms_pedCharge = (
(pedCharge -
PED_CHARGE_MEAN)**2)**0.5
if ts in time_slice["sum_ts"]:
ADC_pedsub += adc
charge += float(ADCtoCharge[shunt][adc])
if ts is time_slice["sum_ts"][-1]:
ADC_pedsub -= pedADC
charge -= pedCharge
# fill a 2d histogram where bins on y-axis are charges and bins on x-axis are GSel ; pedestal was subtracted
book.fill(
(shunt, ADC_pedsub),
"ADC_vs_GSel_%s_%d_%d_%d_%d_%d" %
(errf, fedId, block["Crate"], block["Slot"],
channelData["Fiber"], channelData["FibCh"]),
(MAX_SHUNT, nAdcMax), (0, -0.5),
(MAX_SHUNT, nAdcMax - 0.5),
title="FED %d (ErrF %s 0);GSel;ADC;Counts / bin"
% (fedId, eq))
book.fill(
(shunt, charge),
"Charge_vs_GSel_%s_%d_%d_%d_%d_%d" %
(errf, fedId, block["Crate"], block["Slot"],
channelData["Fiber"], channelData["FibCh"]),
(MAX_SHUNT, 200), (0, 70000),
(MAX_SHUNT, 100000),
title=
"FED %d (ErrF %s 0);GSel;Charge[fC];Counts / bin"
% (fedId, eq))
if shunt == 0:
# fill mean fedestal
book.fill(
pedADC,
"ave_pedADC_%s_%d" % (errf, fedId),
9,
3,
12,
title=
"FED %d (ErrF %s 0);pedCharge;Counts / bin"
% (fedId, eq))
book.fill(
pedCharge,
"ave_pedCharge_%s_%d" % (errf, fedId),
200,
10,
30,
title=
"FED %d (ErrF %s 0);pedCharge;Counts / bin"
% (fedId, eq))
book.fill(
rms_pedADC,
"ave_RMS_pedADC_%s_%d" % (errf, fedId),
20,
0,
1,
title=
"FED %d (ErrF %s 0);pedCharge;Counts / bin"
% (fedId, eq))
print rms_pedCharge
book.fill(
rms_pedCharge,
"ave_RMS_pedCharge_%s_%d" % (errf, fedId),
2500,
0,
25,
title=
"FED %d (ErrF %s 0);pedCharge;Counts / bin"
% (fedId, eq))
def oneHtr(iBlock=None, p={}, dump=None, utca=None,
nonMatchedQie=[], nonMatchedTp=[],
nTsMax=None, perTs=None):
try:
zs = p.get("ZS")
except TypeError as e:
print("iBlock='%s':" % str(iBlock), e)
return
if "nWord16Qie" in p:
col = "nWord16Qie"
elif utca:
col = "DataLength16"
elif p.get("IsTTP"):
col = " "
else:
coords = "crate %2d slot %2d%1s" % (p.get("Crate", -1), p.get("Slot", -1), p.get("Top", "x"))
printer.warning("unpacking did not succeed enough to print more about %s" % coords)
return
columns = ["iWord16",
" EvN",
" OrN5",
" BcN",
"Cr",
"Sl",
" Fl",
"FrmtV",
"nPre",
col,
]
if zs:
columns += ["ZSMask: Thr1, Thr24, ThrTP"]
strings = [" %05d" % p["0Word16"],
" 0x%07x" % p["EvN"],
"0x%02x" % p["OrN5"],
"%4d" % p["BcN"],
"%2d" % p["Crate"],
"%2d%1s" % (p["Slot"], p["Top"]),
"%2d" % p.get("FWFlavor", -1), # absent in uHTR
" 0x%01x" % p["PayloadFormat"],
" %2d" % p["nPreSamples"],
]
if utca:
s = " %4d/%4d" % (p.get(col, -1), p.get(col+"T", -1))
strings.append(s)
else:
strings.append(" %3d " % p.get(col, -1))
if p["IsTTP"]:
strings.append(" TTP ")
elif p["IsIO"]:
strings.append(" uMNio ")
if zs:
strings += [" 0x%04x" % zs["Mask"],
" %3d" % zs["Threshold1"],
" %3d" % zs["Threshold24"],
" %3d" % zs["ThresholdTP"],
]
out = []
if (not iBlock) or 4 <= dump:
out.append(" ".join(columns))
if dump != 4 and dump != 10:
out.append(" ".join(strings))
printer.green("\n".join(out))
if dump <= 3:
return
kargs = {"skipFibers": [0, 1] + list(range(3, 14)) + list(range(15, 24)) if (dump == 4) else [],
"skipFibChs": [0, 2, 3, 4, 5, 6, 7] if (4 <= dump <= 7) else [],
"nonMatched": nonMatchedQie,
"latency": p.get("Latency"),
"zs": p.get("ZS"),
"nTsMax": nTsMax,
"perTs": perTs,
}
if dump in [5, 6, 8]:
kargs["errorsReq"] = False
if dump == 10:
kargs["errorsReq"] = True
if p["IsTTP"]:
cd = ttpData(p["ttpInput"], p["ttpOutput"], p["ttpAlgoDep"])
if p["IsIO"]:
cd = ioData(p)
else:
cd = htrChannelData(p["channelData"].values(),
crate=p["Crate"],
slot=p["Slot"],
top=p["Top"],
**kargs)
if len(cd) >= 2:
printer.yellow(cd[0])
printer.msg("\n".join(cd[1:]))
if 6 <= dump:
kargs = {"crate": p["Crate"],
"slot": p["Slot"],
"top": p["Top"],
"nonMatched": nonMatchedTp,
"dump": dump,
}
if utca:
td = uhtrTriggerData(p["triggerData"], **kargs)
else:
td = htrTriggerData(p["triggerData"], zs=zs, **kargs)
if len(td) >= 2:
printer.yellow(td[0])
printer.msg("\n".join(td[1:]))
def histogramChannelData(book, block, channelData, fedId, caps, ErrF, adcs,
crate2bin, slotCrate, misMatchMapBins, xAxisLabels,
yAxisLabels, fedTime, nTsMax, warn, **other):
nAdcMatch = 0
nAdcMisMatch = 0
ErrF[channelData["ErrF"]] += 1
book.fill(len(channelData["QIE"]),
"nQieSamples_%d" % fedId,
14,
-0.5,
13.5,
title="FED %d;number of QIE samples;Channels / bin" % fedId)
book.fill(slotCrate,
"ErrFAny_vs_slot_crate",
*misMatchMapBins,
title="any;slot;crate;Channels / bin",
xAxisLabels=xAxisLabels,
yAxisLabels=yAxisLabels)
if channelData["ErrF"]:
for name, title in [
("ErrFNZ", "ErrF != 0"),
("ErrF%d" % channelData["ErrF"],
"ErrF == %d" % channelData["ErrF"]),
]:
book.fill(slotCrate,
"%s_vs_slot_crate" % name,
*misMatchMapBins,
title="%s;slot;crate;Channels / bin" % title,
xAxisLabels=xAxisLabels,
yAxisLabels=yAxisLabels)
return nAdcMatch, nAdcMisMatch
book.fill(slotCrate,
"ErrF0_vs_slot_crate",
*misMatchMapBins,
title="ErrF == 0;slot;crate;Channels / bin",
xAxisLabels=xAxisLabels,
yAxisLabels=yAxisLabels)
book.fill((block["Slot"], channelData["Fiber"]),
"fiber_vs_slot_%d" % fedId, (12, 24), (0.5, -0.5), (12.5, 23.5),
title="FED %d;Slot;Fiber;Channels (ErrF == 0) / bin" % fedId)
coords = (block["Crate"], block["Slot"], block["Top"],
channelData["Fiber"], channelData["FibCh"])
if coords in other.get("misMatched", []):
nAdcMisMatch = 1
crate2, slot2, top2 = hw.transformed_qie(*coords)[:3]
for t in [slotCrate, (slot2, crate2bin[crate2])]:
book.fill(t,
"ADC_mismatch_vs_slot_crate",
*misMatchMapBins,
title="ADC mismatch;slot;crate;Channels / bin",
xAxisLabels=xAxisLabels,
yAxisLabels=yAxisLabels)
elif coords in other.get("matched", []):
nAdcMatch = 1
caps[channelData["CapId"][0]] += 1
try:
tsSoi = channelData["SOI"].index(1)
except ValueError:
if warn:
printer.warning("%2d:%2d:%2d:%d SoI not found" %
(block["Crate"], block["Slot"],
channelData["Fiber"], channelData["FibCh"]))
tsSoi = None
if channelData["QIE"]:
errf = "ErrFNZ" if channelData["ErrF"] else "ErrF0"
eq = "!=" if channelData["ErrF"] else "=="
histogramAdcs(book, fedId, block, channelData, adcs, nTsMax, errf, eq)
histogramCaps(book, fedId, block, channelData["CapId"], tsSoi)
if fedTime:
histogramTsVsTime(book, fedTime, fedId, channelData["QIE"])
if channelData.get("TDC"):
histogramTdcs(book, fedId, block, channelData["TDC"], nTsMax, errf, eq,
tsSoi)
return nAdcMatch, nAdcMisMatch
def htrSummary(blocks=[],
book=None,
fedId=None,
fedEvn=None,
fedOrn5=None,
fedBcn=None,
msg="",
warn=True,
fewerHistos=False,
fedTime=None,
nTsMax=None,
**other):
nBadHtrs = 0
caps = {}
ErrF = {}
nEvnMatch = 0
nEvnMisMatch = 0
nAdcMatch = 0
nAdcMisMatch = 0
nTpMatch = 0
nTpMisMatch = 0
adcs = set()
for i in range(4):
caps[i] = 0
ErrF[i] = 0
crate2bin = collections.defaultdict(int)
yAxisLabels = ["?"]
for iCrate, crate in enumerate(sw.crateList()):
crate2bin[crate] = 1 + iCrate
yAxisLabels.append("%2d" % crate)
nSlots = 21
xAxisLabels = ["%d" % s for s in range(1, 1 + nSlots)]
x0 = 0.5
y0 = -0.5
nCrates = len(yAxisLabels)
misMatchMapBins = ((nSlots, nCrates), (x0, y0), (x0 + nSlots,
y0 + nCrates))
for block in blocks:
if type(block) is not dict:
printer.warning("%s block is not dict" % msg)
nBadHtrs += 1
continue
elif "channelData" not in block:
printer.warning("%s block has no channelData" % msg)
nBadHtrs += 1
continue
if block["EvN"] == fedEvn:
nEvnMatch += 1
else:
nEvnMisMatch += 1
if warn:
printer.warning("%s / crate %2d slot %2d%1s has EvN 0x%06x" %
(msg, block["Crate"], block["Slot"],
block["Top"], block["EvN"]))
histogramBlock(book, block, fedId, fedEvn, fedOrn5, fedBcn)
slotCrate = (block["Slot"], crate2bin[block["Crate"]])
histogramBlock2(book, block, fedEvn, fedOrn5, fedBcn, slotCrate,
misMatchMapBins, xAxisLabels, yAxisLabels)
nTpTowerBins = 50
book.fill(len(block["triggerData"]),
"nTpTowers_%d" % fedId,
nTpTowerBins,
-0.5,
nTpTowerBins - 0.5,
title="FED %d;number of TP towers;HTRs / bin" % fedId)
nChannelBins = 75
book.fill(len(block["channelData"]),
"nChannels_%d" % fedId,
nChannelBins,
-0.5,
2 * nChannelBins - 0.5,
title="FED %d;number of channels;HTRs / bin" % fedId)
if fewerHistos:
continue
for otherData in block["otherData"].values():
flavor(book, otherData, fedId)
for techData in block["technicalData"].values():
# remove uHTR pad words from flavor histogram
if techData["technicalDataType"] or techData[
"channelId"] or techData["words"]:
flavor(book, techData, fedId)
for triggerKey, triggerData in block["triggerData"].iteritems():
if "Flavor" in triggerData:
flavor(book, triggerData, fedId)
tpMatch = histogramTriggerData(
book,
block,
triggerData,
triggerKey,
fedId,
crate2bin,
slotCrate,
misMatchMapBins,
xAxisLabels,
yAxisLabels,
tpMatches=other.get("tMatched", []), # tpMismatches,
)
if tpMatch:
nTpMatch += 1
else:
nTpMisMatch += 1
for channelData in block["channelData"].values():
flavor(book, channelData, fedId)
a, b = histogramChannelData(book, block, channelData, fedId, caps,
ErrF, adcs, crate2bin, slotCrate,
misMatchMapBins, xAxisLabels,
yAxisLabels, fedTime, nTsMax, warn,
**other)
nAdcMatch += a
nAdcMisMatch += b
return [
nBadHtrs,
ErrF,
caps,
adcs,
matchFrac(nEvnMatch, nEvnMisMatch),
matchFrac(nAdcMatch, nAdcMisMatch),
matchFrac(nTpMatch, nTpMisMatch),
]
def main(options, args, quiet=False):
if len(args) != 1:
sys.exit("Please provide a run number as the argument.")
try:
run = int(args[0])
except ValueError:
sys.exit("Could not convert %s to int." % args[0])
subdet = "HBHE" if options.hbhe else "HF"
options.feds1 = subdet
options.feds2 = "u%s" % subdet
options.outputFile = "output/%d_%s.root" % (run, subdet)
override(options, quiet, run)
if options.noLoop: # skip file finding
options.file1 = "dummy"
oneRun.main(options)
return
paths = {1: None, 2: None}
if run < 246908: # 2015-06-03 first 13 TeV stable beams
paths.update({3: "/store/data/Commissioning2015/HcalNZS/RAW/v1",
4: "/store/express/Commissioning2015/ExpressPhysics/FEVT/Express-v1",
5: "/store/express/Commissioning2015/ExpressCosmics/FEVT/Express-v1",
6: "/store/data/Commissioning2015/Cosmics/RAW/v1",
7: "/store/data/Commissioning2015/MinimumBias/RAW/v1",
})
elif run < 250985:
paths.update({# 3: "/store/data/Run2015A/HcalNZS/RAW/v1",
4: "/store/express/Run2015A/ExpressPhysics/FEVT/Express-v1/",
})
else:
paths.update({# 3: "/store/data/Run2015B/HcalNZS/RAW/v1",
4: "/store/express/Run2015B/ExpressPhysics/FEVT/Express-v1/",
})
for iFind, grDir in sorted(paths.iteritems()):
if grDir is None:
options.file1 = eval("find%d" % iFind)(run)
else:
options.file1 = find_gr(run, grDir, options.hhmm, quiet)
if not options.file1:
continue
fileNames = options.file1.split(",")
report(fileNames, iFind)
n = len(fileNames)
if 2 <= n and options.hhmm is None:
options.sparseLoop = max(1, options.nEvents / n)
else:
options.sparseLoop = -1
options.file2 = "" # clear from previous iterations
if oneRun.main(options):
continue
else:
return
printer.warning("Did not find a matching file for run %d. Perhaps try 'source env/lxplus6.sh'" % run)
return 1
def pulseRun(raw1={},
raw2={},
book=None,
warnQuality=True,
fewerHistos=False,
**other):
# sanity check
for r, raw in enumerate([raw1, raw2]):
if not raw:
continue
nTsMax = raw[None]["firstNTs"]
#print "nTsMax = ", nTsMax
for fedId, dct in sorted(raw.iteritems()):
if fedId is None:
continue
h = dct["header"]
# Event number
evt = h["EvN"]
# get the important chunks of raw data
blocks = dct["htrBlocks"].values()
#pprint(blocks)
# sanity checks for chunks
#for block in blocks:
for i, block in enumerate(blocks):
if type(block) is not dict:
printer.warning("FED %d block is not dict" % fedId)
continue
elif "channelData" not in block:
printer.warning("FED %d block has no channelData" % fedId)
continue
crate = block["Crate"]
slot = block["Slot"]
#with open("block%d.log"%i, "a+") as f:
# pprint(block, stream=f)
for channelData in block["channelData"].values():
#pprint(channelData)
#print "Fiber %d Ch %d errf = %s"%(channelData["Fiber"], channelData["FibCh"], channelData["ErrF"])
if channelData["QIE"]:
# check error flags
errf = "ErrFNZ" if channelData["ErrF"] else "ErrF0"
# Clean or problematic error flag
eq = "!=" if channelData["ErrF"] else "=="
nAdcMax = 256
# i: time slice
for (i, adc) in enumerate(channelData["QIE"]):
if nTsMax <= i:
break
fib = channelData["Fiber"]
fibCh = channelData["FibCh"]
# Default shunt setting
shunt = 0
book.fill(
(i, adc),
"ADC_vs_TS_2D_%s_%d" % (errf, fedId),
(nTsMax, nAdcMax), (-0.5, -0.5),
(nTsMax - 0.5, nAdcMax - 0.5),
title=
"ADC vs TS FED %d Fib %d Ch %d (ErrF %s 0);time slice;ADC;Counts / bin"
% (fedId, fib, fibCh, eq))
"""
book.fill((i, adc), "ADC_vs_TS_%s_%d_" % (errf, fedId),
nTsMax, -0.5, nTsMax - 0.5,
title="ADC vs TS FED %d (ErrF %s 0) Fib;time slice;ADC;Counts / bin" % (fedId, eq))
book.fill((i, adc),
"ADC_vs_TS_%s_FED_%d_Fib_%d_Ch_%d" % (errf, fedId, fib, fibCh),
nTsMax, -0.5, nTsMax - 0.5,
title="ADC vs TS FED %d Fib %d Ch %d (ErrF %s 0);time slice;ADC;Counts / bin" % (fedId, fib, fibCh, eq))
"""
book.fill(
(i, adc),
"ADC_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d_2D"
% (errf, fedId, crate, slot, fib, fibCh),
(nTsMax, nAdcMax), (-0.5, -0.5),
(nTsMax - 0.5, nAdcMax - 0.5),
title=
"ADC vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;ADC;Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
charge = float(adcCharges[adc])
# Linearized adc (charge vs TS)
book.fill(
(i, charge),
"Charge_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d"
% (errf, fedId, crate, slot, fib, fibCh),
nTsMax,
-0.5,
nTsMax - 0.5,
title=
"Charge vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;Charge [fC];Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
book.fill(
(i, charge),
"Charge_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d_2D"
% (errf, fedId, crate, slot, fib, fibCh),
(nTsMax, 100), (-0.5, -0.5),
(nTsMax - 0.5, 20000),
title=
"Charge vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;Charge [fC];Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
if i > 0:
book.fill(
(i, adc),
"NoTS0_ADC_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d_2D"
% (errf, fedId, crate, slot, fib, fibCh),
(nTsMax, nAdcMax), (-0.5, -0.5),
(nTsMax - 0.5, nAdcMax - 0.5),
title=
"ADC vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;ADC;Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
book.fill(
(i, charge),
"NoTS0_Charge_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d"
% (errf, fedId, crate, slot, fib, fibCh),
nTsMax,
-0.5,
nTsMax - 0.5,
title=
"Charge vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;Charge [fC];Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
book.fill(
(i, charge),
"NoTS0_Charge_vs_TS_%s_FED_%d_Crate_%d_Slot_%d_Fib_%d_Ch_%d_2D"
% (errf, fedId, crate, slot, fib, fibCh),
(nTsMax, 100), (-0.5, -0.5),
(nTsMax - 0.5, 20000),
title=
"Charge vs TS FED %d Crate %d Slot %d Fib %d Ch %d (ErrF %s 0);time slice;Charge [fC];Counts / bin"
% (fedId, crate, slot, fib, fibCh, eq))
def histo_dy(raw1={},
raw2={},
book=None,
warnQuality=True,
fewerHistos=False,
**_):
# sanity check for incoming raw data
for i, raw in enumerate([raw1, raw2]):
if not raw:
continue
# find the number of time samples in the data
nTsMax = raw[None]["firstNTs"]
for fedId, d in sorted(raw.iteritems()): # event loop
if fedId != INSPECT_FEDID:
continue
# get the important chunks of the raw data
blocks = d["htrBlocks"].values()
header = d["header"]
evt = header["EvN"]
shunt = (evt - 1) // SHUNT_EVENT_UNIT
if evt % SHUNT_EVENT_UNIT == 0:
print "EVENT", evt
# sanity checks for these chunks
for block in blocks:
if type(block) is not dict:
printer.warning("FED %d block is not dict" % fedId)
continue
elif "channelData" not in block:
printer.warning("FED %d block has no channelData" % fedId)
continue
# pick a channel >> L48
# if block["Crate"] != CHANNEL["crate"] or block["Slot"] != CHANNEL["slot"]:
# continue
for channelData in block["channelData"].values(
): # channel loop
# pick a channel >> L43
#if channelData["Fiber"] != CHANNEL["fiber"] or channelData["FibCh"] != CHANNEL["fibCh"]:
# continue
# channel information in string form
str_ch = "%d_%d_%d_%d" % (block["Crate"], block["Slot"],
channelData["Fiber"],
channelData["FibCh"])
if channelData["QIE"]:
# check the error flags
errf = "ErrFNZ" if channelData["ErrF"] else "ErrF0"
# some logic for naming two histograms, one for a clean error flag and another with a problematic error flag
eq = "!=" if channelData["ErrF"] else "=="
# delet this if you want to make plots of channel with error
if errf == "ErrFNZ":
continue
nAdcMax = 256
ts_max = 3 # you can find this time slice by seeing pulse shape plot whici is ADC vs TS (or charge vs TS)
for (ts, adc) in enumerate(
channelData["QIE"]
): # TS loop : each channel is consist of 10TS
if nTsMax <= ts:
break
# fill a 2d histogram where bins on y-axis are ADCs and bins on x-axis are time slice index
book.fill(
(ts, adc),
"ADC_vs_TS_%s_%d_%s" % (errf, fedId, str_ch),
#(nTsMax, nAdcMax), (-0.5, -0.5), (nTsMax - 0.5, nAdcMax - 0.5), # active this to make 2D histogram
nTsMax,
-0.5,
nTsMax - 0.5, # active this to make TProfile
title=
"FED %d_%s (ErrF %s 0);time slice;ADC;Counts / bin"
% (fedId, str_ch, eq))
# fill a TProfile where bins on y-axis are Charge
if shunt in SELECTED_SHUNT:
charge = float(ADCtoCharge[shunt][adc])
book.fill(
(ts, charge),
"Charge_vs_TS_%s_%d_%s" %
(errf, fedId, str_ch),
nTsMax,
-0.5,
nTsMax - 0.5,
title=
"FED %d_%s (ErrF %s 0);time slice;Charge[fC];Counts / bin"
% (fedId, str_ch, eq))
if ts is ts_max: # use max time slice
# fill a 2d histogram where bins on y-axis are ADCs and bins on x-axis are shunt setting
book.fill(
(shunt, adc),
"ADC_vs_Shunt_%s_%d_%s" %
(errf, fedId, str_ch),
(MAX_SHUNT, nAdcMax), (0, -0.5),
(MAX_SHUNT, nAdcMax - 0.5),
title=
"FED %d_%s (ErrF %s 0);shunt setting;ADC;Counts / bin"
% (fedId, str_ch, eq))
# pedestal subtraction : for each event, the pedestal was estimated by the average of the first two time samples.
charge_pedsub = 0.0
ADC_pedsub = 0.0
pedCharge = 0.0
pedADC = 0.0
time_slice = {
"ped_ts": [1],
"sum_ts": [3]
} #,4,5,6]} # The total charge collected for each channel per event, q, was q = (TS3+TS4+TS5+TS6)-{(TS0+TS1)/2}
ped_tsNum = len(time_slice["ped_ts"])
sum_tsNum = len(time_slice["sum_ts"])
if shunt in SELECTED_SHUNT and errf == "ErrF0":
for (ts, adc) in enumerate(channelData["QIE"]):
if ts in time_slice["ped_ts"]:
pedADC += adc
pedCharge += float(ADCtoCharge[shunt][adc])
if ts is time_slice["ped_ts"][-1]:
pedADC /= ped_tsNum
pedCharge /= ped_tsNum
if ts in time_slice["sum_ts"]:
ADC_pedsub += adc
charge_pedsub += float(
ADCtoCharge[shunt][adc])
if ts is time_slice["sum_ts"][-1]:
ADC_pedsub -= pedADC
charge_pedsub -= pedCharge
# accumulate mean value of pedestal for each channel, ( ADC(or charge)sum of corresponding TS / #TS )
if not (str_ch in PED):
PED[str_ch] = []
PED[str_ch].append(pedADC)
# fill 2D histograms where bins on y-axis are charges and bins on x-axis are GSel ; pedestal was subtracted
book.fill(
(shunt, ADC_pedsub),
"ADC_vs_GSel_%s_%d_%s" % (errf, fedId, str_ch),
# (MAX_SHUNT, nAdcMax), (0, -0.5), (MAX_SHUNT, nAdcMax-0.5), # active this to make 2D histogram
MAX_SHUNT,
0,
MAX_SHUNT, # active this to make TProfile
title=
"FED %d_%s (ErrF %s 0);GSel;ADC;Counts / bin" %
(fedId, str_ch, eq))
book.fill(
(shunt, charge_pedsub),
"Charge_vs_GSel_%s_%d_%s" %
(errf, fedId, str_ch),
# (MAX_SHUNT, 200), (0, 20000), (MAX_SHUNT, 100000), # active this to make 2D histogram
MAX_SHUNT,
0,
MAX_SHUNT, # active this to make TProfile
title=
"FED %d_%s (ErrF %s 0);GSel;Charge[fC];Counts / bin"
% (fedId, str_ch, eq))
print "# of channel", len(PED)
# pedestal mean and RMS
if evt == SHUNT_EVENT_UNIT:
for ch, adclist in PED.items():
ped_mean = sum(adclist) / len(adclist)
vsum = 0
for x in adclist:
vsum = vsum + (x - ped_mean)**2
var = vsum / len(adclist)
ped_rms = math.sqrt(var)
# fill 1D histograms
book.fill(
ped_mean,
"mean_pedADC_%s_%d" % (errf, fedId),
50,
0,
10,
title="FED %d (ErrF %s 0);pedestal mean;Counts / bin" %
(fedId, eq))
book.fill(
ped_rms,
"RMS_pedADC_%s_%d" % (errf, fedId),
50,
0,
5,
title="FED %d (ErrF %s 0);pedestal RMS;Counts / bin" %
(fedId, eq))
def htrSummary(blocks=[], book=None, fedId=None,
fedEvn=None, fedOrn5=None, fedBcn=None,
msg="", warn=True, fedTime=None,
adcMatches=[], adcMismatches=[],
tpMatches=[], tpMismatches=[]):
nBadHtrs = 0
caps = {}
ErrF = {}
nEvnMatch = 0
nEvnMisMatch = 0
nAdcMatch = 0
nAdcMisMatch = 0
nTpMatch = 0
nTpMisMatch = 0
adcs = set()
for i in range(4):
caps[i] = 0
ErrF[i] = 0
crate2bin = {(37, " "): 1,
(36, " "): 2,
(35, " "): 3,
(34, " "): 4,
(32, " "): 5,
(31, " "): 6,
(30, " "): 7,
(29, " "): 8,
(25, " "): 9,
(24, " "): 10,
(22, " "): 11,
(21, " "): 12,
(20, " "): 13,
(17, "b"): 14,
(17, "t"): 15,
(15, "b"): 16,
(15, "t"): 17,
(14, "b"): 18,
(14, "t"): 19,
(11, "b"): 20,
(11, "t"): 21,
(10, "b"): 22,
(10, "t"): 23,
( 5, "b"): 24,
( 5, "t"): 25,
( 4, "b"): 26,
( 4, "t"): 27,
( 2, "b"): 28,
( 2, "t"): 29,
( 1, "b"): 30,
( 1, "t"): 31,
( 0, "b"): 32,
( 0, "t"): 33,
}
crateFail = 1 + max(crate2bin.values())
yAxisLabels = labels(crate2bin)
misMatchMapBins = ((23, crateFail), (-0.5, 0.5), (22.5, 0.5 + crateFail))
for block in blocks:
if type(block) is not dict:
printer.warning("%s block is not dict" % msg)
nBadHtrs += 1
continue
elif "channelData" not in block:
printer.warning("%s block has no channelData" % msg)
nBadHtrs += 1
continue
if block["EvN"] == fedEvn:
nEvnMatch += 1
else:
nEvnMisMatch += 1
if warn:
printer.warning("%s / crate %2d slot %2d%1s has EvN 0x%06x" % (msg, block["Crate"], block["Slot"], block["Top"], block["EvN"]))
evnMask = 0x7f
book.fill((fedEvn & evnMask, block["EvN"] & evnMask), "EvN_HTR_vs_FED_%d" % fedId,
(evnMask, evnMask), (-0.5, -0.5), (evnMask - 0.5, evnMask - 0.5),
title="FED %d;FED EvN & 0x%x;HTR EvN & 0x%x;HTRs / bin" % (fedId, evnMask, evnMask))
ornMask = 0x1f
book.fill((fedOrn5, block["OrN5"]), "OrN5_HTR_vs_FED_%d" % fedId,
(ornMask, ornMask), (-0.5, -0.5), (ornMask - 0.5, ornMask - 0.5),
title="FED %d;FED OrN5;HTR OrN5;HTRs / bin" % fedId)
book.fill(block["EvN"] - fedEvn, "EvN_HTRs_%d" % fedId,
11, -5.5, 5.5,
title="FED %d;HTR EvN - FED EvN;HTRs / bin" % fedId)
book.fill(block["OrN5"] - fedOrn5, "OrN5_HTRs_%d" % fedId,
11, -5.5, 5.5,
title="FED %d;HTR OrN5 - FED OrN5;HTRs / bin" % fedId)
book.fill(block["BcN"] - fedBcn, "BcN_HTRs_%d" % fedId,
11, -5.5, 5.5,
title="FED %d;HTR BcN - FED BcN;HTRs / bin" % fedId)
slotCrate = (slot2bin(block["Slot"]), crate2bin.get((block["Crate"], block["Top"]), crateFail))
book.fill(slotCrate,
"block_vs_slot_crate",
*misMatchMapBins,
title="any;slot;crate;HTR / bin",
yAxisLabels=yAxisLabels)
for key, fedVar in [("EvN", fedEvn),
("OrN5", fedOrn5),
("BcN", fedBcn),
]:
if (block[key] - fedVar):
book.fill(slotCrate,
"%s_mismatch_vs_slot_crate" % key,
*misMatchMapBins,
title="%s mismatch;slot;crate;HTR - FED mismatches / bin" % key,
yAxisLabels=yAxisLabels)
for otherData in block["otherData"].values():
flavor(book, otherData, fedId)
for techData in block["technicalData"].values():
# remove uHTR pad words from flavor histogram
if techData["technicalDataType"] or techData["channelId"] or techData["words"]:
flavor(book, techData, fedId)
for triggerKey, triggerData in block["triggerData"].iteritems():
if "Flavor" in triggerData:
flavor(book, triggerData, fedId)
book.fill(len(triggerData["TP"]), "nTpSamples_%d" % fedId, 14, -0.5, 13.5,
title="FED %d;number of TP samples;Towers / bin" % fedId)
maxTp = -1
for tp in triggerData["TP"]:
tp8 = tp & 0xff # ignore fine-grain bit
if maxTp < tp8:
maxTp = tp8
if 0 <= maxTp:
book.fill(maxTp, "channel_peak_tp_%d" % fedId, 14, -0.5, 13.5,
title="FED %d;Peak TP E;Towers / bin" % fedId)
tpCoords = (block["Crate"], block["Slot"], block["Top"], triggerKey)
tpCoords2 = hw.transformed_tp(*tpCoords)
if tpCoords2 is None:
continue
crate2, slot2, top2 = tpCoords2[:3]
slotsCrates = [slotCrate,
(slot2bin(slot2), crate2bin.get((crate2, top2), crateFail)),
]
for t in slotsCrates:
book.fill(t, "TP_matchable_vs_slot_crate", *misMatchMapBins,
title="TP matchable;slot;crate;Towers / bin",
yAxisLabels=yAxisLabels)
if tpCoords in tpMatches:
nTpMatch += 1
else:
nTpMisMatch += 1
for t in slotsCrates:
book.fill(t, "TP_mismatch_vs_slot_crate", *misMatchMapBins,
title="TP mismatch;slot;crate;Towers / bin",
yAxisLabels=yAxisLabels)
for channelData in block["channelData"].values():
flavor(book, channelData, fedId)
a, b = histogramChannelData(book, block, channelData, fedId,
caps, ErrF, adcs, adcMatches, adcMismatches,
slotCrate, misMatchMapBins, yAxisLabels,
fedTime)
nAdcMatch += a
nAdcMisMatch += b
return [nBadHtrs, ErrF, caps, adcs,
matchFrac(nEvnMatch, nEvnMisMatch),
matchFrac(nAdcMatch, nAdcMisMatch),
matchFrac(nTpMatch, nTpMisMatch),
]
def htrSummary(blocks=[], book=None, fedId=None,
fedEvn=None, fedOrn5=None, fedBcn=None,
msg="", warn=True, fewerHistos=False, fedTime=None,
**other):
nBadHtrs = 0
caps = {}
ErrF = {}
nEvnMatch = 0
nEvnMisMatch = 0
nAdcMatch = 0
nAdcMisMatch = 0
nTpMatch = 0
nTpMisMatch = 0
adcs = set()
for i in range(4):
caps[i] = 0
ErrF[i] = 0
crate2bin = {
# 3 904 uTCA crates
# 13 USC uTCA crates
(63, " "): 1, (62, " "): 2,
(61, " "): 3,
(37, " "): 4, (36, " "): 5,
(35, " "): 6, (34, " "): 7,
(32, " "): 8, (31, " "): 9,
(30, " "): 10, (29, " "): 11,
(25, " "): 12, (24, " "): 13,
(22, " "): 14, (21, " "): 15,
(20, " "): 16,
# 9 USC VME HBHE crates
# (17, "b"): 14, (17, "t"): 15,
# (15, "b"): 16, (15, "t"): 17,
# (14, "b"): 18, (14, "t"): 19,
# (11, "b"): 20, (11, "t"): 21,
# (10, "b"): 22, (10, "t"): 23,
# ( 5, "b"): 24, ( 5, "t"): 25,
# ( 4, "b"): 26, ( 4, "t"): 27,
# ( 1, "b"): 28, ( 1, "t"): 29,
# ( 0, "b"): 30, ( 0, "t"): 31,
}
crateFail = 1 + max(crate2bin.values())
yAxisLabels = labels(crate2bin)
misMatchMapBins = ((13, crateFail), (-0.5, 0.5), (12.5, 0.5 + crateFail))
for block in blocks:
if type(block) is not dict:
printer.warning("%s block is not dict" % msg)
nBadHtrs += 1
continue
elif "channelData" not in block:
printer.warning("%s block has no channelData" % msg)
nBadHtrs += 1
continue
if block["EvN"] == fedEvn:
nEvnMatch += 1
else:
nEvnMisMatch += 1
if warn:
printer.warning("%s / crate %2d slot %2d%1s has EvN 0x%06x" % (msg, block["Crate"], block["Slot"], block["Top"], block["EvN"]))
histogramBlock(book, block, fedId, fedEvn, fedOrn5, fedBcn)
slotCrate = (slot2bin(block["Slot"]), crate2bin.get((block["Crate"], block["Top"]), crateFail))
histogramBlock2(book, block, fedEvn, fedOrn5, fedBcn, slotCrate, misMatchMapBins, yAxisLabels)
nTpTowerBins = 50
book.fill(len(block["triggerData"]), "nTpTowers_%d" % fedId, nTpTowerBins, -0.5, nTpTowerBins - 0.5,
title="FED %d;number of TP towers;HTRs / bin" % fedId)
nChannelBins = 75
book.fill(len(block["channelData"]), "nChannels_%d" % fedId, nChannelBins, -0.5, 2*nChannelBins - 0.5,
title="FED %d;number of channels;HTRs / bin" % fedId)
if fewerHistos:
continue
for otherData in block["otherData"].values():
flavor(book, otherData, fedId)
for techData in block["technicalData"].values():
# remove uHTR pad words from flavor histogram
if techData["technicalDataType"] or techData["channelId"] or techData["words"]:
flavor(book, techData, fedId)
for triggerKey, triggerData in block["triggerData"].iteritems():
if "Flavor" in triggerData:
flavor(book, triggerData, fedId)
tpMatch = histogramTriggerData(book, block, triggerData, triggerKey, fedId,
crate2bin, crateFail, slotCrate, misMatchMapBins, yAxisLabels,
tpMatches=other.get("tMatched", []), # tpMismatches,
)
if tpMatch:
nTpMatch += 1
else:
nTpMisMatch += 1
for channelData in block["channelData"].values():
flavor(book, channelData, fedId)
a, b = histogramChannelData(book, block, channelData, fedId,
caps, ErrF, adcs, crate2bin, crateFail,
slotCrate, misMatchMapBins,
yAxisLabels, fedTime, **other)
nAdcMatch += a
nAdcMisMatch += b
return [nBadHtrs, ErrF, caps, adcs,
matchFrac(nEvnMatch, nEvnMisMatch),
matchFrac(nAdcMatch, nAdcMisMatch),
matchFrac(nTpMatch, nTpMisMatch),
]
def unpacked(fedData=None, nBytesPer=None, headerOnly=False,
warn=True, skipWords64=[], dump=-99, lastNAmcs=0):
assert fedData
assert nBytesPer in [1, 4, 8], "ERROR: invalid nBytes per index (%s)." % str(nBytesPer)
header = {"iWordPayload0": 6,
"utca": None,
} # modified by decode.header
trailer = {}
other = {}
htrBlocks = {}
nWord64Trailer = 1
nWord64 = fedData.size() * nBytesPer // 8
nWord16Skipped = 0
nToSkip = len(set(skipWords64))
skipped64 = []
for jWord64 in range(nWord64):
word64 = w64(fedData, jWord64, nBytesPer)
if jWord64 in skipWords64:
skipped64.append(word64)
continue
iWord64 = jWord64 - len(skipped64)
if 12 <= dump:
if not iWord64:
print("#iw64 w64")
print("%5d" % iWord64, "%016x" % word64)
if iWord64 < header["iWordPayload0"]:
decode.header(header, iWord64, word64, lastNAmcs)
if header.get("uFoV"):
nWord64Trailer = 2 # accommodate block trailer
iWordTrailer0 = nWord64 - nToSkip - nWord64Trailer
if iWord64 == 1 and not header["OrN"]:
if headerOnly:
break
else:
return unpacked_sw_fed(fedData, header, nBytesPer, dump)
elif headerOnly:
break
elif lastNAmcs and iWord64 < header["iWordPayloadn"]:
continue
elif iWord64 < iWordTrailer0:
for i in range(4):
word16 = (word64 >> (16*i)) & 0xffff
iWord16 = 4*iWord64+i
returnCode = decode.payload(htrBlocks,
iWord16=iWord16,
word16=word16,
word16Counts=header["word16Counts"],
utca=header["utca"],
fedId=header["FEDid"],
dump=dump)
if returnCode is None:
continue
# ignore VME pad words (zero)
if not header["utca"] and iWord64 + 1 == iWordTrailer0:
if 4 * header["iWordPayload0"] + sum(header["word16Counts"]) <= iWord16:
if not word16:
continue
nWord16Skipped += 1
if warn:
printer.warning(" ".join(["skipping",
"FED %d" % header["FEDid"],
"event %d" % header["EvN"],
"iWord16 %d" % iWord16,
"word16 0x%04x" % word16,
]))
else:
if "htrIndex" in htrBlocks:
del htrBlocks["htrIndex"] # fixme
if header["uFoV"] and (iWord64 == nWord64 - nToSkip - 2):
decode.block_trailer_ufov1(trailer, iWord64, word64)
else:
decode.trailer(trailer, iWord64, word64)
decode.other(other, skipped64)
return {"header": header,
"trailer": trailer,
"htrBlocks": htrBlocks,
"other": other,
"nBytesSW": 8*nWord64,
"nWord16Skipped": nWord16Skipped,
}
def htrSummary(blocks=[],
book=None,
fedId=None,
fedEvn=None,
fedOrn5=None,
fedBcn=None,
msg="",
warn=True,
fewerHistos=False,
fedTime=None,
adcMatches=[],
adcMismatches=[],
tpMatches=[],
tpMismatches=[]):
nBadHtrs = 0
caps = {}
ErrF = {}
nEvnMatch = 0
nEvnMisMatch = 0
nAdcMatch = 0
nAdcMisMatch = 0
nTpMatch = 0
nTpMisMatch = 0
adcs = set()
for i in range(4):
caps[i] = 0
ErrF[i] = 0
crate2bin = {
# 3 904 uTCA crates
# 13 USC uTCA crates
(63, " "): 1,
(62, " "): 2,
(61, " "): 3,
(37, " "): 4,
(36, " "): 5,
(35, " "): 6,
(34, " "): 7,
(32, " "): 8,
(31, " "): 9,
(30, " "): 10,
(29, " "): 11,
(25, " "): 12,
(24, " "): 13,
(22, " "): 14,
(21, " "): 15,
(20, " "): 16,
# 9 USC VME HBHE crates
# (17, "b"): 14, (17, "t"): 15,
# (15, "b"): 16, (15, "t"): 17,
# (14, "b"): 18, (14, "t"): 19,
# (11, "b"): 20, (11, "t"): 21,
# (10, "b"): 22, (10, "t"): 23,
# ( 5, "b"): 24, ( 5, "t"): 25,
# ( 4, "b"): 26, ( 4, "t"): 27,
# ( 1, "b"): 28, ( 1, "t"): 29,
# ( 0, "b"): 30, ( 0, "t"): 31,
}
crateFail = 1 + max(crate2bin.values())
yAxisLabels = labels(crate2bin)
misMatchMapBins = ((13, crateFail), (-0.5, 0.5), (12.5, 0.5 + crateFail))
for block in blocks:
if type(block) is not dict:
printer.warning("%s block is not dict" % msg)
nBadHtrs += 1
continue
elif "channelData" not in block:
printer.warning("%s block has no channelData" % msg)
nBadHtrs += 1
continue
if block["EvN"] == fedEvn:
nEvnMatch += 1
else:
nEvnMisMatch += 1
if warn:
printer.warning("%s / crate %2d slot %2d%1s has EvN 0x%06x" %
(msg, block["Crate"], block["Slot"],
block["Top"], block["EvN"]))
histogramBlock(book, block, fedId, fedEvn, fedOrn5, fedBcn)
slotCrate = (slot2bin(block["Slot"]),
crate2bin.get((block["Crate"], block["Top"]), crateFail))
histogramBlock2(book, block, fedEvn, fedOrn5, fedBcn, slotCrate,
misMatchMapBins, yAxisLabels)
nTpTowerBins = 50
book.fill(len(block["triggerData"]),
"nTpTowers_%d" % fedId,
nTpTowerBins,
-0.5,
nTpTowerBins - 0.5,
title="FED %d;number of TP towers;HTRs / bin" % fedId)
nChannelBins = 75
book.fill(len(block["channelData"]),
"nChannels_%d" % fedId,
nChannelBins,
-0.5,
2 * nChannelBins - 0.5,
title="FED %d;number of channels;HTRs / bin" % fedId)
if fewerHistos:
continue
for otherData in block["otherData"].values():
flavor(book, otherData, fedId)
for techData in block["technicalData"].values():
# remove uHTR pad words from flavor histogram
if techData["technicalDataType"] or techData[
"channelId"] or techData["words"]:
flavor(book, techData, fedId)
for triggerKey, triggerData in block["triggerData"].iteritems():
if "Flavor" in triggerData:
flavor(book, triggerData, fedId)
tpMatch = histogramTriggerData(
book,
block,
triggerData,
triggerKey,
fedId,
tpMatches, # tpMismatches,
crate2bin,
crateFail,
slotCrate,
misMatchMapBins,
yAxisLabels)
if tpMatch:
nTpMatch += 1
else:
nTpMisMatch += 1
for channelData in block["channelData"].values():
flavor(book, channelData, fedId)
a, b = histogramChannelData(book, block, channelData, fedId, caps,
ErrF, adcs, adcMatches, adcMismatches,
crate2bin, crateFail, slotCrate,
misMatchMapBins, yAxisLabels, fedTime)
nAdcMatch += a
nAdcMisMatch += b
return [
nBadHtrs,
ErrF,
caps,
adcs,
matchFrac(nEvnMatch, nEvnMisMatch),
matchFrac(nAdcMatch, nAdcMisMatch),
matchFrac(nTpMatch, nTpMisMatch),
]
